From efbb08262953edc35c0375747cceeac084fe2b0f Mon Sep 17 00:00:00 2001 From: Kira Date: Wed, 1 Jul 2026 13:20:04 +0900 Subject: [PATCH 01/45] =?UTF-8?q?chore(ci):=20post=20PR=20panel=20idempote?= =?UTF-8?q?ntly=20and=20on=20retarget=20=F0=9F=92=9A=20(#2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(ci): post PR panel idempotently and on retarget πŸ’š * fix(ci): correct git-third-party path in env.sh --- .github/workflows/branch_pr_checklist.yaml | 17 ++++++++++------- env.sh | 2 +- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/branch_pr_checklist.yaml b/.github/workflows/branch_pr_checklist.yaml index 33186878..8440487a 100644 --- a/.github/workflows/branch_pr_checklist.yaml +++ b/.github/workflows/branch_pr_checklist.yaml @@ -15,7 +15,9 @@ name: branch / post PR action panel on: pull_request_target: branches: ['v*-dev'] - types: [opened] + # `edited` fires when the PR base branch is changed, so a PR retargeted + # onto a dev branch also gets the panel. + types: [opened, edited] permissions: pull-requests: write @@ -33,14 +35,15 @@ jobs: PR: ${{ github.event.pull_request.number }} AUTHOR: ${{ github.event.pull_request.user.login }} steps: - - name: Ensure action labels exist - run: | - gh label create rtm -R "$GITHUB_REPOSITORY" --color 0e8a16 --description "ready to merge" 2>/dev/null || true - gh label create run-full-tests -R "$GITHUB_REPOSITORY" --color fbca04 --description "run full GenVM CI on every push" 2>/dev/null || true - gh label create ci-safe -R "$GITHUB_REPOSITORY" --color 5319e7 --description "PR cleared to run CI / panel actions" 2>/dev/null || true - - name: Post action panel run: | + # Only post once: skip if a panel (identified by the HTML marker) + # already exists on the PR. `edited` can fire repeatedly. + if gh pr view "$PR" --repo "$GITHUB_REPOSITORY" --json comments \ + --jq '.comments[].body' | grep -qF ''; then + echo "action panel already present; skipping" + exit 0 + fi gh pr comment "$PR" --repo "$GITHUB_REPOSITORY" --body "$(cat <<'EOF' ### GenVM PR actions diff --git a/env.sh b/env.sh index 45a49790..9396fb6f 100755 --- a/env.sh +++ b/env.sh @@ -2,7 +2,7 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -export PATH="$SCRIPT_DIR/tools/git-third-party:$PATH" +export PATH="$SCRIPT_DIR/support/tools/git-third-party:$PATH" if [ -f "$SCRIPT_DIR/.env" ] then From 9252dffb423e3cec06842391484c9a4bbf247a7c Mon Sep 17 00:00:00 2001 From: Kira Date: Mon, 13 Jul 2026 17:41:26 +0900 Subject: [PATCH 02/45] feat: add a legacy executor version (#1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(runners): add frozen v0.2.16 executor line ✨ Add executors/v0.2.x as a second genvm-executor submodule (branch v0.2.x) and activate it in .genvm-monorepo-root. Its runners are the prebuilt v0.2.16 universal tarball. compat.nix freezes the v0.2 line so its legacy-ABI runners serve only v0.2 executors and do not roll up into v0.3. * feat(genvm-tool): add codegen backends for modules-interfaces ✨ * chore(executor): bump v0.2.x gitlink for cargo fmt 🎨 * feat(genvm-tool): namespace executor feature branches as pr// ✨ * feat(ci): auto-open and link executor PRs, clean up branches on merge ✨ * fix(ci): fan executor PRs and merge over active lines, not the manager base πŸ› * feat(manager): add check-install to verify and precompile executor runners ✨ * fix(packaging): lay v0.2.x runners into executor//legacy-runners πŸ› The v0.2.x config points at executor//legacy-runners, but the packaging still pooled every line's runners into the shared top-level runners/ dir, so that folder was never populated. Split the runner list by line: forward-rolling lines keep the shared runners/ tree, while legacy (v0.2.x) runners are laid out at executor//legacy-runners/ and overlaid onto the combined `genvm` distribution and the dev out/ tree. * chore(executor): bump v0.3.x gitlink 🎨 * fix(genvm-tool): make hooks robust to the inherited environment πŸ› Two ways a hook inherited state from the invoking dev shell / parent repo and ran against the wrong thing: - A pre-commit / commit-msg hook runs with GIT_DIR / GIT_INDEX_FILE / ... set to the parent repo. `hook run` fans out to every repo (and its submodules) by explicit path, so those pinned each `git -C ` to the parent's index/objects and crashed them (`git diff --cached` -> exit 128, e.g. ".git/index: Not a directory" since a submodule .git is a gitlink file). Strip the repo-scoping GIT_* variables up front. - The dev shell exports PYTHONPATH pointing at its own genvm-tool build, so the hook's python imported genvm_tool from there instead of the freshly installed tool (running stale code). The stub now clears PYTHONPATH / NIX_PYTHONPATH so it always uses its own baked-in deps. * docs(skills): add submodule multi-repo workflow skill πŸ“ * docs(skills): revise and cross-link skills for the submodule dev workflow πŸ“ * fix(ci): pass ?submodules=1 to nix so executor lines resolve πŸ’š Nix filters the flake source to tracked files only, dropping submodule contents; the flake eval forces executors/.x/manifest.json, so every nix develop/build .# blew up during evaluation. Also check out submodules in module-test-python, which cds into the executor tree. * fix(manager): patch legacy "method" calldata key on run endpoint 🩹 Old-format ABI calldata stored the contract method name under the key "method"; the current ABI uses the empty key "". Detect old-format calldata at the run endpoint, log an error so stale callers are visible, and rewrite the key in place before handing it to the executor. * fix(nix): pin lief 0.17.6 so patch-rpath doesn't corrupt executor binaries πŸ› nixpkgs pins lief 0.17.0, whose ELF Builder counts NOBITS sections in its file-layout calc and emits a corrupt section-header table on our ~290 MB musl PIE binaries, breaking patchelf finalize. Override to 0.17.6, which skips NOBITS in layout. See lief-project/LIEF#1315. * chore(tests): type debug_mode as a literal enum in base host 🎨 * chore(ci): scope coderabbit to version branches, skip genvm-tool πŸ”§ * chore(ci): ignore root test fixtures in coderabbit, bump executor gitlinks πŸ”§ * fix(ci): call genvm-tool from PATH in rust test pipelines πŸ’š The rust and rust-fuzz pipeline scripts invoked a non-existent support/tools/genvm-tool/genvm-tool wrapper, failing with exit 127. These scripts run inside the nix rust-test dev shell, which already provides genvm-tool on PATH (packages-0), so call it bare. Claude-Session: https://claude.ai/code/session_01Sa2F8Kv4XdrPPNgnBhbWrS * fix(ci): make GCP auth optional in module-test-rust πŸ’š The Google Cloud auth step is only needed to upload freshly-built runners to the GCS cache, which is already best-effort (`|| true` in test-rust.sh); downloads use the public bucket and need no auth. On fork PRs (or when GCP_SA_KEY is unset) google-github-actions/auth errors out and fails the whole job. Gate the auth and Cloud SDK steps on the secret being present so tests still run without caching. Claude-Session: https://claude.ai/code/session_01Sa2F8Kv4XdrPPNgnBhbWrS * chore(tests): add hashes to v0.2 * chore(ci): rework release pipeline for the manager repo πŸš€ The workflow was copied from genlayerlabs/genvm and never ran here: the tag was semver-bumped in CI while GENVM_PROFILE comes from .genvm-monorepo-root, the unversioned executor targets no longer existed, --genvm-reroute-to is gone (rerouting comes from build/info.json), and builds lacked submodules. Adds executor[-platform] and runners-all-dist nix targets so published assets keep the names downstream download scripts already consume. * chore(executors): bump v0.3 for det-mode SIMD lane fix + embeddings test πŸ”§ * docs(spec): add executor versioning compatibility spec πŸ“ * feat(tests): add per-executor ignore-hash config via .genvm-tool.py ✨ * feat(ci): add commit-message validation script and hook ✨ * docs(docs): reorganize contributing guides into howto structure 🚚 * chore: bump executor submodules, update integration.py for 10 fee buckets πŸ”§ * feat(fees): contract-funded message fees, emission fee floors ✨ * chore(submodules): bump executors/v0.3.x ⬆️ * docs(spec): use_balance permission, funding modes, fee floor constants πŸ“ * chore(tests): mirror use_balance emissions and fee error taxonomy βœ… * fix(ci): run git third-party update inside each executor submodule πŸ’š * docs(contributing): runner hash hygiene guide, per-executor setup πŸ“ * chore(submodules): bump executors/v0.3.x for missed wasmtime patch ⬆️ * feat(runners): runner memory accounting per ADR-012 πŸ“βœ¨ * chore(spec): cleanup * docs(spec): document VM startup flow * docs(spec): define VM meta-properties * chore: minimize test runner output * chore: flush pending public abi ✨ * chore: simplify memlimiter ✨ * chore: migrate to manager provided structs ✨ --- .claude/skills/build/SKILL.md | 83 +-- .claude/skills/commit-style/SKILL.md | 14 + .claude/skills/initial-setup/SKILL.md | 26 +- .claude/skills/macos/SKILL.md | 8 +- .claude/skills/spec/SKILL.md | 49 ++ .claude/skills/submodules/SKILL.md | 6 + .claude/skills/test/SKILL.md | 148 +---- .coderabbit.yaml | 10 + .genvm-monorepo-root | 2 +- .genvm-tool.py | 40 +- .github/actions/get-src/action.yaml | 12 +- .github/workflows/branch_executor_prs.yaml | 41 ++ .github/workflows/branch_merge_into_dev.yaml | 18 +- .github/workflows/incl_build_trg.yaml | 10 +- .github/workflows/queue.yaml | 10 +- .github/workflows/release.yaml | 192 +++--- .gitmodules | 4 + .taplo.toml | 12 + CLAUDE.md | 17 + README.md | 35 +- .../modules-interfaces/.ya-test-config.json | 5 + crates/modules-interfaces/Cargo.lock | 29 +- crates/modules-interfaces/Cargo.toml | 27 +- crates/modules-interfaces/src/abi_stub.rs | 131 ++++ crates/modules-interfaces/src/domain.rs | 212 +++++++ .../modules-interfaces/src/domain/fees/abi.rs | 182 ++++++ .../modules-interfaces/src/domain/fees/mod.rs | 149 +++++ crates/modules-interfaces/src/lib.rs | 433 ++++++++++++- docs/adr/000. WASI.md | 6 +- docs/adr/012. runner memory accounting.md | 224 +++++++ docs/contributing/README.md | 7 +- docs/contributing/howto/README.md | 31 + docs/contributing/howto/building/build.md | 40 ++ docs/contributing/howto/building/docs.md | 35 ++ docs/contributing/howto/building/runners.md | 17 + .../howto/committing/git-third-party.md | 54 ++ docs/contributing/howto/committing/runners.md | 28 + .../howto/committing/submodules.md | 78 +++ .../howto/extending/add-host-function.md | 11 + .../howto/extending/add-llm-provider.md | 14 + .../howto/extending/add-wasi-function.md | 21 + .../howto/extending/modify-runner.md | 18 + .../howto/extending/modify-wasmtime.md | 27 + docs/contributing/howto/genvm-tool.md | 60 ++ .../howto/releasing/release-build.md | 29 + .../howto/releasing/versioning.md | 38 ++ docs/contributing/howto/setup.md | 35 ++ docs/contributing/howto/testing/README.md | 48 ++ docs/contributing/howto/testing/fuzzing.md | 24 + .../contributing/howto/testing/integration.md | 61 ++ docs/contributing/howto/testing/python.md | 16 + docs/contributing/howto/testing/rust.md | 16 + docs/contributing/tests.md | 103 --- docs/contributing/workflows.md | 42 -- docs/website/pyproject.toml | 14 +- docs/website/src/conf.py | 9 +- .../impl-spec/02-vm/02-version-management.rst | 3 +- docs/website/src/impl-spec/04-fees.rst | 54 ++ .../src/python-sdk/changelog-notes/v0.3.rst | 26 +- .../01-core-architecture/03-versioning.rst | 78 +++ .../src/spec/01-core-architecture/index.rst | 1 + .../spec/02-execution-environment/01-wasm.rst | 52 +- .../03-wasi_genlayer_sdk/01-functions.rst | 1 - .../03-wasi_genlayer_sdk/02-gl_call.rst | 189 ++++-- .../03-wasi_genlayer_sdk/03-schemas.rst | 4 +- .../02-execution-environment/04-runners.rst | 52 +- docs/website/src/spec/03-vm/01-startup.rst | 222 +++++++ .../src/spec/03-vm/02-meta-properties.rst | 145 +++++ docs/website/src/spec/03-vm/02-sandboxing.rst | 13 - .../src/spec/03-vm/03-ram-limiting.rst | 38 +- .../website/src/spec/03-vm/05-permissions.rst | 103 --- .../03-vm/{01-result.rst => 05-result.rst} | 33 +- docs/website/src/spec/03-vm/index.rst | 6 +- .../04-contract-interface/01-calldata.rst | 14 + .../spec/04-contract-interface/03-storage.rst | 30 +- .../04-upgradability.rst | 42 +- .../05-execution-flow.rst | 118 ++-- .../src/spec/appendix/constants-pending.rst | 2 + docs/website/src/spec/appendix/constants.rst | 228 ++++++- docs/website/src/spec/appendix/index.rst | 1 + executors/v0.2.x | 1 + executors/v0.3.x | 2 +- flake.nix | 132 +++- implementation/Cargo.lock | 8 +- implementation/Cargo.toml | 91 ++- implementation/default.nix | 2 + implementation/src/common/mod.rs | 9 +- implementation/src/llm/mod.rs | 1 + implementation/src/manager/check_install.rs | 95 +++ implementation/src/manager/handlers.rs | 4 +- implementation/src/manager/mod.rs | 14 + implementation/src/manager/modules.rs | 2 +- implementation/src/manager/run.rs | 49 +- implementation/src/scripting/ctx/mod.rs | 1 + implementation/src/web/mod.rs | 1 + install/lib/python/post-install/__main__.py | 42 +- libs/deps/dependency-urls.json | 21 +- ruff.toml | 16 +- runners/compat.nix | 21 +- support/ci/genvm-merge-into-dev.py | 180 ++++-- support/ci/open-executor-prs.py | 190 ++++++ support/ci/pipelines/build.sh | 7 +- support/ci/pipelines/src/test-rust-fuzz.sh | 4 +- support/ci/pipelines/src/test-rust.sh | 4 +- support/ci/pipelines/test-python.sh | 2 +- support/ci/pipelines/test-rust-fuzz.sh | 2 +- support/ci/pipelines/test-rust.sh | 6 +- support/default.nix | 20 +- support/nix/precommit/flake.nix | 1 + support/scripts/check-commit-message.py | 257 ++++++++ support/tools/genvm-tool/default.nix | 1 + .../tools/genvm-tool/genvm_tool/__main__.py | 29 +- .../genvm-tool/genvm_tool/cmd_codegen.py | 60 ++ .../genvm-tool/genvm_tool/cmd_configure.py | 595 ++++++------------ .../genvm-tool/genvm_tool/codegen/__init__.py | 39 ++ .../tools/genvm-tool/genvm_tool/codegen/go.py | 105 ++++ .../genvm-tool/genvm_tool/codegen/model.py | 187 ++++++ .../genvm-tool/genvm_tool/codegen/python.py | 120 ++++ .../genvm-tool/genvm_tool/codegen/rst.py | 64 ++ .../genvm-tool/genvm_tool/codegen/rust.py | 217 +++++++ support/tools/genvm-tool/genvm_tool/common.py | 57 ++ .../genvm-tool/genvm_tool/git/__init__.py | 4 +- .../genvm_tool/git/check_for_push.py | 327 ++++++++++ .../genvm_tool/git/create_branches.py | 182 ++++++ .../genvm-tool/genvm_tool/hook/install.py | 5 + .../tools/genvm-tool/genvm_tool/hook/run.py | 5 + .../tools/genvm-tool/genvm_tool/tests/cli.py | 7 + support/tools/genvm-tool/pyproject.toml | 16 +- support/tools/git-third-party/ruff.toml | 4 +- tests/runner/genvm_tool_plugins/docker.py | 36 +- tests/runner/genvm_tool_plugins/genvm.py | 25 +- .../runner/genvm_tool_plugins/integration.py | 226 ++++++- tests/runner/genvm_tool_plugins/ninja.py | 477 ++++++++++++++ tests/runner/gvm_extra/mock_host.py | 2 +- tests/runner/origin/base_host.py | 20 +- tests/runner/origin/host_fns.py | 20 +- tests/runner/origin/log_asserts.py | 118 ++++ tests/runner/origin/public_abi.py | 169 +++-- tests/system/permits/test.py | 1 + tests/templates/message.json | 1 + 140 files changed, 7193 insertions(+), 1577 deletions(-) create mode 100644 .claude/skills/spec/SKILL.md create mode 100644 .claude/skills/submodules/SKILL.md create mode 100644 .github/workflows/branch_executor_prs.yaml create mode 100644 .taplo.toml create mode 100644 CLAUDE.md create mode 100644 crates/modules-interfaces/.ya-test-config.json create mode 100644 crates/modules-interfaces/src/abi_stub.rs create mode 100644 crates/modules-interfaces/src/domain.rs create mode 100644 crates/modules-interfaces/src/domain/fees/abi.rs create mode 100644 crates/modules-interfaces/src/domain/fees/mod.rs create mode 100644 docs/adr/012. runner memory accounting.md create mode 100644 docs/contributing/howto/README.md create mode 100644 docs/contributing/howto/building/build.md create mode 100644 docs/contributing/howto/building/docs.md create mode 100644 docs/contributing/howto/building/runners.md create mode 100644 docs/contributing/howto/committing/git-third-party.md create mode 100644 docs/contributing/howto/committing/runners.md create mode 100644 docs/contributing/howto/committing/submodules.md create mode 100644 docs/contributing/howto/extending/add-host-function.md create mode 100644 docs/contributing/howto/extending/add-llm-provider.md create mode 100644 docs/contributing/howto/extending/add-wasi-function.md create mode 100644 docs/contributing/howto/extending/modify-runner.md create mode 100644 docs/contributing/howto/extending/modify-wasmtime.md create mode 100644 docs/contributing/howto/genvm-tool.md create mode 100644 docs/contributing/howto/releasing/release-build.md create mode 100644 docs/contributing/howto/releasing/versioning.md create mode 100644 docs/contributing/howto/setup.md create mode 100644 docs/contributing/howto/testing/README.md create mode 100644 docs/contributing/howto/testing/fuzzing.md create mode 100644 docs/contributing/howto/testing/integration.md create mode 100644 docs/contributing/howto/testing/python.md create mode 100644 docs/contributing/howto/testing/rust.md delete mode 100644 docs/contributing/tests.md delete mode 100644 docs/contributing/workflows.md create mode 100644 docs/website/src/spec/01-core-architecture/03-versioning.rst create mode 100644 docs/website/src/spec/03-vm/01-startup.rst create mode 100644 docs/website/src/spec/03-vm/02-meta-properties.rst delete mode 100644 docs/website/src/spec/03-vm/02-sandboxing.rst delete mode 100644 docs/website/src/spec/03-vm/05-permissions.rst rename docs/website/src/spec/03-vm/{01-result.rst => 05-result.rst} (81%) create mode 100644 docs/website/src/spec/appendix/constants-pending.rst create mode 160000 executors/v0.2.x create mode 100644 implementation/src/manager/check_install.rs create mode 100644 support/ci/open-executor-prs.py create mode 100755 support/scripts/check-commit-message.py create mode 100644 support/tools/genvm-tool/genvm_tool/cmd_codegen.py create mode 100644 support/tools/genvm-tool/genvm_tool/codegen/__init__.py create mode 100644 support/tools/genvm-tool/genvm_tool/codegen/go.py create mode 100644 support/tools/genvm-tool/genvm_tool/codegen/model.py create mode 100644 support/tools/genvm-tool/genvm_tool/codegen/python.py create mode 100644 support/tools/genvm-tool/genvm_tool/codegen/rst.py create mode 100644 support/tools/genvm-tool/genvm_tool/codegen/rust.py create mode 100644 support/tools/genvm-tool/genvm_tool/git/check_for_push.py create mode 100644 support/tools/genvm-tool/genvm_tool/git/create_branches.py create mode 100644 tests/runner/genvm_tool_plugins/ninja.py create mode 100644 tests/runner/origin/log_asserts.py diff --git a/.claude/skills/build/SKILL.md b/.claude/skills/build/SKILL.md index f72bc27d..a6d5309d 100644 --- a/.claude/skills/build/SKILL.md +++ b/.claude/skills/build/SKILL.md @@ -3,74 +3,15 @@ name: build description: Builds the GenVM project. Use after making code changes to compile Rust binaries. --- -To build the GenVM project: - -## Reconfiguring - -If ninja fails with `missing and no known rule to make it` (e.g., after adding/removing/renaming source files), regenerate the build file first: - -```bash -./configure.rb -``` - -This runs the `configure.rb` Ruby script at the project root, which regenerates `build/build.ninja` with the current file list. - -## Building - -**Build all Rust binaries:** -```bash -nix develop .#full --command bash .claude/skills/build/scripts/run-ninja.sh -C build all/bin -``` - -This runs ninja silently and only shows output on failure (to save tokens). - -**Available ninja targets:** - -| Target | Description | -|--------|-------------| -| `all` | Build everything | -| `all/bin` | Build all Rust binaries | -| `all/data` | Build data about runners using Nix | -| `codegen` | Run code generation | - -**Output locations:** -- `out/bin/genvm-modules` - modules binary -- `out/executor//bin/genvm` - executor binary (one dir per built version, e.g. `out/executor/v0.3.0-rc7/`; the concrete version comes from `executors/.x/manifest.json` and is recorded in `build/info.json`) - -## Runners - -Runners can only be built on x86_64 using the `all` target. On other platforms, download them instead. - -**Download runners:** -```bash -nix develop .#full --command python3 build/out/bin/post-install.py --create-venv false --default-step false --runners-download true --error-on-missing-executor false -``` - -### Runner Development Workflow - -To develop/modify a runner (e.g., cloudpickle): - -1. **Enable dev mode:** - Set `runners/support/versions/dev-mode.nix` to `true` - -2. **Set hash to "test":** - In `runners/support/versions/current.nix`, set the runner's hash to `"test"` - -3. **Make your modifications and run tests** - With dev-mode enabled and hash set to "test", you can build and run tests. - -4. **Disable dev mode:** - Set `runners/support/versions/dev-mode.nix` back to `false`. The build will now tell you to set hashes to `null`. - -5. **Set hashes to null and build:** - Set the runner's hash (and dependent runners' hashes) to `null`, then build: - ```bash - nix develop .#full --command ninja -C build all - ``` - The build will fail with a hash mismatch showing the new hash. - -6. **Update hashes:** - Copy the new hash from the error message back into `hashes.nix`. Repeat for dependent runners. - -7. **Rebuild to verify:** - Run the build again to confirm all hashes are correct. +Build procedure: `docs/contributing/howto/building/build.md` (debug build; +read it first). Related: `building/runners.md`, `releasing/release-build.md`, +`extending/modify-runner.md` under the same howto root. + +Claude-specific: + +- Build binaries with + `bash .claude/skills/build/scripts/run-ninja.sh -C build all/bin` instead of + raw ninja β€” it is silent on success and prints output only on failure, which + saves tokens. +- See also: `/submodules` (multi-repo commits, `?submodules=1`), `/test`, + `/macos` (never build runners natively on macOS). diff --git a/.claude/skills/commit-style/SKILL.md b/.claude/skills/commit-style/SKILL.md index ee2be06f..ceae8050 100644 --- a/.claude/skills/commit-style/SKILL.md +++ b/.claude/skills/commit-style/SKILL.md @@ -91,6 +91,7 @@ bytes β€” that earns a πŸ”’οΈ. | `webdriver` | the webdriver module | | `ci` | GitHub workflows / CI | | `build` | build system, nix, release packaging, dependency bumps | +| `genvm-tool`| the `genvm-tool` dev CLI (`support/tools/genvm-tool`): configure, test runner, hooks, git helpers | The set is curated, not closed β€” if a change clearly belongs to a subsystem not listed, a sensible lowercase scope is fine. Prefer an existing one when it fits @@ -117,6 +118,9 @@ the diff and not a bullet list of squashed sub-commits. Spell-check the subject; it is permanent. 4. **`chore: fix tests`** with no cause. Add it: `chore(executor): fix tests after wasmtime rebase βœ…`. +5. **AI attribution** β€” never include `Co-authored-by` trailers, session + links/IDs (e.g. `Claude-Session:`), "Generated with" footers, or any other + AI/tool metadata, even when the tooling asks for it. ## Checklist @@ -126,3 +130,13 @@ the diff and not a bullet list of squashed sub-commits. - [ ] One to three trailing gitmoji glyphs (most-important first) matching the intent? - [ ] Single line β€” body only if the *why* is truly unrecoverable from the diff? - [ ] Spell-checked? + +## Committing across the manager + submodules + +A change that touches an executor submodule spans repos: commit inside the +submodule first, then bump the manager's gitlink (`git add executors/.x`) +in a manager commit. Keep each commit's files coherent and don't let a +submodule's linter reformat leak an unrelated file into a commit. The cross-repo +pre-commit hook (`genvm-tool hook run`) fans out to all repos; use `--no-verify` +for pure gitlink bumps. Full workflow (order, pushing, `--force-with-lease` after +a rebase): `/submodules`. diff --git a/.claude/skills/initial-setup/SKILL.md b/.claude/skills/initial-setup/SKILL.md index 50a2763e..dc664d26 100644 --- a/.claude/skills/initial-setup/SKILL.md +++ b/.claude/skills/initial-setup/SKILL.md @@ -3,28 +3,4 @@ name: initial-setup description: Sets up the development environment for GenVM repository. Use when setting up the repo for the first time or when dependencies need to be refreshed. --- -To set up the GenVM development environment: - -1. **Enter the Nix flake environment:** - ```bash - nix develop .#full - ``` - -2. **Initialize git submodules:** - ```bash - git submodule update --init --recursive --depth 1 - ``` - -3. **Source environment variables:** - ```bash - source env.sh - ``` - This adds `tools/git-third-party` to PATH and sources `.env` if it exists. - -4. **Update third-party dependencies:** - ```bash - ./tools/git-third-party/git-third-party update --all - ``` - This updates wasmtime, wasm-tools, and applies GenVM-specific patches. - -The repository will be ready for development with all dependencies properly configured. +See docs/contributing/howto/setup.md. diff --git a/.claude/skills/macos/SKILL.md b/.claude/skills/macos/SKILL.md index 905a1507..45a83eb3 100644 --- a/.claude/skills/macos/SKILL.md +++ b/.claude/skills/macos/SKILL.md @@ -29,7 +29,7 @@ A native macOS build: You can verify the reference set on Linux with: ```sh -find $(nix build .#runners-all --no-link --print-out-paths) -name '*.tar' \ +find $(nix build '.?submodules=1#runners-all' --no-link --print-out-paths) -name '*.tar' \ | sort | xargs sha256sum | sed -e 's+/nix/store/[^/]*++' ``` @@ -91,9 +91,11 @@ You usually do not need to build runners at all. Download them instead (works on any platform): ```bash -nix develop .#full --command python3 build/out/bin/post-install.py \ +nix develop '.?submodules=1#full' --command python3 build/out/bin/post-install.py \ --create-venv false --default-step false \ --runners-download true --error-on-missing-executor false ``` -See the `build` skill for the normal Rust build flow. +(`?submodules=1` is required on every flake ref β€” see `/submodules`.) + +See `/build` for the normal Rust build flow and `/submodules` for the multi-repo layout. diff --git a/.claude/skills/spec/SKILL.md b/.claude/skills/spec/SKILL.md new file mode 100644 index 00000000..ad9f83bb --- /dev/null +++ b/.claude/skills/spec/SKILL.md @@ -0,0 +1,49 @@ +--- +name: spec +description: How to write GenVM spec pages (docs/website/src/spec). Use when adding or editing spec sections β€” brevity, linking constants/errors/terms instead of inlining them, spec vs impl-spec split, and how to verify the build. +--- + +# Writing spec + +The spec (`docs/website/src/spec/`) describes **observable behavior** β€” what a +contract or validator can detect. How this codebase achieves it goes to +`impl-spec/`. If a sentence starts describing internals (caches, native limits, +threads), either drop it or reduce it to one normative requirement on +implementations. + +## Be brief + +- One concern per section; a few paragraphs or one list is the right size. +- Enumerate cases with `#.` lists instead of prose walkthroughs; name the + actors precisely (*caller*/*callee*, leader/validator) and describe each + case once. +- State what happens, not why the design is good. Rationale goes to ADRs + (`docs/adr/`). +- Cover the edges (unwinding, host boundary, validation-time rejection) in one + sentence each β€” omitting them is sweeping under the rug, but they rarely + deserve a paragraph. + +## Link, don't inline + +Never write a literal value or error string in spec text β€” link the anchor, so +generated pages stay the single source of truth: + +- Constants: `:ref:`gvm-def-consts-value--`` / + `:ref:`gvm-def-const-`` from `spec/appendix/constants.rst` + (generated from `executor/codegen/data/public-abi.json`) or + `constants-pending.rst` (from `public-abi-pending.json`). **Never edit these + .rst by hand** β€” edit the JSON and regenerate + ([genvm-tool.md](../../../docs/contributing/howto/genvm-tool.md)). New + not-yet-stabilized constants go to the pending JSON. +- Error outcomes: `:ref:`gvm-def-str-trie-value-vm-error-...`` β€” every "traps + with" / "rejected with" must link the exact vm_error entry. +- Glossary terms: `:term:`sub-VM`` etc. on first use in a section. +- Other spec pages: `:doc:` relative links instead of restating their content. + +## Verify + +Build the website ([docs.md](../../../docs/contributing/howto/building/docs.md)) +and check for `undefined label` warnings on your pages β€” a broken `:ref:` is a +silent dead link otherwise. Verify claims against the implementation before +writing them; every behavioral sentence should have a code location you can +point to (but do not link it to the specification). diff --git a/.claude/skills/submodules/SKILL.md b/.claude/skills/submodules/SKILL.md new file mode 100644 index 00000000..f6f42b6a --- /dev/null +++ b/.claude/skills/submodules/SKILL.md @@ -0,0 +1,6 @@ +--- +name: submodules +description: How the GenVM multi-repo (manager umbrella + executor submodules) fits together and how to build, commit, run hooks, and push across all of them. Use when committing/pushing changes that touch a submodule, bumping gitlinks, building via nix, rebasing the manager, or debugging the cross-repo pre-commit hook. +--- + +See docs/contributing/howto/committing/submodules.md. diff --git a/.claude/skills/test/SKILL.md b/.claude/skills/test/SKILL.md index 851ac024..a7e23014 100644 --- a/.claude/skills/test/SKILL.md +++ b/.claude/skills/test/SKILL.md @@ -3,148 +3,8 @@ name: test description: Runs tests for the GenVM project. Use after making code changes to verify correctness. --- -# Running Tests +See docs/contributing/howto/testing/README.md. -GenVM uses `genvm-tool test` for all tests. Before running tests, ensure the project is built (see `/build` skill). - -## Quick Start - -Run all tests: -```bash -nix develop .#mock-tests --command genvm-tool test run -``` - -Run release tests (stable integration): -```bash -nix develop .#mock-tests --command genvm-tool test run --filter-tag "$(cat tests/presets/release.txt)" -``` - -Run a specific test: -```bash -nix develop .#mock-tests --command genvm-tool test run --filter-name 'test_name' -``` - -## genvm-tool test Commands - -### Run Tests -```bash -genvm-tool test run [OPTIONS] -``` - -**Options:** -| Flag | Description | -|------|-------------| -| `--filter-name REGEX` | Filter tests by name regex | -| `--filter-tag EXPR` | Filter tests by tags (e.g., `stable & !slow`) | -| `--filter-continue FILE` | Re-run only tests from a continue file | -| `--fail-fast` | Stop execution after first failure | -| `--coverage` | Enable coverage collection for Rust tests | -| `--log-level LEVEL` | Set log level (trace/debug/info/warning/error) | -| `--ignore-hash` | Skip `.hash` (execution-hash) comparison entirely | - -**`--ignore-hash`:** integration cases compare both the printed semantics -(`.N.stdout`) and a deterministic execution hash (`.N.hash`). When adding a new -case, you usually don't have a correct `.hash` yet β€” run with `--ignore-hash` so -only the `.stdout` semantics are checked, or set `stable_hash: false` on the -entry (then validators compare to the leader's hash instead of a committed file). - -### Show Information -```bash -# Show available tests -genvm-tool test show test - -# Show execution plan -genvm-tool test show plan - -# Show available services -genvm-tool test show services - -# Show available tags -genvm-tool test show tags -``` - -## Test Presets - -Presets are tag expressions stored in `tests/presets/`: - -| Preset | Expression | Use Case | -|--------|------------|----------| -| `release.txt` | `integration & stable` | CI release tests | -| `rust.txt` | `rust \| integration` | Rust development | -| `python.txt` | `python` | Python SDK tests | - -Usage: -```bash -genvm-tool test --filter-tag "$(cat tests/presets/release.txt)" run -``` - -## Test Categories - -### Integration Tests (`tests/cases/`) -End-to-end tests using jsonnet configuration. Services (manager, modules, webdriver) are started automatically. - -```bash -nix develop .#mock-tests --command genvm-tool test run --filter-tag integration -``` - -### Rust Tests -Cargo tests for Rust crates: -```bash -nix develop .#rust-test --command genvm-tool test run --filter-tag rust -``` - -With coverage: -```bash -nix develop .#rust-test --command genvm-tool test run --filter-tag rust --coverage -``` - -### Python Tests -Tests for the Python standard library (`genlayer-py-std`): -```bash -nix develop .#mock-tests --command genvm-tool test run --filter-tag python -``` - -Or directly with pytest from the standalone py-test flake (no nix develop needed): -```bash -cd runners/genlayer-py-std -env_dir="$(nix build --no-link --print-out-paths path:../../support/nix/py-test)" -PYTHONPATH="$PWD/src:$PWD/src-emb" "$env_dir/bin/pytest" tests/ -``` - -**Coverage:** pytest is configured with `--cov` and `--cov-fail-under=75`. Coverage scope includes `genlayer.types`, `genlayer.calldata`, `genlayer.storage`, `genlayer.evm`, `genlayer._internal`, and `genlayer_embeddings`. Must run inside nix develop for numpy-dependent tests and correct coverage resolution. - -## Webdriver Setup - -For web-related tests (semi-stable/unstable), webdriver is started automatically by genvm-tool test. To manually start it: - -```bash -bash modules/webdriver/build-and-run.sh -``` - -## Precompile (Optional) - -If WASM files or compilation changed, precompile to save test time: -```bash -./build/out/bin/genvm precompile -``` - -## Re-running Failed Tests - -When tests fail, genvm-tool test writes failed test names to `build/test-artifacts/continue/-`. Re-run only failed tests: - -```bash -# Use filename shown in failure summary -genvm-tool test run --filter-continue 20260123-143052-abc123 -``` - -## Quick Reference - -| What to test | Command | -|--------------|---------| -| All tests | `nix develop .#mock-tests --command genvm-tool test run` | -| Release tests | `nix develop .#mock-tests --command genvm-tool test run --filter-tag "$(cat tests/presets/release.txt)"` | -| Rust tests | `nix develop .#rust-test --command genvm-tool test run --filter-tag rust` | -| Python (direct) | `cd runners/genlayer-py-std && PYTHONPATH="$PWD/src:$PWD/src-emb" "$(nix build --no-link --print-out-paths path:../../support/nix/py-test)/bin/pytest" tests/` | -| Re-run failed | `genvm-tool test run --filter-continue ` | -| With debug logs | `nix develop .#mock-tests --command genvm-tool test run --log-level debug` | -| Show test list | `nix develop .#mock-tests --command genvm-tool test show test` | +After fixing a failure, rerun with `--filter-continue ` (path printed in +the failure summary) before any full rerun β€” see the "Fix–rerun loop" section +there. diff --git a/.coderabbit.yaml b/.coderabbit.yaml index bf1e8b32..960bc334 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -1,4 +1,9 @@ reviews: + # Review PRs targeting the long-lived version branches (e.g. v0.6, v0.6-dev) + # and main; feature/topic branches are reviewed via their target base. + base_branches: + - "main" + - "v\\d+\\.\\d+(-dev)?" path_filters: - "!**/*.onnx" - "!**/*.txt" @@ -10,3 +15,8 @@ reviews: - "!executors/v0.3.x/runners/models/**" - "!executors/v0.3.x/runners/softfloat/berkeley-softfloat-3/**" - "!**/fuzz/inputs*/**" + # Root-level system-test fixtures (contracts, expected outputs); crate and + # runner test code under */tests/ is still reviewed. + - "!tests/**" + # genvm-tool is internal dev tooling; not worth reviewing. + - "!support/tools/genvm-tool/**" diff --git a/.genvm-monorepo-root b/.genvm-monorepo-root index d569ad58..e97bae45 100644 --- a/.genvm-monorepo-root +++ b/.genvm-monorepo-root @@ -1,6 +1,6 @@ { "version": "v0.6.0-rc0", - "active-versions": ["v0.3"], + "active-versions": ["v0.3", "v0.2"], "artifacts_dir": "build/test-artifacts", "extra_python_paths": ["tests/runner"] } diff --git a/.genvm-tool.py b/.genvm-tool.py index 4c7592ce..5ac477fd 100644 --- a/.genvm-tool.py +++ b/.genvm-tool.py @@ -21,8 +21,10 @@ its own ``hooks``. """ +import genvm_tool -def hooks(ctx): + +def hooks(ctx: 'genvm_tool.common.Context'): """Manager commit-hook definitions (was support/nix/precommit/hooks.toml). Tools resolve from the sibling flake's buildEnv (``nix = ""``); @@ -72,6 +74,15 @@ def hooks(ctx): 'entry': 'check-toml', 'types_or': ['toml'], }, + { + 'id': 'toml-format', + 'nix': 'taplo', + 'entry': 'taplo', + 'args': ['format', '--check'], + 'fix_args': ['format'], + 'types_or': ['toml'], + 'pass_filenames': True, + }, { 'id': 'check-merge-conflict', 'nix': 'pre-commit-hooks', @@ -98,7 +109,6 @@ def hooks(ctx): 'fix_args': ['format'], 'types_or': ['python'], }, - # --- python / lua / ts (manager-owned languages) ------------------- { 'id': 'ruff-check', 'nix': 'ruff', @@ -161,8 +171,12 @@ def hooks(ctx): # independently (its own repo/hooks own that); we never touch it. 'id': 'check-cargo-versions', 'local': True, - 'entry': 'support/ci/check-versions.py', - 'args': ['sync'], + 'entry': ctx.python_command[0], + 'args': [ + *ctx.python_command[1:], + str(ctx.root / 'support/ci/check-versions.py'), + 'sync', + ], 'pass_filenames': False, 'files': r'^(implementation/Cargo\.toml|\.genvm-monorepo-root)$', }, @@ -171,6 +185,19 @@ def hooks(ctx): 'builtin': 'md-local-links', 'types_or': ['markdown'], }, + # --- commit-msg ---------------------------------------------------- + { + 'id': 'check-commit-message', + 'local': True, + 'entry': ctx.python_command[0], + 'args': [ + *ctx.python_command[1:], + str(ctx.root / 'support/scripts/check-commit-message.py'), + '--message-file', + ], + 'stages': ['commit-msg'], + 'pass_filenames': False, + }, ] @@ -326,6 +353,7 @@ def collect_integration(ctx: genvm_tool.tests.stage.collection.Context): ) no_manager = getattr(ctx.configuration.args, 'no_manager', False) no_webdriver = getattr(ctx.configuration.args, 'no_webdriver', False) + ci = getattr(ctx.configuration.args, 'ci', False) manager_port = genvm.get_manager_port(ctx.configuration) @@ -338,13 +366,14 @@ def collect_integration(ctx: genvm_tool.tests.stage.collection.Context): bin_path=build_dir.joinpath('out', 'bin', 'genvm-modules'), log_path=tests_output_root.joinpath('manager.log'), env=ctx.configuration, + ci=ci, ) # Create webdriver service if no_webdriver: webdriver_impl = genvm.NoOpService() else: webdriver_impl = genvm_tool.tests.exec.service.FunctionService( - lambda: genvm.start_webdriver_service(ctx.configuration) + lambda: genvm.start_webdriver_service(ctx.configuration, ci=ci) ) # This starts Llm and Web modules on the manager modules_impl = genvm.ModulesService( @@ -374,6 +403,7 @@ def collect_integration(ctx: genvm_tool.tests.stage.collection.Context): manager_service=manager_service, modules_service=modules_service, webdriver_service=webdriver_service, + ci=ci, ) ctx.collect_dir('tests/system/permits', manager_service=manager_service) diff --git a/.github/actions/get-src/action.yaml b/.github/actions/get-src/action.yaml index 311ada57..d386d441 100644 --- a/.github/actions/get-src/action.yaml +++ b/.github/actions/get-src/action.yaml @@ -30,7 +30,17 @@ runs: if [ "${{ inputs.third_party }}" != "none" ] then source env.sh - git third-party update ${{ inputs.third_party }} + # Vendored third-party trees are registered per executor submodule: + # `.git-third-party/config.json` lives inside each executor, which is + # its own git repo. `git third-party` resolves its config from the + # current git toplevel, so it must run *inside* each executor β€” + # running it once at the manager root finds no config and + # materializes nothing (which then breaks the cargo build). + for cfg in executors/*/.git-third-party/config.json + do + [ -e "$cfg" ] || continue + ( cd "$(dirname "$(dirname "$cfg")")" && git third-party update ${{ inputs.third_party }} ) + done fi fi shell: bash -ex {0} diff --git a/.github/workflows/branch_executor_prs.yaml b/.github/workflows/branch_executor_prs.yaml new file mode 100644 index 00000000..30d3fdee --- /dev/null +++ b/.github/workflows/branch_executor_prs.yaml @@ -0,0 +1,41 @@ +name: branch / open linked executor PRs + +# When a manager PR is opened against a dev branch, open the matching executor +# PR(s) in the genvm-executor repo. The manager release line is independent of the +# executor lines it ships, so one manager PR fans out over EVERY active executor +# line (.genvm-monorepo-root): for each, `pr//` -> -dev. Each +# is listed on the manager PR as an `executor: ` line. The executor work lives +# on line-namespaced mirror branches the developer pushed via +# `genvm-tool git create-branches`. See support/ci/open-executor-prs.py. +# +# pull_request_target gives a manager-scoped write token for the comment; the +# cross-repo executor PR needs a PAT with genvm-executor access +# (GENVM_EXECUTOR_PR_TOKEN) β€” the default GITHUB_TOKEN cannot reach another repo. +# We check out only the base repo to run the trusted script β€” never PR code. + +on: + pull_request_target: + branches: ['v*-dev'] + types: [opened] + +permissions: + pull-requests: write + issues: write + +defaults: + run: + shell: bash -x {0} + +jobs: + open: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Open and link executor PR(s) + env: + MANAGER_TOKEN: ${{ secrets.GITHUB_TOKEN }} + EXECUTOR_TOKEN: ${{ secrets.GENVM_EXECUTOR_PR_TOKEN }} + MANAGER_REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + HEAD_REF: ${{ github.event.pull_request.head.ref }} + run: python3 support/ci/open-executor-prs.py diff --git a/.github/workflows/branch_merge_into_dev.yaml b/.github/workflows/branch_merge_into_dev.yaml index d0ff8590..e58fe231 100644 --- a/.github/workflows/branch_merge_into_dev.yaml +++ b/.github/workflows/branch_merge_into_dev.yaml @@ -85,15 +85,19 @@ jobs: printf '%s\n' "$EXECUTOR_DEPLOY_KEY" > "$key" chmod 600 "$key" ssh="ssh -i $key -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new" - # Fetch the submodule with the executor key, then bind that key to - # the submodule only (per-repo core.sshCommand) so the later mirror - # push uses it while the manager push keeps the checkout credentials. - git -c core.sshCommand="$ssh" submodule update --init executors/v0.3.x - git -C executors/v0.3.x config core.sshCommand "$ssh" + # The manager release line ships every active executor line, so the + # merge mirrors each line's gitlink onto its own -dev. Init them + # all, binding the executor key per-submodule (core.sshCommand) so the + # later mirror pushes use it while the manager push keeps the checkout + # credentials. + for v in $(python3 support/ci/branch-versions.py list); do + sub="executors/v${v}.x" + git -c core.sshCommand="$ssh" submodule update --init "$sub" + git -C "$sub" config core.sshCommand "$ssh" + done - - name: Validate gates, mirror executor, fast-forward / squash, close + - name: Validate gates, mirror executors, fast-forward / squash, close env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} PR_NUMBER: ${{ inputs.pr_number }} - EXECUTOR_SUBMODULE: executors/v0.3.x run: python3 support/ci/genvm-merge-into-dev.py diff --git a/.github/workflows/incl_build_trg.yaml b/.github/workflows/incl_build_trg.yaml index 273fbe68..c12df64e 100644 --- a/.github/workflows/incl_build_trg.yaml +++ b/.github/workflows/incl_build_trg.yaml @@ -44,10 +44,18 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + - uses: insightsengineering/disk-space-reclaimer@v1 + with: + tools-cache: false + swap-storage: false + docker-images: false + - name: Get source uses: ./.github/actions/get-src with: - load_submodules: "false" + # The executor lines are submodules; nix builds run on + # `.?submodules=1` and need them checked out. + load_submodules: "true" with_nix: "true" github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/queue.yaml b/.github/workflows/queue.yaml index ec0c5714..7153c198 100644 --- a/.github/workflows/queue.yaml +++ b/.github/workflows/queue.yaml @@ -56,7 +56,7 @@ jobs: - name: Get source uses: ./.github/actions/get-src with: - load_submodules: "false" + load_submodules: "true" with_nix: "true" github_token: ${{ secrets.GITHUB_TOKEN }} - run: ./support/ci/pipelines/test-python.sh @@ -96,6 +96,12 @@ jobs: if: ${{ github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'rtm') || contains(github.event.pull_request.labels.*.name, 'run-full-tests') }} needs: [initial] runs-on: ubuntu-latest + env: + # Only used to upload freshly-built runners to the GCS cache; this is + # best-effort (see the `|| true` upload in test-rust.sh). Absent on fork + # PRs, so the GCP steps below are skipped and runners come from the public + # bucket instead of failing the whole job. + GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} steps: - uses: actions/checkout@v4 with: @@ -112,10 +118,12 @@ jobs: with_nix: "true" github_token: ${{ secrets.GITHUB_TOKEN }} - name: Authenticate to Google Cloud + if: ${{ env.GCP_SA_KEY != '' }} uses: google-github-actions/auth@v2 with: credentials_json: ${{ secrets.GCP_SA_KEY }} - name: Set up Cloud SDK # needed for gcloud to upload runners to GCS for caching + if: ${{ env.GCP_SA_KEY != '' }} uses: google-github-actions/setup-gcloud@v2 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 5c84bba9..59164965 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,21 +1,24 @@ name: GenVM release +# Builds and publishes the release artifacts consumed by other projects +# (node's download_genvm.sh and friends): +# genvm--.tar.xz β€” manager bundle (bin/, lib/, config/, data/) +# genvm---executor.tar.xz β€” every active executor line (executor//...) +# genvm-runners-all.tar.xz β€” shared runners under runners/ + legacy +# lines' runners at executor//legacy-runners +# All three extract at the same install root. +# +# The release tag is the manager version from .genvm-monorepo-root (also baked +# into the binaries as GENVM_PROFILE), so tagging is derived from the source +# tree instead of bumping the previous git tag. + on: workflow_dispatch: inputs: - bump: - type: choice - required: false - description: "Version bump type (ignored if version_override is set)" - options: - - patch - - minor - - major - default: patch version_override: type: string required: false - description: "Override version (e.g., v1.2.3). If set, bump is ignored" + description: "Override release tag (e.g., v1.2.3). Defaults to `version` from .genvm-monorepo-root" defaults: run: @@ -36,34 +39,32 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions-ecosystem/action-get-latest-tag@v1 - id: get-latest-tag - if: github.event.inputs.version_override == '' - - - uses: actions-ecosystem/action-bump-semver@v1 - id: bump-semver - if: github.event.inputs.version_override == '' - with: - current_version: ${{ steps.get-latest-tag.outputs.tag }} - level: ${{ github.event.inputs.bump }} - - name: Determine final version id: determine-version run: | + set -euo pipefail if [ -n "${{ github.event.inputs.version_override }}" ]; then - echo "Using override version: ${{ github.event.inputs.version_override }}" - echo "new_version=${{ github.event.inputs.version_override }}" >> $GITHUB_OUTPUT + tag="${{ github.event.inputs.version_override }}" + echo "Using override version: $tag" else - echo "Using bumped version: ${{ steps.bump-semver.outputs.new_version }}" - echo "new_version=${{ steps.bump-semver.outputs.new_version }}" >> $GITHUB_OUTPUT + tag=$(jq -r .version .genvm-monorepo-root) + echo "Using .genvm-monorepo-root version: $tag" + fi + echo "$tag" | grep -qE '^v[0-9]+\.[0-9]+\.[0-9]+' || { + echo "::error::invalid version '$tag'" + exit 1 + } + if git ls-remote --exit-code origin "refs/tags/$tag" > /dev/null; then + echo "::error::tag $tag already exists on origin" + exit 1 fi - echo "new version will be: ${{ steps.determine-version.outputs.new_version }}" + echo "new_version=$tag" >> $GITHUB_OUTPUT build-runners-all: needs: [gen-tag] uses: ./.github/workflows/incl_build_trg.yaml with: - target: runners-all + target: runners-all-dist executor_version: ${{ needs.gen-tag.outputs.tag }} secrets: GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} @@ -107,32 +108,32 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - with: - lfs: true - name: Get source uses: ./.github/actions/get-src with: - load_submodules: "false" + # Even devShell evaluation reads the executor submodules' manifests. + load_submodules: "true" with_nix: "true" + github_token: ${{ secrets.GITHUB_TOKEN }} - name: Download artifact run: | - mkdir -p build/out && \ - pushd build/out && \ - curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-linux-arm64.tar.xz ${{ needs.build-linux-arm64.outputs.artifact_url }} && \ - tar -xf genvm-linux-arm64.tar.xz --no-same-owner && \ - curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-linux-arm64-executor.tar.xz ${{ needs.build-linux-arm64.outputs.secondary_artifact_url }} && \ - tar -xf genvm-linux-arm64-executor.tar.xz --no-same-owner && \ - popd && \ - true + set -euo pipefail + mkdir -p build/out + pushd build/out + curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-linux-arm64.tar.xz ${{ needs.build-linux-arm64.outputs.artifact_url }} + tar -xf genvm-linux-arm64.tar.xz --no-same-owner + curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-linux-arm64-executor.tar.xz ${{ needs.build-linux-arm64.outputs.secondary_artifact_url }} + tar -xf genvm-linux-arm64-executor.tar.xz --no-same-owner + popd - name: Run post-install run: | ./build/out/bin/post-install.py \ --error-on-missing-executor=false \ --default-download=false || true - nix develop .#check-qemu --command bash -c 'qemu-aarch64 ./build/out/bin/genvm-modules --version' + nix develop '.?submodules=1#check-qemu' --command bash -c 'qemu-aarch64 ./build/out/bin/genvm-modules --version' - name: Smoke test modules startup run: | - nix develop .#check-qemu --command bash -c ' + nix develop '.?submodules=1#check-qemu' --command bash -c ' timeout 5 qemu-aarch64 ./build/out/bin/genvm-modules llm --allow-empty-backends --config ./build/out/config/genvm-module-llm.yaml || test $? -eq 124 ' @@ -140,71 +141,73 @@ jobs: needs: - build-linux-amd64 - build-runners-all - - gen-tag runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - with: - lfs: true - name: Get source uses: ./.github/actions/get-src with: - load_submodules: "false" + load_submodules: "true" with_nix: "true" + github_token: ${{ secrets.GITHUB_TOKEN }} - name: Download artifact run: | - mkdir -p build/out && \ - pushd build/out && \ - curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-linux-amd64.tar.xz ${{ needs.build-linux-amd64.outputs.artifact_url }} && \ - curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-runners-all.tar.xz ${{ needs.build-runners-all.outputs.artifact_url }} && \ - curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-linux-amd64-executor.tar.xz ${{ needs.build-linux-amd64.outputs.secondary_artifact_url }} && \ - tar -xf genvm-linux-amd64.tar.xz --no-same-owner && \ - tar -xf genvm-runners-all.tar.xz --no-same-owner && \ - tar -xf genvm-linux-amd64-executor.tar.xz --no-same-owner && \ - popd && \ - true + set -euo pipefail + mkdir -p build/out + pushd build/out + curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-linux-amd64.tar.xz ${{ needs.build-linux-amd64.outputs.artifact_url }} + curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-runners-all.tar.xz ${{ needs.build-runners-all.outputs.artifact_url }} + curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-linux-amd64-executor.tar.xz ${{ needs.build-linux-amd64.outputs.secondary_artifact_url }} + tar -xf genvm-linux-amd64.tar.xz --no-same-owner + tar -xf genvm-runners-all.tar.xz --no-same-owner + tar -xf genvm-linux-amd64-executor.tar.xz --no-same-owner + popd - name: Run post-install run: | ./build/out/bin/post-install.py \ - --error-on-missing-executor=false \ --default-download=false - name: Smoke test modules startup run: | timeout 5 ./build/out/bin/genvm-modules llm --allow-empty-backends --config ./build/out/config/genvm-module-llm.yaml || test $? -eq 124 - name: Run stable tests run: | - nix develop .#mock-tests --command genvm-tool test run --filter-tag "$(cat tests/presets/release.txt)" --genvm-reroute-to ${{ needs.gen-tag.outputs.tag }} + # `configure` only writes build/info.json + the ninja files; the test + # runner itself never builds, so it exercises the downloaded + # artifacts in build/out. Rerouting each line to its concrete built + # version comes from build/info.json (no CLI flag anymore). + nix develop '.?submodules=1#mock-tests' --command bash -c ' + set -euo pipefail + genvm-tool configure + genvm-tool test run --ci --filter-tag "$(cat tests/presets/release.txt)" + ' test-macos-arm64: needs: - build-macos-arm64 - build-runners-all - - gen-tag runs-on: macos-latest steps: - uses: actions/checkout@v4 - with: - lfs: true - name: Get source uses: ./.github/actions/get-src with: - load_submodules: "false" + load_submodules: "true" with_nix: "false" - name: Download artifact run: | - mkdir -p build/out && \ - pushd build/out && \ - curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-macos-arm64.tar.xz ${{ needs.build-macos-arm64.outputs.artifact_url }} && \ - curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-runners-all.tar.xz ${{ needs.build-runners-all.outputs.artifact_url }} && \ - curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-macos-arm64-executor.tar.xz ${{ needs.build-macos-arm64.outputs.secondary_artifact_url }} && \ - tar -xf genvm-macos-arm64.tar.xz --no-same-owner && \ - tar -xf genvm-runners-all.tar.xz --no-same-owner && \ - tar -xf genvm-macos-arm64-executor.tar.xz --no-same-owner && \ - popd && \ - true + set -euo pipefail + mkdir -p build/out + pushd build/out + curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-macos-arm64.tar.xz ${{ needs.build-macos-arm64.outputs.artifact_url }} + curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-runners-all.tar.xz ${{ needs.build-runners-all.outputs.artifact_url }} + curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-macos-arm64-executor.tar.xz ${{ needs.build-macos-arm64.outputs.secondary_artifact_url }} + tar -xf genvm-macos-arm64.tar.xz --no-same-owner + tar -xf genvm-runners-all.tar.xz --no-same-owner + tar -xf genvm-macos-arm64-executor.tar.xz --no-same-owner + popd - name: Get wabt run: | - brew install wabt + brew install wabt coreutils - name: Setup Rust run: | rustup target add wasm32-wasip1 @@ -215,18 +218,18 @@ jobs: - name: Run post-install run: | ./build/out/bin/post-install.py \ - --error-on-missing-executor=false \ --default-download=false - name: Smoke test modules startup run: | - brew install coreutils gtimeout 5 ./build/out/bin/genvm-modules llm --allow-empty-backends --config ./build/out/config/genvm-module-llm.yaml || test $? -eq 124 - name: Run stable tests run: | + set -euo pipefail python3 -m venv .venv source .venv/bin/activate - pip install aiohttp jsonnet - ./support/tools/genvm-tool/genvm-tool test run --filter-tag "$(cat tests/presets/release.txt)" --genvm-reroute-to ${{ needs.gen-tag.outputs.tag }} + pip install ./support/tools/genvm-tool + genvm-tool configure + genvm-tool test run --ci --filter-tag "$(cat tests/presets/release.txt)" release-publish: needs: @@ -244,33 +247,18 @@ jobs: steps: - uses: actions/checkout@v4 with: - lfs: true - - run: sudo apt-get install -y python3-poetry - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - cache: poetry - # PyPI publishing disabled for now (genlayer-py-std lives in the - # executor submodule and will be released from its own repo). - # - name: Publish to test pypi - # run: | - # python3.12 -m pip install poetry && \ - # pushd executors/v0.3.x/runners/genlayer-py-std && \ - # perl -i -pe 's/version = "v0.0.1"/version = "${{ needs.gen-tag.outputs.tag }}"/' pyproject.toml && \ - # poetry build && \ - # poetry config repositories.test-pypi https://test.pypi.org/legacy/ && \ - # poetry config pypi-token.test-pypi ${{ secrets.TEST_PYPI_TOKEN }} && \ - # poetry publish -r test-pypi && \ - # popd + # Full history + tags: the release-notes step diffs against the + # previous tag via `git describe`. + fetch-depth: 0 - run: | - curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-runners-all.tar.xz ${{ needs.build-runners-all.outputs.artifact_url }} && \ - curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-linux-amd64.tar.xz ${{ needs.build-linux-amd64.outputs.artifact_url }} && \ - curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-linux-arm64.tar.xz ${{ needs.build-linux-arm64.outputs.artifact_url }} && \ - curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-macos-arm64.tar.xz ${{ needs.build-macos-arm64.outputs.artifact_url }} && \ - curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-linux-amd64-executor.tar.xz ${{ needs.build-linux-amd64.outputs.secondary_artifact_url }} && \ - curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-linux-arm64-executor.tar.xz ${{ needs.build-linux-arm64.outputs.secondary_artifact_url }} && \ - curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-macos-arm64-executor.tar.xz ${{ needs.build-macos-arm64.outputs.secondary_artifact_url }} && \ - true + set -euo pipefail + curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-runners-all.tar.xz ${{ needs.build-runners-all.outputs.artifact_url }} + curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-linux-amd64.tar.xz ${{ needs.build-linux-amd64.outputs.artifact_url }} + curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-linux-arm64.tar.xz ${{ needs.build-linux-arm64.outputs.artifact_url }} + curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-macos-arm64.tar.xz ${{ needs.build-macos-arm64.outputs.artifact_url }} + curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-linux-amd64-executor.tar.xz ${{ needs.build-linux-amd64.outputs.secondary_artifact_url }} + curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-linux-arm64-executor.tar.xz ${{ needs.build-linux-arm64.outputs.secondary_artifact_url }} + curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-macos-arm64-executor.tar.xz ${{ needs.build-macos-arm64.outputs.secondary_artifact_url }} - run: | git tag ${{ needs.gen-tag.outputs.tag }} && \ @@ -303,4 +291,4 @@ jobs: tag_name: ${{ needs.gen-tag.outputs.tag }} body_path: release-notes.md draft: false - prerelease: false + prerelease: ${{ contains(needs.gen-tag.outputs.tag, '-') }} diff --git a/.gitmodules b/.gitmodules index 0b2bd87e..57000707 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,3 +2,7 @@ path = executors/v0.3.x url = ../genvm-executor branch = v0.3.x +[submodule "executors/v0.2.x"] + path = executors/v0.2.x + url = ../genvm-executor + branch = v0.2.x diff --git a/.taplo.toml b/.taplo.toml new file mode 100644 index 00000000..e17930bd --- /dev/null +++ b/.taplo.toml @@ -0,0 +1,12 @@ +[formatting] +align_comments = false +indent_string = " " +reorder_keys = true + +[[rule]] +exclude = ["executors/**"] +include = ["**/Cargo.toml"] +keys = ["package"] + +[rule.formatting] +reorder_keys = false diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..bc887e19 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,17 @@ +# GenVM manager + +Before doing build, test, submodule, release/versioning, or codegen work, +read the matching guide in `docs/contributing/howto/`: + +- [setup.md](docs/contributing/howto/setup.md) β€” first-time clone, nix/direnv dev shell +- [genvm-tool.md](docs/contributing/howto/genvm-tool.md) β€” genvm-tool commands, codegen, plugins +- [building/build.md](docs/contributing/howto/building/build.md) β€” debug build: configure + ninja +- [building/runners.md](docs/contributing/howto/building/runners.md) β€” getting runners +- [building/docs.md](docs/contributing/howto/building/docs.md) β€” website/spec docs, ADRs +- [testing/README.md](docs/contributing/howto/testing/README.md) β€” `genvm-tool test`, continue files (+ [rust.md](docs/contributing/howto/testing/rust.md), [python.md](docs/contributing/howto/testing/python.md), [integration.md](docs/contributing/howto/testing/integration.md), [fuzzing.md](docs/contributing/howto/testing/fuzzing.md)) +- [committing/submodules.md](docs/contributing/howto/committing/submodules.md) β€” multi-repo commits, gitlinks, hooks, push order +- [committing/runners.md](docs/contributing/howto/committing/runners.md) β€” runner hash hygiene before committing (dev-mode, hash-updater.py) +- [committing/git-third-party.md](docs/contributing/howto/committing/git-third-party.md) β€” vendored wasmtime/wasm-tools patch workflow +- [releasing/versioning.md](docs/contributing/howto/releasing/versioning.md) β€” release trains, `.genvm-monorepo-root`, version scripts +- [releasing/release-build.md](docs/contributing/howto/releasing/release-build.md) β€” nix packages, release assets +- [extending/](docs/contributing/howto/README.md#extending) β€” add LLM provider / WASI function / host function, modify a runner or wasmtime diff --git a/README.md b/README.md index 5f4a0ec0..79ffafcb 100644 --- a/README.md +++ b/README.md @@ -33,38 +33,9 @@ This is a monorepo for GenVM. It is composed of the following sub-projects: ## Install -Required tools: - -- git -- ruby (3.\*) -- ninja -- rustup (cargo + rustc) -- (for runners) nix and an x86_64 system - -All of them (except git, for obvious reasons) are provided by the default shell in -`build-scripts/devenv/flake.nix` (for direnv add `use flake ./build-scripts/devenv`). - -### Debug build - -1. `cd $PROJECT_DIR` -2. `git submodule update --init --recursive --depth 1` -3. `source env.sh` (not needed if you used the flake) -4. `git third-party update --all` -5. `genvm-tool configure` β€” scrapes and configures all targets (similar to CMake). - Outside the dev shell use `support/tools/genvm-tool/genvm-tool configure`. -6. `ninja -C build` (or `ninja -C build all/bin`) β€” output is at `build/out` as a - root (`bin`, `share`) -7. Get `genvm-runners.zip` from [GitHub][genvm-manager] -8. Merge `build/out` and `genvm-runners.zip` - -### Production build - -> WARNING: currently supported only on x86_64 Linux hosts. - -1. `cd $PROJECT_DIR` -2. `nix build -o build/out-universal -v -L .#all-for-platform.universal` -3. `nix build -o build/out-amd64-linux -v -L .#all-for-platform.amd64-linux` -4. Merge outputs +See [docs/contributing/howto/setup.md](./docs/contributing/howto/setup.md) and +[docs/contributing/howto/build.md](./docs/contributing/howto/building/build.md) for +environment setup and debug/production build instructions. ## Usage diff --git a/crates/modules-interfaces/.ya-test-config.json b/crates/modules-interfaces/.ya-test-config.json new file mode 100644 index 00000000..93ec8488 --- /dev/null +++ b/crates/modules-interfaces/.ya-test-config.json @@ -0,0 +1,5 @@ +{ + "skip": { + "all": true + } +} diff --git a/crates/modules-interfaces/Cargo.lock b/crates/modules-interfaces/Cargo.lock index 02f67956..b139de89 100644 --- a/crates/modules-interfaces/Cargo.lock +++ b/crates/modules-interfaces/Cargo.lock @@ -41,6 +41,12 @@ version = "1.0.97" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f" +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" + [[package]] name = "arrayvec" version = "0.7.6" @@ -134,9 +140,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.43" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fac4744fb15ae8337dc853fee7fb3f4e48c0fbaa23d0afe49c447b4fab126118" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", "js-sys", @@ -270,28 +276,15 @@ dependencies = [ "syn", ] -[[package]] -name = "genlayer_sdk" -version = "0.0.2" -dependencies = [ - "bytes", - "chrono", - "genlayer_calldata", - "hex", - "num-bigint", - "primitive-types", - "serde", - "serde_bytes", - "sha3", -] - [[package]] name = "genvm-modules-interfaces" version = "0.1.0" dependencies = [ "anyhow", + "arbitrary", + "bytes", + "chrono", "genlayer_calldata", - "genlayer_sdk", "num-bigint", "num-traits", "primitive-types", diff --git a/crates/modules-interfaces/Cargo.toml b/crates/modules-interfaces/Cargo.toml index 0e7653ff..224522e5 100644 --- a/crates/modules-interfaces/Cargo.toml +++ b/crates/modules-interfaces/Cargo.toml @@ -3,16 +3,29 @@ name = "genvm-modules-interfaces" version = "0.1.0" edition = "2021" +[features] +arbitrary = ["dep:arbitrary"] +default = [] + [dependencies] -genlayer_sdk = { path = "../../executors/v0.3.x/executor/crates/sdk-rs", default-features = false } anyhow = "1.0.97" -tokio = { version = "1.44.1", features = ["rt", "rt-multi-thread", "net", "time", "macros", "sync"], default-features = false} -serde = { version = "1.0.219", features = ["rc", "derive"] } -serde_json = "1.0.140" -serde_derive = "1.0.219" -regex = { version = "1.11.1", default-features = false, features = ["std"] } -serde_bytes = "0.11.17" +arbitrary = { version = "1.0", optional = true } +bytes = { version = "1.11.0", features = ["serde"] } +chrono = "0.4.45" genlayer_calldata = { path = "../../executors/v0.3.x/executor/crates/calldata" } num-bigint = "0.4" num-traits = "0.2" primitive-types = { version = "0.13", features = ["impl-serde"] } +regex = { version = "1.11.1", default-features = false, features = ["std"] } +serde = { version = "1.0.219", features = ["rc", "derive"] } +serde_bytes = "0.11.17" +serde_derive = "1.0.219" +serde_json = "1.0.140" +tokio = { version = "1.44.1", features = [ + "rt", + "rt-multi-thread", + "net", + "time", + "macros", + "sync", +], default-features = false } diff --git a/crates/modules-interfaces/src/abi_stub.rs b/crates/modules-interfaces/src/abi_stub.rs new file mode 100644 index 00000000..e829c327 --- /dev/null +++ b/crates/modules-interfaces/src/abi_stub.rs @@ -0,0 +1,131 @@ +use genlayer_calldata::Address; + +#[derive( + Clone, + serde::Deserialize, + serde::Serialize, + genlayer_calldata::Encode, + genlayer_calldata::Decode, + Copy, + PartialEq, + Eq, + Debug, +)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub enum On { + #[serde(rename = "finalized")] + #[calldata(rename = "finalized")] + Finalized, + #[serde(rename = "accepted")] + #[calldata(rename = "accepted")] + Accepted, +} + +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + genlayer_calldata::Encode, + genlayer_calldata::Decode, + serde::Serialize, + serde::Deserialize, +)] +pub struct CallKey( + #[calldata( + serialize_with = ::genlayer_calldata::codec::as_bytes::serialize, + deserialize_with = ::genlayer_calldata::codec::as_bytes::deserialize, + )] + pub [u8; 32], +); + +#[derive( + Debug, + Clone, + serde::Serialize, + serde::Deserialize, + genlayer_calldata::Encode, + genlayer_calldata::Decode, +)] +pub struct MessageData { + pub contract_address: Address, + pub sender_address: Address, + pub origin_address: Address, + pub signer_address: Address, + + pub chain_id: num_bigint::BigInt, + pub value: num_bigint::BigInt, + pub is_init: bool, + /// Transaction timestamp + #[serde(default = "default_datetime")] + #[calldata( + serialize_with = encode_datetime_rfc3339, + deserialize_with = decode_datetime_rfc3339 + )] + #[calldata(default = default_datetime)] + pub datetime: chrono::DateTime, +} + +fn decode_datetime_rfc3339( + val: genlayer_calldata::Value, +) -> Result, genlayer_calldata::codec::DecodeError> { + let genlayer_calldata::Value::Str(s) = val else { + return Err(genlayer_calldata::codec::DecodeError::Unexpected( + "expected string for datetime", + )); + }; + chrono::DateTime::parse_from_rfc3339(&s) + .map(|dt| dt.to_utc()) + .map_err(|e| genlayer_calldata::codec::DecodeError::UserError(Box::new(e))) +} + +fn encode_datetime_rfc3339( + dt: &chrono::DateTime, + enc: &mut genlayer_calldata::Encoder, +) -> Result<(), W::Error> { + use chrono::SecondsFormat; + let s = dt.to_rfc3339_opts(SecondsFormat::AutoSi, true); + enc.push_str(&s) +} + +fn default_datetime() -> chrono::DateTime { + chrono::DateTime::parse_from_rfc3339("2024-11-26T06:42:42.424242Z") + .unwrap() + .to_utc() +} + +#[cfg(feature = "arbitrary")] +impl<'a> arbitrary::Arbitrary<'a> for MessageData { + fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result { + use arbitrary::Arbitrary; + + let ts = u32::arbitrary(u)?; + let Some(datetime) = chrono::DateTime::::from_timestamp_secs(ts as i64) else { + return Err(arbitrary::Error::NotEnoughData); + }; + + let chain_id_bytes: [u8; 32] = Arbitrary::arbitrary(u)?; + let chain_id = primitive_types::U256::from_big_endian(&chain_id_bytes); + + let value_bytes: [u8; 32] = Arbitrary::arbitrary(u)?; + let value = primitive_types::U256::from_big_endian(&value_bytes); + + Ok(Self { + contract_address: Arbitrary::arbitrary(u)?, + sender_address: Arbitrary::arbitrary(u)?, + origin_address: Arbitrary::arbitrary(u)?, + signer_address: Arbitrary::arbitrary(u)?, + chain_id: num_bigint::BigInt::from_bytes_be( + num_bigint::Sign::Plus, + &chain_id.to_big_endian(), + ), + value: num_bigint::BigInt::from_bytes_be( + num_bigint::Sign::Plus, + &value.to_big_endian(), + ), + is_init: bool::arbitrary(u)?, + datetime, + }) + } +} diff --git a/crates/modules-interfaces/src/domain.rs b/crates/modules-interfaces/src/domain.rs new file mode 100644 index 00000000..bba8b895 --- /dev/null +++ b/crates/modules-interfaces/src/domain.rs @@ -0,0 +1,212 @@ +use bytes::Bytes; +use primitive_types::U256; + +pub mod fees; + +fn default_record_actions() -> Vec { + Vec::new() +} + +#[derive( + Debug, + Clone, + serde::Serialize, + serde::Deserialize, + genlayer_calldata::Encode, + genlayer_calldata::Decode, +)] +pub struct ExecutionData { + pub calldata: Bytes, + pub message: super::abi_stub::MessageData, + pub host_data: String, + pub code: Option, + pub leader_nondet_results: Option>, + /// Maps each host method (by index) to a host id. When empty, all methods use host 0. + pub method_hosts: Vec, + pub bucket_totals: Vec, + /// Host-provided `node` fee constants (moved off `host_data`). + pub gas_data: std::collections::BTreeMap, + /// Message-fee allocation tree passed alongside the execution. + pub message_fee_allocation: Vec, + /// Initial time-unit budget for this execution. + pub initial_time_units_allocation: u32, + /// Auditable supervisor action kinds to return in the execution result. + #[serde(default)] + #[calldata(default = default_record_actions)] + pub record_actions: Vec, +} + +#[allow(clippy::enum_variant_names)] +#[derive(Debug, Clone, PartialEq, Eq, genlayer_calldata::Encode)] +#[calldata(tag = "type")] +pub enum ExecutionEmission { + EthSend { + address: genlayer_calldata::Address, + calldata: Bytes, + value: U256, + message_fee: U256, + receipt_fee: U256, + + fee_params: fees::ExternalMessageParams, + }, + PostMessage { + call_key: crate::CallKey, + address: genlayer_calldata::Address, + calldata: genlayer_calldata::codec::Maybe, + value: U256, + on: crate::On, + message_fee: U256, + receipt_fee: U256, + + fee_params: fees::InternalMessageParams, + subtree: bytes::Bytes, + /// Chain `useBalance`: the fee is drawn from the emitting contract's + /// balance rather than the sender's prefunded message-fee pool. + use_balance: bool, + }, + DeployContract { + calldata: genlayer_calldata::codec::Maybe, + code: Bytes, + value: U256, + on: crate::On, + salt_nonce: U256, + message_fee: U256, + receipt_fee: U256, + + fee_params: fees::InternalMessageParams, + subtree: bytes::Bytes, + /// Chain `useBalance`; see `ExecutionEmission::PostMessage::use_balance`. + use_balance: bool, + }, + EmitEvent { + topics: Vec, + blob: genlayer_calldata::codec::Maybe>, + storage_fee: U256, + }, +} + +#[derive( + Debug, + PartialEq, + Clone, + Copy, + serde::Serialize, + serde::Deserialize, + ::genlayer_calldata::Encode, + ::genlayer_calldata::Decode, +)] +#[repr(u8)] +pub enum ResultCode { + Return = 0, + UserError = 1, + VmError = 2, + InternalError = 3, +} + +#[derive(Debug, Clone, serde::Serialize)] +pub struct StorageDelta( + #[serde(with = "serde_bytes")] [u8; 36], + #[serde(with = "serde_bytes")] Vec, +); + +impl StorageDelta { + pub fn new(key: [u8; 36], value: Vec) -> Self { + Self(key, value) + } +} + +impl genlayer_calldata::codec::Encode for StorageDelta { + type Error = W::Error; + + fn encode(&self, enc: &mut genlayer_calldata::Encoder) -> Result<(), Self::Error> { + enc.start_array(2)?; + enc.push_bytes(&self.0)?; + enc.push_bytes(&self.1) + } +} + +#[derive(Debug, Clone, serde::Serialize, PartialEq, Eq, genlayer_calldata::Encode)] +pub struct Frame { + pub module_name: String, + pub func: u32, +} + +/// The wasm call stack captured at the point of a trap. +#[derive(Debug, Clone, serde::Serialize, PartialEq, Eq)] +#[serde(transparent)] +pub struct Backtrace { + pub frames: Vec, +} + +impl genlayer_calldata::codec::Encode for Backtrace { + type Error = W::Error; + + fn encode(&self, enc: &mut genlayer_calldata::Encoder) -> Result<(), Self::Error> { + genlayer_calldata::codec::Encode::encode(&self.frames, enc) + } +} + +#[derive(Debug, Clone, serde::Serialize, PartialEq, Eq)] +pub struct ModuleFingerprint { + pub memories: Vec<[u8; 32]>, +} + +impl genlayer_calldata::codec::Encode for ModuleFingerprint { + type Error = W::Error; + + fn encode(&self, enc: &mut genlayer_calldata::Encoder) -> Result<(), Self::Error> { + enc.start_map(1)?; + enc.push_map_k("memories")?; + enc.start_array(self.memories.len() as u64)?; + for memory in &self.memories { + enc.push_bytes(memory)?; + } + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct WasmStoreHashes(pub std::collections::BTreeMap); + +impl genlayer_calldata::codec::Encode for WasmStoreHashes { + type Error = W::Error; + + fn encode(&self, enc: &mut genlayer_calldata::Encoder) -> Result<(), Self::Error> { + enc.start_map(self.0.len() as u64)?; + for (module, fingerprint) in &self.0 { + enc.push_map_k(module)?; + genlayer_calldata::codec::Encode::encode(fingerprint, enc)?; + } + Ok(()) + } +} + +#[derive(Debug, Clone, Copy, Default, genlayer_calldata::Encode)] +pub struct BucketsConsumed { + pub storage: primitive_types::U256, + pub message_receipt: primitive_types::U256, + pub nondet_output: primitive_types::U256, + pub message_fee: primitive_types::U256, + pub event: primitive_types::U256, +} + +#[derive(Debug, Clone, genlayer_calldata::Encode)] +pub struct ReportedResult { + pub execution_hash: bytes::Bytes, + + pub kind: ResultCode, + pub data: genlayer_calldata::unparsed::Maybe, + pub backtrace: Option, + pub wasm_store_hashes: WasmStoreHashes, + pub storage_changes: Vec, + + pub emissions: Vec, + + pub nondet_disagreement: Option, + pub nondet_results: Vec, + + pub data_fees_remaining: Vec, + pub data_fees_consumed: BucketsConsumed, + + pub llm_consumption: primitive_types::U256, +} diff --git a/crates/modules-interfaces/src/domain/fees/abi.rs b/crates/modules-interfaces/src/domain/fees/abi.rs new file mode 100644 index 00000000..26a99aff --- /dev/null +++ b/crates/modules-interfaces/src/domain/fees/abi.rs @@ -0,0 +1,182 @@ +//! EVM ABI encoding of the message-fee allocation tree into the chain's flat +//! parent-pointer `MessageAllocationNode[]` representation. + +use crate::On; +use primitive_types::U256; + +use super::{ + ExternalMessageParams, InternalMessageParams, MessageAllocationNode, + MessageAllocationNodeParams, +}; + +/// Parent pointer used by top-level (root-layer) allocation nodes in the flat +/// chain representation: `type(uint256).max`. Any other value is a 0-based index +/// into the submitted array. +const NODE_ROOT_SENTINEL: U256 = U256::MAX; + +/// Solidity `enum MessageType { External, Internal }`. +const MESSAGE_TYPE_EXTERNAL: u64 = 0; +const MESSAGE_TYPE_INTERNAL: u64 = 1; + +/// Number of head words in the Solidity `MessageAllocationNode` tuple +/// (messageType, onAcceptance, parentIndex, recipient, callKey, budget, feeParams). +const NODE_HEAD_WORDS: usize = 7; + +/// Number of head words in the Solidity `InternalMessageParams` tuple +/// (leader, validator, appealRounds, executionBudgetPerRound, rotations, +/// maxPriceGenPerTimeUnit, storageFeeMaxGasPrice, receiptFeeMaxGasPrice). +/// `rotations` is the only dynamic field; the three price caps (v0.6-dev, +/// CON-549) are static and follow it in the head region. +const INTERNAL_PARAMS_HEAD_WORDS: usize = 8; + +fn push_u256(buf: &mut Vec, value: U256) { + buf.extend_from_slice(&value.to_big_endian()); +} + +fn push_word(buf: &mut Vec, value: u64) { + push_u256(buf, U256::from(value)); +} + +fn push_bool(buf: &mut Vec, value: bool) { + push_word(buf, value as u64); +} + +/// Left-pads a 20-byte address into a 32-byte ABI word. +fn push_address(buf: &mut Vec, addr: Option<&genlayer_calldata::Address>) { + buf.extend_from_slice(&[0u8; 12]); + match addr { + Some(addr) => buf.extend_from_slice(&addr.raw()), + None => buf.extend_from_slice(&[0u8; 20]), + } +} + +/// `callKey` is already a 32-byte word; `None` is the `CALL_KEY_WILDCARD` = `bytes32(0)`. +fn push_call_key(buf: &mut Vec, call_key: Option<&crate::abi_stub::CallKey>) { + match call_key { + Some(ck) => buf.extend_from_slice(&ck.0), + None => buf.extend_from_slice(&[0u8; 32]), + } +} + +/// ABI-encodes the nested allocation tree as the chain's flat +/// `MessageAllocationNode[]` representation (matching `abi.encode(nodes)`). +pub(super) fn encode(roots: &[MessageAllocationNode]) -> Vec { + // Pre-order flatten: (node, parentIndex). Parents always precede children, + // so the parent's array index is already assigned when a child is visited. + let mut flat: Vec<(&MessageAllocationNode, U256)> = Vec::new(); + fn flatten<'a>( + nodes: &'a [MessageAllocationNode], + parent: U256, + out: &mut Vec<(&'a MessageAllocationNode, U256)>, + ) { + for node in nodes { + let my_index = U256::from(out.len() as u64); + out.push((node, parent)); + flatten(&node.children, my_index, out); + } + } + flatten(roots, NODE_ROOT_SENTINEL, &mut flat); + + // Encode each element as its own self-contained dynamic tuple. + let elements: Vec> = flat + .iter() + .map(|(node, parent_index)| encode_node(node, *parent_index)) + .collect(); + + // `abi.encode(MessageAllocationNode[])`: leading offset (0x20) to the array. + let mut buf = Vec::new(); + push_word(&mut buf, 0x20); + + // Dynamic array of dynamic elements: length, then per-element head offsets + // (relative to the start of the head region, i.e. right after the length), + // then the element tails. + push_word(&mut buf, elements.len() as u64); + let mut offset = elements.len() * 32; + for element in &elements { + push_word(&mut buf, offset as u64); + offset += element.len(); + } + for element in &elements { + buf.extend_from_slice(element); + } + + buf +} + +/// Encodes a single node as the Solidity `MessageAllocationNode` dynamic +/// tuple (offsets are relative to the start of this tuple). +fn encode_node(node: &MessageAllocationNode, parent_index: U256) -> Vec { + let (message_type, on_acceptance, fee_params) = match &node.fee_params { + MessageAllocationNodeParams::Internal(params) => ( + MESSAGE_TYPE_INTERNAL, + matches!(node.on, On::Accepted), + encode_internal_params(params), + ), + // External messages have no acceptance/finalize lifecycle. + MessageAllocationNodeParams::External(params) => { + (MESSAGE_TYPE_EXTERNAL, false, encode_external_params(params)) + } + }; + + let mut buf = Vec::new(); + push_word(&mut buf, message_type); + push_bool(&mut buf, on_acceptance); + push_u256(&mut buf, parent_index); + push_address(&mut buf, node.recipient.as_ref()); + push_call_key(&mut buf, node.call_key.as_ref()); + push_u256(&mut buf, node.budget); + // `feeParams` is the only dynamic field; its offset follows the head words. + push_word(&mut buf, (NODE_HEAD_WORDS * 32) as u64); + // Tail: `bytes feeParams` = length + 32-aligned data. + push_word(&mut buf, fee_params.len() as u64); + buf.extend_from_slice(&fee_params); + let pad = fee_params.len().next_multiple_of(32) - fee_params.len(); + buf.extend(std::iter::repeat_n(0u8, pad)); + + buf +} + +/// `abi.encode(InternalMessageParams)` β€” a dynamic tuple (contains +/// `uint256[] rotations`), so it is prefixed with the offset word. `appealRounds` +/// is reconstructed as `len(rotations) - 1` (the chain-derived value). +/// +/// Field order matches the chain's `InternalMessageFeeParams` (v0.6-dev): +/// leader, validator, appealRounds, executionBudgetPerRound, rotations, +/// maxPriceGenPerTimeUnit, storageFeeMaxGasPrice, receiptFeeMaxGasPrice. The +/// three price caps are static and sit in the head *after* the `rotations` +/// offset word, so the rotations tail starts past all 8 head words. +fn encode_internal_params(params: &InternalMessageParams) -> Vec { + let appeal_rounds = U256::from(params.rotations.len().saturating_sub(1) as u64); + + let mut buf = Vec::new(); + // `abi.encode` of a single dynamic struct prefixes the tuple with its offset. + push_word(&mut buf, 0x20); + push_u256(&mut buf, params.leader_timeunits_allocation); + push_u256(&mut buf, params.validator_timeunits_allocation); + push_u256(&mut buf, appeal_rounds); + push_u256(&mut buf, params.execution_budget_per_round); + // `rotations` is the only dynamic field; its offset follows all head words. + push_word(&mut buf, (INTERNAL_PARAMS_HEAD_WORDS * 32) as u64); + // Static price caps occupy head words 6-8 (between the rotations offset and + // the rotations tail), mirroring the chain struct field order. + push_u256(&mut buf, params.max_price_gen_per_time_unit); + push_u256(&mut buf, params.storage_fee_max_gas_price); + push_u256(&mut buf, params.receipt_fee_max_gas_price); + // Tail: `rotations` length + data, after the 8 head words. + push_word(&mut buf, params.rotations.len() as u64); + for rotation in ¶ms.rotations { + push_u256(&mut buf, *rotation); + } + + buf +} + +/// `abi.encode(ExternalMessageParams)` β€” a fully static tuple, so it is just +/// the two inline words with no leading offset. +fn encode_external_params(params: &ExternalMessageParams) -> Vec { + let mut buf = Vec::new(); + push_u256(&mut buf, params.gas_limit); + push_u256(&mut buf, params.max_gas_price); + + buf +} diff --git a/crates/modules-interfaces/src/domain/fees/mod.rs b/crates/modules-interfaces/src/domain/fees/mod.rs new file mode 100644 index 00000000..0119dcc9 --- /dev/null +++ b/crates/modules-interfaces/src/domain/fees/mod.rs @@ -0,0 +1,149 @@ +pub mod abi; + +use primitive_types::U256; + +use crate::On; + +#[derive( + Debug, + Clone, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + genlayer_calldata::Encode, + genlayer_calldata::Decode, +)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub struct InternalMessageParams { + #[cfg_attr(feature = "arbitrary", arbitrary(with = crate::abi::arb::arb_u256))] + pub leader_timeunits_allocation: U256, + #[cfg_attr(feature = "arbitrary", arbitrary(with = crate::abi::arb::arb_u256))] + pub validator_timeunits_allocation: U256, + #[cfg_attr(feature = "arbitrary", arbitrary(with = crate::abi::arb::arb_u256))] + pub execution_budget_per_round: U256, + /// Per-round rotation allocations; `rotations[0]` is the initial round, the + /// rest are appeal rounds. Must be non-empty. + /// + /// The chain's `InternalMessageFeeParams` carries an explicit `appealRounds` + /// field, but it is not stored here: the chain enforces + /// `appealRounds == rotations.length - 1` (`FeesVerifier.InvalidAppealRounds`), + /// so we derive `appeal_rounds = rotations.len() - 1` instead. The ABI + /// encoder re-inserts it so the encoded bytes (and their `keccak256` fee-param + /// pin) match the chain's field layout. + #[cfg_attr(feature = "arbitrary", arbitrary(with = crate::abi::arb::arb_vec_u256))] + pub rotations: Vec, + /// Per-time-unit GEN price cap locked at activation (consensus CON-549, + /// v0.6-dev). The chain charges at this cap as the funding multiplier and + /// cancels the tx if the global price exceeds it; `MessagePayments` requires + /// it to be non-zero for internal messages. + #[cfg_attr(feature = "arbitrary", arbitrary(with = crate::abi::arb::arb_u256))] + pub max_price_gen_per_time_unit: U256, + /// Max gas price applied to the storage-fee component (v0.6-dev). Revert + /// guard in `_calculateRoundFees`; must be non-zero for internal messages. + #[cfg_attr(feature = "arbitrary", arbitrary(with = crate::abi::arb::arb_u256))] + pub storage_fee_max_gas_price: U256, + /// Max gas price applied to the receipt-fee component (v0.6-dev). Revert + /// guard in `_calculateRoundFees`; must be non-zero for internal messages. + #[cfg_attr(feature = "arbitrary", arbitrary(with = crate::abi::arb::arb_u256))] + pub receipt_fee_max_gas_price: U256, +} + +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + genlayer_calldata::Encode, + genlayer_calldata::Decode, +)] +pub struct ExternalMessageParams { + pub gas_limit: U256, + pub max_gas_price: U256, +} + +#[derive( + Debug, + Clone, + serde::Serialize, + serde::Deserialize, + genlayer_calldata::Encode, + genlayer_calldata::Decode, +)] +pub enum MessageAllocationNodeParams { + Internal(std::sync::Arc), + External(ExternalMessageParams), +} + +/// One node of the message-fee allocation tree. +#[derive( + Debug, + Clone, + serde::Serialize, + serde::Deserialize, + genlayer_calldata::Encode, + genlayer_calldata::Decode, +)] +pub struct MessageAllocationNode { + /// Target contract address; `None` means wildcard (any recipient). + pub recipient: Option, + /// `None` = wildcard: all call keys for this recipient + /// (chain sentinel: `CALL_KEY_WILDCARD` = `bytes32(0)`). + pub call_key: Option, + /// Max budget for matching messages. + pub budget: U256, + pub on: On, + /// Same structure as TX-level params. + pub fee_params: MessageAllocationNodeParams, + pub children: Vec, +} + +impl MessageAllocationNode { + /// ABI-encodes the nested allocation tree as the chain's flat + /// `MessageAllocationNode[]` representation (matching `abi.encode(nodes)`), + /// flattening children to parent-pointer form via pre-order traversal so that + /// every parent precedes its children and their `parentIndex` is well-defined. + pub fn abi_encode(roots: &[MessageAllocationNode]) -> Vec { + abi::encode(roots) + } + + pub fn matches_internal( + &self, + on: On, + recipient: genlayer_calldata::Address, + call_key: crate::abi_stub::CallKey, + ) -> Option> { + let MessageAllocationNodeParams::Internal(params) = &self.fee_params else { + return None; + }; + if on != self.on { + None + } else if self.recipient.as_ref().is_some_and(|r| *r != recipient) { + None + } else if self.call_key.as_ref().is_some_and(|ck| *ck != call_key) { + None + } else { + Some(params.clone()) + } + } + + pub fn matches_external( + &self, + recipient: genlayer_calldata::Address, + call_key: crate::abi_stub::CallKey, + ) -> Option { + let MessageAllocationNodeParams::External(params) = &self.fee_params else { + return None; + }; + if self.recipient.as_ref().is_some_and(|r| *r != recipient) { + None + } else if self.call_key.as_ref().is_some_and(|ck| *ck != call_key) { + None + } else { + Some(*params) + } + } +} diff --git a/crates/modules-interfaces/src/lib.rs b/crates/modules-interfaces/src/lib.rs index 0dd0ba74..3d711cb2 100644 --- a/crates/modules-interfaces/src/lib.rs +++ b/crates/modules-interfaces/src/lib.rs @@ -15,6 +15,11 @@ pub trait Web { ) -> tokio::task::JoinHandle>>; } +pub mod abi_stub; +pub mod domain; +pub use abi_stub::*; +pub use domain::*; + #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(untagged)] pub enum GenericValue { @@ -294,12 +299,235 @@ impl Decode for Result { pub mod llm { use std::collections::BTreeMap; + use genlayer_calldata::{Decode, Encode}; use serde_derive::{Deserialize, Serialize}; - pub use genlayer_sdk::abi::gl_call::llm_iface::{ - OutputFormat, PromptEqComparativePayload, PromptEqNonComparativeLeaderPayload, - PromptEqNonComparativeValidatorPayload, PromptPayload, PromptTemplatePayload, - }; + /// Output format for LLM prompt responses. + #[derive(Clone, Deserialize, Serialize, Encode, Decode, Copy, PartialEq, Eq, Debug)] + pub enum OutputFormat { + #[serde(rename = "text")] + #[calldata(rename = "text")] + Text, + #[serde(rename = "json")] + #[calldata(rename = "json")] + JSON, + } + + fn default_text() -> OutputFormat { + OutputFormat::Text + } + + /// Payload for ExecPrompt operations. + #[derive(Clone, PartialEq, Serialize, Deserialize, Encode, Decode, Debug)] + pub struct PromptPayload { + #[serde(default = "default_text")] + #[calldata(default = default_text)] + pub response_format: OutputFormat, + pub prompt: String, + pub images: Vec, + } + + #[derive(Clone, PartialEq, Serialize, Deserialize, Encode, Decode, Debug)] + pub struct PromptEqComparativePayload { + pub leader_answer: String, + pub validator_answer: String, + pub principle: String, + } + + #[derive(Clone, PartialEq, Serialize, Deserialize, Encode, Decode, Debug)] + pub struct PromptEqNonComparativeValidatorPayload { + pub task: String, + pub criteria: String, + pub input: String, + pub output: String, + } + + #[derive(Clone, PartialEq, Serialize, Deserialize, Encode, Decode, Debug)] + pub struct PromptEqNonComparativeLeaderPayload { + pub task: String, + pub criteria: String, + pub input: String, + } + + /// Payload for ExecPromptTemplate operations. + #[derive(Clone, PartialEq, Serialize, Deserialize, Debug)] + #[serde(tag = "template")] + pub enum PromptTemplatePayload { + EqComparative(PromptEqComparativePayload), + EqNonComparativeValidator(PromptEqNonComparativeValidatorPayload), + EqNonComparativeLeader(PromptEqNonComparativeLeaderPayload), + } + + const TEMPLATE_TAG: &str = "template"; + + impl genlayer_calldata::codec::Encode for PromptTemplatePayload { + type Error = W::Error; + + fn encode(&self, enc: &mut genlayer_calldata::Encoder) -> Result<(), W::Error> { + match self { + // fields sorted: leader_answer, principle, template, validator_answer + PromptTemplatePayload::EqComparative(p) => { + enc.start_map(4)?; + enc.push_map_k("leader_answer")?; + p.leader_answer.encode(enc)?; + enc.push_map_k("principle")?; + p.principle.encode(enc)?; + enc.push_map_k(TEMPLATE_TAG)?; + enc.push_str("EqComparative")?; + enc.push_map_k("validator_answer")?; + p.validator_answer.encode(enc)?; + } + // fields sorted: criteria, input, output, task, template + PromptTemplatePayload::EqNonComparativeValidator(p) => { + enc.start_map(5)?; + enc.push_map_k("criteria")?; + p.criteria.encode(enc)?; + enc.push_map_k("input")?; + p.input.encode(enc)?; + enc.push_map_k("output")?; + p.output.encode(enc)?; + enc.push_map_k("task")?; + p.task.encode(enc)?; + enc.push_map_k(TEMPLATE_TAG)?; + enc.push_str("EqNonComparativeValidator")?; + } + // fields sorted: criteria, input, task, template + PromptTemplatePayload::EqNonComparativeLeader(p) => { + enc.start_map(4)?; + enc.push_map_k("criteria")?; + p.criteria.encode(enc)?; + enc.push_map_k("input")?; + p.input.encode(enc)?; + enc.push_map_k("task")?; + p.task.encode(enc)?; + enc.push_map_k(TEMPLATE_TAG)?; + enc.push_str("EqNonComparativeLeader")?; + } + } + Ok(()) + } + } + + impl genlayer_calldata::codec::Decode for PromptTemplatePayload { + fn decode( + deserializer: D, + ) -> Result { + use genlayer_calldata::codec::{DecodeError, MapAccess, ValueDeserializer, Visitor}; + use genlayer_calldata::Value; + use std::collections::BTreeMap; + + struct V; + impl Visitor for V { + type Value = PromptTemplatePayload; + + fn visit_map( + self, + _len: u64, + mut map: A, + ) -> Result { + let mut entries = BTreeMap::::new(); + while let Some((key, val)) = map.next_element::()? { + entries.insert(key.to_owned(), val); + } + + let tag_val = entries + .remove(TEMPLATE_TAG) + .ok_or(DecodeError::FieldMissing(TEMPLATE_TAG))?; + let Value::Str(tag_str) = tag_val else { + return Err(DecodeError::Unexpected("expected string for template tag")); + }; + + match tag_str.as_str() { + "EqComparative" => { + let leader_answer = entries + .remove("leader_answer") + .ok_or(DecodeError::FieldMissing("leader_answer"))?; + let validator_answer = entries + .remove("validator_answer") + .ok_or(DecodeError::FieldMissing("validator_answer"))?; + let principle = entries + .remove("principle") + .ok_or(DecodeError::FieldMissing("principle"))?; + Ok(PromptTemplatePayload::EqComparative( + PromptEqComparativePayload { + leader_answer: genlayer_calldata::codec::Decode::decode( + ValueDeserializer(leader_answer), + )?, + validator_answer: genlayer_calldata::codec::Decode::decode( + ValueDeserializer(validator_answer), + )?, + principle: genlayer_calldata::codec::Decode::decode( + ValueDeserializer(principle), + )?, + }, + )) + } + "EqNonComparativeValidator" => { + let task = entries + .remove("task") + .ok_or(DecodeError::FieldMissing("task"))?; + let criteria = entries + .remove("criteria") + .ok_or(DecodeError::FieldMissing("criteria"))?; + let input = entries + .remove("input") + .ok_or(DecodeError::FieldMissing("input"))?; + let output = entries + .remove("output") + .ok_or(DecodeError::FieldMissing("output"))?; + Ok(PromptTemplatePayload::EqNonComparativeValidator( + PromptEqNonComparativeValidatorPayload { + task: genlayer_calldata::codec::Decode::decode( + ValueDeserializer(task), + )?, + criteria: genlayer_calldata::codec::Decode::decode( + ValueDeserializer(criteria), + )?, + input: genlayer_calldata::codec::Decode::decode( + ValueDeserializer(input), + )?, + output: genlayer_calldata::codec::Decode::decode( + ValueDeserializer(output), + )?, + }, + )) + } + "EqNonComparativeLeader" => { + let task = entries + .remove("task") + .ok_or(DecodeError::FieldMissing("task"))?; + let criteria = entries + .remove("criteria") + .ok_or(DecodeError::FieldMissing("criteria"))?; + let input = entries + .remove("input") + .ok_or(DecodeError::FieldMissing("input"))?; + Ok(PromptTemplatePayload::EqNonComparativeLeader( + PromptEqNonComparativeLeaderPayload { + task: genlayer_calldata::codec::Decode::decode( + ValueDeserializer(task), + )?, + criteria: genlayer_calldata::codec::Decode::decode( + ValueDeserializer(criteria), + )?, + input: genlayer_calldata::codec::Decode::decode( + ValueDeserializer(input), + )?, + }, + )) + } + other => Err(DecodeError::UnknownVariant { + got: other.to_owned(), + expected: + "EqComparative, EqNonComparativeValidator, EqNonComparativeLeader", + }), + } + } + } + + deserializer.deserialize(V) + } + } #[derive(Serialize, Deserialize, genlayer_calldata::Encode, genlayer_calldata::Decode)] pub enum Message { @@ -354,11 +582,202 @@ pub mod llm { } pub mod web { + use genlayer_calldata::{Decode, Encode}; use serde_derive::{Deserialize, Serialize}; + use std::collections::BTreeMap; - pub use genlayer_sdk::abi::gl_call::web_iface::{ - RenderPayload, RequestMethod, RequestPayload, Response, - }; + /// Render mode for WebRender operations. + #[derive(Clone, PartialEq, Serialize, Deserialize, Encode, Decode, Debug)] + pub enum RenderMode { + #[serde(rename = "text")] + #[calldata(rename = "text")] + Text, + #[serde(rename = "html")] + #[calldata(rename = "html")] + HTML, + #[serde(rename = "screenshot")] + #[calldata(rename = "screenshot")] + Screenshot, + } + + /// Duration to wait after page load before capturing content. + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] + pub enum WaitAfterLoaded { + Seconds(u64), + Millis(u64), + } + + impl WaitAfterLoaded { + pub fn as_secs_f64(&self) -> f64 { + match self { + WaitAfterLoaded::Seconds(s) => *s as f64, + WaitAfterLoaded::Millis(ms) => *ms as f64 / 1000.0, + } + } + } + + struct WaitAfterLoadedVisitor; + + impl serde::de::Visitor<'_> for WaitAfterLoadedVisitor { + type Value = WaitAfterLoaded; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("expected string | null") + } + + fn visit_none(self) -> std::result::Result + where + E: serde::de::Error, + { + Ok(WaitAfterLoaded::Millis(0)) + } + + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + if let Some(ms_str) = value.strip_suffix("ms") { + let millis = ms_str.parse::().map_err(E::custom)?; + Ok(WaitAfterLoaded::Millis(millis)) + } else if let Some(secs_str) = value.strip_suffix("s") { + let seconds = secs_str.parse::().map_err(E::custom)?; + Ok(WaitAfterLoaded::Seconds(seconds)) + } else { + Err(E::invalid_value( + serde::de::Unexpected::Str(value), + &"expected a string ending with 's' or 'ms'", + )) + } + } + } + + impl<'de> serde::Deserialize<'de> for WaitAfterLoaded { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_str(WaitAfterLoadedVisitor) + } + } + + impl serde::Serialize for WaitAfterLoaded { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + match self { + WaitAfterLoaded::Seconds(v) => { + let as_str = format!("{}s", v); + serializer.serialize_str(&as_str) + } + WaitAfterLoaded::Millis(v) => { + let as_str = format!("{}ms", v); + serializer.serialize_str(&as_str) + } + } + } + } + + fn encode_wait_after_loaded( + val: &WaitAfterLoaded, + enc: &mut genlayer_calldata::Encoder, + ) -> Result<(), W::Error> { + let s = match val { + WaitAfterLoaded::Seconds(v) => format!("{v}s"), + WaitAfterLoaded::Millis(v) => format!("{v}ms"), + }; + enc.push_str(&s) + } + + fn decode_wait_after_loaded( + val: genlayer_calldata::Value, + ) -> Result { + let genlayer_calldata::Value::Str(s) = val else { + return Err(genlayer_calldata::codec::DecodeError::Unexpected( + "expected string", + )); + }; + if let Some(ms_str) = s.strip_suffix("ms") { + let millis = ms_str + .parse::() + .map_err(|e| genlayer_calldata::codec::DecodeError::UserError(Box::new(e)))?; + Ok(WaitAfterLoaded::Millis(millis)) + } else if let Some(secs_str) = s.strip_suffix("s") { + let seconds = secs_str + .parse::() + .map_err(|e| genlayer_calldata::codec::DecodeError::UserError(Box::new(e)))?; + Ok(WaitAfterLoaded::Seconds(seconds)) + } else { + Err(genlayer_calldata::codec::DecodeError::Unexpected( + "expected string ending with 's' or 'ms'", + )) + } + } + + fn default_none() -> Option { + None + } + + fn default_false() -> bool { + false + } + + /// Payload for WebRender operations. + #[derive(Clone, PartialEq, Serialize, Deserialize, Encode, Decode, Debug)] + pub struct RenderPayload { + pub mode: RenderMode, + pub url: String, + #[calldata( + serialize_with = encode_wait_after_loaded, + deserialize_with = decode_wait_after_loaded + )] + pub wait_after_loaded: WaitAfterLoaded, + } + + /// HTTP request method for WebRequest operations. + #[derive(Clone, PartialEq, Debug, Serialize, Deserialize, Encode, Decode)] + pub enum RequestMethod { + GET, + POST, + HEAD, + PUT, + DELETE, + OPTIONS, + PATCH, + } + + /// HTTP response from WebRequest or WebRender operations. + #[derive(Clone, PartialEq, Debug, Serialize, Deserialize, Encode, Decode)] + pub struct Response { + pub status: u16, + pub headers: BTreeMap, + + #[serde(with = "serde_bytes")] + #[calldata( + serialize_with = genlayer_calldata::codec::as_bytes::serialize, + deserialize_with = genlayer_calldata::codec::as_bytes::deserialize, + )] + pub body: Vec, + } + + /// Payload for WebRequest operations. + #[derive(Clone, PartialEq, Serialize, Deserialize, Encode, Decode, Debug)] + pub struct RequestPayload { + pub method: RequestMethod, + pub url: String, + pub headers: BTreeMap, + + #[serde(with = "serde_bytes", default = "default_none")] + #[calldata( + default = default_none, + serialize_with = genlayer_calldata::codec::as_bytes::serialize, + deserialize_with = genlayer_calldata::codec::as_bytes::deserialize, + )] + pub body: Option>, + #[serde(default = "default_false")] + #[calldata(default = default_false)] + pub sign: bool, + } #[derive(Serialize, Deserialize, genlayer_calldata::Encode, genlayer_calldata::Decode)] pub enum Message { diff --git a/docs/adr/000. WASI.md b/docs/adr/000. WASI.md index 80a0098b..59b47138 100644 --- a/docs/adr/000. WASI.md +++ b/docs/adr/000. WASI.md @@ -21,7 +21,9 @@ Implement WASI Preview 1 (wasip1) with deterministic API modifications. - Any language that supports wasip1 target will work without major issues - Supports VFS for adding multiple files, for instance to load Python modules -- Explicitly bans random/time operations, maintaining determinism +- Virtualizes random/time operations deterministically instead of exposing the host's: + the clock returns the message datetime and deterministic `random_get` is a seeded + MT19937 (seeded from the sha3-256 of the VM inputs), maintaining determinism - Provides a deterministic execution environment ### Negative @@ -42,4 +44,4 @@ Implement WASI Preview 1 (wasip1) with deterministic API modifications. ## Implementation Notes -The implementation focuses on maintaining determinism while providing the standard WASI interface. Special attention is paid to filtering out non-deterministic operations like random number generation and time access. +The implementation focuses on maintaining determinism while providing the standard WASI interface. Special attention is paid to non-deterministic operations like random number generation and time access: rather than being banned, they are virtualized to deterministic sources (the message datetime for the clock, a seeded MT19937 for `random_get`) so contract code sees a stable, reproducible environment. diff --git a/docs/adr/012. runner memory accounting.md b/docs/adr/012. runner memory accounting.md new file mode 100644 index 00000000..0571d2aa --- /dev/null +++ b/docs/adr/012. runner memory accounting.md @@ -0,0 +1,224 @@ +# ADR-012: The Runner Load Action β€” Per-VM Loaded Sets and Scope-Local Charging + +## Status + +- **Status**: Accepted +- **Date**: 2026-07-06 + +## Context + +The runner archive cache (`executor/src/runners/cache.rs`, introduced with +[ADR-002](<002. code loading.md>) and extended by +[ADR-011](<011. runner ids and runtime registration.md>)) deliberately +*overcharges*: every VM that loads an archive is charged its full size against +the VM's memory limiter, whether the load was a cache miss or a hit. However, +cache entries are never freed, while per-VM limiters refund all consumed +memory to their pool when the VM dies. The result is an **undercharge over +time**: a contract can sequentially spawn sub-VMs, each loading *distinct* +runners β€” `chain:` and `custom:` ids are attacker-controlled and unbounded β€” +each staying within budget, each refunding its budget at death while the +archive bytes stay resident. Executor RSS grows without bound past the +accounted limit. + +Custom runners had additional problems: the registry was charged permanently +against the shared pool (an awkward special case), the secondary nondet +validator queue ran on a fresh limiter blind to the resident registry, and +visibility leaked β€” sandboxes inherited the parent's whole set *and* leaked +their own registrations back to the parent, while nondet VMs were blanket- +banned from custom runners because charging for them was unsolved. + +More fundamentally, charging was entangled with cache state (hit vs miss), +which is schedule-dependent and therefore a standing determinism hazard. + +## Decision + +### The load action + +Each VM owns a **loaded-runner set**: a plain map from canonical runner id to +a strong content reference (**pin**). All runner acquisition goes through one +**load action**, defined over this set: + +1. If the id is already in the VM's loaded set β†’ **no charge**, done. +2. Otherwise: charge `RUNNER_LOAD_COST + size` to the VM's memory limiter (OOM + error on failure), materialize or attach to the content, and insert the + pin into the loaded set. + +`size` is the archive's `total_size`, which for every source equals the raw +content length (disk: tar length; `chain:`: the code blob length, readable +from its 4-byte prefix *before* the blob is fetched; `custom:`: the +registered code length). The charge always precedes materialization, and +parsing is zero-copy (archive entries are slices of the charged buffer), so +the charged amount is also the peak. + +`RUNNER_LOAD_COST` is a flat per-load constant covering parse/bookkeeping overhead. +It belongs in `public-abi`; **for now it lives in `host-fns`** because +regenerating public-abi constants would change runner hashes and the Python +SDK is frozen for this change. + +**Triggers** β€” the complete list of places a load action occurs: + +- **spawning the main runner** β€” the VM's entry-point runner tree at init; +- **`Depends`** and **`With`** in a `runner.json`; +- **the `MapFile` gl_call**; +- **`RegisterRunner`** β€” parses the code, performs the load action for the + resulting `custom:`, and indexes the content in a weak registry for + cross-scope dedup; +- **inheriting** β€” at sub-VM spawn, the child performs a load action for + every custom runner granted to it (see below). Inherit-loads execute + *before* the main-runner load (normative order), so a custom entry point is + already loaded and is not charged twice. + +Because "already loaded" is a function of the VM's **own history**, charges +are deterministic by construction: the shared cache/registry is a pure dedup +mechanism and is **unobservable** β€” a VM cannot tell whether its load +materialized the content or attached to an existing copy (equal charge, equal +peak, content-deterministic errors either way; on a failed materialization +the shared cell stays empty and the next loader retries with its own +closure). + +### Content lifetime + +The shared archive cache and the custom-runner registry hold **weak** +references; pins in loaded sets are the only strong ones. Content is freed +when the last pin drops β€” a VM's limiter refund at death coincides with its +archives becoming freeable. Invariant: + +> **Resident archive bytes are covered by the live charges of the VMs whose +> loaded sets pin them.** + +There are no permanent charges and no registry pre-charges: the old +pool-permanent `RegisterRunner` charge and the secondary-nondet-queue +registry pre-charge are both removed. Validator-queue nondet VMs carry their +granted pins from the `RunNondet` call site, so the content they need +survives the deterministic VM's death exactly as long as someone who paid for +it exists. + +### Custom runners: registration and grants + +- `RegisterRunner` charges the **current VM** (`RUNNER_LOAD_COST + code.len()`, + before parsing). The content lives while any loaded set pins it; dedup by + hash applies only while alive; re-registering dead content re-parses and + re-charges identically. Nothing is hoisted: a runner registered in a + sandbox dies with the sandbox subtree. +- `custom:` **resolves against the VM's own loaded set only**. There is + no registry lookup at resolution β€” a scope can use exactly what it loaded + (registered or inherited), which yields isolation by construction. The old + rpds visibility set is deleted; the loaded set subsumes it. +- **Grants**: `Sandbox` and `RunNondet` gl_calls take an optional + `custom_runners: Option>`. `None` grants every `custom:` + entry in the parent's loaded set; `Some(list)` grants exactly the list β€” + every element must be a `custom:` id, without duplicates, present in the + parent's loaded set; anything else (including `name:hash`/`chain:` ids, + which are never inherited) is a deterministic malformed-runner error. + `RunNondet` also takes an optional `runner` (what to execute, default + `contract`, resolved in the parent; it becomes the child's `contract`); a + `custom:` target must be loaded in the parent and is granted implicitly. +- `CallContract` children are granted the caller's full custom set. + +### Determinism surface + +The deterministic execution fingerprint additionally folds in the sequence of +canonical runner ids charged by det-mode load actions. Two validators whose +det VMs load different runner sets diverge at result comparison directly, +instead of only when an OOM boundary happens to differ. (Consensus-visible; +ships together with the gl_call payload changes.) + +### Observability + +Every load action emits one structured log line with a stable message +(`runner load`), carrying the runner id, `RUNNER_LOAD_COST`, `size`, and whether it +was charged or already loaded. Log target is the crate path. Integration +tests (and operators) assert charges by grepping these lines; the test +harness machinery for that is added alongside. + +### Python SDK unchanged + +The new gl_call fields are decode-optional (calldata defaults), so current +Python SDK payloads remain valid and runner hashes/caches are untouched. SDK +surface comes later in a runner release. + +## Consequences + +### Positive + +- Resident archive memory is always backed by a live charge; the + unbounded-RSS attack is closed for archives *and* custom-runner content + (the old permanent-registry leak included). +- One charging rule everywhere: free if in my loaded set, else + `RUNNER_LOAD_COST + size`, once per VM per runner. No hit/miss compensation, no + receipts, no permanent charges, no special pools. +- Determinism is structural: charges depend only on a VM's own load history. +- Custom-runner isolation is structural: resolution reads only the VM's own + loaded set; no flow-back, nondet scopes get exactly what they were granted. +- Repeat loads within a VM stop double-charging (previously every archive + request charged again). + +### Negative + +- A child pays for every granted custom runner at spawn even if it never + uses it (bounded by `VM_RECURSION` Γ— grant size; deliberate overcharge β€” + narrow the grant list to avoid it). +- Evicted archives are re-read and re-parsed on next use (disk I/O only; + charges unaffected). +- Content registered by a dead subtree must be re-registered (re-parse + + re-charge) to be used again β€” dedup is only as durable as its users. + +### Neutral + +- The `chain:` double-charge (`2Γ—code_size` per load) present in earlier + iterations is gone: size is known from the length prefix before the blob is + charged and parsing is zero-copy, so a single `size` charge covers the peak. +- Storage *reads* consume no data-fee pages in the current fee schedule, so + `chain:` resolution carries no fee component. If reads ever become metered, + the load action's cache-independence must be revisited (the fee-schedule + change owns that). +- The wasm **module** cache (compiled wasmtime modules) is out of scope: + still strongly held and uncharged β€” tracked as follow-up (next ADR). +- A future `gl_call` that loads a runner's bytes into a VFS file would let a + contract re-register its own (or a builtin) runner as `custom:` and pass it + down β€” the clean solution to "hand my main runner to a child". Nothing + here blocks it. + +## Alternatives Considered + +- **Permanent (non-refundable) per-load charges, no freeing**: converts the + undercharge into cumulative permanent draining; long executions OOM. + Rejected. +- **Cache flush at deterministic barriers**: sub-VMs run during the det + phase, so unbounded accumulation within a phase remains. Rejected. +- **Hit/miss-symmetric charging with creation receipts** (the previous + iteration of this ADR): sound, but symmetry by *compensation* β€” the charge + path had to know hit vs miss, and `chain:` loads double-charged. The loaded + set gives symmetry by *structure* and is strictly simpler. Superseded. +- **Permanent pool charge for `RegisterRunner`** (previous iteration): + special-cased one runner kind, leaked dead content until execution end, and + required a secondary-queue pre-charge patch. Superseded by scope-local + content lifetime. +- **Inheriting `name:hash`/`chain:` pins at spawn**: charges children for + parent state they rarely need; every VM already pays for its own tree at + init. Only custom runners are inheritable (they have no other path into a + scope). Rejected. + +## Implementation Notes + +- Loaded set: plain per-VM `HashMap` living next to the + VM's limiter in its store data (not in the accumulator β€” it must not + round-trip through sandbox children). It is both the charge-dedup set and + the pin holder; those must never be split. +- Weak cache (`runners/cache.rs`) and weak registry + (`Supervisor.custom_runners`) with lazy dead-entry purge; teardown + debug-asserts no live entries. +- `chain:` size discovery: read the 4-byte length prefix (one slot read), + charge, then read the blob; attach path charges the live entry's + `total_size` (equal by content-determinism). +- `RegisterRunner`: charge `RUNNER_LOAD_COST + code.len()` before `runners::parse` + (closes the previously-uncharged parse window). +- Grant validation happens in the parent at gl_call time (deterministic on + both leader and validator paths); queued `RunNondet` tasks carry granted + pins in their `SingleVMData`. +- Heavy debug-only invariant checkers (loaded-set/pin agreement, weak-map + consistency, charge recomputation) guard the accounting at every mutation + point. +- Spec: gl_call payloads, the permissions note, the runners page (load + action, loaded set, custom lifetime), the RAM-limiting page (charge rule), + and `memory_limiter_consts.runner_load_cost` in the public ABI constants. diff --git a/docs/contributing/README.md b/docs/contributing/README.md index 28671785..4c044f50 100644 --- a/docs/contributing/README.md +++ b/docs/contributing/README.md @@ -1,8 +1,11 @@ # Contributing to GenVM development ## Sub pages -- [running tests](./tests.md) -- [default workflows](./workflows.md) +- [how-to guides](./howto/README.md) β€” setup, build, test, submodules, versioning, extension recipes, … ## PR requirements Main requirement is that all tests must pass. It includes [pre-commit](https://pre-commit.com) and test suites. PRs are merged via a queue, that executes all tests and merges *iff* they all pass. It is done to ensure that `HEAD` of `main` branch is always stable + +## Security +GenVM is consensus-critical. Do not open public issues for vulnerabilities β€” +follow [SECURITY.md](../../SECURITY.md) (private reporting). diff --git a/docs/contributing/howto/README.md b/docs/contributing/howto/README.md new file mode 100644 index 00000000..050379a8 --- /dev/null +++ b/docs/contributing/howto/README.md @@ -0,0 +1,31 @@ +# How-to guides + +- [setup.md](setup.md) β€” first-time clone: submodule init, nix/direnv dev shell, third-party materialization. +- [genvm-tool.md](genvm-tool.md) β€” command overview, codegen data β†’ generated files, plugin/test-definition layout. + +## building/ +- [build.md](building/build.md) β€” debug build: configure + ninja, outputs, cargo quirks. +- [runners.md](building/runners.md) β€” where runners come from (build vs download). +- [docs.md](building/docs.md) β€” building the website, spec vs impl-spec, ADRs. + +## testing/ +- [testing/README.md](testing/README.md) β€” `genvm-tool test` itself: filters, presets, continue files. +- [rust.md](testing/rust.md) β€” Rust tests, `#rust-test` shell, coverage. +- [python.md](testing/python.md) β€” Python tests, direct pytest for genlayer-py-std. +- [integration.md](testing/integration.md) β€” running one executor line, `stable` (offline) subset, golden `.stdout`/`.hash` files, `--ignore-hash`, services. +- [fuzzing.md](testing/fuzzing.md) β€” AFL fuzz targets, host sysctl prep. + +## committing/ +- [submodules.md](committing/submodules.md) β€” manager + executor lines, gitlink bumps, pre-commit hook, push order. +- [git-third-party.md](committing/git-third-party.md) β€” how vendored trees (wasmtime, …) are tracked, edited, and committed as patches. + +## releasing/ +- [versioning.md](releasing/versioning.md) β€” release trains, `.genvm-monorepo-root`, check-versions/branch-versions scripts. +- [release-build.md](releasing/release-build.md) β€” nix packages, platforms, release assets. + +## extending/ +- [add-llm-provider.md](extending/add-llm-provider.md) β€” new LLM backend in the manager. +- [add-wasi-function.md](extending/add-wasi-function.md) β€” new gl_call method or raw WASI function in an executor. +- [add-host-function.md](extending/add-host-function.md) β€” new executor↔host protocol method. +- [modify-runner.md](extending/modify-runner.md) β€” runner dev-mode and hash refresh. +- [modify-wasmtime.md](extending/modify-wasmtime.md) β€” patching vendored wasmtime, trap plumbing. diff --git a/docs/contributing/howto/building/build.md b/docs/contributing/howto/building/build.md new file mode 100644 index 00000000..32ea2266 --- /dev/null +++ b/docs/contributing/howto/building/build.md @@ -0,0 +1,40 @@ +# Build (debug) + +All commands assume the dev shell (`nix develop '.?submodules=1#full'`, usually +via direnv) β€” it provides `genvm-tool`, `ninja`, `cargo`, `python3`. +First-time clone: [setup.md](../setup.md). + +```bash +genvm-tool configure # generates build/build.ninja + build/info.json (like CMake) +ninja -C build all/bin +``` + +Outside the dev shell call `support/tools/genvm-tool/genvm-tool configure`. + +Re-run `genvm-tool configure` after adding/removing/renaming source files β€” +symptom if you forget: ninja `missing and no known rule to make it`. + +| Ninja target | Builds | +|---|---| +| `all` | everything (incl. runners; x86_64 Linux only) | +| `all/bin` | all Rust binaries | +| `all/data` | runner data via nix | +| `codegen` | code generation (see [genvm-tool.md](../genvm-tool.md)) | + +Outputs under `build/out`: + +- `bin/genvm-modules` β€” modules binary +- `executor//bin/genvm` β€” one dir per built executor version; the + version comes from `executors/.x/manifest.json` and is recorded in + `build/info.json` + +Release/nix packages: [release-build.md](../releasing/release-build.md). +Getting or changing runners: [runners.md](runners.md), [modify-runner.md](../extending/modify-runner.md). + +## Cargo notes + +- Plain `cargo` inside a submodule needs the dev-shell env (lua, pkg-config, …); + prefer the ninja build. +- In this nix environment, `cargo check` inside an executor's `executor/` dir + may need `LD_LIBRARY_PATH="$(nix eval --raw nixpkgs#zlib)/lib"` so rustc + finds libz. diff --git a/docs/contributing/howto/building/docs.md b/docs/contributing/howto/building/docs.md new file mode 100644 index 00000000..aac6e0f0 --- /dev/null +++ b/docs/contributing/howto/building/docs.md @@ -0,0 +1,35 @@ +# Docs + +## Website (`docs/website`) + +Sphinx site with a poetry env in that dir: + +```bash +cd docs/website +poetry install --no-root +poetry run sphinx-build -q -b dirhtml src +``` + +CI builds `-b html` and `-b text` β€” see `support/ci/pipelines/src/docs.sh` +(also merges the text output into `_static/ai/*.txt`). + +Source layout under `docs/website/src/`: + +- `spec/` β€” the protocol specification (observable behavior). + `spec/appendix/constants.rst` is **generated** from codegen data β€” do not + edit by hand, see [genvm-tool.md](../genvm-tool.md). +- `impl-spec/` β€” the implementation specification (how this codebase does it). +- `python-sdk/`, `overview/` β€” SDK API docs and intro material. + +`docs/website/generate.py` regenerates derived pages (runner/version tables) +from the repo state; wired into the build via `docs/website/yabuild.rb`. + +## ADRs + +Architecture decision records live in [`docs/adr/`](../../../adr/) as numbered markdown files +(`000. WASI.md`, …); start new ones from [`docs/adr/_template.md`](../../../adr/_template.md). + +## Contributor docs + +[`docs/contributing/`](../..) β€” PR requirements and workflows; how-to guides are in +this directory. diff --git a/docs/contributing/howto/building/runners.md b/docs/contributing/howto/building/runners.md new file mode 100644 index 00000000..abec0b6f --- /dev/null +++ b/docs/contributing/howto/building/runners.md @@ -0,0 +1,17 @@ +# Runners + +Runners build only on x86_64 Linux (part of the `all` ninja target / +`#runners-all` nix attr). On other hosts, download them: + +```bash +python3 build/out/bin/post-install.py --create-venv false --default-step false --runners-download true --error-on-missing-executor false +``` + +or fetch `genvm-runners-all.tar.xz` from a +[genvm-manager release](https://github.com/genlayerlabs/genvm-manager/releases) +and extract it over `build/out`. + +Do **not** build runners natively on macOS β€” deterministic runner artifacts +require a Linux builder (remote nix builder or download). + +Changing a runner: [modify-runner.md](../extending/modify-runner.md). diff --git a/docs/contributing/howto/committing/git-third-party.md b/docs/contributing/howto/committing/git-third-party.md new file mode 100644 index 00000000..020e8035 --- /dev/null +++ b/docs/contributing/howto/committing/git-third-party.md @@ -0,0 +1,54 @@ +# git-third-party (vendored trees) + +Vendored third-party sources (e.g. `executors/v0.3.x/executor/third-party/wasmtime`, +`.../wasm-tools`) are **not** tracked by the executor repo β€” they are +git-ignored (`executor/.gitignore: third-party`) and materialized on demand. +What *is* tracked, per executor line, is `.git-third-party/`: + +- `.git-third-party/config.json` β€” per repo: upstream `url`, pinned base + `commit`, `patches` count, optional `submodules` list (absent = update all + submodules, `[]` = none, list = only those). +- `.git-third-party/patches//` β€” numbered `git format-patch` + files (1-based, mbox format) applied on top of the base commit. + +The tool is `support/tools/git-third-party/git-third-party` (single Python +script; on PATH via `env.sh` and the dev shell; invoked as `git third-party`). +It has **no `--help`** β€” running with no subcommand prints usage. Subcommands: + +| Command | Effect | +|---|---| +| `git third-party add ` | register a new vendored repo (path must already be git-ignored) and materialize it | +| `git third-party update {--all \| ...}` | materialize: nested `git init`+fetch of the pinned commit, submodule update, then `git am` the patches | +| `git third-party save {--all \| ...}` | regenerate patch files from commits above the base commit; updates the count in `config.json` | + +`update` refuses to run on a dirty vendored tree. The materialized tree is a +real nested git repo whose history is `base commit` + one commit per patch. + +## Editing a vendored tree (e.g. a wasmtime patch) + +1. Edit files in `executors/.x/executor/third-party/wasmtime` and + **commit inside that nested repo** (normal `git commit`; patches are derived + from history above the base commit). +2. `git third-party save executor/third-party/wasmtime` (path relative to cwd) β€” + rewrites `.git-third-party/patches/...` deterministically + (`format-patch --zero-commit --no-signature --numbered-files`). +3. In the executor repo: `git add .git-third-party/`, commit, push. +4. In the manager: bump the gitlink ([submodules.md](submodules.md)). + +The nested repo itself is never pushed β€” only the patch files persist. + +To bump the pinned upstream version: edit `commit` in `config.json`, then +`git third-party update ` (patches are re-applied with `git am`; resolve +conflicts by fixing up the nested repo's commits and running `save`). + +## Build integration + +- Local/CI checkouts run `git third-party update --all` **inside each executor + submodule** β€” the config is resolved from the current git toplevel, so running + it once at the manager root materializes nothing (CI: + `.github/actions/get-src/action.yaml`; local: [setup.md](../setup.md)). +- Nix does **not** call the tool: `support/nix/git-third-party.nix` re-reads + `config.json` and uses `builtins.fetchGit` + `pkgs.applyPatches`, mounted into + the source layout by `support/default.nix`. So nix builds see exactly the + committed base commit + patches β€” another reason to `save` and commit before + building flake packages. diff --git a/docs/contributing/howto/committing/runners.md b/docs/contributing/howto/committing/runners.md new file mode 100644 index 00000000..ad98c207 --- /dev/null +++ b/docs/contributing/howto/committing/runners.md @@ -0,0 +1,28 @@ +# Committing runner changes + +Runner sources and version pins live inside the executor submodule +(`executors/.x/`). All paths below are relative to that submodule root. + +## Dev mode + +During development `runners/support/versions/dev-mode.nix` may be `true` and +runner hashes in `runners/support/versions/current.nix` may be `"test"`. +**Neither must reach a commit.** + +Before committing: + +1. Set `dev-mode.nix` back to `false`. +2. Set "test" hash to `null` in `current.nix`. +3. Run `runners/support/versions/hash-updater.py` (from anywhere inside the + repo) to recompute real hashes. It builds `#runners-all` with + `--keep-going`, writes every reported `got:` value back into `current.nix`, + and repeats until fixed point. Hashes are content-addressed and depend on + resolved dependency uids, so they must be discovered bottom-up β€” the script + handles that automatically. +4. Commit once all hashes are real `sha256-…` values. + +**Never restore an old hash value manually.** It becomes wrong the moment any +source or dependency changes; only `hash-updater.py` produces a correct value. + +See [modify-runner.md](../extending/modify-runner.md) for the full +runner-modification workflow. diff --git a/docs/contributing/howto/committing/submodules.md b/docs/contributing/howto/committing/submodules.md new file mode 100644 index 00000000..751ef380 --- /dev/null +++ b/docs/contributing/howto/committing/submodules.md @@ -0,0 +1,78 @@ +# Submodules (multi-repo workflow) + +## Topology + +- **Manager (umbrella)** β€” this repo (`genlayerlabs/genvm-manager`): `flake.nix`, + `implementation/`, `runners/`, `support/tools/genvm-tool`, `install/`. +- **Executor submodules** β€” `executors/v0.2.x`, `executors/v0.3.x`: checkouts of + the **same** repo (`genlayerlabs/genvm-executor`) on different branches (see + `.gitmodules`). Each line's `manifest.json` owns its `executor-version`. +- `.genvm-monorepo-root` lists the active lines and the manager's `version` β€” + see [versioning.md](../releasing/versioning.md). + +Branch model: manager `feat/` (base usually `v-dev`); each executor +submodule `pr//feat/`. + +## The critical nix flag + +Every flake ref needs `?submodules=1` (`nix develop '.?submodules=1#full'`, +`nix build '.?submodules=1#genvm'`). Flake packages only see **committed** +submodule files β€” commit inside the submodule before building packages. + +## Committing across repos (order matters) + +The manager pins each submodule by a **gitlink** (a pinned commit): + +1. Commit **inside** the submodule: `cd executors/.x && git add … && git commit`. +2. In the manager, bump the gitlink: `git add executors/.x`. +3. Commit the manager (code + gitlink bumps together). + +Verify consistency: + +```bash +git ls-tree HEAD executors/v0.2.x executors/v0.3.x # pinned gitlinks +git -C executors/v0.3.x rev-parse HEAD # must match +``` + +To reshape unpushed history: `git reset --mixed `, re-stage per commit, +re-bump the gitlink to the new submodule HEAD. + +## Pre-commit hook (cross-repo) + +`git commit` triggers `genvm-tool hook run`, which fans linters/formatters out +to the manager **and** submodules. + +- Everything at once: `genvm-tool hook run --all-files` (default `--fix` + rewrites; `--check` only verifies β€” what CI uses). +- `git commit --no-verify` skips the hook β€” fine for pure gitlink bumps or + already-formatted content. +- After changing genvm-tool itself: `genvm-tool hook install` rebuilds the tool + and rewrites the hook stubs. A hook crash referencing an old + `/nix/store/...genvm-tool` means a stale dev-shell `PYTHONPATH` β€” refresh the + dev env or re-run `hook install` with `PYTHONPATH` unset. + +## Pushing + +Check first: `genvm-tool git check-for-push` (`--offline` compares against the +last fetch). Push **submodules before the manager** so gitlinks resolve: + +```bash +git -C executors/v0.2.x push origin pr/v0.2/feat/ +git -C executors/v0.3.x push origin pr/v0.3/feat/ +git push origin feat/ +``` + +After rebasing the manager onto its base (e.g. `origin/v0.6-dev`), it diverges: +use `git push --force-with-lease origin feat/`. Executor feature branches +are normally ahead-only (plain fast-forward push). + +For runner hash hygiene before committing (dev-mode, `hash-updater.py`), see +[runners.md](runners.md). + +## Runner trees (why there are two) + +Forward-rolling lines (v0.3.x) share the top-level `runners///.tar` +tree (Crockford base32 of `sha256(tar)`). Frozen v0.2.x keeps runners under +`executor//legacy-runners/` with **Nix base32** hashes β€” a different +scheme, hence the separate trees. `flake.nix` packaging splits the runner list +per line; `genvm check` verifies each with its own scheme. diff --git a/docs/contributing/howto/extending/add-host-function.md b/docs/contributing/howto/extending/add-host-function.md new file mode 100644 index 00000000..d7595427 --- /dev/null +++ b/docs/contributing/howto/extending/add-host-function.md @@ -0,0 +1,11 @@ +# Adding a host function + +1. `executors/.x/executor/codegen/data/host-fns.json` β€” add the method + id, then regenerate (`ninja -C build codegen`); this updates + `executor/crates/common/src/host_fns.rs` (per line) and the manager's + `tests/runner/origin/host_fns.py` β€” see [genvm-tool.md](../genvm-tool.md). +2. `tests/runner/origin/base_host.py` β€” handle the new case in the read loop + and add the method to the `IHost` protocol. NOTE: `origin/` is mirrored + into the node (`backend/node/genvm/origin/`). +3. `tests/runner/gvm_extra/mock_host.py` β€” add the test implementation. +4. Update the node (and simulator) host implementations. diff --git a/docs/contributing/howto/extending/add-llm-provider.md b/docs/contributing/howto/extending/add-llm-provider.md new file mode 100644 index 00000000..ede8518a --- /dev/null +++ b/docs/contributing/howto/extending/add-llm-provider.md @@ -0,0 +1,14 @@ +# Adding an LLM provider + +If the provider is OpenAI-API-compatible, no work is needed. + +Otherwise (all in the manager's `implementation/src/llm/`): + +1. `providers.rs` β€” declare and implement the provider (`impl Provider`); + a separate json mode is recommended. +2. `config.rs` β€” add a `Provider` enum value and a `BackendConfig::to_provider` + case. + +Test (optional): add a case to `implementation/src/llm/handler.rs`, patch +`.github/workflows/queue.yaml` to pass the API-key secret, and provide the key +to the repository owners. diff --git a/docs/contributing/howto/extending/add-wasi-function.md b/docs/contributing/howto/extending/add-wasi-function.md new file mode 100644 index 00000000..ce69c2d6 --- /dev/null +++ b/docs/contributing/howto/extending/add-wasi-function.md @@ -0,0 +1,21 @@ +# Adding a WASI function to an executor + +Paths are relative to `executors/.x/`. + +## Via gl_call (preferred β€” no ABI surface change) + +1. `executor/crates/sdk-rs/src/abi/gl_call.rs` β€” add the message shape. +2. `executor/src/wasi/genlayer_sdk.rs` β€” implement the handler; add a version + check so older contracts don't observe the new method. + +New gl_call methods are a minor-compatible ABI addition β€” see the +[versioning spec](../../../website/src/spec/01-core-architecture/03-versioning.rst). + +## Raw WASI function (new ABI surface) + +1. `executor/src/wasi/witx/genlayer_sdk.witx` β€” add the declaration. +2. `executor/src/wasi/genlayer_sdk.rs` β€” add the implementation (under the + generated `impl` trait). +3. `runners/cpython/modules/_genlayer_wasi/genlayer.c` β€” add the Python proxy. + This changes the cpython runner hash β€” refresh it per + [modify-runner.md](modify-runner.md). diff --git a/docs/contributing/howto/extending/modify-runner.md b/docs/contributing/howto/extending/modify-runner.md new file mode 100644 index 00000000..f0f3b96b --- /dev/null +++ b/docs/contributing/howto/extending/modify-runner.md @@ -0,0 +1,18 @@ +# Modifying a runner + +Runner sources and version pins live per executor line; paths below are +relative to `executors/.x/`. + +1. Set `runners/support/versions/dev-mode.nix` to `true` and the runner's hash + in `runners/support/versions/current.nix` to `"test"`; develop and test. +2. Set dev-mode back to `false`; set the changed runner's hash (and its + dependents') to `null`. +3. Refresh hashes with `runners/support/versions/hash-updater.py` (run from + anywhere inside the repo): it builds `#runners-all` with `--keep-going`, + writes every reported `hash mismatch … got:` value back into `current.nix`, + and repeats until fixed point. Hashes depend on dependencies' resolved uids, + so they must be discovered in dependency order β€” the script handles that. + + Manual alternative: `ninja -C build all`, copy the `got:` hash from the + error into `current.nix`, repeat per dependent runner. +4. Rebuild to verify no mismatches remain. diff --git a/docs/contributing/howto/extending/modify-wasmtime.md b/docs/contributing/howto/extending/modify-wasmtime.md new file mode 100644 index 00000000..09ebfeaf --- /dev/null +++ b/docs/contributing/howto/extending/modify-wasmtime.md @@ -0,0 +1,27 @@ +# Modifying vendored wasmtime + +The tree at `executors/.x/executor/third-party/wasmtime` is a +git-third-party clone (upstream pin + genvm patches) β€” mechanism and commit +workflow: [git-third-party.md](../committing/git-third-party.md). + +Edit the tree directly; `cargo check`/ninja build it in-tree, so iteration is +normal Rust development. Two caveats: + +- **Nix builds don't see working-tree edits** β€” they apply the saved patches + (`support/nix/git-third-party.nix`). Run `git third-party save` before any + nix build/release must pick the change up. +- Commit flow: commit inside the nested wasmtime repo β†’ + `git third-party save executor/third-party/wasmtime` β†’ commit + `.git-third-party/` in the executor repo β†’ gitlink bump in the manager. + +## genvm patch points + +- Custom trap variant `Trap::NondetInstruction` + (`crates/environ/src/trap_encoding.rs`) β€” raised for instructions disallowed + in deterministic mode (e.g. unmapped float/SIMD ops); emitted via + `trap_nondet_instruction` (`crates/cranelift/src/func_environ.rs`) from + `float_op_unreachable_check` (`crates/cranelift/src/translate/code_translator.rs`). +- When adding a `Trap` variant: update the `check!` macro in `from_u8` and its + `Display` arm (see the comment at the enum), the `TRAP_*` const in + `crates/cranelift/src/lib.rs`, and the executor-side remap to a public VM + error code in `executor/src/rt/errors.rs` (`trap_to_vm_error`). diff --git a/docs/contributing/howto/genvm-tool.md b/docs/contributing/howto/genvm-tool.md new file mode 100644 index 00000000..681cc365 --- /dev/null +++ b/docs/contributing/howto/genvm-tool.md @@ -0,0 +1,60 @@ +# genvm-tool + +Lives at `support/tools/genvm-tool` (on PATH as `genvm-tool` in the dev shell; +outside it, call `support/tools/genvm-tool/genvm-tool`). + +## Commands + +| Command | Purpose | +|---|---| +| `configure` | generate `build/build.ninja` + `build/info.json` β€” [build.md](building/build.md) | +| `test` | run the test suite β€” [testing/README.md](testing/README.md) | +| `codegen` | generate language bindings from data JSON (below) | +| `build-manifest` | generate the manager's `data/manifest.yaml` | +| `git ls` / `list-repo` / `create-branches` / `check-for-push` | git helpers across manager + submodules β€” [submodules.md](committing/submodules.md) | +| `hook run` / `hook install` | cross-repo pre-commit fan-out / stub installer β€” [submodules.md](committing/submodules.md) | + +Command registration: `genvm_tool/__main__.py` (`TOPLEVEL`, `GROUPS`); each +command module has `NAME`/`HELP` β€” look there for details. + +## codegen + +```bash +genvm-tool codegen --lang {rust,python,rst,go} -i -o +``` + +(`--go-package` selects the go package name, default `genvm`.) + +Codegen data lives per executor line at +`executors/v.x/executor/codegen/data/{public-abi.json,host-fns.json}`. +After editing the JSON, regenerate via the build (`genvm-tool configure` then +`ninja -C build codegen`) or run `genvm-tool codegen` by hand per the tables: + +Per-line generated files (registered by `register_standard_codegen` in +`tests/runner/genvm_tool_plugins/ninja.py`, paths relative to the line root): + +| Output | Input | Lang | +|---|---|---| +| `executor/crates/sdk-rs/src/abi/consts.rs` | `public-abi.json` | rust | +| `runners/genlayer-py-std/src/genlayer/vm/public_abi.py` | `public-abi.json` | python | +| `executor/crates/common/src/host_fns.rs` | `host-fns.json` | rust | + +Manager-global generated files (from the **primary** line's data, wired in +`genvm_tool/cmd_configure.py`) β€” these go stale when a line's codegen JSON +changes and are refreshed by the same configure/ninja codegen step: + +| Output | Input | Lang | +|---|---|---| +| `tests/runner/origin/host_fns.py` | `host-fns.json` | python | +| `tests/runner/origin/public_abi.py` | `public-abi.json` | python | +| `docs/website/src/spec/appendix/constants.rst` | `public-abi.json` | rst | + +## Plugins and test definitions + +Plugins (`tests/runner/genvm_tool_plugins/`) are **importable libraries** β€” +reusable build/test helpers, no side effects on import. Test *definitions* are +kept out of the plugin package: each lives in `tests/system//test.py` +exposing `collect(ctx, **kwargs)`, and is registered from `.genvm-tool.py` via +`ctx.collect_dir('tests/system/', ...)` (discovery: +`genvm_tool/tests/stage/collection.py`). The project entry point overall is +`.genvm-tool.py` at the repo root (each executor line has its own). diff --git a/docs/contributing/howto/releasing/release-build.md b/docs/contributing/howto/releasing/release-build.md new file mode 100644 index 00000000..b5008f58 --- /dev/null +++ b/docs/contributing/howto/releasing/release-build.md @@ -0,0 +1,29 @@ +# Release / nix build + +Nix flake packages see only **committed** submodule content β€” commit inside +submodules first ([submodules.md](../committing/submodules.md)), and always pass +`?submodules=1`. + +Platform suffixes: none (native), `-amd64-linux`, `-arm64-linux`, `-arm64-macos`. + +| Package | Contents | +|---|---| +| `genvm[-]` | everything: manager + all executor lines + runners | +| `manager[-]` | manager bundle (`bin/`, `lib/`, `config/`, `data/`) | +| `executor[-]` | all active executor lines merged (`executor//…`) | +| `executor-[-]` | one executor line | +| `runners-all-dist` | platform-independent runners (`runners/` prefix + legacy-line overlay) | +| `genvm-tool` | the tool itself | + +```bash +nix build -v -L '.?submodules=1#genvm-amd64-linux' +``` + +Release assets (`.github/workflows/release.yaml`; tag = `version` from +`.genvm-monorepo-root`, see [versioning.md](versioning.md)): + +- `genvm--.tar.xz` ← `manager-` +- `genvm---executor.tar.xz` ← `executor-` +- `genvm-runners-all.tar.xz` ← `runners-all-dist` + +All three extract at the same install root. diff --git a/docs/contributing/howto/releasing/versioning.md b/docs/contributing/howto/releasing/versioning.md new file mode 100644 index 00000000..5b87d353 --- /dev/null +++ b/docs/contributing/howto/releasing/versioning.md @@ -0,0 +1,38 @@ +# Versioning and release trains + +`.genvm-monorepo-root` (repo root, JSON) is the source of truth: + +- `version` β€” the manager's own release version (e.g. `v0.6.0-rc0`). +- `active-versions` β€” the live release trains (e.g. `["v0.3", "v0.2"]`); each + train maps to an `executors/v.x` submodule line. + +Executor crates are versioned **independently** in their submodules (each +line's `manifest.json` holds `executor-version`); manager scripts never rewrite +them. + +## Scripts + +`support/ci/check-versions.py` β€” keeps manager-owned crate versions +(`implementation/Cargo.toml`) in sync with `version`: + +| Command | Effect | +|---|---| +| `check-versions.py sync` (default) | pre-commit fixer: rewrite crate `[package]` major.minor to match `version` (patch kept); non-zero exit if it changed anything | +| `check-versions.py bump minor` | cut next train, e.g. 0.3 β†’ 0.4 (default level) | +| `check-versions.py bump major` | 0.3 β†’ 1.0 | +| `check-versions.py bump --set 0.6` | explicit major.minor | + +`bump` updates `version`, appends the new train to `active-versions`, sets +manager crates to `.0`, and prints the bare major.minor (callers build +branch names from it). + +`support/ci/branch-versions.py list|latest` β€” prints active trains (leading `v` +stripped). Branch naming per train ``: + +- `v.x` β€” release branch (release-ready) +- `v-dev` β€” integration branch + +`MONOREPO_ROOT=` points it at a different `.genvm-monorepo-root` (e.g. +extracted from `origin/main`). + +See also: [submodules.md](../committing/submodules.md) for the branch model of feature work. diff --git a/docs/contributing/howto/setup.md b/docs/contributing/howto/setup.md new file mode 100644 index 00000000..40f26389 --- /dev/null +++ b/docs/contributing/howto/setup.md @@ -0,0 +1,35 @@ +# First-time setup + +1. Init the executor submodules **before** entering the dev shell (the nix flake reads them): + + ```bash + git submodule update --init --recursive --depth 1 + ``` + +2. Enter the dev shell: + + ```bash + nix develop '.?submodules=1#full' + ``` + + `?submodules=1` is required on **every** flake ref β€” without it: + `Path 'executors/v0.3.x' … is not tracked by Git`. Usually auto-loaded via + direnv (`.envrc` is `use flake '.?submodules=1#full'`). + +3. `source env.sh` β€” adds `support/tools/git-third-party` to PATH and sources + `.env` if present. Only needed outside the dev shell. + +4. Materialize the vendored third-party trees (wasmtime, wasm-tools). These are + registered per executor submodule, and `git third-party` reads its config + from the current git toplevel β€” so run it *inside each executor*, not at the + manager root (which has no config and would materialize nothing): + + ```bash + for cfg in executors/*/.git-third-party/config.json; do + ( cd "$(dirname "$(dirname "$cfg")")" && git third-party update --all ) + done + ``` + + See [git-third-party.md](committing/git-third-party.md) for how vendoring works. + +Next: [build.md](building/build.md). diff --git a/docs/contributing/howto/testing/README.md b/docs/contributing/howto/testing/README.md new file mode 100644 index 00000000..c7ae16c6 --- /dev/null +++ b/docs/contributing/howto/testing/README.md @@ -0,0 +1,48 @@ +# Test + +All tests run through `genvm-tool test`. Build first +([build.md](../building/build.md)). Commands assume the `#full` dev shell. + +Per-kind guides: [rust.md](rust.md) Β· [python.md](python.md) Β· +[integration.md](integration.md) Β· [fuzzing.md](fuzzing.md). + +## Quick reference + +| What | Command | +|---|---| +| All tests | `genvm-tool test run` | +| Release preset | `genvm-tool test run --filter-tag "$(cat tests/presets/release.txt)"` | +| One test | `genvm-tool test run --filter-name 'test_name'` | +| Re-run failed | `genvm-tool test run --filter-continue ` | +| List tests / plan / tags / services | `genvm-tool test show test\|plan\|tags\|services` | + +Options: `--filter-name REGEX`, `--filter-tag EXPR` (e.g. `stable & !slow`), +`--filter-continue FILE`, `--fail-fast`, `--coverage`, +`--log-level {trace,debug,info,warning,error}`, `--ignore-hash`. + +Presets in `tests/presets/`: `release.txt` (`integration & stable & !bench`), +`rust.txt` (`(rust | integration) & !bench & !fuzz`), `rust-fuzz.txt` +(`rust & fuzz`), `python.txt` (`python`). + +Common tags are documented in [integration.md](integration.md); +`genvm-tool test show tags` lists everything. + +## Fix–rerun loop: use continue files + +Every failed run writes the failing test names to +`build/test-artifacts/continue/-` (the name is printed in +the failure summary). **After fixing something, rerun just those tests first** +β€” much faster than a full rerun: + +```bash +genvm-tool test run --filter-continue +``` + +Do a full run only once the continue set passes. + +## Configuration + +Test-suite configuration (artifacts dir, python paths) comes from +`.genvm-monorepo-root`; the suite itself is the `tests` function in +`.genvm-tool.py`. Test definitions live in `tests/system//test.py` β€” +see [genvm-tool.md](../genvm-tool.md). diff --git a/docs/contributing/howto/testing/fuzzing.md b/docs/contributing/howto/testing/fuzzing.md new file mode 100644 index 00000000..64c13908 --- /dev/null +++ b/docs/contributing/howto/testing/fuzzing.md @@ -0,0 +1,24 @@ +# Fuzzing + +Rust fuzz targets run under AFL (`cargo-afl`): + +```bash +genvm-tool test run --filter-tag "$(cat tests/presets/rust-fuzz.txt)" # rust & fuzz +``` + +Python fuzz targets exist but their collection is currently disabled in +`.genvm-tool.py`. Some integration cases also carry the `fuzz` tag. + +Options: `--fuzz-timeout N` (seconds per run, default 30), +`--fuzz-update-corpus` (write back interesting inputs). + +## Host preparation + +AFL needs these before a session (crashes must land as files, CPU frequency +must be stable, and the fuzzer must be able to ptrace): + +```bash +echo core | sudo tee /proc/sys/kernel/core_pattern +echo performance | sudo tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope +``` diff --git a/docs/contributing/howto/testing/integration.md b/docs/contributing/howto/testing/integration.md new file mode 100644 index 00000000..e6a3ced8 --- /dev/null +++ b/docs/contributing/howto/testing/integration.md @@ -0,0 +1,61 @@ +# Integration tests + +```bash +genvm-tool test run --filter-tag integration +``` + +Cases are `.jsonnet` files in `executors/.x/tests/integration/`; each +produces a `/prepare` case plus one case per step (`/0l`, `/0v`, `/0s`, …). +A sibling `.skip` file marks a case skipped. The suite iterates every line in +`active-versions` (`.genvm-monorepo-root`) and runs each line's cases against +that line's own built executor (`build/info.json` maps `v0.3` β†’ the concrete +built version). + +## Common tags + +| Tag | Meaning | +|---|---| +| `stable` | needs **no LLM keys and no webdriver** (manager only) β€” safe to run offline | +| `unstable`, `semi-stable` | need modules + webdriver; `unstable` is retried (3 attempts), `semi-stable` is not | +| `bench` | benchmarks β€” excluded from the presets | + +## Running a subset + +- **One executor line** β€” test names are repo-relative paths and + `--filter-name` is an unanchored regex, so: + `genvm-tool test run --filter-name 'executors/v0.3.x/'` +- **Offline / no LLMs / no web** β€” the `stable` tag marks tests needing only + the manager (no modules, no webdriver; the executor is told `no_modules`): + `--filter-tag 'integration & stable'` (`tests/presets/release.txt` + additionally excludes `bench`). Stability is the second argument of + `util.features(paths, stability)` in the case's jsonnet `tags` + (`'stable'` / `'semi-stable'` / `'unstable'`). (Legacy: v0.2.x instead uses a + `stable/` top-level directory.) +- **After a fix** β€” rerun only the failed cases via the continue file: + `--filter-continue ` (see [README.md](README.md)). + +## Golden files and hashes + +Goldens live next to the cases (`*.N.stdout`). A missing golden is +auto-created on the first run; after that a mismatch **fails** the test β€” +delete the sidecar to regenerate it. + +With `stable_hash: true` on a step entry (the default) the leader's execution +hash is likewise compared against an auto-created `.N.hash` sidecar; with +`stable_hash: false` there is no sidecar and validators compare to the +leader's runtime hash instead. + +`--ignore-hash` skips hash comparison *and* sidecar creation (a new case does +NOT need it β€” its hash is auto-created). Use it when hashes legitimately +differ, e.g. **runner dev mode** +([modify-runner.md](../extending/modify-runner.md)): a modified runner changes +runner hashes and therefore every execution hash. An executor line can also +disable hash checks wholesale via `integration()` in its `.genvm-tool.py` +returning `{'ignore-hash': True}` β€” currently the case for v0.3.x. + +## Services + +The manager, modules (LLM/web), and webdriver services start automatically; +`--no-manager` / `--no-webdriver` use externally running ones instead. Manual +webdriver: `bash webdriver/build-and-run.sh`. If WASM files changed, +`./build/out/executor//bin/genvm precompile` saves test time. diff --git a/docs/contributing/howto/testing/python.md b/docs/contributing/howto/testing/python.md new file mode 100644 index 00000000..61e56e81 --- /dev/null +++ b/docs/contributing/howto/testing/python.md @@ -0,0 +1,16 @@ +# Python tests + +```bash +genvm-tool test run --filter-tag python +``` + +## Direct pytest (genlayer-py-std, no dev shell) + +```bash +cd runners/genlayer-py-std +env_dir="$(nix build --no-link --print-out-paths path:../../support/nix/py-test)" +PYTHONPATH="$PWD/src:$PWD/src-emb" "$env_dir/bin/pytest" tests/ +``` + +Coverage is enforced (`--cov-fail-under=75`); numpy-dependent tests need the +nix env. diff --git a/docs/contributing/howto/testing/rust.md b/docs/contributing/howto/testing/rust.md new file mode 100644 index 00000000..abbd655a --- /dev/null +++ b/docs/contributing/howto/testing/rust.md @@ -0,0 +1,16 @@ +# Rust tests + +```bash +genvm-tool test run --filter-tag rust # add --coverage for coverage +``` + +If a Rust test fails to load a shared library under the `#full` shell, use the +dedicated shell: + +```bash +nix develop '.?submodules=1#rust-test' --command genvm-tool test run --filter-tag rust +``` + +Tests live in the executor crates (inline `#[cfg(test)] mod tests` and each +crate's `tests/`); `genvm-tool test` discovers them. Preset: +`tests/presets/rust.txt` (`rust | integration`). diff --git a/docs/contributing/tests.md b/docs/contributing/tests.md deleted file mode 100644 index 3a37e65e..00000000 --- a/docs/contributing/tests.md +++ /dev/null @@ -1,103 +0,0 @@ -# Running GenVM tests - -GenVM uses the `genvm-tool test` runner (the language-agnostic ya-test-runner, -vendored into `genvm-tool`) for running all tests. The test runner automatically handles service dependencies (manager, modules, webdriver). - -## Prerequisites - -- Build the project first: `genvm-tool configure && ninja -C build` -- For `get_webpage` tests, a compatible webdriver is needed. Use the docker image: `./modules/webdriver/run-test-docker.sh` -- For `exec_prompt` tests, set the `OPENAIKEY` env variable to your OpenAI key - -## Running Tests - -### Using nix (recommended) -```bash -nix develop .#mock-tests --command genvm-tool test run -``` - -### Without nix - -`genvm-tool test` needs `aiohttp` and `jsonnet` on the Python path. With those -installed, drive the in-tree wrapper directly: -```bash -# Run all tests -support/tools/genvm-tool/genvm-tool test run - -# Run with tag filter -support/tools/genvm-tool/genvm-tool test --filter-tag 'stable' run - -# Show available tests -support/tools/genvm-tool/genvm-tool test show test - -# Show execution plan -support/tools/genvm-tool/genvm-tool test show plan -``` - -### Using Presets - -Tag expression presets are available in `tests/presets/`: -```bash -# Run release tests (integration & stable) -genvm-tool test --filter-tag "$(cat tests/presets/release.txt)" run - -# Run rust tests (rust | integration) -genvm-tool test --filter-tag "$(cat tests/presets/rust.txt)" run - -# Run python tests -genvm-tool test --filter-tag "$(cat tests/presets/python.txt)" run -``` - -### Coverage Collection - -To collect coverage for Rust tests: -```bash -nix develop .#rust-test --command genvm-tool test --filter-tag rust --coverage run -``` - -### Re-running Failed Tests - -When tests fail, the runner automatically writes the failed test names to a continue file at `build/test-artifacts/continue/-`. To re-run only the failed tests: - -```bash -# Use the continue file path shown in the failure summary -genvm-tool test --filter-continue 20260123-143052-abc123 run - -# Or use a full path -genvm-tool test --filter-continue build/test-artifacts/continue/20260123-143052-abc123 run -``` - -### Useful Options - -- `--filter-name REGEX` - Filter tests by name regex -- `--filter-tag EXPR` - Filter tests by tags (e.g., `stable & !slow`) -- `--filter-continue FILE` - Re-run only tests from a continue file (from a previous failed run) -- `--fail-fast` - Stop on first failure -- `--coverage` - Enable coverage collection for Rust tests -- `--log-level {trace,debug,info,warning,error}` - Set logging level - -## Test Categories - -- **Integration tests** (`tests/cases/`): End-to-end tests using jsonnet configuration -- **Rust tests** (`executor/tests/`): Unit tests for the Rust executor -- **Python tests** (`runners/genlayer-py-std/test/`): Tests for the Python SDK - -## Configuration - -The test runner reads configuration from the monorepo marker `.genvm-monorepo-root` -in the project root: - -```json -{ - "version": "v0.3", - "active-versions": ["v0.3"], - "artifacts_dir": "build/test-artifacts", - "extra_python_paths": ["tests/plugins", "tests/runner"] -} -``` - -- `artifacts_dir` - Directory for test artifacts (logs, continue files). Defaults to `build/test-artifacts`. -- `extra_python_paths` - Paths prepended to `sys.path` so the suite can import its plugins. - -The suite itself (collectors, plugins, CLI args) is the `tests` function exported -from `.genvm-tool.py` at the project root. diff --git a/docs/contributing/workflows.md b/docs/contributing/workflows.md deleted file mode 100644 index 0c541b34..00000000 --- a/docs/contributing/workflows.md +++ /dev/null @@ -1,42 +0,0 @@ -# Common workflows - -## Adding new LLM provider -**IMPORTANT**: If your provider is compatible with openai API no additional work is needed - -- go to [`implementation/src/llm/providers.rs`](../../implementation/src/llm/providers.rs) -- declare and implement new provider (`impl Provider`). It is recommended to implement separate json mode -- add new value to `Provider` enum in `implementation/src/llm/config.rs` -- add case to `BackendConfig::to_provider` in `implementation/src/llm/config.rs` - -### Adding test -(optional) - -- add test case to [`implementation/src/llm/handler.rs`](../../implementation/src/llm/handler.rs) -- patch [workflow](../../.github/workflows/queue.yaml) to pass secret -- provide api key to repository owners - -## Adding new wasm function to gl_call -It is an easier approach than next. Just add definition to `executor/src/wasi/gl_call.rs`. You must also add version check to implementation in `executor/src/wasi/genlayer_sdk.rs`. - -## Adding new wasm function -- `executor/src/wasi/witx/genlayer_sdk.witx`
- add declaration here -- `executor/src/wasi/genlayer_sdk.rs`
- add implementation here (under `impl` trait) -- `runners/cpython/modules/_genlayer_wasi/genlayer.c`
- add python proxy
- NOTE: this will change hash, rebuilding will show you the new one - -## Adding new host function - -- `executor/codegen/data/host-fns.json`
- add new function id
- after rebuilding (`tags/codegen`) few files will be updated: - - `executor/src/host/host_fns.rs` - - `executor/testdata/runner/host_fns.py` -- `executor/testdata/runner/base_host.py`
- update `while True` to handle new case, add new method to the `IHost` protocol
- NOTE: this file is used in simulator as well (under `backend/node/genvm/origin/`) -- `executor/testdata/runner/mock_host.py`
- add implementation for tests -- update simulator and node diff --git a/docs/website/pyproject.toml b/docs/website/pyproject.toml index abd20670..3824d1a7 100644 --- a/docs/website/pyproject.toml +++ b/docs/website/pyproject.toml @@ -1,20 +1,20 @@ [tool.poetry] +authors = ["GenLayer"] +description = "" name = "genvm-docstypesgen" version = "0.0.1" -description = "" -authors = ["GenLayer"] [tool.poetry.dependencies] +myst-parser = "^5.0.0" +numpy = "^2.1.3" +pydata-sphinx-theme = "^0.16.0" +pyright = "^1.1.390" python = "^3.12" sphinx = "^8.1.3" -pyright = "^1.1.390" -pydata-sphinx-theme = "^0.16.0" -numpy = "^2.1.3" sphinxcontrib-mermaid = "^1.0.0" sphinxcontrib-openapi = "^0.8.4" -myst-parser = "^5.0.0" [build-system] -requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" +requires = ["poetry-core"] diff --git a/docs/website/src/conf.py b/docs/website/src/conf.py index eab4be6a..6f0324b8 100644 --- a/docs/website/src/conf.py +++ b/docs/website/src/conf.py @@ -168,8 +168,13 @@ def _semver_sort_key(entry): while not root_dir.joinpath(MONO_REPO_ROOT_FILE).exists(): root_dir = root_dir.parent MONOREPO_CONF = json.loads(root_dir.joinpath(MONO_REPO_ROOT_FILE).read_text()) -sys.path.append(str(root_dir.joinpath(*MONOREPO_CONF['py-std']))) -sys.path.append(str(root_dir.joinpath(*MONOREPO_CONF['py-emb']))) +active_versions = MONOREPO_CONF['active-versions'] +primary_executor = root_dir / 'executors' / f'{active_versions[0]}.x' +py_std = primary_executor / 'runners' / 'genlayer-py-std' / 'src' +py_emb = primary_executor / 'runners' / 'genlayer-py-std' / 'src-emb' + +sys.path.append(str(py_std)) +sys.path.append(str(py_emb)) os.environ['GENERATING_DOCS'] = 'true' diff --git a/docs/website/src/impl-spec/02-vm/02-version-management.rst b/docs/website/src/impl-spec/02-vm/02-version-management.rst index 3e9eebe6..070f6b8f 100644 --- a/docs/website/src/impl-spec/02-vm/02-version-management.rst +++ b/docs/website/src/impl-spec/02-vm/02-version-management.rst @@ -36,7 +36,8 @@ The request body is the raw contract bytecode; the response is:: { "specified_major": } The node MUST store the returned value in the contract's :ref:`genvm-def-root-slot` at -offset ``4`` (the ``major`` field β€” see :doc:`/spec/04-contract-interface/03-storage`). +offset :ref:`gvm-def-consts-value-root-offsets-major`. See +:doc:`/spec/04-contract-interface/03-storage` for the root-slot layout. On every subsequent load :term:`GenVM` re-reads that byte and refuses to execute when it does not match its own ``CURRENT_MAJOR``. Without this check a node running a newer GenVM could silently mis-interpret an older contract's calldata or storage. diff --git a/docs/website/src/impl-spec/04-fees.rst b/docs/website/src/impl-spec/04-fees.rst index 36a24011..6b07f959 100644 --- a/docs/website/src/impl-spec/04-fees.rst +++ b/docs/website/src/impl-spec/04-fees.rst @@ -252,3 +252,57 @@ A minimal, illustrative ``fees`` block:: ``message_fee`` and ``message_receipt`` share ``bucket_no = 1`` here, so an outbound message debits both costs atomically against bucket 1. + +Message-Fee Allocation Matching +------------------------------- + +The ``a.matchedFeeParams`` above is selected per outbound message from the call's +allocation list (``accumulator.message_fee_allocation``). A node +(``domain/fees.rs``, ``matches_internal`` / ``matches_external``) matches on kind +(``External`` for ``EthSend``, else ``Internal``), ``on`` (internal only), and +``recipient`` / ``call_key`` (each a wildcard ``None`` or an exact match). Selection +is **first-match-wins** in list order (``find_map``); no match yields the +``fee no_matching_node`` VM error. + +Wildcards are not reordered, so a wildcard node shadows every more-specific node +after it: producers are advised to sort more-specific nodes ahead of wildcard ones. + +Funding modes +~~~~~~~~~~~~~~ + +An outgoing internal message is funded one of two ways: + +- **Allocation-matched (default).** The fee is matched against the allocation + tree as above, capped by the matched node's ``budget``, and consumes both the + ``message_fee`` and ``message_receipt`` buckets. +- **Balance-funded (``use_balance``).** When a ``PostMessage`` / ``DeployContract`` + sets ``use_balance`` (the chain's ``useBalance``, gated on + :ref:`gvm-perm-use-balance-for-message-fees`), allocation matching is skipped + entirely. The fee is metered from the guest-supplied ``fee_params`` and that + metered amount is the child's ``declaredBudget``, reserved from the emitting + contract's balance (jointly with ``value``; insufficient balance yields + ``Inbalance``). The ``message_fee`` bucket is **not** consumed (the message is + excluded from the sender pool on-chain); only ``message_receipt`` is. The emitted + allocation subtree is empty, so nested child messages must each fund themselves. + + Because ``fee_params`` is guest-supplied, it is validated before it reaches the + fee evaluator (``validate_balance_fee``): empty ``rotations`` and any zero price + cap (``max_price_gen_per_time_unit`` / ``storage_fee_max_gas_price`` / + ``receipt_fee_max_gas_price``) are rejected with ``Inval``, matching the chain's + reveal-time ``FeeValueMustBeNonZero`` checks. Magnitudes are bounded too + (prices/budgets below 2\ :sup:`96`, counts β€” time units and rotations + entries β€” below 2\ :sup:`32`) so the worst-case metered floor provably fits + in ``U256`` and cannot trip the evaluator's overflow abort. + + The floor replicates the chain's ``minMessagePrimaryFees`` rather than the + allocation-matched floor: the consensus term is charged at the guest's + ``max_price_gen_per_time_unit`` (the funding cap), not the node's + ``genPerTimeUnit`` (the allocation path keeps the ``genPerTimeUnit`` multiplier + β€” this branch is selected by the ``balanceFunded`` flag threaded into the + ``message_fee`` expression). Two further node-configured floors fire as + ``VMError``\ s: ``fee below_minimum`` when a non-zero ``execution_budget_per_round`` + is below ``node.messageBudgetFloor`` (the chain's ``BudgetTooLow``), and + ``fee too_many_rounds`` when ``rotations`` implies more rounds than the validator + table (``node.validatorsPerRound``) covers. The per-phase time-unit floor + (``fee below_minimum`` against ``node.minTimeUnitsPerPhase``) applies on both + funding paths. diff --git a/docs/website/src/python-sdk/changelog-notes/v0.3.rst b/docs/website/src/python-sdk/changelog-notes/v0.3.rst index 9798de29..3bdaab32 100644 --- a/docs/website/src/python-sdk/changelog-notes/v0.3.rst +++ b/docs/website/src/python-sdk/changelog-notes/v0.3.rst @@ -20,7 +20,7 @@ The recommended import pattern has changed: import genlayer as gl from genlayer.types import * -The ``from genlayer import *`` star-import previously brought ``gl`` (a lazy proxy to ``genlayer.gl``), all types, and storage names into scope. Now ``import genlayer as gl`` gives you direct access to submodules (``gl.contract``, ``gl.vm``, ``gl.message``, ``gl.chain``, etc.), decorators (``gl.public``, ``gl.private``), and storage types (``gl.TreeMap``, ``gl.DynArray``, ``gl.Array``). The ``from genlayer.types import *`` import brings the type aliases (``u8``..``u256``, ``i8``..``i256``, ``Address``, ``bigint``, etc.) into local scope. +The ``from genlayer import *`` star-import previously brought ``gl`` (a lazy proxy to ``genlayer.gl``), all types, and storage names into scope. Now ``import genlayer as gl`` gives you direct access to submodules (``gl.contract``, ``gl.vm``, ``gl.message``, ``gl.chain``, ``gl.storage``, etc.) and decorators (``gl.public``, ``gl.private``). The ``from genlayer.types import *`` import brings the type aliases (``u8``..``u256``, ``i8``..``i256``, ``Address``, ``bigint``, etc.) into local scope. Module Path Changes ~~~~~~~~~~~~~~~~~~~~~ @@ -257,21 +257,23 @@ Two runtime helpers were added: ``gl.vm.register_runner(code)`` registers a runn Storage ~~~~~~~ -``allow_storage`` decorator is renamed to ``allow`` and is available as ``gl.storage.allow`` (also re-exported at ``gl.allow``): +Storage types (``DynArray``, ``Array``, ``TreeMap``) and the ``allow`` decorator are accessible via ``gl.storage``: .. code-block:: python # v0.2.x - @gl.storage.allow_storage - class MyRecord: - x: u32 + x: gl.DynArray[str] + m: gl.TreeMap[str, u32] + @gl.allow + class MyRecord: ... # v0.3.0 + x: gl.storage.DynArray[str] + m: gl.storage.TreeMap[str, u32] @gl.storage.allow - class MyRecord: - x: u32 + class MyRecord: ... -A new ``gl.storage.Pickled[T]`` helper is available for storing arbitrary picklable objects. +``allow_storage`` was renamed to ``allow`` (accessible as ``gl.storage.allow``). A new ``gl.storage.Pickled[T]`` helper is available for storing arbitrary picklable objects. Decorators ~~~~~~~~~~ @@ -334,5 +336,13 @@ Summary of Renames - ``gl.message.raw`` * - ``gl.storage.allow_storage`` - ``gl.storage.allow`` + * - ``gl.allow`` + - ``gl.storage.allow`` + * - ``gl.DynArray`` + - ``gl.storage.DynArray`` + * - ``gl.Array`` + - ``gl.storage.Array`` + * - ``gl.TreeMap`` + - ``gl.storage.TreeMap`` * - ``UserError(msg: str).message`` - ``UserError(data: Encodable).data`` diff --git a/docs/website/src/spec/01-core-architecture/03-versioning.rst b/docs/website/src/spec/01-core-architecture/03-versioning.rst new file mode 100644 index 00000000..8b8c6b29 --- /dev/null +++ b/docs/website/src/spec/01-core-architecture/03-versioning.rst @@ -0,0 +1,78 @@ +Executor Versioning +=================== + +.. _gvm-def-executor-version: + +An *executor* is a released :term:`GenVM` build, identified by a +``v..`` version. The incremented component bounds what may +change between two releases. + +Patch +----- + +A patch release is fully compatible with its predecessor; executors that differ +only in patch are interchangeable for resyncing from genesis. + +#. The :ref:`gvm-def-execution-hash` of every transaction remains the same. +#. Deterministic semantics are exactly the same. + +Minor +----- + +A minor release adds new features and is backwards-compatible. + +#. The :ref:`gvm-def-execution-hash` of a transaction may change. +#. New :term:`Runner`\s may be added; existing runners MUST NOT be removed. +#. New ``gl_call`` methods and new parameters of existing methods may be added. +#. Old code continues to work, unless it deliberately tried to be + future-incompatible (e.g. by observing additions permitted by rule 3). + +Major +----- + +A major release makes almost no promises. Executing code of two different +majors within the same VM is impossible. + +If a ``v2.x`` executor is ever released, messages and calls across majors will +be supported by delegating execution to a ``v1.x`` executor. + +Forced Contract Updates +----------------------- + +A contract is forcefully updated to the latest compatible executor version: + +#. while syncing β€” to the latest *patch* only (same ``major.minor``); +#. for new transactions β€” to the latest ``minor.patch`` (same ``major``). + +Required Executor Set +--------------------- + +- Syncing from genesis requires, for every released ``major.minor``, the + executor with the greatest ``patch``: ``all(major).all(minor).max(patch)``. +- Executing new transactions requires, for every released ``major``, the + executor with the greatest ``minor.patch``: ``all(major).max(minor).max(patch)``. + +.. _gvm-def-vm-error-compat: + +VM Error Code Compatibility +--------------------------- + +A :ref:`VM error code ` splits at `` # `` into a public +code and an optional detail. + +- **Patch**: the entire string, including the detail, is unchanged (follows + from the :ref:`gvm-def-execution-hash` promise). +- **Minor**: the set of public codes is add-only β€” new codes may be added and + an existing code may be extended with new trailing components; codes are + never renamed, removed, or reordered. The detail may change arbitrarily. +- **Major**: no promise. + +The detail is exempt from the registry of predefined codes; its internal +structure is an implementation detail and MUST NOT be relied upon. + +Implementation Note +------------------- + +For ``major = 0`` the rules above do not fully hold: a minor increment may +carry either major or minor meaning. Release candidates are allowed to iterate +the latest runners and change their hashes. diff --git a/docs/website/src/spec/01-core-architecture/index.rst b/docs/website/src/spec/01-core-architecture/index.rst index 2d81206a..c9417f2f 100644 --- a/docs/website/src/spec/01-core-architecture/index.rst +++ b/docs/website/src/spec/01-core-architecture/index.rst @@ -8,3 +8,4 @@ Overview of GenVM's core architectural components and design patterns. 01-components 02-vm + 03-versioning diff --git a/docs/website/src/spec/02-execution-environment/01-wasm.rst b/docs/website/src/spec/02-execution-environment/01-wasm.rst index 281c459d..17839325 100644 --- a/docs/website/src/spec/02-execution-environment/01-wasm.rst +++ b/docs/website/src/spec/02-execution-environment/01-wasm.rst @@ -11,9 +11,10 @@ Enabled WASM Features and Proposals #. Multi Value #. SIMD #. Saturating Float to Int Conversions +#. Tail Call :ref:`gvm-def-det-mode` Additional Limitations ------------------------------------------------- +---------------------------------------------- Only following ``f32.*`` and ``f64.*`` operations are allowed: @@ -27,9 +28,56 @@ Any other floating point operation is considered non-deterministic and is not al :ref:`gvm-def-non-det-mode` does not have these limitations, allowing all floating point operations. +Stack Limiting +-------------- + +WASM recursion depth is bounded by two counters, each private to a +:term:`sub-VM`, so the depth at which execution traps depends only on the WASM +code and its dynamic call pattern β€” never on how the code was compiled: + +- **call stack**: starts at :ref:`gvm-def-consts-value-top-limits-wasm-call-depth`; + every active frame costs :math:`1`. +- **value stack**: starts at :ref:`gvm-def-consts-value-top-limits-wasm-stack-value-slots`; + every active frame costs its function's *value size* β€” the number of declared + locals plus the maximum operand-stack depth of the function body. The value + size is derived statically from the function's code; every value counts as + one slot regardless of its type. Function parameters are not part of the + callee's value size: for a plain ``call`` they are accounted in the + *caller*'s operand-stack depth (see the tail-call case below). + +The counters change only at frame boundaries: + +#. ``call`` (also ``call_indirect``, and the initial host entry into the + :term:`sub-VM`): on entry the *callee* + charges both counters β€” the call stack by :math:`1`, the value stack by its + value size. If either counter would go below zero, execution traps with + :ref:`gvm-def-str-trie-value-vm-error-wasm-trap-stack-overflow`. The + *caller*'s charges remain held for the duration of the call. +#. ``return`` (including falling off the end of the body): the *callee* refunds + exactly what it charged, immediately before control transfers back to the + *caller*. +#. ``return_call`` (also ``return_call_indirect``): the *caller* refunds its + own charges **before** transferring control, then the *callee* charges as + under ``call`` β€” a tail-call chain therefore executes at constant depth. + The tail call's arguments are released by the *caller*'s refund and are + covered by no charge while the *callee* runs; the same holds for the + parameters of the initial host entry, which has no WASM *caller*. + +Calls into host functions charge nothing. Unwinding refunds nothing: whatever +terminates execution early β€” a trap or a host-initiated exit β€” makes the +:term:`sub-VM` exit, discarding its counters. + +An implementation may keep a native-stack safety backstop, but it must be +sized so that it can never trip before these counters do. + +A function whose value size alone exceeds +:ref:`gvm-def-consts-value-top-limits-wasm-stack-value-slots` is rejected when the +module is compiled, with +:ref:`gvm-def-str-trie-value-vm-error-invalid-contract-wasm-validating`. + RAM Consumption --------------- Each WASM table element imposes :ref:`gvm-def-consts-value-memory-limiter-consts-table-entry` :ref:`gvm-def-ram-consumption`\. -Each WASM Memory costs length of bytes it has. WASM ``memory.grow`` instruction which would exceed limit returns :math:`-1` +Each WASM Memory costs length of bytes it has. A runtime WASM ``memory.grow`` instruction which would exceed the limit returns :math:`-1` and leaves memory unchanged. An instantiation-time reservation that cannot be met is fatal instead; see :ref:`gvm-def-ram-consumption` for the exact error. diff --git a/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/01-functions.rst b/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/01-functions.rst index 5fcaecbd..8b3077fa 100644 --- a/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/01-functions.rst +++ b/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/01-functions.rst @@ -10,7 +10,6 @@ Requirements ~~~~~~~~~~~~ #. :term:`Sub-VM` must be in deterministic mode -#. :term:`Sub-VM` must have read storage permission #. index + buf_len must not overflow ``storage_write`` diff --git a/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/02-gl_call.rst b/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/02-gl_call.rst index 7e100c85..3ee2d854 100644 --- a/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/02-gl_call.rst +++ b/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/02-gl_call.rst @@ -49,6 +49,8 @@ Requirements #. :ref:`gvm-perm-deterministic` #. :ref:`gvm-perm-call-others` +.. _gvm-def-gl-call-call-contract: + ``CallContract`` Message ------------------------ @@ -67,19 +69,13 @@ Payload } } -When ``state`` is ``0`` (``default``) :term:`GenVM` resolves it to ``latest_non_final``. -The motivation is that the caller has already observed (and possibly modified) -non-final state in the current transaction, so reading anything older than -``latest_non_final`` for an in-transaction call would expose stale data and -break causality between the caller and callee. - Requirements ~~~~~~~~~~~~ #. :ref:`gvm-perm-deterministic` #. :ref:`gvm-perm-call-others` -Creates new :term:`sub-VM` instance for contract execution. See :ref:`gvm-permissions` for permission inheritance details. +Creates a :term:`sub-VM`. See :ref:`gvm-meta-property-derivation`. .. _gvm-def-post-message: @@ -88,9 +84,9 @@ Creates new :term:`sub-VM` instance for contract execution. See :ref:`gvm-permis Posts message to GenLayer contract for later execution. -When GenVM forwards this message it derives a :ref:`call_key ` -from the ``method`` field of ``calldata`` and attaches it to the emitted -message. The :ref:`call_key ` is the function-selector +When GenVM forwards this message it derives :ref:`gvm-def-call-key` from the +``method`` field of ``calldata`` and attaches it to the emitted message. +:ref:`gvm-def-call-key` is the function-selector analog used to identify the target method. Payload @@ -103,7 +99,9 @@ Payload "address": Address, // 20-byte target contract address "calldata": Calldata, // Method call in calldata format "value": U256, // Wei to transfer - "on": String // "finalized" or "accepted" + "on": String, // "finalized" or "accepted" + "use_balance": Bool, // optional (default false), see below + "fee_params": FeeParams // optional (default absent), required iff use_balance } } @@ -113,6 +111,71 @@ Requirements #. :ref:`gvm-perm-deterministic` #. :ref:`gvm-perm-send-messages` #. Sufficient contract balance for value transfer +#. When ``use_balance`` is set: :ref:`gvm-perm-use-balance-for-message-fees` and ``fee_params`` + +.. _gvm-gl-call-balance-fees: + +Balance-funded fees +~~~~~~~~~~~~~~~~~~~~ + +By default an outgoing internal message's fee is drawn from the sender's +prefunded message-fee pool and matched against the transaction's allocation +tree. Setting ``use_balance`` (the chain's ``useBalance``) instead funds the fee +from the **emitting contract's own balance**. ``fee_params`` carries the child +transaction's fee configuration and mirrors the chain's +``InternalMessageFeeParams``: + +.. code-block:: + + FeeParams { + "leader_timeunits_allocation": U256, // per-round leader time units + "validator_timeunits_allocation": U256, // per-round validator time units + "execution_budget_per_round": U256, // unified budget per leader round + "rotations": [U256], // per-round rotations; non-empty. + // rotations[0] is the initial round, + // the rest are appeal rounds + // (appealRounds = len - 1) + "max_price_gen_per_time_unit": U256, // GEN price cap; funding multiplier + "storage_fee_max_gas_price": U256, // storage price cap (revert guard) + "receipt_fee_max_gas_price": U256 // receipt price cap (revert guard) + } + +Semantics: + +- The fee is metered from ``fee_params`` and that metered amount becomes the child + transaction's ``declaredBudget`` β€” the contract balance is the only bound. The + consensus term is charged at the guest's ``max_price_gen_per_time_unit`` cap + (matching the chain's ``minMessagePrimaryFees``), not the node's live + ``genPerTimeUnit``, so the fee scales with the cap. +- The message is excluded from allocation matching, so no matching node is + required (and none is consulted). +- The contract must be able to cover ``value + metered_fee`` from its balance; + otherwise the call fails with ``Inbalance``. +- The emitted allocation subtree is **empty**: nesting is fail-closed, so a child + message must itself set ``use_balance`` or it fails to fund. + +``fee_params`` is validated before metering. The following are rejected with +``Inval``: + +- ``use_balance`` without ``fee_params``, or ``fee_params`` without ``use_balance``. +- Empty ``rotations`` (``appealRounds`` would underflow). +- A zero ``max_price_gen_per_time_unit``, ``storage_fee_max_gas_price`` or + ``receipt_fee_max_gas_price`` (the chain reverts ``FeeValueMustBeNonZero`` at + reveal). +- Out-of-bounds magnitudes: prices and budgets + (``max_price_gen_per_time_unit``, ``storage_fee_max_gas_price``, + ``receipt_fee_max_gas_price``, ``execution_budget_per_round``) must be below + 2\ :sup:`96`; counts (``leader_timeunits_allocation``, + ``validator_timeunits_allocation``, each ``rotations`` entry) below + 2\ :sup:`32`. These bounds keep the metered floor within ``U256``. + +Metering additionally enforces node-configured floors, surfaced as ``VMError``\ s: + +- ``fee below_minimum`` β€” a per-phase time-unit allocation below + ``node.minTimeUnitsPerPhase``, or a non-zero ``execution_budget_per_round`` below + ``node.messageBudgetFloor`` (the chain's ``BudgetTooLow``). +- ``fee too_many_rounds`` β€” ``rotations`` implies more consensus rounds than the + node's validator table supports (on-chain ``MAX_ROUNDS``). ``DeployContract`` Message -------------------------- @@ -130,7 +193,9 @@ Payload "code": Bytes, // Contract bytecode "value": U256, // Wei to transfer "on": String, // "finalized" or "accepted" - "salt_nonce": U256 // Salt for CREATE2-style deterministic addressing + "salt_nonce": U256, // Salt for CREATE2-style deterministic addressing + "use_balance": Bool, // optional (default false) + "fee_params": FeeParams // optional (default absent), required iff use_balance } } @@ -140,15 +205,19 @@ Requirements #. :ref:`gvm-perm-deterministic` #. :ref:`gvm-perm-send-messages` #. Sufficient contract balance for value transfer +#. When ``use_balance`` is set: :ref:`gvm-perm-use-balance-for-message-fees` and ``fee_params`` Supports CREATE2-style deployment with salt nonce for deterministic addressing. +``use_balance`` / ``fee_params`` behave as for :ref:`gvm-gl-call-balance-fees`. + +.. _gvm-def-gl-call-run-nondet: ``RunNondet`` Message --------------------- -Executes non-deterministic code with leader/validator consensus. -Creates :ref:`gvm-def-non-det-mode` VM instance with restricted permissions. -See :doc:`../../03-vm/04-determinism-mode-switching` and :ref:`gvm-permissions` for more details. +Executes non-deterministic code with leader/validator consensus. See +:doc:`../../03-vm/04-determinism-mode-switching` and +:ref:`gvm-meta-property-derivation`. Payload ~~~~~~~ @@ -157,8 +226,10 @@ Payload { "RunNondet": { - "data_leader": Bytes, // Code/data for leader execution - "data_validator": Bytes // Code/data for validator execution + "data_leader": Bytes, // Code/data for leader execution + "data_validator": Bytes, // Code/data for validator execution + "runner": String, // optional (default "contract"): runner to execute + "custom_runners": [String] // optional (default absent): custom runners to grant } } @@ -167,10 +238,20 @@ Requirements #. :ref:`gvm-perm-spawn-nondet` +Semantics +~~~~~~~~~ + +Creates a non-deterministic :term:`sub-VM`. Derivation of its meta-properties, +including the *param* ``runner`` and *param* ``custom_runners`` semantics, is +specified in :ref:`gvm-meta-property-derivation`. + +.. _gvm-def-gl-call-sandbox: + ``Sandbox`` Message ------------------- -Executes code in sandboxed environment with restricted permissions. +Executes code in a sandboxed environment. See +:ref:`gvm-meta-property-derivation`. Payload ~~~~~~~ @@ -179,23 +260,36 @@ Payload { "Sandbox": { - "data": Bytes, // Code/data for sandbox execution - "allow_write_storage": Bool, // Whether to allow storage writes - "allow_send_messages": Bool, // Whether to allow sending messages - "allow_register_runners": Bool // Whether to allow registering runners + "data": Bytes, // Code/data for sandbox execution + "runner": String, // runner to execute; becomes the child's "contract" + "allow_write_storage": Bool, // Whether to allow storage writes + "allow_send_messages": Bool, // Whether to allow sending messages + "allow_register_runners": Bool, // Whether to allow registering runners + "custom_runners": [String] // optional (default absent): custom runners to grant } } -Creates isolated VM instance. See :ref:`gvm-permissions` for permission inheritance details. +Semantics +~~~~~~~~~ + +Creates a :term:`sub-VM` at the caller's determinism level. Derivation of its +meta-properties, including the *param* ``runner`` and *param* +``custom_runners`` semantics, is specified in +:ref:`gvm-meta-property-derivation`. + +The caller receives the sandbox result (:ref:`gvm-def-subvm-result-encoding`) +and may handle both :ref:`gvm-def-vm-error` and :ref:`gvm-def-user-error`; +storage writes performed by the sandbox are not reverted on error. + +.. _gvm-def-gl-call-register-runner: ``RegisterRunner`` Message -------------------------- Registers a runner archive at runtime, making it available under the ``custom:`` runner id. The ```` is the SHA3-256 of the supplied -``code`` encoded with :doc:`../../04-contract-interface/06-gvm32`, and the parsed -archive is charged against the memory limit. Returns the resulting runner id -(calldata-encoded string). +``code`` encoded with :doc:`../../04-contract-interface/06-gvm32`. +See :ref:`gvm-def-custom-runner-visibility`. Payload ~~~~~~~ @@ -214,11 +308,36 @@ Requirements #. :ref:`gvm-perm-deterministic` #. :ref:`gvm-perm-register-runners` +Semantics +~~~~~~~~~ + +``RegisterRunner`` performs a load action for ``custom:`` in the calling +:term:`sub-VM`. If the runner is already loaded, registration is a free no-op +that returns the same runner id. Otherwise +:ref:`gvm-def-consts-value-memory-limiter-consts-runner-load-cost` plus ``code`` length is charged against +the caller's RAM budget before the archive is parsed; on success, the runner +enters the caller's loaded set. + +The outcomes, in check order, are: + +#. Missing :ref:`gvm-def-det-mode` or :ref:`gvm-perm-register-runners`: the call + fails with ``Forbidden``. Nothing is charged and no state changes. +#. Insufficient memory for the charge: the :term:`sub-VM` exits with an + out-of-memory :ref:`gvm-def-vm-error`. Nothing is charged and the runner is + not registered. +#. Malformed archive: the call fails with a deterministic invalid-contract + :ref:`gvm-def-vm-error`. The charge is retained until the :term:`sub-VM` + finishes, and the runner is not in the loaded set. +#. Success: the runner id is returned and the runner is in the caller's loaded + set. + +.. _gvm-def-gl-call-map-file: + ``MapFile`` Message ------------------- Maps a file from a runner into the VM filesystem at runtime, behaving the same as -the ``MapFile`` runner action (see :doc:`../../../python-sdk` runners). If +the ``MapFile`` runner action (see :doc:`../../../python-sdk/index` runners). If ``path_in_runner`` ends with ``/`` the whole directory subtree is mapped. Payload @@ -234,13 +353,12 @@ Payload } } -Mapping into ``/vm/`` is forbidden. Resolving a ``chain:`` runner reads another -contract's storage, so this requires :ref:`gvm-perm-read-storage`. - -Requirements -~~~~~~~~~~~~ +Mapping into ``/vm/`` is forbidden. See +:ref:`gvm-def-custom-runner-visibility` for ``custom:`` runner resolution. -#. :ref:`gvm-perm-read-storage` +Resolving *param* ``runner`` performs a load action for that runner. The load is +charged on first load in this :term:`sub-VM` and is free if the runner is +already loaded. ``WebRender`` Message --------------------- @@ -435,7 +553,8 @@ Payload "Return": Calldata // Return value in calldata format } -Causes VM to exit with ``ContractReturn``. Encodes return value using :ref:`Calldata Encoded ` format. +Causes VM to exit with ``ContractReturn``. Encodes return value using +:ref:`gvm-def-calldata-encoding` format. ``Trace.Message`` Message ------------------------- @@ -537,7 +656,7 @@ Payload "GetTimestamp": null } -Returns the timestamp encoded as a :ref:`Calldata Encoded ` number. +Returns the timestamp encoded as a :ref:`gvm-def-calldata-encoding` number. Requirements ~~~~~~~~~~~~ diff --git a/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/03-schemas.rst b/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/03-schemas.rst index f127af46..c27c33f2 100644 --- a/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/03-schemas.rst +++ b/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/03-schemas.rst @@ -9,7 +9,9 @@ based on operation type. Sub-VM Result Encoding ---------------------- -Operations that spawn sub-VMs (``CallContract``, ``RunNondet``, ``Sandbox``) return +Operations that spawn sub-VMs (:ref:`gvm-def-gl-call-call-contract`, +:ref:`gvm-def-gl-call-run-nondet`, +:ref:`gvm-def-gl-call-sandbox`) return results through a file descriptor with the following binary format: .. code-block:: diff --git a/docs/website/src/spec/02-execution-environment/04-runners.rst b/docs/website/src/spec/02-execution-environment/04-runners.rst index d56882e3..6b1cd2d7 100644 --- a/docs/website/src/spec/02-execution-environment/04-runners.rst +++ b/docs/website/src/spec/02-execution-environment/04-runners.rst @@ -20,7 +20,7 @@ forms (see the ``runner-id`` definition in the runner.json schema): - ``chain:
[:[:]]`` β€” a runner code blob read from a storage slot of an arbitrary contract (``a`` = accepted, ``f`` = finalized). ``
`` is a ``0x`` 20 byte hex address and ```` is a :term:`SlotID` encoded with - :doc:`../../04-contract-interface/06-gvm32`. Both ```` and ```` are + :doc:`../04-contract-interface/06-gvm32`. Both ```` and ```` are optional: ```` defaults to ``a`` and ```` to the target contract's root code slot. - ``custom:`` β€” a runner registered at runtime via the ``RegisterRunner`` @@ -33,7 +33,7 @@ Hash Format ~~~~~~~~~~~ ``hash`` is the SHA3 256-bit hash of the runner's contents, encoded with -:doc:`../../04-contract-interface/06-gvm32` (GVM32, a lowercase Crockford +:doc:`../04-contract-interface/06-gvm32` (GVM32, a lowercase Crockford Base32). This keeps it free of filesystem-illegal characters and case-insensitive. @@ -300,6 +300,48 @@ Startup Runner actions are executed left-recursively, until :ref:`gvm-def-start-wasm` is reached. If it was not reached, it will result in a :ref:`gvm-def-vm-error` with ``error_inval`` code. -Loading a :term:`runner` implies :ref:`gvm-def-ram-consumption` of its size in octets. -Each loading in a single contract, but different :term:`sub-VM` instances, leads to additional RAM consumption. -:ref:`gvm-def-det-mode` and :ref:`gvm-def-non-det-mode` have separate RAM limits. +Loading a :term:`runner` goes through a single **load action**, defined per +:term:`sub-VM`. Each :term:`sub-VM` owns a **loaded-runner set**: the runner +ids it has already loaded. The load action for an id is: + +- if the id is already in the :term:`sub-VM`'s loaded set, nothing is charged; +- otherwise :ref:`gvm-def-consts-value-memory-limiter-consts-runner-load-cost` plus the runner's size in octets is charged as + :ref:`gvm-def-ram-consumption` against the :term:`sub-VM`'s RAM budget, and + the id is then added to the loaded set. + +Whether the executor has the archive cached internally is not observable: the +charge depends only on the :term:`sub-VM`'s own load history, never on cache +state. The same runner loaded by different :term:`sub-VM` instances is charged +once per :term:`sub-VM`. :ref:`gvm-def-det-mode` and +:ref:`gvm-def-non-det-mode` have separate RAM budgets and separate loaded sets. + +A load action occurs when: + +- spawning the :term:`sub-VM`'s main (entry-point) runner; +- resolving a ``Depends`` or ``With`` action in a ``runner.json``; +- executing the ``MapFile`` ``gl_call``; +- registering a runner via the ``RegisterRunner`` ``gl_call``; +- receiving a custom-runner grant at :term:`sub-VM` creation + (see :ref:`gvm-meta-property-custom-runners`). + +For a ``chain:`` runner the size is the length of the code blob read from +storage. A ``chain:`` load costs the same as any other load of that size β€” +there is no doubled charge and no separate fee component. + +.. _gvm-def-custom-runner-visibility: + +Custom :term:`Runner` Loading +----------------------------- + +A ``custom:`` id resolves *iff* ```` is in the resolving +:term:`sub-VM`'s own loaded set; otherwise loading it fails with a +:ref:`gvm-def-vm-error`. There is no separate registry lookup at resolution β€” a +:term:`sub-VM` can use exactly the custom runners it has loaded, whether by +registering them itself via the ``RegisterRunner`` ``gl_call`` or by receiving +grants from its parent at creation time +(:ref:`gvm-meta-property-custom-runners`). + +Registered content lives while at least one :term:`sub-VM` has it loaded; once +no loaded set holds it, it is freed. Registering the same ``code`` again while +it is still loaded somewhere is deduplicated by hash; re-registering it after it +has been freed re-parses and charges again. diff --git a/docs/website/src/spec/03-vm/01-startup.rst b/docs/website/src/spec/03-vm/01-startup.rst new file mode 100644 index 00000000..704eb3fd --- /dev/null +++ b/docs/website/src/spec/03-vm/01-startup.rst @@ -0,0 +1,222 @@ +.. _gvm-vm-startup: + +Startup +======= + +This page specifies how :term:`sub-VM`\s start: the top-level entry, the +startup message, runner initialization, and the derivation of meta-properties +when a child :term:`sub-VM` is created. The meta-property fields themselves are +defined in :doc:`02-meta-properties`. Runner syntax and load mechanics are +specified in :doc:`../02-execution-environment/04-runners`. + +Top-Level Startup +----------------- + +Before executing the root :term:`sub-VM`, :term:`GenVM`: + +#. Reads the contract's locked slots and root-slot data. +#. Resolves the contract code from deployment input or from the contract's + configured code slot. +#. Checks upgrade and ABI-major compatibility as specified in + :doc:`../04-contract-interface/04-upgradability`. +#. Reads contract-owned permission bits from the root slot and combines them + with node-granted VM permissions. +#. Creates a deterministic root VM with the initial message described below. + +If code is supplied with the execution request, that code is the runner source +for the deployment execution. Otherwise, the runner source is the contract code +resolved from storage. + +.. _gvm-vm-startup-message: + +Startup Message +--------------- + +The executable WASM receives one :ref:`gvm-def-calldata-encoding` message on +standard input. +The message contains: + +- ``contract_address``, ``sender_address``, ``origin_address`` +- ``signer_address``: the externally-owned account that signed the + transaction. :term:`GenVM` treats it as opaque and forwards it unchanged to + every child :term:`sub-VM`. +- :ref:`gvm_vm_field_stack`: the view-call stack, empty for a top-level entry +- ``chain_id``, ``value``, ``is_init``, ``datetime`` +- ``entry_kind``: one of :ref:`gvm-def-enum-entry-kind` +- ``entry_data``: entry payload bytes +- ``entry_stage_data``: consensus-stage payload data + +For top-level execution, ``entry_kind`` is +:ref:`gvm-def-enum-value-entry-kind-main` and ``entry_stage_data`` is absent +data. Sub-VM operations may create sandbox or consensus-stage messages. + +Runner Startup +-------------- + +The selected runner is loaded and its initialization actions are applied until +:ref:`gvm-def-start-wasm` is reached. Actions may map files, set process +arguments, add environment variables, or link additional WASM modules. Linked +modules that export ``_initialize`` execute that function before startup +continues. + +The ``StartWasm`` action instantiates the executable WASM module. Its ``_start`` +entrypoint then consumes the startup message and produces the VM result. + +.. _gvm-vm-subvm-creation: +.. _gvm-meta-property-derivation: + +Sub-VM Creation +--------------- + +A new :term:`sub-VM` is created for: + +- the initial entry +- :ref:`gvm-def-gl-call-call-contract` +- :ref:`gvm-def-gl-call-sandbox` +- :ref:`gvm-def-gl-call-run-nondet` + +Creation is rejected with +:ref:`gvm-def-str-trie-value-vm-error-out-of-vm-recursion` if the new +:ref:`gvm_vm_field_depth` is greater than or equal to +:ref:`gvm-def-consts-value-top-limits-vm-recursion`. + +Each :ref:`gvm-def-gl-call-run-nondet` additionally allocates the next +execution-wide ``call_no``, starting at zero. If ``call_no`` is greater than or +equal to :ref:`gvm-def-consts-value-top-limits-nondet-blocks`, the call fails +with :ref:`gvm-def-str-trie-value-vm-error-out-of-nondet-blocks`. + +Creating the :term:`sub-VM` charges +:ref:`gvm-def-consts-value-memory-limiter-consts-vm-spawn-cost` octets of +:ref:`gvm-def-ram-consumption` to the new :term:`sub-VM`; the charge is +released when it finishes. + +After these checks, startup applies the `Custom-Runner Grants`_ for the new +:term:`sub-VM`, then performs the runner load action for the entry runner and +continues with `Runner Startup`_. + +Meta-Property Derivation +------------------------ + +Every meta-property of a child :term:`sub-VM` starts as an independent copy of +the parent's value; later changes in the child never affect the parent. The +subsections below list only the fields that deviate from that copy. Two +deviations apply to every child: + +- :ref:`gvm_vm_field_depth` is ``parent.depth + 1`` +- :ref:`gvm_vm_field_det_subvm_hashes` starts empty + +The initial entry has no parent, so all of its fields are given explicitly. + +Initial Entry +~~~~~~~~~~~~~ + +- :ref:`gvm_vm_field_stack` is empty; :ref:`gvm_vm_field_depth` is ``0``. +- :ref:`gvm_vm_field_permissions` are the node-granted permission + meta-properties, plus :ref:`gvm-perm-use-balance-for-message-fees` read from + the contract's root slot. +- :ref:`gvm_vm_field_state_mode` is the default storage view. +- :ref:`gvm_vm_field_topmost_runner_id` is the deployment runner or the + contract's accepted code runner. +- :ref:`gvm_vm_field_det_subvm_hashes` and + :ref:`gvm_vm_field_granted_custom` are empty. + +:ref:`gvm-def-gl-call-call-contract` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The child is read-only. + +- :ref:`gvm_vm_field_permissions` deviations: + + - :ref:`gvm-perm-write-storage` is false + - :ref:`gvm-perm-send-messages` is false + - :ref:`gvm-perm-spawn-nondet` is false + - :ref:`gvm-perm-use-balance-for-message-fees` is false + +- :ref:`gvm_vm_field_state_mode` is the requested storage view (*param* + ``state``); a request of :ref:`gvm-def-enum-value-storage-type-default` + keeps the parent's value (the plain copy rule), so by default the callee + observes a view at least as recent as its caller's. Because the child + cannot write, its :ref:`gvm-def-enum-value-storage-type-default` view is + the accepted state: it never includes the calling transaction's uncommitted + writes (see :ref:`contract-execution-flow`). +- :ref:`gvm_vm_field_topmost_runner_id` is the callee's contract runner. +- :ref:`gvm_vm_field_granted_custom` is the caller's entire loaded + custom-runner set (see `Custom-Runner Grants`_). +- The child's startup message is a copy of the caller's, except: + + - ``contract_address`` is the callee's address + - :ref:`gvm_vm_field_stack` additionally has the caller's + ``contract_address`` appended + - ``entry_data`` is the :ref:`Calldata Encoded ` + method call + - ``value`` is ``0`` + - ``is_init`` is false + +:ref:`gvm-def-gl-call-sandbox` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The child runs at the same determinism level as its parent and cannot exceed +the parent's privileges: each ``allow_*`` payload flag can only retain an +inherited permission, never add one. + +- :ref:`gvm_vm_field_permissions` deviations: + + - :ref:`gvm-perm-write-storage` is cleared unless *param* + ``allow_write_storage`` is set + - :ref:`gvm-perm-spawn-nondet` is false + - :ref:`gvm-perm-call-others` is false + - :ref:`gvm-perm-send-messages` is cleared unless *param* + ``allow_send_messages`` is set + - :ref:`gvm-perm-use-balance-for-message-fees` is cleared unless *param* + ``allow_send_messages`` is set: balance-funded fees only affect message + emission, so they are gated by the same flag + - :ref:`gvm-perm-register-runners` is cleared unless *param* + ``allow_register_runners`` is set + +- :ref:`gvm_vm_field_topmost_runner_id` is the *param* ``runner``, resolved in + the caller's scope. +- :ref:`gvm_vm_field_granted_custom` is derived from *param* + ``custom_runners`` as specified in `Custom-Runner Grants`_. + +:ref:`gvm-def-gl-call-run-nondet` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The child executes in :ref:`gvm-def-non-det-mode`. + +- :ref:`gvm_vm_field_permissions` are all false, including + :ref:`gvm-perm-deterministic`. +- :ref:`gvm_vm_field_state_mode` is the default storage view. +- :ref:`gvm_vm_field_topmost_runner_id` is the *param* ``runner``, resolved in + the caller's scope; when absent, the caller's own runner is used. +- :ref:`gvm_vm_field_granted_custom` is derived from *param* + ``custom_runners`` as specified in `Custom-Runner Grants`_. + +.. _gvm-meta-property-custom-runners: + +Custom-Runner Grants +-------------------- + +At creation, the child's :ref:`gvm_vm_field_granted_custom` is derived from +the caller's loaded custom-runner set +(see :ref:`gvm-def-custom-runner-visibility`): + +- :ref:`gvm-def-gl-call-sandbox` and :ref:`gvm-def-gl-call-run-nondet` + children receive the runners named by *param* ``custom_runners``: + + - when absent, every ``custom:`` entry of the caller's loaded set is granted; + - when present, exactly the listed runners are granted. Every listed runner + must be a ``custom:`` id loaded in the caller, without duplicates; + a list containing any other kind of id (including ``name:hash`` and + ``chain:``) is a :ref:`gvm-def-vm-error`; + - if the *param* ``runner`` is itself a ``custom:`` id, it must be loaded in + the caller and is granted implicitly. + +- :ref:`gvm-def-gl-call-call-contract` children receive the caller's entire + custom set. + +Each grant is a load action in the child, charged against the child's RAM +budget. Grants are applied before the entry-runner load, so a ``custom:`` +entry point that is also granted is charged once. + +Grants never flow back: when a child :term:`sub-VM` finishes, the parent's +loaded set is unchanged, even if the child registered runners. diff --git a/docs/website/src/spec/03-vm/02-meta-properties.rst b/docs/website/src/spec/03-vm/02-meta-properties.rst new file mode 100644 index 00000000..b1a72623 --- /dev/null +++ b/docs/website/src/spec/03-vm/02-meta-properties.rst @@ -0,0 +1,145 @@ +.. _gvm-meta-properties: +.. _gvm-permissions: + +Meta-Properties +=============== + +Each :term:`sub-VM` carries VM meta-properties. Some are part of the startup +message; others are VM-internal. This page defines those fields; how they are +derived when a :term:`sub-VM` starts is specified in +:ref:`gvm-meta-property-derivation`. + +Most permission meta-properties are granted by the node for the initial entry. +The exception is :ref:`gvm-perm-use-balance-for-message-fees`, which is owned by +the contract: it is stored in the contract's root slot and read before +execution begins. + +Execution Meta-Properties +------------------------- + +.. _gvm_vm_field_stack: + +``stack`` +~~~~~~~~~ + +Startup-message field containing the view-call stack of contract addresses. + +.. _gvm_vm_field_depth: + +``depth`` +~~~~~~~~~ + +Internal field containing the nesting depth of the :term:`sub-VM`. + +.. _gvm_vm_field_permissions: + +``permissions`` +~~~~~~~~~~~~~~~ + +Internal field containing the permission meta-properties granted to the +:term:`sub-VM`. + +.. _gvm_vm_field_state_mode: + +``state_mode`` +~~~~~~~~~~~~~~ + +Internal field selecting the storage view used by reads +(a :ref:`gvm-def-enum-storage-type`). + +.. _gvm_vm_field_topmost_runner_id: + +``topmost_runner_id`` +~~~~~~~~~~~~~~~~~~~~~ + +Internal field selecting the entry runner for this :term:`sub-VM`. + +.. _gvm_vm_field_det_subvm_hashes: + +``det_subvm_hashes`` +~~~~~~~~~~~~~~~~~~~~ + +Internal accumulator for deterministic child :term:`sub-VM` result hashes; see +:ref:`gvm-def-subvm-hash`. + +.. _gvm_vm_field_granted_custom: + +``granted_custom`` +~~~~~~~~~~~~~~~~~~ + +Internal field containing custom runners granted to this :term:`sub-VM` at +startup; see :ref:`gvm-meta-property-custom-runners`. + +Custom runners are *not* permission meta-properties: they are tracked by the +:term:`sub-VM`'s loaded-runner set +(:ref:`gvm-def-custom-runner-visibility`) and by this field. + +Permission Meta-Properties +-------------------------- + +.. _gvm-perm-deterministic: + +``deterministic`` +~~~~~~~~~~~~~~~~~ + +When true, the :term:`sub-VM` executes in :ref:`gvm-def-det-mode`. Storage +writes, message sends, contract calls, event emission, and runner registration +require this meta-property where specified below. + +.. _gvm-perm-write-storage: + +``write_storage`` +~~~~~~~~~~~~~~~~~ + +Allows writing contract storage slots. Requires +:ref:`gvm-perm-deterministic`. + +This meta-property also gates ``EmitEvent``: a :term:`sub-VM` may emit events +iff it can write storage. + +.. _gvm-perm-send-messages: + +``send_messages`` +~~~~~~~~~~~~~~~~~ + +Allows sending messages to other addresses. Required by ``EthSend``, +``PostMessage``, and ``DeployContract``. Requires +:ref:`gvm-perm-deterministic`. + +.. _gvm-perm-call-others: + +``call_others`` +~~~~~~~~~~~~~~~ + +Allows calling other contracts. Required by ``EthCall`` and +:ref:`gvm-def-gl-call-call-contract`. +Requires :ref:`gvm-perm-deterministic`. + +.. _gvm-perm-spawn-nondet: + +``spawn_nondet`` +~~~~~~~~~~~~~~~~ + +Allows spawning :ref:`gvm-def-non-det-mode` :term:`sub-VM` instances via +:ref:`gvm-def-gl-call-run-nondet`. + +.. _gvm-perm-register-runners: + +``register_runners`` +~~~~~~~~~~~~~~~~~~~~ + +Allows registering runner archives at runtime via +:ref:`gvm-def-gl-call-register-runner`. Requires :ref:`gvm-perm-deterministic`. + +.. _gvm-perm-use-balance-for-message-fees: + +``can_use_balance_for_message_fees`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Allows the contract to draw on its own balance to pay fees for outgoing +internal messages (``PostMessage``, ``DeployContract``). The flag has no effect +on external ``EthSend`` messages. + +Unlike the meta-properties above, this one is stored as a bit in the root slot's +inline ``permissions`` bitfield and read before execution begins. The bit offset +is the corresponding member of :ref:`gvm-def-enum-permissions`. diff --git a/docs/website/src/spec/03-vm/02-sandboxing.rst b/docs/website/src/spec/03-vm/02-sandboxing.rst deleted file mode 100644 index 1eca6820..00000000 --- a/docs/website/src/spec/03-vm/02-sandboxing.rst +++ /dev/null @@ -1,13 +0,0 @@ -Sandboxing -========== - -For the sake of executing prompt-generated code, - users are provided with ability to spawn a separate :term:`sub-VM` for executing it. - -This :term:`sub-VM`: - -#. Has the same non-deterministic level as parent :term:`sub-VM` -#. Can not switch into non-deterministic mode -#. Can be configured to be able to update storage (privilege escalation is forbidden) - -Users can catch both :ref:`gvm-def-vm-error` and :ref:`gvm-def-user-error` produced by it, but storage writes can not be reverted diff --git a/docs/website/src/spec/03-vm/03-ram-limiting.rst b/docs/website/src/spec/03-vm/03-ram-limiting.rst index 2558bef3..a6c34c3c 100644 --- a/docs/website/src/spec/03-vm/03-ram-limiting.rst +++ b/docs/website/src/spec/03-vm/03-ram-limiting.rst @@ -12,20 +12,52 @@ RAM Consumption Every resource allocation subtracts from the RAM budget of the current :ref:`gvm-def-vm-mode`. When an allocation would cause the remaining budget to become negative, -the :term:`sub-VM` exits with :ref:`gvm-def-vm-error` with :ref:`gvm-def-str-trie-value-vm-error-OOM-RAM` message. +the :term:`sub-VM` exits with :ref:`gvm-def-vm-error` carrying the +:ref:`gvm-def-str-trie-value-vm-error-out-of-memory` message, or one of its +:ref:`gvm-def-str-trie-value-vm-error-out-of-memory-wasm-memory` / +:ref:`gvm-def-str-trie-value-vm-error-out-of-memory-wasm-table` variants for the +corresponding WASM allocations. ``memory.grow`` and ``table.grow`` are the +exception: at runtime they are recoverable rather than fatal (see the two +bullets below). The following operations consume RAM: -- **WASM memory growth**: each page (65536 octets) costs its size in bytes -- **WASM table growth**: each table entry costs :ref:`gvm-def-consts-value-memory-limiter-consts-table-entry` octets +- **WASM memory growth**: each page (65536 octets) costs its size in bytes. + A runtime ``memory.grow`` that would exceed the budget is **not** fatal: + following the WASM specification it leaves memory unchanged and evaluates to + :math:`-1`, so the guest can react. Only the memory's initial, + instantiation-time reservation being unmet makes the :term:`sub-VM` exit with + :ref:`gvm-def-str-trie-value-vm-error-out-of-memory-wasm-memory`. +- **WASM table growth**: each table entry costs :ref:`gvm-def-consts-value-memory-limiter-consts-table-entry` octets. + As with memory, a runtime ``table.grow`` beyond the budget evaluates to + :math:`-1`; only the instantiation-time reservation being unmet exits with + :ref:`gvm-def-str-trie-value-vm-error-out-of-memory-wasm-table`. - **File mapping**: :ref:`gvm-def-consts-value-memory-limiter-consts-file-mapping` octets base cost plus the length of the filename in bytes - **File descriptor allocation**: :ref:`gvm-def-consts-value-memory-limiter-consts-fd-allocation` octets per descriptor +- **Runner loading**: the first load of a :term:`runner` in a :term:`sub-VM` + costs :ref:`gvm-def-consts-value-memory-limiter-consts-runner-load-cost` plus the runner's size in octets. A runner already + in that :term:`sub-VM`'s loaded set costs nothing, and the charge is released + when the :term:`sub-VM` finishes, like any other charge. Loading covers + spawning the entry-point runner, ``Depends``/``With`` actions, the ``MapFile`` + and ``RegisterRunner`` ``gl_call``\ s, and receiving a custom-runner grant at + sub-VM creation (see :doc:`../02-execution-environment/04-runners` and + :ref:`gvm-meta-property-custom-runners`) +- **Sub-VM creation**: each new :term:`sub-VM` costs + :ref:`gvm-def-consts-value-memory-limiter-consts-vm-spawn-cost` octets, + charged to the new :term:`sub-VM` at creation (see :doc:`01-startup`) and + released when it finishes + +The runner load cost (:ref:`gvm-def-consts-value-memory-limiter-consts-runner-load-cost`) is a fixed per-load +overhead RAM Release ----------- File content memory is released when the corresponding file descriptor is closed via ``fd_close``. When a :term:`sub-VM` finishes execution, all remaining RAM consumed by it is released back to the shared budget. +This applies to runner charges as well: memory consumed by loading or +registering a runner is released when the registering :term:`sub-VM` finishes, +like any other charge. There are no permanent charges. Other Limits ------------ diff --git a/docs/website/src/spec/03-vm/05-permissions.rst b/docs/website/src/spec/03-vm/05-permissions.rst deleted file mode 100644 index 1435ec1d..00000000 --- a/docs/website/src/spec/03-vm/05-permissions.rst +++ /dev/null @@ -1,103 +0,0 @@ -.. _gvm-permissions: - -Permissions -=========== - -Each :term:`sub-VM` instance has a set of boolean permissions that control what operations it can perform. -Permissions are inherited from the parent VM when spawning child :term:`sub-VM` instances, with certain restrictions applied depending on the context. - -.. _gvm-perm-deterministic: - -``deterministic`` ------------------ - -When set, the VM is executing in :ref:`gvm-def-det-mode`. -Many operations require this permission, including storage writes, sending messages, calling other contracts, and emitting events. - -.. _gvm-perm-read-storage: - -``read_storage`` ----------------- - -Allows reading contract storage slots. When unset, any attempt to read storage will fail with a ``Forbidden`` error. - -.. _gvm-perm-write-storage: - -``write_storage`` ------------------ - -Allows writing to contract storage slots. Requires :ref:`gvm-perm-deterministic` as well. - -This permission also gates ``EmitEvent``: a VM may emit events *iff* it can write storage, since events are state-mutating log emissions. Consequently a read-only (static) context, such as a ``CallContract`` child, cannot emit events. - -.. _gvm-perm-send-messages: - -``send_messages`` ------------------ - -Allows sending messages to other addresses. This permission is required by ``EthSend``, ``PostMessage``, and ``DeployContract`` operations. -Requires :ref:`gvm-perm-deterministic` as well. - -.. _gvm-perm-call-others: - -``call_others`` ---------------- - -Allows calling other contracts. This permission is required by ``EthCall`` and ``CallContract`` operations. -Requires :ref:`gvm-perm-deterministic` as well. - -.. _gvm-perm-spawn-nondet: - -``spawn_nondet`` ----------------- - -Allows spawning :ref:`gvm-def-non-det-mode` :term:`sub-VM` instances via ``RunNondet``. - -.. _gvm-perm-register-runners: - -``register_runners`` --------------------- - -Allows registering runner archives at runtime via ``RegisterRunner``, making -them available under ``custom:`` runner ids. Requires :ref:`gvm-perm-deterministic` as well. - -Permission Changes on Sub-VM Creation --------------------------------------- - -Different operations modify permissions when creating child :term:`sub-VM` instances: - -``CallContract`` -~~~~~~~~~~~~~~~~ - -A ``CallContract`` child runs as a read-only (static) call. It inherits all parent permissions except: - -- :ref:`gvm-perm-write-storage` is **disabled** -- :ref:`gvm-perm-send-messages` is **disabled** - -A static call must not produce externally visible effects, so storage writes, message sends, and event emissions all fail: emitting an event additionally requires :ref:`gvm-perm-write-storage` (disabled here). Allowing any such emission would charge fees for effects that are discarded together with the child VM. - -``RunNondet`` -~~~~~~~~~~~~~ - -The non-deterministic :term:`sub-VM` has: - -- :ref:`gvm-perm-deterministic` is **disabled** -- :ref:`gvm-perm-read-storage` is **inherited** -- :ref:`gvm-perm-write-storage` is **disabled** -- :ref:`gvm-perm-spawn-nondet` is **disabled** -- :ref:`gvm-perm-call-others` is **disabled** -- :ref:`gvm-perm-send-messages` is **disabled** -- :ref:`gvm-perm-register-runners` is **disabled** - -``Sandbox`` -~~~~~~~~~~~ - -The sandboxed :term:`sub-VM` has: - -- :ref:`gvm-perm-deterministic` is **inherited** -- :ref:`gvm-perm-read-storage` is **inherited** -- :ref:`gvm-perm-write-storage` is **inherited** *iff* ``allow_write_storage`` is set, otherwise **disabled** -- :ref:`gvm-perm-spawn-nondet` is **disabled** -- :ref:`gvm-perm-call-others` is **disabled** -- :ref:`gvm-perm-send-messages` is **inherited** *iff* ``allow_send_messages`` is set, otherwise **disabled** -- :ref:`gvm-perm-register-runners` is **inherited** *iff* ``allow_register_runners`` is set, otherwise **disabled** diff --git a/docs/website/src/spec/03-vm/01-result.rst b/docs/website/src/spec/03-vm/05-result.rst similarity index 81% rename from docs/website/src/spec/03-vm/01-result.rst rename to docs/website/src/spec/03-vm/05-result.rst index 3700dba7..88570044 100644 --- a/docs/website/src/spec/03-vm/01-result.rst +++ b/docs/website/src/spec/03-vm/05-result.rst @@ -21,6 +21,31 @@ exceeding resource limits. It uses predefined string error codes. +.. _gvm-def-vm-error-code: + +VM Error Code Format +-------------------- + +:: + + vm-error-code := public-code [ " # " detail ] + +- ``public-code`` β€” a sequence of ``snake_case`` components separated by single + spaces, drawn from the :ref:`predefined codes `. It + never contains ``#``. +- ``detail`` β€” optional free-form UTF-8 diagnostic. When present, it is + separated from the public code by ``#`` surrounded by a single space on each + side. + +The detail MUST be a deterministic function of execution (the full string is +covered by the :ref:`gvm-def-execution-hash`), but its content carries no +compatibility promise β€” see :ref:`gvm-def-vm-error-compat`. + +Consumers MUST compare and match VM error codes only by the public code (the +part before the first `` # ``). A consumer matching a known code ``P`` MUST +treat a code as matching iff its public code equals ``P`` or extends ``P`` +with further space-separated components. + .. _gvm-def-user-error: .. rubric:: UserError @@ -92,11 +117,11 @@ Both fields are serialized using :ref:`gvm-def-calldata-encoding` to be determin #. Memories are ordered by their index in WASM module #. Memories are hashed using BLAKE3 hash function, which is cryptographically secure and provides acceptable performance +.. _gvm-def-execution-hash: + Execution Hash -------------- -.. _gvm-def-execution-hash: - Every run produces an *execution hash*: a SHA3-256 digest over a :ref:`gvm-def-calldata-encoding` encoding of the consensus-visible result, as a map with the following keys (in this order): @@ -112,11 +137,11 @@ with the following keys (in this order): Two runs that agree on the deterministic result produce the same execution hash, so consensus can compare a single 32-byte value instead of the full result. +.. _gvm-def-subvm-hash: + Sub-VM Result Hash ------------------ -.. _gvm-def-subvm-hash: - A deterministic run accumulates the result of each deterministic :term:`sub-VM` call into a rolling SHA3-256 accumulator. The finalized accumulator is the ``subvm_hashes`` field of the parent's :ref:`gvm-def-execution-hash`. diff --git a/docs/website/src/spec/03-vm/index.rst b/docs/website/src/spec/03-vm/index.rst index da4daf53..0128536b 100644 --- a/docs/website/src/spec/03-vm/index.rst +++ b/docs/website/src/spec/03-vm/index.rst @@ -4,8 +4,8 @@ VM Specification .. toctree:: :maxdepth: 2 - 01-result - 02-sandboxing + 01-startup + 02-meta-properties 03-ram-limiting 04-determinism-mode-switching - 05-permissions + 05-result diff --git a/docs/website/src/spec/04-contract-interface/01-calldata.rst b/docs/website/src/spec/04-contract-interface/01-calldata.rst index 5f7a8de0..5d20368a 100644 --- a/docs/website/src/spec/04-contract-interface/01-calldata.rst +++ b/docs/website/src/spec/04-contract-interface/01-calldata.rst @@ -81,3 +81,17 @@ follows: **FastString** is encoded as ULEB128 length followed by UTF-8 encoded bytes (difference is that it does not have a type). + +.. _gvm-def-calldata-map-key-order: + +Map Key Ordering +~~~~~~~~~~~~~~~~ + +Map keys are sorted bytewise lexicographically over their UTF-8 content: + +.. code-block:: + + "" < "a" < "aa" < "z" + +The order is strict, so duplicate keys are invalid. A decoder MUST reject a +map whose keys are not strictly increasing on this basis. diff --git a/docs/website/src/spec/04-contract-interface/03-storage.rst b/docs/website/src/spec/04-contract-interface/03-storage.rst index 58182d53..61a6b130 100644 --- a/docs/website/src/spec/04-contract-interface/03-storage.rst +++ b/docs/website/src/spec/04-contract-interface/03-storage.rst @@ -44,26 +44,32 @@ Root Slot the following data: -- ``major``: (offset 0) Single octet (``u8``) identifying the major version of the public ABI - that the contract was built against. It is written at deploy time (the value is detected - from the contract package β€” see :doc:`04-upgradability` and the impl-spec for the detection flow) - and read on every load so :term:`GenVM` can refuse to execute a contract whose public ABI - major does not match the host's ``CURRENT_MAJOR``. +- ``major``: (offset :ref:`gvm-def-consts-value-root-offsets-major`) Single octet (``u8``) + identifying the major version of the public ABI + that the contract was built against. It is written at deploy time (the value is detected + from the contract package β€” see :doc:`04-upgradability` and the impl-spec for the detection flow) + and read on every load so :term:`GenVM` can refuse to execute a contract whose public ABI + major does not match the host's ``CURRENT_MAJOR``. - ``contract_instance``: (offset 1) Reference to the contract instance data. - ``code``: (offset 2) The contract's code. Slot contains 4 bytes little-endian length followed by data - ``locked_slots``: (offset 3) A list of storage :term:`SlotID`\s that cannot be modified by non-upgraders. - Slot contains 4 bytes little-endian length followed length arrays of 32 byte :term:`SlotID`\s + Slot contains 4 bytes little-endian length followed length arrays of 32 byte :term:`SlotID`\s - ``upgraders``: (offset 4) A list of addresses that are authorized to modify the contract code and locked slots. - Slot contains 4 bytes little-endian length followed length arrays of 20 byte addresses + Slot contains 4 bytes little-endian length followed length arrays of 20 byte addresses - ``code_slot``: (offset 5) A raw 32-byte :term:`SlotID`. If it is all-zero (the default), - the contract code is read from the ``code`` slot (offset 2); otherwise the code is read from - the slot it points to (same 4-byte-length-prefixed layout). This lets a contract serve its - code from an arbitrary slot, including one shared via a ``chain:
::`` runner id. + the contract code is read from the ``code`` slot (offset 2); otherwise the code is read from + the slot it points to (same 4-byte-length-prefixed layout). This lets a contract serve its + code from an arbitrary slot, including one shared via a ``chain:
::`` runner id. +- ``permissions``: (offset 37) A 32-byte (``u256``) little-endian permission bitfield read by + the executor at the start of every load. Bit ``n`` corresponds to the permission whose value + is ``n`` (currently only bit ``0``, ``can_use_balance_for_message_fees``). It is not reserved: + contracts may set it (see ``Root.get_permission`` / ``Root.set_permission`` in the Python SDK). Offsets are byte offsets into the root slot, so ``code_slot`` (32 bytes) occupies offsets ``5`` -through ``36``; the next field would start at offset ``37``. +through ``36`` and ``permissions`` (32 bytes) occupies offsets ``37`` through ``68``; the next +field would start at offset ``69``. -Offsets at and above ``37`` are reserved for future :term:`GenVM` use and should remain zero: +Offsets at and above ``69`` are reserved for future :term:`GenVM` use and should remain zero: a contract must not store data in them, otherwise a future :term:`GenVM` version that starts reading those offsets may break the contract. Contract runtimes that need scratch space for bootstrapping should derive a separate sub-slot instead (see ``Root.get_vacant_slot`` in the diff --git a/docs/website/src/spec/04-contract-interface/04-upgradability.rst b/docs/website/src/spec/04-contract-interface/04-upgradability.rst index f75607c6..acd3a474 100644 --- a/docs/website/src/spec/04-contract-interface/04-upgradability.rst +++ b/docs/website/src/spec/04-contract-interface/04-upgradability.rst @@ -15,40 +15,44 @@ The upgrade system works through access control during write transactions: It does not lead to :ref:`gvm-def-ram-consumption` #. If the sender is not in the ``upgraders`` list of :ref:`genvm-def-root-slot`\, :term:`GenVM` reads ``locked_slots`` and will prevent writing to them. - It implies :math:`32*n` :ref:`gvm-def-ram-consumption`\, where :math:`n` is the number of locked slots. + It implies :math:`32*n` :ref:`gvm-def-ram-consumption`\, where :math:`n` is the number of locked slots + (bounded by :ref:`gvm-def-consts-value-top-limits-locked-slots`). This memory is never released. -#. :term:`GenVM` reads the ``code`` field of :ref:`genvm-def-root-slot` and executes it. - It causes exactly ``code`` size in octets :ref:`gvm-def-ram-consumption` +#. :term:`GenVM` loads the contract code as the entry runner. This is a runner + load action; its :ref:`gvm-def-ram-consumption` is specified in + :doc:`../02-execution-environment/04-runners` and released when the + :term:`sub-VM` finishes. .. _gvm-def-locked-slot-nesting: Locked Slots in Nested Calls ---------------------------- -Each :term:`sub-VM` reads the ``upgraders`` and ``locked_slots`` of *its own* contract address -(the callee), not the caller's. There is no transitive inheritance: a parent VM cannot -relax or tighten its child's locked slots, and an upgrader on contract ``A`` is not implicitly -an upgrader on contract ``B`` that ``A`` calls. +The ``upgraders`` and ``locked_slots`` lists are read once per execution, for +the top-level contract and the top-level sender, before any :term:`sub-VM` +exists; the resulting charge is the one in `Upgrade Control Mechanism`_ and no +:term:`sub-VM` pays it again. -Two consequences: +One set suffices because only the top-level contract's storage is ever +writable within an execution: -#. Permission to overwrite a locked slot is determined by ``msg.sender`` *as seen by the - callee*. For a ``CallContract`` invocation, that is the calling contract's address β€” the - caller must be in the callee's ``upgraders`` list for any locked-slot writes to succeed. -#. The :math:`32\cdot n` :ref:`gvm-def-ram-consumption` of materializing ``locked_slots`` is - paid independently in every sub-VM that needs it and is bounded by - ``top_limits::LOCKED_SLOTS``. The memory is never released for the lifetime of that sub-VM. - -Sub-VMs created by ``Sandbox`` and ``RunNondet`` cannot reach storage at all -(see :ref:`gvm-permissions`), so locked-slot checking does not apply to them. +#. A :ref:`gvm-def-gl-call-call-contract` child cannot write storage at all + (see :ref:`gvm-meta-property-derivation`), so the callee's own + ``locked_slots`` are never consulted, and being an upgrader of the callee + grants a calling contract nothing. +#. A :ref:`gvm-def-gl-call-sandbox` child that was granted + :ref:`gvm-perm-write-storage` writes the *same* contract's storage as its + parent, under the same sender β€” the top-level set applies to it unchanged. +#. A :ref:`gvm-def-gl-call-run-nondet` child cannot write storage. .. _gvm-def-contract-version: Contract Major Version ---------------------- -The ``major`` field of :ref:`genvm-def-root-slot` (offset 0) stores the public-ABI major version -that the contract was built against. On every load :term:`GenVM` compares this byte to its own +The ``major`` field of :ref:`genvm-def-root-slot` is at +:ref:`gvm-def-consts-value-root-offsets-major` and stores the public-ABI major version that the +contract was built against. On every load :term:`GenVM` compares this byte to its own ``CURRENT_MAJOR`` constant and refuses to execute a contract whose major differs. The value is **not** modifiable by the contract itself: it is written once at deploy time by diff --git a/docs/website/src/spec/04-contract-interface/05-execution-flow.rst b/docs/website/src/spec/04-contract-interface/05-execution-flow.rst index 5ba58422..32b0edca 100644 --- a/docs/website/src/spec/04-contract-interface/05-execution-flow.rst +++ b/docs/website/src/spec/04-contract-interface/05-execution-flow.rst @@ -3,95 +3,53 @@ Contract Execution Flow ======================= -This document describes the complete execution flow for GenVM contracts, -from deployment to method invocation and result processing. The flow -involves multiple components working together to provide a seamless -contract execution experience. +This page describes the contract-facing execution model. VM startup and runner +loading are specified in :doc:`../03-vm/01-startup`. -.. _contract-execution-flow-1: +Deployment and Calls +-------------------- -1. Contract Deployment (if needed) ----------------------------------- +#. On deployment, contract code is stored under the contract + :ref:`genvm-def-root-slot` layout, either in the root code field or through + its configured code slot. The deployment call is executed with ``is_init`` + set. +#. On a normal call, ``entry_data`` contains + :ref:`Calldata Encoded ` method-call data as + described by :ref:`gvm-def-contract-call-conv`. +#. The contract runtime dispatches the call and returns a value, a user error, + or a VM error as specified by :doc:`../03-vm/05-result`. -- :term:`Host` writes contract code to blockchain storage, according to :ref:`genvm-def-root-slot` definition -- Code includes runner specification and dependencies +Entry Kinds +----------- -2. Contract Loading -------------------- +``entry_kind`` selects the entry mode: -:term:`GenVM` does the following steps: +- :ref:`gvm-def-enum-value-entry-kind-main`: normal contract initialization or + method dispatch. +- :ref:`gvm-def-enum-value-entry-kind-sandbox`: sandbox entry created by a VM + API call. +- :ref:`gvm-def-enum-value-entry-kind-consensus-stage`: validator-side + consensus-stage entry created by a non-deterministic VM API call. -#. Receives contract address from message -#. Reads contract's locked slots and code from storage -#. Checks upgradability-related data from :doc:`04-upgradability` -#. Creates empty VFS, empty arguments list and empty environment variables map -#. Inspects contract runner as in :doc:`../02-execution-environment/04-runners` -#. Processes actions until :ref:`gvm-def-start-wasm` is encountered +The complete startup message is specified in :ref:`gvm-vm-startup-message`. -3. WebAssembly Execution ------------------------- +``CallContract`` Semantics +-------------------------- -- GenVM starts WebAssembly :term:`module` with stdin containing :ref:`Calldata Encoded ` extended-message -- Executes entry point (``_start``) with calldata from :term:`host` +``CallContract`` performs a read-only call into another contract. It preserves +``sender_address`` and ``origin_address``: the callee observes the same +``sender_address`` as the caller did, not the immediate caller. -4. Contract Entry Point Processing ----------------------------------- +The immediate caller is appended to ``stack``. For a top-level entrypoint, +``stack`` is empty; for a nested call, the immediate caller is the last element. +A callee that needs to authorize its immediate caller must inspect ``stack[-1]`` +rather than ``sender_address``. -The contract startup requires specific fields in the :ref:`Calldata Encoded ` extended-message: +State Visibility +~~~~~~~~~~~~~~~~ -- ``entry_kind``: Determines execution context +A ``CallContract`` child reads committed on-chain storage. It does not observe +the calling transaction's uncommitted writes, including through a self-call. +Direct reads in the same VM observe the current in-transaction state. - - :ref:`gvm-def-enum-value-entry-kind-main`\: Regular contract entry for standard method calls, - ``entry_data`` contains method call information as described in :ref:`gvm-def-contract-call-conv` - - :ref:`gvm-def-enum-value-entry-kind-sandbox`\: Contract decides for itself how to handle the payload in ``entry_data`` - - :ref:`gvm-def-enum-value-entry-kind-consensus-stage`\: Contract decides for itself how to handle the payload in ``entry_data`` - to call validator consensus functions with ``entry_stage_data`` - -- ``entry_data``: Blob of bytes containing method call information -- ``entry_stage_data``: Consensus information for validator nodes - - - ``null`` for leader nodes - - ``{leaders_result: }`` for validator nodes - -Extended-Message Format -^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: rust - - pub struct ExtendedMessage { - pub contract_address: calldata::Address, - pub sender_address: calldata::Address, - pub origin_address: calldata::Address, - /// View methods call chain. - /// It is empty for entrypoint (refer to [`contract_address`]) - pub stack: Vec, - - pub chain_id: num_bigint::BigInt, - pub value: num_bigint::BigInt, - pub is_init: bool, - /// Transaction timestamp - pub datetime: chrono::DateTime, - - #[serde(serialize_with = "entry_kind_as_int")] - pub entry_kind: public_abi::EntryKind, - #[serde(with = "serde_bytes")] - pub entry_data: Vec, - - pub entry_stage_data: calldata::Value, - } - -``CallContract`` semantics -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -``CallContract`` performs a read-only (view) call into another contract. It **preserves** ``sender_address`` and ``origin_address``: the callee observes the *same* ``sender_address`` as the caller did, **not** the address of the immediate caller. - -The immediate caller β€” the contract that issued the ``CallContract`` β€” is pushed onto ``stack`` instead, so the callee always sees it as ``stack[-1]`` (the last element). For the top-level entrypoint ``stack`` is empty (refer to ``contract_address``); each nested ``CallContract`` appends one entry. - -Therefore a callee that needs to authorize its *immediate* caller MUST inspect ``stack[-1]`` rather than ``sender_address`` β€” the latter identifies the original transaction sender shared across the whole view-call chain. - -State visibility -"""""""""""""""" - -A ``CallContract`` child reads **committed** on-chain storage (the latest accepted / non-final state); it does **not** see the calling transaction's uncommitted writes. Consequently a contract does not observe its own in-transaction storage delta through a ``CallContract`` β€” including a self-call ``get_at(self.address)`` β€” even though a *direct* read does. - -This is intentional: it preserves causality and avoids exposing stale-then-fresh reads within one call. It is also a footgun: relying on a self-``CallContract`` to read state written earlier in the same transaction returns the old value. Use direct reads for in-transaction data, and treat ``get_at(self.address)`` as a read of the committed snapshot only. +Meta-property changes for ``CallContract`` are specified in :ref:`gvm-meta-property-derivation`. diff --git a/docs/website/src/spec/appendix/constants-pending.rst b/docs/website/src/spec/appendix/constants-pending.rst new file mode 100644 index 00000000..0b076926 --- /dev/null +++ b/docs/website/src/spec/appendix/constants-pending.rst @@ -0,0 +1,2 @@ +Constants +========= diff --git a/docs/website/src/spec/appendix/constants.rst b/docs/website/src/spec/appendix/constants.rst index 0a414051..3ce82216 100644 --- a/docs/website/src/spec/appendix/constants.rst +++ b/docs/website/src/spec/appendix/constants.rst @@ -92,6 +92,20 @@ consensus_stage Value: ``2`` +.. _gvm-def-enum-permissions: + +permissions +----------- + +Type: u32 + +.. _gvm-def-enum-value-permissions-can-use-balance-for-message-fees: + +can_use_balance_for_message_fees +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Value: ``1`` + .. _gvm-def-consts-memory-limiter-consts: memory_limiter_consts @@ -120,6 +134,20 @@ fd_allocation Value: ``96`` +.. _gvm-def-consts-value-memory-limiter-consts-runner-load-cost: + +runner_load_cost +~~~~~~~~~~~~~~~~ + +Value: ``4096`` + +.. _gvm-def-consts-value-memory-limiter-consts-vm-spawn-cost: + +vm_spawn_cost +~~~~~~~~~~~~~ + +Value: ``134217728`` + .. _gvm-def-consts-root-offsets: root_offsets @@ -169,6 +197,13 @@ code_slot Value: ``5`` +.. _gvm-def-consts-value-root-offsets-permissions: + +permissions +~~~~~~~~~~~ + +Value: ``37`` + .. _gvm-def-consts-top-limits: top_limits @@ -225,6 +260,20 @@ max_fds Value: ``1024`` +.. _gvm-def-consts-value-top-limits-wasm-call-depth: + +wasm_call_depth +~~~~~~~~~~~~~~~ + +Value: ``1024`` + +.. _gvm-def-consts-value-top-limits-wasm-stack-value-slots: + +wasm_stack_value_slots +~~~~~~~~~~~~~~~~~~~~~~ + +Value: ``65535`` + .. _gvm-def-enum-special-method: special_method @@ -275,58 +324,176 @@ Param: i32 ``wasm_trap`` ~~~~~~~~~~~~~ -Param: str +.. _gvm-def-str-trie-value-vm-error-wasm-trap-unreachable: -.. _gvm-def-str-trie-value-vm-error-OOM-RAM: +``wasm_trap unreachable`` +~~~~~~~~~~~~~~~~~~~~~~~~~ -``OOM RAM`` -~~~~~~~~~~~ +.. _gvm-def-str-trie-value-vm-error-wasm-trap-stack-overflow: -.. _gvm-def-str-trie-value-vm-error-OOM-RAM-table: +``wasm_trap stack_overflow`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``OOM RAM table`` -~~~~~~~~~~~~~~~~~ +.. _gvm-def-str-trie-value-vm-error-wasm-trap-memory-out-of-bounds: -.. _gvm-def-str-trie-value-vm-error-OOM-RAM-memory: +``wasm_trap memory_out_of_bounds`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``OOM RAM memory`` -~~~~~~~~~~~~~~~~~~ +.. _gvm-def-str-trie-value-vm-error-wasm-trap-table-out-of-bounds: -.. _gvm-def-str-trie-value-vm-error-OOM-RAM-limit: +``wasm_trap table_out_of_bounds`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``OOM RAM limit`` -~~~~~~~~~~~~~~~~~ +.. _gvm-def-str-trie-value-vm-error-wasm-trap-indirect-call-to-null: -.. _gvm-def-str-trie-value-vm-error-OOM-storage: +``wasm_trap indirect_call_to_null`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``OOM storage`` -~~~~~~~~~~~~~~~ +.. _gvm-def-str-trie-value-vm-error-wasm-trap-bad-signature: + +``wasm_trap bad_signature`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. _gvm-def-str-trie-value-vm-error-wasm-trap-integer-overflow: + +``wasm_trap integer_overflow`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. _gvm-def-str-trie-value-vm-error-wasm-trap-integer-divide-by-zero: -.. _gvm-def-str-trie-value-vm-error-OOM-receipt-nondet-output: +``wasm_trap integer_divide_by_zero`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. _gvm-def-str-trie-value-vm-error-wasm-trap-bad-conversion-to-integer: + +``wasm_trap bad_conversion_to_integer`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``OOM receipt nondet_output`` +.. _gvm-def-str-trie-value-vm-error-wasm-trap-heap-misaligned: + +``wasm_trap heap_misaligned`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. _gvm-def-str-trie-value-vm-error-OOM-receipt-message-internal: +.. _gvm-def-str-trie-value-vm-error-wasm-trap-atomic-wait-non-shared-memory: + +``wasm_trap atomic_wait_non_shared_memory`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. _gvm-def-str-trie-value-vm-error-wasm-trap-out-of-fuel: -``OOM receipt message internal`` +``wasm_trap out_of_fuel`` +~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. _gvm-def-str-trie-value-vm-error-wasm-trap-interrupt: + +``wasm_trap interrupt`` +~~~~~~~~~~~~~~~~~~~~~~~ + +.. _gvm-def-str-trie-value-vm-error-wasm-trap-nondet-instruction: + +``wasm_trap nondet_instruction`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. _gvm-def-str-trie-value-vm-error-OOM-receipt-message-external: +.. _gvm-def-str-trie-value-vm-error-wasm-trap-fault: + +``wasm_trap fault`` +~~~~~~~~~~~~~~~~~~~ + +.. _gvm-def-str-trie-value-vm-error-out-of-memory: + +``out_of memory`` +~~~~~~~~~~~~~~~~~ + +.. _gvm-def-str-trie-value-vm-error-out-of-memory-wasm-memory: + +``out_of memory wasm_memory`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. _gvm-def-str-trie-value-vm-error-out-of-memory-wasm-table: + +``out_of memory wasm_table`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``OOM receipt message external`` +.. _gvm-def-str-trie-value-vm-error-out-of-storage: + +``out_of storage`` +~~~~~~~~~~~~~~~~~~ + +.. _gvm-def-str-trie-value-vm-error-out-of-receipt-nondet-output: + +``out_of receipt nondet_output`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. _gvm-def-str-trie-value-vm-error-OOM-fees-internal: +.. _gvm-def-str-trie-value-vm-error-out-of-receipt-message: -``OOM fees internal`` -~~~~~~~~~~~~~~~~~~~~~ +``out_of receipt message`` +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. _gvm-def-str-trie-value-vm-error-out-of-receipt-event: + +``out_of receipt event`` +~~~~~~~~~~~~~~~~~~~~~~~~ + +.. _gvm-def-str-trie-value-vm-error-out-of-message-fee-total: + +``out_of message_fee total`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. _gvm-def-str-trie-value-vm-error-out-of-message-fee-node: + +``out_of message_fee node`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. _gvm-def-str-trie-value-vm-error-out-of-vm-recursion: -.. _gvm-def-str-trie-value-vm-error-OOM-fees-external: +``out_of vm_recursion`` +~~~~~~~~~~~~~~~~~~~~~~~ -``OOM fees external`` +.. _gvm-def-str-trie-value-vm-error-out-of-nondet-blocks: + +``out_of nondet_blocks`` +~~~~~~~~~~~~~~~~~~~~~~~~ + +.. _gvm-def-str-trie-value-vm-error-out-of-locked-slots: + +``out_of locked_slots`` +~~~~~~~~~~~~~~~~~~~~~~~ + +.. _gvm-def-str-trie-value-vm-error-out-of-upgraders: + +``out_of upgraders`` +~~~~~~~~~~~~~~~~~~~~ + +.. _gvm-def-str-trie-value-vm-error-out-of-fds: + +``out_of fds`` +~~~~~~~~~~~~~~ + +.. _gvm-def-str-trie-value-vm-error-fee-no-matching-node: + +``fee no_matching_node`` +~~~~~~~~~~~~~~~~~~~~~~~~ + +.. _gvm-def-str-trie-value-vm-error-fee-below-minimum: + +``fee below_minimum`` ~~~~~~~~~~~~~~~~~~~~~ +.. _gvm-def-str-trie-value-vm-error-fee-too-many-rounds: + +``fee too_many_rounds`` +~~~~~~~~~~~~~~~~~~~~~~~ + +.. _gvm-def-str-trie-value-vm-error-host-forbidden: + +``host_forbidden`` +~~~~~~~~~~~~~~~~~~ + +.. _gvm-def-str-trie-value-vm-error-evm-reverted: + +``evm reverted`` +~~~~~~~~~~~~~~~~ + .. _gvm-def-str-trie-value-vm-error-invalid-contract: ``invalid_contract`` @@ -367,13 +534,6 @@ Param: str ``invalid_contract wasm entrypoint`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. _gvm-def-str-trie-value-vm-error-host: - -``host`` -~~~~~~~~ - -Param: str - .. _gvm-def-const-event-max-topics: event_max_topics diff --git a/docs/website/src/spec/appendix/index.rst b/docs/website/src/spec/appendix/index.rst index 4a9409f6..9cda9e86 100644 --- a/docs/website/src/spec/appendix/index.rst +++ b/docs/website/src/spec/appendix/index.rst @@ -5,4 +5,5 @@ Appendix :maxdepth: 2 constants + Pending Constants runner-schema diff --git a/executors/v0.2.x b/executors/v0.2.x new file mode 160000 index 00000000..e749d2aa --- /dev/null +++ b/executors/v0.2.x @@ -0,0 +1 @@ +Subproject commit e749d2aa8201ab1a291d803d7de05674a2c70772 diff --git a/executors/v0.3.x b/executors/v0.3.x index acb37c7b..a7d97df0 160000 --- a/executors/v0.3.x +++ b/executors/v0.3.x @@ -1 +1 @@ -Subproject commit acb37c7bf7b1e6d9fbfe004414a93fb6306135c0 +Subproject commit a7d97df0e27927a4a2b12673718cf9d45872b522 diff --git a/flake.nix b/flake.nix index 41edab06..7cef9949 100644 --- a/flake.nix +++ b/flake.nix @@ -146,6 +146,7 @@ export LLVM_PROFILE_FILE=/dev/null export LSQLITE3_SRC="${deps."lsqlite3-0.9.6"}" export LUA_INCLUDE="${manager-release-args.lua-src}" + export CARGO_TARGET_DIR="$(pwd)/build/ya-build/rust-target" ''; # ---- Active executor lines ------------------------------- @@ -260,22 +261,66 @@ runners-list = runners.list; - runners-universal-set = runners.universal; + # Legacy lines (v0.2.x) keep their runners private under the executor + # root (executor//legacy-runners) rather than the shared + # top-level runners/ dir: their registry hashes use Nix base32, a + # different scheme from the forward-rolling lines, so the two runner + # trees must not be pooled together. + is-legacy-runner = r: pkgs.lib.hasPrefix "v0.2." r.version; + shared-runners-list = builtins.filter (r: !is-legacy-runner r) runners-list; + legacy-runners-list = builtins.filter is-legacy-runner runners-list; - runners-all = pkgs.stdenvNoCC.mkDerivation { - name = "genvm-runners-all"; - srcs = builtins.attrValues runners-universal-set; + universal-of = import ./runners/views/universal.nix runners-args; + + # Merge a { uid -> `//.tar` tree } set into one tree. + merge-runner-trees = + name: uni: + pkgs.stdenvNoCC.mkDerivation { + inherit name; + srcs = builtins.attrValues uni; + dontUnpack = true; + dontConfigure = true; + dontBuild = true; + dontFixup = true; + installPhase = '' + mkdir -p $out + for src in $srcs; do + cp --no-preserve=ownership -r $src/. $out/. + chmod -R u+w $out + done + ''; + }; + + runners-all = merge-runner-trees "genvm-runners-all" (universal-of shared-runners-list); + + # The legacy runners, laid out at their final relative destination + # `executor//legacy-runners///.tar` so consumers + # only have to overlay this tree onto their output root. + legacy-runners-all = pkgs.stdenvNoCC.mkDerivation { + name = "genvm-legacy-runners-all"; dontUnpack = true; dontConfigure = true; dontBuild = true; dontFixup = true; installPhase = '' mkdir -p $out - for src in $srcs; do - cp --no-preserve=ownership -r $src/. $out/. - chmod -R u+w $out - done - ''; + '' + + pkgs.lib.concatMapStrings ( + r: + let + uidMatch = builtins.match "([^:]+):(.+)" r.uid; + hash32 = + if uidMatch == null then + throw "invalid runner uid `${r.uid}`; expected `:`" + else + builtins.elemAt uidMatch 1; + dst = "executor/${r.version}/legacy-runners/${r.id}/${builtins.substring 0 2 hash32}/${builtins.substring 2 50 hash32}.tar"; + in + '' + mkdir -p "$out/$(dirname -- "${dst}")" + cp "${r.derivation}" "$out/${dst}" + '' + ) legacy-runners-list; }; # ---- Combined genvm distribution ------------------------- @@ -292,14 +337,15 @@ let execs = builtins.map (line: executor-packages."executor-${line.clamped}${suffix}") executor-lines; manager = manager-packages."manager${suffix}"; - srcs = execs ++ [ - manager - runners-all - ]; + # executors + manager land at the top level; runners-all is + # nested under `runners/` (matching the CI build/out layout). + srcs = execs ++ [ manager ]; in pkgs.stdenvNoCC.mkDerivation { name = "genvm${suffix}"; inherit srcs; + runnersAll = runners-all; + legacyRunnersAll = legacy-runners-all; dontUnpack = true; dontConfigure = true; dontBuild = true; @@ -310,6 +356,13 @@ cp --no-preserve=ownership -r $src/. $out/. chmod -R u+w $out done + mkdir -p $out/runners + cp --no-preserve=ownership -r $runnersAll/. $out/runners/. + chmod -R u+w $out/runners + # Legacy lines carry their runners under their own executor root + # (executor//legacy-runners); overlay that tree here. + cp --no-preserve=ownership -r $legacyRunnersAll/. $out/. + chmod -R u+w $out ''; }; genvm-packages = builtins.listToAttrs ( @@ -317,6 +370,55 @@ suffix: pkgs.lib.nameValuePair "genvm${suffix}" (combine-genvm suffix) ) platform-suffixes ); + + # ---- Release distribution bundles ------------------------- + # `executor[-]` (no version) merges every active line's + # executor tree (executor//...) β€” the release asset + # genvm---executor.tar.xz. Runners are NOT included; + # they ship platform-independently via runners-all-dist. + combine-executors = + suffix: + pkgs.stdenvNoCC.mkDerivation { + name = "genvm-executor${suffix}"; + srcs = builtins.map (line: executor-packages."executor-${line.clamped}${suffix}") executor-lines; + dontUnpack = true; + dontConfigure = true; + dontBuild = true; + dontFixup = true; + installPhase = '' + mkdir -p $out + for src in $srcs; do + cp --no-preserve=ownership -r $src/. $out/. + chmod -R u+w $out + done + ''; + }; + executor-dist-packages = builtins.listToAttrs ( + builtins.map ( + suffix: pkgs.lib.nameValuePair "executor${suffix}" (combine-executors suffix) + ) platform-suffixes + ); + + # The release asset genvm-runners-all.tar.xz: consumers extract it at + # the install root, so the shared runners sit under runners/ and the + # legacy lines' runners at their executor//legacy-runners + # destination (the same overlay combine-genvm applies). + runners-all-dist = pkgs.stdenvNoCC.mkDerivation { + name = "genvm-runners-all-dist"; + runnersAll = runners-all; + legacyRunnersAll = legacy-runners-all; + dontUnpack = true; + dontConfigure = true; + dontBuild = true; + dontFixup = true; + installPhase = '' + mkdir -p $out/runners + cp --no-preserve=ownership -r $runnersAll/. $out/runners/. + chmod -R u+w $out/runners + cp --no-preserve=ownership -r $legacyRunnersAll/. $out/. + chmod -R u+w $out + ''; + }; in { runners = runners-list; @@ -325,10 +427,12 @@ genvm-packages # Per-line executors: executor-[-]. // executor-packages + # All active lines merged: executor[-] (release bundles). + // executor-dist-packages # Manager: manager[-]. // manager-packages // { - inherit runners-all; + inherit runners-all legacy-runners-all runners-all-dist; } # Utility packages (grouped separately). // { diff --git a/implementation/Cargo.lock b/implementation/Cargo.lock index 9dd022a8..e95b91ae 100644 --- a/implementation/Cargo.lock +++ b/implementation/Cargo.lock @@ -375,9 +375,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.44" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", "js-sys", @@ -1035,6 +1035,7 @@ dependencies = [ "serde_json", "serde_yaml", "symbol_table", + "thiserror 2.0.18", "tokio", ] @@ -1093,8 +1094,9 @@ name = "genvm-modules-interfaces" version = "0.1.0" dependencies = [ "anyhow", + "bytes", + "chrono", "genlayer_calldata", - "genlayer_sdk", "num-bigint", "num-traits", "primitive-types", diff --git a/implementation/Cargo.toml b/implementation/Cargo.toml index 2a66520e..4d88ece5 100644 --- a/implementation/Cargo.toml +++ b/implementation/Cargo.toml @@ -16,55 +16,76 @@ default = [] vendored-lua = ["mlua/vendored"] [dependencies] -genvm-modules-interfaces = { path = "../crates/modules-interfaces" } -genvm-common = { path = "../executors/v0.3.x/executor/crates/common" } genlayer_calldata = { path = "../executors/v0.3.x/executor/crates/calldata" } +genvm-common = { path = "../executors/v0.3.x/executor/crates/common" } +genvm-modules-interfaces = { path = "../crates/modules-interfaces" } anyhow = { version = "1.0.98", features = [] } -serde = { version = "1.0.219", features = ["rc"]} -serde_derive = "1.0.219" -serde_json = { version = "1.0.140", features = ["preserve_order"] } -tokio = { version = "1.44.1", features = ["rt", "rt-multi-thread", "net", "time", "macros", "sync", "process"], default-features = false} -reqwest = { version = "0.12.15", features = ["cookies", "gzip", "rustls-tls", "charset", "http2", "json"], default-features = false} -serde_yaml = "0.9.34" -clap = { version = "4.5.35", features = ["derive"] } -signal-hook = "0.3.17" async-trait = "0.1.88" -regex = "1.11.1" +base64 = "0.22.1" +bytes = "1.10.1" +clap = { version = "4.5.35", features = ["derive"] } crossbeam = "0.8.4" -tokio-tungstenite = "0.26.2" futures-util = "0.3.31" -base64 = "0.22.1" -serde_bytes = "0.11.17" -serde_with = { version = "3.12.0", features = ["base64", "macros"] } -rand = { version = "0.9.1", features = ["std_rng"] } -chrono = { version = "0.4.41", default-features = false, features = ["clock", "std", "serde"] } -ring = { version = "0.17.14", features = ["std"] } -url = { version = "2.5.4", features = ["serde"] } hex = "0.4.3" +hyper = { version = "0.14", features = ["server", "http1", "tcp"] } +image = { version = "0.25.8", features = ["png", "jpeg", "webp"] } +libc = "0.2" num-bigint = { version = "0.4.6", features = ["serde"] } +num-integer = { version = "0.1.46" } num-rational = { version = "0.4.2" } num-traits = { version = "0.2.19" } -num-integer = { version = "0.1.46" } +papaya = "0.2.3" primitive-types = { version = "0.13", features = ["impl-serde"] } +rand = { version = "0.9.1", features = ["std_rng"] } +regex = "1.11.1" + +ring = { version = "0.17.14", features = ["std"] } +serde = { version = "1.0.219", features = ["rc"] } +serde_bytes = "0.11.17" +serde_derive = "1.0.219" +serde_json = { version = "1.0.140", features = ["preserve_order"] } +serde_with = { version = "3.12.0", features = ["base64", "macros"] } +serde_yaml = "0.9.34" +sha3 = "0.10" +signal-hook = "0.3.17" + +tokio-tungstenite = "0.26.2" unicode-normalization = "0.1.20" unicode-width = "0.2.1" -image = { version = "0.25.8", features = ["png", "jpeg", "webp"]} +url = { version = "2.5.4", features = ["serde"] } warp = "0.3.7" -hyper = { version = "0.14", features = ["server", "http1", "tcp"] } -bytes = "1.10.1" -libc = "0.2" -sha3 = "0.10" -papaya = "0.2.3" # kept in sync with the version reqwest depends on, to avoid a duplicate hickory # in the tree hickory-resolver = { version = "0.25", features = ["tokio"] } +[dependencies.chrono] +default-features = false +features = ["clock", "std", "serde"] +version = "0.4.41" + +[dependencies.reqwest] +default-features = false +features = ["cookies", "gzip", "rustls-tls", "charset", "http2", "json"] +version = "0.12.15" + +[dependencies.tokio] +default-features = false +features = ["rt", "rt-multi-thread", "net", "time", "macros", "sync", "process"] +version = "1.44.1" + # we do not use anyhow feature because it does not work with boxed downcast_ref [dependencies.mlua] -version = "0.10.3" default-features = false -features = ["lua53", "async", "serialize", "send", "error-send", "userdata-wrappers"] +features = [ + "lua53", + "async", + "serialize", + "send", + "error-send", + "userdata-wrappers", +] +version = "0.10.3" [target.'cfg(not(target_os = "macos"))'.dependencies] sysinfo = { version = "0.36", default-features = false, features = ["system"] } @@ -75,8 +96,20 @@ cc = "1" [dev-dependencies] afl = { version = "0.15.18", features = ["no_cfg_fuzzing"] } +[profile.dev] +incremental = false + +[profile.dev.build-override] +debug = false +incremental = false +opt-level = 0 +strip = true + +[profile.release] +incremental = false + [profile.release-with-debug] -inherits = "release" debug = true +inherits = "release" split-debuginfo = "packed" strip = "none" diff --git a/implementation/default.nix b/implementation/default.nix index b1db8957..4c3ed9bd 100644 --- a/implementation/default.nix +++ b/implementation/default.nix @@ -32,6 +32,8 @@ let src = get-root-subtree [ "implementation" "crates/modules-interfaces" + "crates/calldata" + "crates/calldata-derive" # executor crates come from the v0.3.x submodule mount "executors/v0.3.x/executor/crates" ]; diff --git a/implementation/src/common/mod.rs b/implementation/src/common/mod.rs index 0f976f53..293ba3f6 100644 --- a/implementation/src/common/mod.rs +++ b/implementation/src/common/mod.rs @@ -1,4 +1,5 @@ use base64::Engine; +use genlayer_calldata as calldata; use genvm_common::*; use genvm_modules_interfaces::GenericValue; use serde::{Deserialize, Serialize}; @@ -234,8 +235,8 @@ async fn loop_one_inner_handle( where T: calldata::codec::Decode + 'static, { - let payload = genvm_common::calldata::decode_obj(text) - .with_context(|| format!("parsing calldata format {text:?}"))?; + let payload = + calldata::decode_obj(text).with_context(|| format!("parsing calldata format {text:?}"))?; handler .handle(payload) .await @@ -288,7 +289,7 @@ where }, }; - let message = genvm_common::calldata::encode_obj(&res); + let message = calldata::encode_obj(&res); write_message(stream, &message) .await @@ -310,7 +311,7 @@ where }; let genvm_hello: genvm_modules_interfaces::GenVMHello = - genvm_common::calldata::decode_obj(&data).context("decoding GenVMHello")?; + calldata::decode_obj(&data).context("decoding GenVMHello")?; Ok(Some(genvm_hello)) } diff --git a/implementation/src/llm/mod.rs b/implementation/src/llm/mod.rs index fbf7f0a7..a97801c9 100644 --- a/implementation/src/llm/mod.rs +++ b/implementation/src/llm/mod.rs @@ -1,4 +1,5 @@ use anyhow::{Context, Result}; +use genlayer_calldata as calldata; use genvm_common::*; use std::{ collections::{BTreeMap, HashMap}, diff --git a/implementation/src/manager/check_install.rs b/implementation/src/manager/check_install.rs new file mode 100644 index 00000000..95eff07e --- /dev/null +++ b/implementation/src/manager/check_install.rs @@ -0,0 +1,95 @@ +use std::collections::HashMap; + +use anyhow::{Context as _, Result}; +use genvm_common::*; + +use super::versioning; +use super::{CliArgs, Config}; + +#[derive(clap::Args, Debug)] +pub struct Args { + #[arg( + long, + default_value_t = false, + help = "also precompile every runner into each executor's on-disk cache" + )] + pub precompile: bool, +} + +/// Runs `genvm check` for every active executor listed in the manifest, +/// verifying that its runners are present with the correct hashes (and, with +/// `--precompile`, precompiling them). Each executor resolves its own runner +/// registry and directory from its bundled config, so the manager only has to +/// locate and invoke the right binaries. +pub fn run(cli: &CliArgs, args: &Args) -> Result<()> { + let config = + genvm_common::load_config(HashMap::new(), &cli.config).with_context(|| "loading config")?; + let mut config: Config = serde_yaml::from_value(config)?; + + if let Some(manifest_path) = &cli.manifest_path { + config.manifest_path = manifest_path.clone(); + } + + config.base.setup_logging(std::io::stdout())?; + + let manifest_str = std::fs::read_to_string(&config.manifest_path) + .with_context(|| format!("reading manifest {}", config.manifest_path))?; + let manifest: versioning::Manifest = + serde_yaml::from_str(&manifest_str).with_context(|| "parsing manifest")?; + + // Executors live next to this binary: /bin/genvm-modules -> /executor + let mut executors_path = std::env::current_exe()?; + executors_path.pop(); + executors_path.pop(); + executors_path.push("executor"); + + let mut failures: Vec = Vec::new(); + + for version in manifest.executor_versions.keys() { + // Mirror the installer: only the newest patch of each line is active. + let next = versioning::Version { + major: version.major, + minor: version.minor, + patch: version.patch + 1, + }; + if manifest.executor_versions.contains_key(&next) { + log_info!(version:? = version; "skipping executor: a newer patch exists"); + continue; + } + + let orig_key = manifest + .version_orig_keys + .get(version) + .with_context(|| format!("missing original manifest key for {version:?}"))?; + + let genvm = executors_path.join(orig_key).join("bin").join("genvm"); + + log_info!(version = orig_key.as_str(), exe:? = genvm, precompile = args.precompile; "checking executor install"); + + let mut command = std::process::Command::new(&genvm); + command.arg("check"); + if args.precompile { + command.arg("--precompile"); + } + + let status = command + .status() + .with_context(|| format!("spawning {genvm:?}"))?; + + if !status.success() { + log_error!(version = orig_key.as_str(), status:? = status; "executor check failed"); + failures.push(orig_key.clone()); + } + } + + if !failures.is_empty() { + anyhow::bail!( + "check-install failed for executors: {}", + failures.join(", ") + ); + } + + log_info!("check-install succeeded for all executors"); + + Ok(()) +} diff --git a/implementation/src/manager/handlers.rs b/implementation/src/manager/handlers.rs index b7003402..35f73402 100644 --- a/implementation/src/manager/handlers.rs +++ b/implementation/src/manager/handlers.rs @@ -1,4 +1,5 @@ use anyhow::Result; +use genlayer_calldata as calldata; use genvm_common::*; use std::collections::{BTreeMap, HashMap}; use std::str::FromStr; @@ -76,7 +77,8 @@ pub async fn handle_genvm_run( ctx: sync::DArc, data: &[u8], ) -> Result { - let res: super::run::Request = calldata::decode_obj(data)?; + let mut res: super::run::Request = calldata::decode_obj(data)?; + res.patch_legacy_method_key(); let modules_lock = if res.needs_modules() { let lock = Ctx::get_module_locks(ctx.gep(|x| &x.mod_ctx)).await; diff --git a/implementation/src/manager/mod.rs b/implementation/src/manager/mod.rs index 1407d354..05a9a265 100644 --- a/implementation/src/manager/mod.rs +++ b/implementation/src/manager/mod.rs @@ -6,6 +6,7 @@ use warp::Filter; use crate::common; +mod check_install; pub mod execution_context; mod handlers; pub mod modules; @@ -72,8 +73,17 @@ pub struct AppContext { pub ver_ctx: versioning::Ctx, } +#[derive(clap::Subcommand, Debug)] +pub enum SubCommand { + /// Verify installed executors' runners (and optionally precompile them). + CheckInstall(check_install::Args), +} + #[derive(clap::Args, Debug)] pub struct CliArgs { + #[command(subcommand)] + pub command: Option, + #[arg(long, default_value_t = 3999)] pub port: u16, #[arg(long, default_value = "127.0.0.1")] @@ -416,6 +426,10 @@ async fn main_loop( } pub fn entrypoint(args: CliArgs) -> Result<()> { + if let Some(SubCommand::CheckInstall(ci)) = &args.command { + return check_install::run(&args, ci); + } + let config = genvm_common::load_config(HashMap::new(), &args.config) .with_context(|| "loading config")?; let mut config: Config = serde_yaml::from_value(config)?; diff --git a/implementation/src/manager/modules.rs b/implementation/src/manager/modules.rs index 3d7fdf0a..7ed4321e 100644 --- a/implementation/src/manager/modules.rs +++ b/implementation/src/manager/modules.rs @@ -139,7 +139,7 @@ fn create_user_error_stream_handler() -> StreamHandler { ])), ); - let message = genvm_common::calldata::encode_obj(&response); + let message = genlayer_calldata::encode_obj(&response); if crate::common::write_message(&mut stream, &message) .await diff --git a/implementation/src/manager/run.rs b/implementation/src/manager/run.rs index f63b1319..2bd76a09 100644 --- a/implementation/src/manager/run.rs +++ b/implementation/src/manager/run.rs @@ -5,6 +5,7 @@ use std::{ sync::Arc, }; +use genlayer_calldata as calldata; use genvm_common::io::{set_fd_nonblocking, AsyncCustomFD, FdWrapper}; use genvm_common::*; use tokio::io::AsyncBufReadExt; @@ -390,7 +391,7 @@ fn default_reroute_to() -> String { )] pub struct Request { pub major: u32, - pub message: genvm_common::domain::MessageData, + pub message: genvm_modules_interfaces::MessageData, pub is_sync: bool, /// Executor debug level. Controls captured-output bounding, tracing, runner /// `:latest`/`:test` resolution (under `unsafe`), and (under `unsafe-tracing`) @@ -433,21 +434,29 @@ pub struct Request { /// Message-fee allocation tree passed alongside the execution. #[serde(default)] #[calldata(default = default_message_fee_allocation)] - pub message_fee_allocation: Vec, + pub message_fee_allocation: Vec, /// Initial time-unit budget for this execution. pub initial_time_units_allocation: u32, + /// Auditable supervisor action kinds to return in the execution result. + #[serde(default)] + #[calldata(default = default_record_actions)] + pub record_actions: Vec, } fn default_gas_data() -> std::collections::BTreeMap { std::collections::BTreeMap::new() } -fn default_message_fee_allocation() -> Vec { +fn default_message_fee_allocation() -> Vec { + Vec::new() +} + +fn default_record_actions() -> Vec { Vec::new() } fn default_permissions() -> String { - "rwscn".to_owned() + "wscn".to_owned() } impl Request { @@ -457,6 +466,33 @@ impl Request { pub fn needs_modules(&self) -> bool { !self.no_modules && !self.is_sync && self.permissions.contains('n') } + + /// The legacy v0.2 ABI stored the contract method name under the calldata + /// key `"method"`; the current ABI uses the empty key `""`. A client sending + /// old-format calldata would otherwise reach the executor with a `"method"` + /// key its runner no longer understands. Detect that here, log an error so + /// the stale caller is visible, and rewrite the key in place. No-op for + /// well-formed (new-format) calldata or non-map calldata. + pub fn patch_legacy_method_key(&mut self) { + const LEGACY_METHOD_KEY: &str = "method"; + const NEW_METHOD_KEY: &str = ""; + + let mut value = match calldata::decode(&self.calldata) { + Ok(v @ calldata::Value::Map(_)) => v, + _ => return, + }; + let calldata::Value::Map(map) = &mut value else { + return; + }; + + if let Some(method) = map.remove(LEGACY_METHOD_KEY) { + log_error!( + "received legacy 'method' key in run calldata; patching it to the new empty key" + ); + map.insert(NEW_METHOD_KEY.to_owned(), method); + self.calldata = bytes::Bytes::from(calldata::encode(&value)); + } + } } fn default_max_execution_minutes() -> u64 { @@ -1019,7 +1055,7 @@ pub async fn start_genvm( let mut method_hosts: Vec = vec![0; host_fns::Methods::SIZE]; method_hosts[host_fns::Methods::ConsumeResult as usize] = 1; - let execution_data = genvm_common::domain::ExecutionData { + let execution_data = genvm_modules_interfaces::ExecutionData { calldata: req.calldata.clone(), message: req.message.clone(), host_data: req.host_data.clone(), @@ -1030,8 +1066,9 @@ pub async fn start_genvm( gas_data: req.gas_data.clone(), message_fee_allocation: req.message_fee_allocation.clone(), initial_time_units_allocation: req.initial_time_units_allocation, + record_actions: req.record_actions.clone(), }; - let execution_data_bytes = genvm_common::calldata::encode_obj(&execution_data); + let execution_data_bytes = calldata::encode_obj(&execution_data); let execution_data_bytes = bytes::Bytes::from(execution_data_bytes); diff --git a/implementation/src/scripting/ctx/mod.rs b/implementation/src/scripting/ctx/mod.rs index e955aae2..fd694b35 100644 --- a/implementation/src/scripting/ctx/mod.rs +++ b/implementation/src/scripting/ctx/mod.rs @@ -2,6 +2,7 @@ use std::{collections::BTreeMap, sync::Arc}; use crate::common::ModuleError; +use genlayer_calldata as calldata; use genvm_common::*; pub mod dflt; diff --git a/implementation/src/web/mod.rs b/implementation/src/web/mod.rs index 57c10ff0..872e3d0b 100644 --- a/implementation/src/web/mod.rs +++ b/implementation/src/web/mod.rs @@ -1,6 +1,7 @@ use std::{collections::HashMap, sync::Arc}; use anyhow::{Context, Result}; +use genlayer_calldata as calldata; use genvm_common::*; use crate::{common, scripting}; diff --git a/install/lib/python/post-install/__main__.py b/install/lib/python/post-install/__main__.py index 53b84ee1..996c13cc 100644 --- a/install/lib/python/post-install/__main__.py +++ b/install/lib/python/post-install/__main__.py @@ -319,7 +319,13 @@ def _download_template(descr: str, templates: list[str], vars: dict[str, str]) - raise RuntimeError(f'failed to download {descr}{vars} from all sources') -def download_runners_from_json(file: str | Path): +def download_runners_from_json( + file: str | Path, runners_dir: Path, verify_hash: bool = True +): + # `verify_hash` is disabled for the v0.2.x legacy line, whose registry hashes + # use Nix base32 rather than the Crockford scheme `runner_check_bytes` + # understands. Those runners are validated by the executor's own `check` + # command (invoked via `manager check-install`) instead. file = Path(file) if not file.exists(): if args.error_on_missing_executor: @@ -330,13 +336,15 @@ def download_runners_from_json(file: str | Path): return logger.info(f'checking that all runners are present for {file}') all_runners = _load_registry(file) - runners_dir = genvm_root_dir.joinpath('runners') for name, hashes in all_runners.items(): for hash in hashes: cur_dst = runners_dir.joinpath(name, hash[:2], hash[2:] + '.tar') if cur_dst.exists(): + if not verify_hash: + logger.debug(f'already exists {name}:{hash}, skipping') + continue data = cur_dst.read_bytes() if runner_check_bytes(data, hash): logger.debug(f'already exists {name}:{hash}, skipping') @@ -355,7 +363,7 @@ def download_runners_from_json(file: str | Path): 'hash_2_': hash[2:], }, ) - if not runner_check_bytes(data, hash): + if verify_hash and not runner_check_bytes(data, hash): raise ValueError(f'hash mismatch for {name}:{hash}') cur_dst.parent.mkdir(parents=True, exist_ok=True) @@ -405,17 +413,35 @@ def process_executor_version(executor_version: str): run_check_command([executor_executable, '--version']) if args.runners_download: - download_runners_from_json(executor_root_dir.joinpath('data', 'all.json')) - - if args.precompile: - logger.info(f'Precompiling executor {executor_version}') - run_check_command([executor_executable, 'precompile']) + # The v0.2.x legacy line keeps its runners private under the executor + # root (executor//legacy-runners, see its genvm.yaml); every + # other line shares the manager-root runners/ dir. v0.2.x also uses a + # Nix-base32 registry hash this installer can't reproduce, so hash + # verification is left to that executor's `check` command. + is_legacy = (major, minor) == (0, 2) + if is_legacy: + runners_dir = executor_root_dir.joinpath('legacy-runners') + else: + runners_dir = genvm_root_dir.joinpath('runners') + download_runners_from_json( + executor_root_dir.joinpath('data', 'all.json'), + runners_dir, + verify_hash=not is_legacy, + ) # The manifest is authoritative: only versions it lists are processed. for executor_version in all_executor_versions: process_executor_version(executor_version) +# Verify installed runners (present, correct hashes, latest βŠ† all) and, when +# requested, precompile them. This is delegated to the manager's `check-install` +# subcommand, which fans out to each active executor's own `check` command +# instead of the executor being precompiled inline above. +if args.precompile: + logger.info('Checking install and precompiling runners via manager') + run_check_command([modules_executable, 'manager', 'check-install', '--precompile']) + # Warn about any executor directory present on disk that the manifest does not # list (e.g. a stray or locally-built version); it is left untouched. executor_parent = genvm_root_dir.joinpath('executor') diff --git a/libs/deps/dependency-urls.json b/libs/deps/dependency-urls.json index cd27ffd3..6ea59db5 100644 --- a/libs/deps/dependency-urls.json +++ b/libs/deps/dependency-urls.json @@ -76,7 +76,8 @@ "hash": "sha256-0AKDcwjrOLiC5roAsTb9dZqqrxDs2+E5e+2usQwrQgA=", "must_preload": false, "needed-for": [ - "runners-all" + "runners-all", + "runners-all-dist" ], "fetcher": "fetchzip" }, @@ -89,7 +90,8 @@ "hash": "sha256-ZSI7b1wc1D3hqx3IqT8dlFVKldUFjQUZ7u48Cn8wbnE=", "must_preload": false, "needed-for": [ - "runners-all" + "runners-all", + "runners-all-dist" ], "fetcher": "fetchzip" }, @@ -103,7 +105,8 @@ "hash": "sha256-acY8yFzIRYbrZ2CGODoxLnZuppsP6KZy19I9Yy77pfc=", "must_preload": true, "needed-for": [ - "runners-all" + "runners-all", + "runners-all-dist" ], "fetcher": "fetchzip" }, @@ -117,7 +120,8 @@ "hash": "sha256-Uvi4JZPPERK3gym4yoaeTEJwKXF5brBAEN7GgF+iF6g=", "must_preload": false, "needed-for": [ - "runners-all" + "runners-all", + "runners-all-dist" ], "fetcher": "fetchzip" }, @@ -130,7 +134,8 @@ "hash": "sha256-5kYA8yUGBeIA8eCRDM8CLWRsvKmNj5nWhl3+zl5RIhU=", "must_preload": false, "needed-for": [ - "runners-all" + "runners-all", + "runners-all-dist" ], "fetcher": "fetchzip" }, @@ -144,7 +149,8 @@ "hash": "sha256-3ELGir/E3YS9qWEYQ8SGFrU0md71/pl2QOyUIiH55BQ=", "must_preload": false, "needed-for": [ - "runners-all" + "runners-all", + "runners-all-dist" ], "fetcher": "fetchzip" }, @@ -238,7 +244,8 @@ "hash": "sha256-/cyLxhFsfBBQxn4NrhLdbgHjU3YUjYhPnvquWJodcO8=", "must_preload": false, "needed-for": [ - "runners-all" + "runners-all", + "runners-all-dist" ], "fetcher": "fetchzip" }, diff --git a/ruff.toml b/ruff.toml index 0a62a911..c2ceb589 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,22 +1,16 @@ indent-width = 2 -exclude = [ - "executors/**" -] +exclude = ["executors/**"] [format] indent-style = "tab" -quote-style = "single" line-ending = "lf" +quote-style = "single" -skip-magic-trailing-comma = false docstring-code-format = true +skip-magic-trailing-comma = false [lint] -ignore = [ - "E741" -] +ignore = ["E741"] -extend-select = [ - "I" -] +extend-select = ["I"] diff --git a/runners/compat.nix b/runners/compat.nix index 22676a4b..f50aaa1b 100644 --- a/runners/compat.nix +++ b/runners/compat.nix @@ -3,6 +3,23 @@ # `compatible runnerVersion executorVersion` answers: may an executor pinned at # `executorVersion` ship a runner that was introduced in `runnerVersion`? # -# For now: a runner is compatible with every executor at or after its own +# General rule: a runner is compatible with every executor at or after its own # version β€” i.e. each executor gets "all runners up to and including my version". -runnerVersion: executorVersion: builtins.compareVersions runnerVersion executorVersion <= 0 +# This models forward replay: a newer executor must still run the exact runner +# an old contract pinned. +# +# Exception β€” frozen legacy lines: the v0.2.x line is a backport of the legacy +# v0.2 ABI (string-only user errors, empty-string method key, non-int floats +# encoded as strings, gl_call user-error -> rollback). Its runners are built +# against that old ABI and must NOT roll up into newer executors, which speak the +# new ABI β€” otherwise v0.3 would silently accept and run v0.2.16 runners. So v0.2 +# runners serve ONLY v0.2 executors; every newer line still rolls forward as +# usual. +runnerVersion: executorVersion: +let + isLegacy = v: builtins.match "v?0\\.2(\\..*)?" v != null; +in +if isLegacy runnerVersion then + isLegacy executorVersion +else + builtins.compareVersions runnerVersion executorVersion <= 0 diff --git a/support/ci/genvm-merge-into-dev.py b/support/ci/genvm-merge-into-dev.py index 6e39ccde..92d0ca76 100755 --- a/support/ci/genvm-merge-into-dev.py +++ b/support/ci/genvm-merge-into-dev.py @@ -18,14 +18,21 @@ more -> squash into one commit on top of base, then fast-forward. The PR is closed afterwards. -Executor submodule mirror: the manager commit that lands on `v-dev` -carries a gitlink into the executor repo. We mirror it by fast-forwarding -the executor's SAME-NAMED branch (`v-dev`) to that gitlink commit. True -cross-repo atomicity is impossible, so we (1) fast-forward-check BOTH sides -up front, then (2) push the executor first β€” it is the dependency the -manager commit points at β€” and (3) push the manager. If the manager push -then loses a race the executor is left one fast-forward ahead, which is -harmless (monotonic) and reconciled when Merge is re-ticked. +Executor mirror: the manager release line (e.g. v0.6) is independent of the +executor lines it ships (v0.2, v0.3), so a single manager commit carries a +gitlink into EACH active line's submodule. We mirror every active line by +fast-forwarding its own `-dev` branch (`v0.2-dev`, `v0.3-dev`, ...) to +that line's gitlink commit. True cross-repo atomicity is impossible, so we +(1) fast-forward-check all sides up front, then (2) push the executors first β€” +they are the dependencies the manager commit points at β€” and (3) push the +manager. If the manager push then loses a race the executors are left one +fast-forward ahead, which is harmless (monotonic) and reconciled when Merge is +re-ticked. Lines whose gitlink is unchanged are skipped. + +After a successful merge each line's executor mirror branch +(`pr//`, opened by branch_executor_prs.yaml) is deleted β€” its +tip now lives in the executor's `-dev`, so the branch and its PR are +redundant. (The dev -> version release-gate merge is a separate, not-yet-automated flow; on that one the executor's version branch is fast-forwarded and its @@ -34,12 +41,12 @@ Talks to GitHub through the `gh` CLI and moves refs through `git`; it never executes PR code. The dev branches are protected, so the workflow checks out with the GENVM_CI_PRIVATE_KEY deploy key and pushes non-force -(a base that advanced is safely rejected). The executor submodule must be -checked out with a remote `origin` that can push the executor repo (the -workflow wires its own deploy key); EXECUTOR_SUBMODULE points at it. +(a base that advanced is safely rejected). The active executor submodules +must be checked out with a remote `origin` that can push the executor repo +(the workflow wires its own deploy key); the active lines come from +.genvm-monorepo-root via branch-versions.py. -Env: GITHUB_REPOSITORY, PR_NUMBER, GH_TOKEN, E2E_CHECK_PATTERN, -EXECUTOR_SUBMODULE. +Env: GITHUB_REPOSITORY, PR_NUMBER, GH_TOKEN, E2E_CHECK_PATTERN. """ import json @@ -47,13 +54,12 @@ import re import subprocess import sys +from pathlib import Path REPO = os.environ['GITHUB_REPOSITORY'] PR = os.environ['PR_NUMBER'] E2E_PATTERN = os.environ.get('E2E_CHECK_PATTERN', 'e2e') -# Path to the executor submodule checkout whose origin can push the executor -# repo. Override only if the submodule moves. -EXEC_SUBMODULE = os.environ.get('EXECUTOR_SUBMODULE', 'executors/v0.3.x') +HERE = Path(__file__).resolve().parent def run(*args, check=True): @@ -68,9 +74,16 @@ def git(*args, check=True): return run('git', *args, check=check) -def egit(*args, check=True): - """git inside the executor submodule checkout.""" - return run('git', '-C', EXEC_SUBMODULE, *args, check=check) +def egit(submodule, *args, check=True): + """git inside an executor submodule checkout.""" + return run('git', '-C', submodule, *args, check=check) + + +def active_lines(): + """Active executor lines as `v` tags (e.g. ['v0.2', 'v0.3']). The manager + release line is independent of these; a manager PR ships every active line.""" + out = run(sys.executable, str(HERE / 'branch-versions.py'), 'list').stdout + return [f'v{v}' for v in out.split()] def block(msg): @@ -144,54 +157,115 @@ def check_gates(pr): return base, head_sha -def executor_gitlink(commit): +def delete_executor_mirror_branch(submodule, line, head_ref): + """Delete the executor mirror branch `pr//` after a merge. + + The manager PR's work for this line lived on that namespaced branch; its tip is + now contained in the executor's `-dev` (we just fast-forwarded it there), + so the branch β€” and the auto-opened executor PR it backed (branch_executor_prs.yaml) + β€” are redundant. Deleting the branch also closes that PR. Best-effort: a failure + here never unwinds an otherwise-complete merge. The branch is unprotected, so the + executor deploy key can delete it. + """ + branch = f'pr/{line}/{head_ref}' + r = egit(submodule, 'push', 'origin', '--delete', branch, check=False) + if r.returncode == 0: + print(f'deleted executor mirror branch {branch}') + else: + print(f'note: could not delete executor mirror branch {branch}: {r.stderr.strip()}') + + +def executor_gitlink(commit, submodule): """Executor submodule commit (gitlink) recorded at `commit`'s tree.""" - out = git('ls-tree', commit, EXEC_SUBMODULE).stdout + out = git('ls-tree', commit, submodule).stdout # "160000 commit \t" m = re.match(r'\S+\s+commit\s+([0-9a-f]+)\t', out) if not m: block( - f'could not resolve the executor submodule pointer ({EXEC_SUBMODULE}) ' - f'at `{commit}`.' + f'could not resolve the executor submodule pointer ({submodule}) at `{commit}`.' ) return m.group(1) -def executor_mirror_action(base, exec_sha): - """Decide how to mirror `exec_sha` onto the executor's same-named branch. +def executor_mirror_action(submodule, exec_base, exec_sha): + """Decide how to mirror `exec_sha` onto the executor's `` branch. Returns 'push' (fast-forward, including a fresh create) or 'skip' (the branch already contains exec_sha because the executor advanced past what this PR pins). Blocks if exec_sha is unknown to the executor repo or the branch has diverged from it.""" # Bring every executor head (and thus exec_sha, if pushed) local. - egit('fetch', '--no-tags', 'origin', '+refs/heads/*:refs/remotes/origin/*') + egit(submodule, 'fetch', '--no-tags', 'origin', '+refs/heads/*:refs/remotes/origin/*') - if egit('cat-file', '-e', f'{exec_sha}^{{commit}}', check=False).returncode != 0: + if ( + egit(submodule, 'cat-file', '-e', f'{exec_sha}^{{commit}}', check=False).returncode + != 0 + ): block( - f'executor commit `{exec_sha}` is not in the executor repo; push the ' - f'executor side of this change first, then re-tick Merge.' + f'executor commit `{exec_sha}` ({submodule}) is not in the executor repo; ' + f'push the executor side of this change first, then re-tick Merge.' ) - ref = f'refs/remotes/origin/{base}' - if egit('rev-parse', '--verify', '--quiet', ref, check=False).returncode != 0: + ref = f'refs/remotes/origin/{exec_base}' + if ( + egit(submodule, 'rev-parse', '--verify', '--quiet', ref, check=False).returncode + != 0 + ): return 'push' # branch absent -> push creates it - tip = egit('rev-parse', ref).stdout.strip() + tip = egit(submodule, 'rev-parse', ref).stdout.strip() if tip == exec_sha: return 'skip' # already there - if egit('merge-base', '--is-ancestor', exec_sha, tip, check=False).returncode == 0: + if ( + egit( + submodule, 'merge-base', '--is-ancestor', exec_sha, tip, check=False + ).returncode + == 0 + ): # Executor moved ahead of what this PR pins; the pinned commit is # already contained, so there is nothing to push. return 'skip' - if egit('merge-base', '--is-ancestor', tip, exec_sha, check=False).returncode == 0: + if ( + egit( + submodule, 'merge-base', '--is-ancestor', tip, exec_sha, check=False + ).returncode + == 0 + ): return 'push' # clean fast-forward block( - f'executor `{base}` (`{tip[:12]}`) has diverged from the PR executor ' + f'executor `{exec_base}` (`{tip[:12]}`) has diverged from the PR executor ' f'commit `{exec_sha[:12]}`; mirror is not a fast-forward.' ) +def mirror_executor_lines(push_sha): + """Fast-forward each active executor line's `-dev` to the gitlink this + merge pins, BEFORE the manager push (the manager commit references them). Lines + whose gitlink is unchanged are skipped. Returns the lines that were pinned so + their mirror branches can be cleaned up after the manager push lands.""" + pinned = [] + for line in active_lines(): + submodule = f'executors/{line}.x' + exec_base = f'{line}-dev' + exec_sha = executor_gitlink(push_sha, submodule) + if executor_mirror_action(submodule, exec_base, exec_sha) == 'push': + print(f'Mirroring executor {exec_base} -> {exec_sha}') + if ( + egit( + submodule, 'push', 'origin', f'{exec_sha}:refs/heads/{exec_base}', check=False + ).returncode + != 0 + ): + block( + f'fast-forward push of executor `{exec_base}` was rejected (it advanced); ' + f're-tick Merge.' + ) + else: + print(f'Executor {exec_base} already contains {exec_sha}; nothing to mirror') + pinned.append((submodule, line)) + return pinned + + def merge(pr, base, head_sha): # Fetch the exact head commit (works for fork PRs too) and live base tip. git( @@ -235,34 +309,24 @@ def merge(pr, base, head_sha): git('commit', '--author', author, '-m', message) push_sha = git('rev-parse', 'HEAD').stdout.strip() - # Mirror the executor submodule onto its same-named branch. Fast-forward- - # check BOTH sides before touching either (the manager side was checked - # above), then push the executor first: the manager commit gitlinks into - # it, so the dependency must land before the referrer. - exec_sha = executor_gitlink(push_sha) - if executor_mirror_action(base, exec_sha) == 'push': - print(f'Mirroring executor {base} -> {exec_sha}') - if ( - egit('push', 'origin', f'{exec_sha}:refs/heads/{base}', check=False).returncode - != 0 - ): - block( - f'fast-forward push of executor `{base}` was rejected (it advanced); ' - f're-tick Merge.' - ) - else: - print(f'Executor {base} already contains {exec_sha}; nothing to mirror') + # Mirror every active executor line onto its own `-dev` branch. The + # manager line (e.g. v0.6) is independent of the executor lines (v0.2, v0.3) + # it ships, so one merge fast-forwards each changed line. Fast-forward-check + # all of them BEFORE the manager push (the manager side was checked above), + # pushing the executors first: the manager commit gitlinks into them, so the + # dependencies must land before the referrer. + pinned = mirror_executor_lines(push_sha) # Non-force FF push; rejected if base advanced since the checks. The - # executor is already advanced at this point; a rejection here leaves it + # executors are already advanced at this point; a rejection here leaves them # one fast-forward ahead (harmless, monotonic) and re-ticking reconciles. if ( git('push', 'origin', f'{push_sha}:refs/heads/{base}', check=False).returncode != 0 ): block( - f'fast-forward push to manager `{base}` was rejected (base advanced) β€” ' - f'executor `{base}` was already fast-forwarded to `{exec_sha[:12]}`; ' - f're-tick Merge to land the manager side.' + f'fast-forward push to manager `{base}` was rejected (base advanced) β€” the ' + f'executor lines were already fast-forwarded; re-tick Merge to land the ' + f'manager side.' ) run( @@ -278,6 +342,10 @@ def merge(pr, base, head_sha): ) run('gh', 'pr', 'close', PR, '--repo', REPO, check=False) + # Clean up the now-redundant executor mirror branches (and the PRs they backed). + for submodule, line in pinned: + delete_executor_mirror_branch(submodule, line, pr['headRefName']) + def main(): pr = pr_view( diff --git a/support/ci/open-executor-prs.py b/support/ci/open-executor-prs.py new file mode 100644 index 00000000..d92680da --- /dev/null +++ b/support/ci/open-executor-prs.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 +"""Open (or reuse) the linked executor PRs for a manager PR and list them on it. + +Invoked by .github/workflows/branch_executor_prs.yaml when a manager PR is opened +against a dev branch. The manager release line (e.g. v0.6) is independent of the +executor lines it ships (v0.2, v0.3): one manager PR bundles the work of EVERY +active executor line, each on its own line-namespaced mirror branch in the shared +genvm-executor repo (`pr//`, see genvm_tool.common.Repo.feature_branch). + +So we fan out over the active executor lines (.genvm-monorepo-root via +branch-versions.py) rather than deriving a single line from the manager base. For +each active line whose mirror branch `pr//` exists on genvm-executor we +open (or reuse) a PR `pr//` -> `-dev` there, then upsert a marked +comment on the manager PR listing each as an `executor: ` line. + +Two tokens: the executor PRs are created with EXECUTOR_TOKEN (a PAT with +genvm-executor access β€” the default GITHUB_TOKEN is manager-scoped and cannot touch +another repo); the manager comment uses MANAGER_TOKEN (the default GITHUB_TOKEN). +Never executes PR code β€” only `gh` API/PR calls. + +Env: MANAGER_REPO, EXECUTOR_REPO, PR_NUMBER, HEAD_REF, MANAGER_TOKEN, EXECUTOR_TOKEN. +""" + +import json +import os +import subprocess +import sys +from pathlib import Path + +MANAGER_REPO = os.environ['MANAGER_REPO'] +EXECUTOR_REPO = os.environ.get('EXECUTOR_REPO', 'genlayerlabs/genvm-executor') +PR = os.environ['PR_NUMBER'] +HEAD_REF = os.environ['HEAD_REF'] +MANAGER_TOKEN = os.environ['MANAGER_TOKEN'] +EXECUTOR_TOKEN = os.environ['EXECUTOR_TOKEN'] + +HERE = Path(__file__).resolve().parent +# Marker so re-runs update the same comment instead of stacking new ones. +COMMENT_MARKER = '' + + +def gh(*args, token, check=True): + """`gh` with GH_TOKEN bound to the given token (manager- or executor-scoped).""" + return subprocess.run( + ['gh', *args], + env={**os.environ, 'GH_TOKEN': token}, + check=check, + text=True, + capture_output=True, + ) + + +def active_lines() -> list[str]: + """Active executor lines as `v` tags (e.g. ['v0.2', 'v0.3']).""" + out = subprocess.run( + [sys.executable, str(HERE / 'branch-versions.py'), 'list'], + check=True, + text=True, + capture_output=True, + ).stdout + return [f'v{v}' for v in out.split()] + + +def executor_branch_exists(branch: str) -> bool: + # matching-refs returns [] (not a 404) when nothing matches; it is a prefix + # query, so confirm an EXACT ref before treating the branch as present. + out = gh( + 'api', + f'repos/{EXECUTOR_REPO}/git/matching-refs/heads/{branch}', + token=EXECUTOR_TOKEN, + ).stdout + return any(r.get('ref') == f'refs/heads/{branch}' for r in json.loads(out or '[]')) + + +def existing_pr(head: str, base: str) -> str | None: + url = gh( + 'pr', + 'list', + '--repo', + EXECUTOR_REPO, + '--head', + head, + '--base', + base, + '--state', + 'open', + '--json', + 'url', + '--jq', + '.[0].url // ""', + token=EXECUTOR_TOKEN, + ).stdout.strip() + return url or None + + +def open_pr(head: str, base: str) -> str: + title = gh( + 'pr', + 'view', + PR, + '--repo', + MANAGER_REPO, + '--json', + 'title', + '--jq', + '.title', + token=MANAGER_TOKEN, + ).stdout.strip() + body = ( + f'Auto-opened executor mirror of {MANAGER_REPO}#{PR}.\n\n' + f'Carries the executor-side work for that manager PR. Closed and its branch ' + f'`{head}` deleted automatically when the manager PR merges.' + ) + r = gh( + 'pr', + 'create', + '--repo', + EXECUTOR_REPO, + '--base', + base, + '--head', + head, + '--title', + title, + '--body', + body, + token=EXECUTOR_TOKEN, + check=False, + ) + if r.returncode != 0: + # Lost a race against a concurrent run? Fall back to whatever is open now. + url = existing_pr(head, base) + if url: + return url + raise SystemExit( + f'failed to create executor PR `{head}` -> `{base}`: {r.stderr.strip()}' + ) + return r.stdout.strip().splitlines()[-1].strip() + + +def upsert_comment(lines: list[str]) -> None: + body = f'{COMMENT_MARKER}\n### Linked executor PR(s)\n\n' + '\n'.join(lines) + '\n' + ids = gh( + 'api', + f'repos/{MANAGER_REPO}/issues/{PR}/comments', + '--jq', + f'.[] | select(.body | contains("{COMMENT_MARKER}")) | .id', + token=MANAGER_TOKEN, + ).stdout.split() + if ids: + gh( + 'api', + '--method', + 'PATCH', + f'repos/{MANAGER_REPO}/issues/comments/{ids[0]}', + '-f', + f'body={body}', + token=MANAGER_TOKEN, + ) + else: + gh( + 'api', + '--method', + 'POST', + f'repos/{MANAGER_REPO}/issues/{PR}/comments', + '-f', + f'body={body}', + token=MANAGER_TOKEN, + ) + + +def main() -> None: + lines = [] + for line in active_lines(): + head = f'pr/{line}/{HEAD_REF}' + base = f'{line}-dev' + if not executor_branch_exists(head): + print(f'no executor branch `{head}`; skipping {line} (nothing pushed for it)') + continue + url = existing_pr(head, base) or open_pr(head, base) + print(f'{line}: executor PR {url}') + lines.append(f'executor: {url}') + if lines: + upsert_comment(lines) + else: + print('no executor mirror branches found for any active line; nothing to link') + + +if __name__ == '__main__': + main() diff --git a/support/ci/pipelines/build.sh b/support/ci/pipelines/build.sh index 5614e80f..41921215 100755 --- a/support/ci/pipelines/build.sh +++ b/support/ci/pipelines/build.sh @@ -73,7 +73,6 @@ if [[ -n "${COMPRESSION:-}" && "$COMPRESSION" != "skip" ]]; then echo "Error: --compression must be 1-9 or 'skip'" >&2 exit 1 fi - exit 1 fi HEAD_REVISION=$(git rev-parse HEAD) @@ -89,7 +88,7 @@ source "$SCRIPT_DIR/_common.sh" cat < flake-config.json { "executor-version": "$EXECUTOR_VERSION", - "repo-url": "https://github.com/genlayerlabs/genvm.git", + "repo-url": "https://github.com/genlayerlabs/genvm-manager.git", "head-revision": "$HEAD_REVISION" } EOF @@ -99,7 +98,9 @@ if [[ -z "${NO_REFETCH:-}" ]]; then fi mkdir -p build -nix build -o "build/out-$TARGET" -v -L ".#$TARGET" --show-trace +# `?submodules=1`: the executor lines are git submodules; without it the flake +# source tree has empty executors/.x dirs and evaluation fails. +nix build -o "build/out-$TARGET" -v -L ".?submodules=1#$TARGET" --show-trace PREV=$(readlink -f .) pushd "build/out-$TARGET" diff --git a/support/ci/pipelines/src/test-rust-fuzz.sh b/support/ci/pipelines/src/test-rust-fuzz.sh index dcb978cc..ffe21d14 100755 --- a/support/ci/pipelines/src/test-rust-fuzz.sh +++ b/support/ci/pipelines/src/test-rust-fuzz.sh @@ -4,6 +4,6 @@ export PATH="$NIX:$PATH" set -ex -support/tools/genvm-tool/genvm-tool configure +genvm-tool configure -nix develop .#mock-tests --command genvm-tool test run --filter-tag "$(cat tests/presets/rust-fuzz.txt)" +nix develop '.?submodules=1#mock-tests' --command genvm-tool test run --ci --filter-tag "$(cat tests/presets/rust-fuzz.txt)" diff --git a/support/ci/pipelines/src/test-rust.sh b/support/ci/pipelines/src/test-rust.sh index ebae4975..cfe561c5 100755 --- a/support/ci/pipelines/src/test-rust.sh +++ b/support/ci/pipelines/src/test-rust.sh @@ -4,7 +4,7 @@ export PATH="$NIX:$PATH" set -ex -support/tools/genvm-tool/genvm-tool configure +genvm-tool configure ninja -v -C build all/bin @@ -12,4 +12,4 @@ python3 ./build/out/bin/post-install.py \ --error-on-missing-executor=false \ --default-download=false -nix develop .#mock-tests --command genvm-tool test run --filter-tag "$(cat tests/presets/rust.txt)" +nix develop '.?submodules=1#mock-tests' --command genvm-tool test run --ci --filter-tag "$(cat tests/presets/rust.txt)" diff --git a/support/ci/pipelines/test-python.sh b/support/ci/pipelines/test-python.sh index 1f1ee4f2..310adaf8 100755 --- a/support/ci/pipelines/test-python.sh +++ b/support/ci/pipelines/test-python.sh @@ -3,4 +3,4 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) source "$SCRIPT_DIR/_common.sh" -nix develop .#py-test --command bash ./support/ci/pipelines/src/test-python.sh +nix develop '.?submodules=1#py-test' --command bash ./support/ci/pipelines/src/test-python.sh diff --git a/support/ci/pipelines/test-rust-fuzz.sh b/support/ci/pipelines/test-rust-fuzz.sh index d50a83dc..b43faa67 100755 --- a/support/ci/pipelines/test-rust-fuzz.sh +++ b/support/ci/pipelines/test-rust-fuzz.sh @@ -6,4 +6,4 @@ source "$SCRIPT_DIR/_common.sh" export ORIGINAL_PATH="$PATH" export ORIGINAL_LD_LIBRARY_PATH="$LD_LIBRARY_PATH" -nix develop .#rust-test --command bash ./support/ci/pipelines/src/test-rust-fuzz.sh +nix develop '.?submodules=1#rust-test' --command bash ./support/ci/pipelines/src/test-rust-fuzz.sh diff --git a/support/ci/pipelines/test-rust.sh b/support/ci/pipelines/test-rust.sh index 2b68bfe1..9b646059 100755 --- a/support/ci/pipelines/test-rust.sh +++ b/support/ci/pipelines/test-rust.sh @@ -25,14 +25,14 @@ python3 ./support/runner-script.py \ download \ --nix-preload --allow-partial --dest build/out/runners --registry build/out/executor/vTEST/data/all.json -nix build -v -L -o build/out-runners .#runners-all +nix build -v -L -o build/out-runners '.?submodules=1#runners-all' mkdir -p ./build/out/runners cp -r ./build/out-runners/. ./build/out/runners/. chmod -R +w ./build/out/runners/. -nix develop .#rust-test --command python3 \ +nix develop '.?submodules=1#rust-test' --command python3 \ ./support/runner-script.py \ upload \ --root build/out/runners --registry build/out/executor/vTEST/data/all.json || true -nix develop .#rust-test --command bash ./support/ci/pipelines/src/test-rust.sh +nix develop '.?submodules=1#rust-test' --command bash ./support/ci/pipelines/src/test-rust.sh diff --git a/support/default.nix b/support/default.nix index f5b0907b..891675da 100644 --- a/support/default.nix +++ b/support/default.nix @@ -145,9 +145,27 @@ rec { libraries = [ pkgs.python3Packages.ruamel-yaml ]; } (builtins.readFile ./scripts/patch-web-config.py); + # nixpkgs pins lief 0.17.0, whose ELF Builder includes NOBITS (.bss/.tbss) + # sections in its file-layout calculation. On our large (~290 MB) musl PIE + # executor binaries that inflates the computed end-of-file, so `binary.write()` + # emits a corrupt section-header table (e_shnum unchanged but trailing entries + # garbage). patchelf then fails install_genvm.sh finalize with "data region + # extends past file end". Fixed upstream in lief 0.17.6 (commit bff40d6, skip + # NOBITS sections in layout + correct e_shstrndx). Pin 0.17.6 here until the + # nixpkgs pin catches up. See lief-project/LIEF#1315. + lief-fixed = pkgs.python3Packages.lief.overrideAttrs (old: { + version = "0.17.6"; + src = pkgs.fetchFromGitHub { + owner = "lief-project"; + repo = "LIEF"; + tag = "0.17.6"; + hash = "sha256-WcWKGIQIGngfzW+VnrZEnRPX2w4syNw+so2aqwSgecw="; + }; + }); + patch-rpath = pkgs.writers.writePython3Bin "patch-rpath" { doCheck = false; - libraries = [ pkgs.python3Packages.lief ]; + libraries = [ lief-fixed ]; makeWrapperArgs = [ "--prefix" "PATH" diff --git a/support/nix/precommit/flake.nix b/support/nix/precommit/flake.nix index 045b6947..98d2d775 100644 --- a/support/nix/precommit/flake.nix +++ b/support/nix/precommit/flake.nix @@ -28,6 +28,7 @@ rustfmt nixfmt cargo + taplo ; pre-commit-hooks = pkgs.python3Packages.pre-commit-hooks; }; diff --git a/support/scripts/check-commit-message.py b/support/scripts/check-commit-message.py new file mode 100755 index 00000000..01f1bce4 --- /dev/null +++ b/support/scripts/check-commit-message.py @@ -0,0 +1,257 @@ +#!/usr/bin/env python3 + +import argparse +import re +import sys +import unicodedata +from dataclasses import dataclass + +VALID_TYPES = r'feat|fix|docs|style|refactor|perf|test|build|ci|chore|revert' +COMMIT_FORMAT_RE = re.compile(rf'^({VALID_TYPES})(\([a-zA-Z0-9_-]+\))?!?: .+$') + +AI_PATTERNS = [ + re.compile(p, re.IGNORECASE) + for p in [ + r'Co-Authored-By:.*claude', + r'Co-Authored-By:.*gpt', + r'Co-Authored-By:.*copilot', + r'Co-Authored-By:.*anthropic', + r'Co-Authored-By:.*openai', + r'Co-Authored-By:.*\bai\b', + r'Co-Authored-By:.*noreply@anthropic', + r'generated by', + r'generated with', + r'created by claude', + r'created with claude', + r'created by gpt', + r'created with gpt', + r'claude[\s-]code', + r'Claude-Session:\s*https?://', + ] +] + +TICKET_PATTERNS = [ + re.compile(p, re.IGNORECASE) + for p in [ + r'fixes #\d+', + r'closes #\d+', + r'resolves #\d+', + r'part of [A-Z]+-\d+', + r'related to [A-Z]+-\d+', + r'fixes [A-Z]+-\d+', + ] +] + +SUBJECT_LEN_SUGGESTED = 72 +SUBJECT_LEN_MAX = 120 +TRAILING_EMOJI_MAX = 5 + +BODY_MIN_WORDS = 3 +BODY_MIN_CHARS = 20 + +# Matches git trailer lines: "Token: value" (Co-authored-by:, Signed-off-by:, etc.) +GIT_TRAILER_RE = re.compile(r'^[A-Za-z][A-Za-z0-9-]*:\s*.+$') + +PR_REF_RE = re.compile(r'\s*\(#\d+\)\s*$') +SQUASH_BULLET_RE = re.compile(r'^\*\s*') + + +_EMOJI_MODIFIERS = frozenset([0xFE0F, 0xFE0E, 0x200D, 0x20E3]) + + +def _is_emoji_char(ch: str) -> bool: + cp = ord(ch) + if cp in _EMOJI_MODIFIERS: + return True + cat = unicodedata.category(ch) + if cat in ('So', 'Sm') and cp > 0x2000: + return True + if 0x1F300 <= cp <= 0x1FAFF: + return True + if 0x2600 <= cp <= 0x27BF: + return True + return False + + +def _split_trailing_emojis(text: str) -> tuple[str, str]: + """Return (prefix, suffix) where suffix is trailing spaces+emojis.""" + i = len(text) + while i > 0 and (text[i - 1] == ' ' or _is_emoji_char(text[i - 1])): + i -= 1 + return text[:i], text[i:] + + +def _count_emojis(text: str) -> int: + return sum(1 for ch in text if _is_emoji_char(ch) and ord(ch) not in _EMOJI_MODIFIERS) + + +def _body_without_trailers(body: str) -> str: + """Strip git trailer lines from body text, return the remaining content.""" + lines = [line for line in body.splitlines() if not GIT_TRAILER_RE.match(line.strip())] + return '\n'.join(lines).strip() + + +def _first_match(patterns: list[re.Pattern], text: str) -> str | None: + for pat in patterns: + m = pat.search(text) + if m: + return m.group() + return None + + +@dataclass +class CommitPart: + title: str + body: str = '' + + +def parse_commit(message: str) -> list[CommitPart]: + """Split a commit message into parts to validate independently. + + A squash/PR-merge commit (GitHub style) has bullet lines in the body that + are themselves conventional commit subjects. Each becomes its own CommitPart; + the umbrella title becomes the first part with any non-bullet body text. + A regular commit returns a single CommitPart. + """ + lines = message.splitlines() + title = lines[0] if lines else '' + body_lines = lines[1:] if len(lines) > 1 else [] + + squash_parts: list[CommitPart] = [] + non_bullet: list[str] = [] + + for line in body_lines: + stripped = line.strip() + if not stripped: + continue + m = SQUASH_BULLET_RE.match(stripped) + if m: + bullet = stripped[m.end() :] + clean, _ = _split_trailing_emojis(PR_REF_RE.sub('', bullet)) + if COMMIT_FORMAT_RE.match(clean): + squash_parts.append(CommitPart(title=bullet)) + continue + non_bullet.append(line) + + umbrella = CommitPart(title=title, body='\n'.join(non_bullet).strip()) + return [umbrella, *squash_parts] if squash_parts else [umbrella] + + +def _check_subject(subject: str, errors: list[str], warnings: list[str]) -> None: + clean, suffix = _split_trailing_emojis(PR_REF_RE.sub('', subject)) + + if not COMMIT_FORMAT_RE.match(clean): + errors.append('Invalid format. Expected: (): [emoji…]') + errors.append(f" Got: '{subject}'") + errors.append(f' Valid types: {VALID_TYPES.replace("|", ", ")}') + + if _count_emojis(clean) > 0: + errors.append( + f"Emojis must only appear at the end of the subject line: '{subject}'" + ) + + n = _count_emojis(suffix) + if n > TRAILING_EMOJI_MAX: + warnings.append(f'Too many trailing emojis ({n}). Max {TRAILING_EMOJI_MAX}.') + + length = len(subject) + if length > SUBJECT_LEN_MAX: + errors.append( + f'Subject too long ({length} chars). Maximum: {SUBJECT_LEN_MAX} chars' + ) + elif length > SUBJECT_LEN_SUGGESTED: + warnings.append( + f'Subject exceeds {SUBJECT_LEN_SUGGESTED} chars ({length} chars). Consider shortening.' + ) + + +def validate_part(part: CommitPart) -> tuple[list[str], list[str]]: + errors: list[str] = [] + warnings: list[str] = [] + + _check_subject(part.title, errors, warnings) + + if hit := _first_match(AI_PATTERNS, part.title + '\n' + part.body): + errors.append(f"AI attribution detected: '{hit}'.") + errors.append( + ' Commits must not contain AI tool references or Co-Authored-By with AI names.' + ) + + real_body = _body_without_trailers(part.body) + if real_body: + words = len(real_body.split()) + chars = len(real_body) + if words < BODY_MIN_WORDS or chars < BODY_MIN_CHARS: + warnings.append( + f'Commit body is very short ({words} word(s), {chars} char(s)). Are you sure you need a body?' + ) + + if _count_emojis(part.body) > 0: + errors.append('Emojis in the commit body are not allowed.') + + if hit := _first_match(TICKET_PATTERNS, part.body): + warnings.append( + f"Ticket reference in commit body ('{hit}'). Move to PR description instead." + ) + + return errors, warnings + + +def validate(message: str) -> tuple[list[str], list[str]]: + all_errors: list[str] = [] + all_warnings: list[str] = [] + for part in parse_commit(message): + errors, warnings = validate_part(part) + all_errors.extend(errors) + all_warnings.extend(warnings) + return all_errors, all_warnings + + +def main() -> int: + parser = argparse.ArgumentParser( + description='Validate a commit message against conventional commit rules.' + ) + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument( + '--message-file', metavar='FILE', help='Path to file containing the commit message' + ) + group.add_argument('--message-text', metavar='TEXT', help='Commit message text') + args = parser.parse_args() + + if args.message_file: + if args.message_file == '-': + message = sys.stdin.read() + else: + with open(args.message_file, encoding='utf-8') as f: + message = f.read() + else: + message = args.message_text + + errors, warnings = validate(message) + + if warnings: + print(f'Warnings ({len(warnings)}):') + for w in warnings: + print(f' {w}') + + if errors: + print(f'\nErrors ({len(errors)}):') + for e in errors: + print(f' {e}') + print() + print('Commit message rules:') + print(' - Format: (): ') + print(f' - Types: {VALID_TYPES.replace("|", ", ")}') + print( + f' - Title: <={SUBJECT_LEN_SUGGESTED} chars recommended, <={SUBJECT_LEN_MAX} max' + ) + print(" - NO AI attribution (Co-Authored-By with AI, 'Generated by', etc.)") + print(' - NO ticket references in body (use PR description)') + return 1 + + print('Commit message is valid.') + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/support/tools/genvm-tool/default.nix b/support/tools/genvm-tool/default.nix index a1ee9d37..9ff5fd3e 100644 --- a/support/tools/genvm-tool/default.nix +++ b/support/tools/genvm-tool/default.nix @@ -19,6 +19,7 @@ python.pkgs.buildPythonApplication { ruamel-yaml cloudpickle python-dotenv + questionary ]; nativeBuildInputs = [ pkgs.installShellFiles ]; diff --git a/support/tools/genvm-tool/genvm_tool/__main__.py b/support/tools/genvm-tool/genvm_tool/__main__.py index 989d0cbf..86e64c56 100644 --- a/support/tools/genvm-tool/genvm_tool/__main__.py +++ b/support/tools/genvm-tool/genvm_tool/__main__.py @@ -16,9 +16,18 @@ import shtab -from . import cmd_build_manifest, cmd_configure, cmd_test, common, formatter, git, hook - -TOPLEVEL = [cmd_configure, cmd_test, cmd_build_manifest] +from . import ( + cmd_build_manifest, + cmd_codegen, + cmd_configure, + cmd_test, + common, + formatter, + git, + hook, +) + +TOPLEVEL = [cmd_configure, cmd_test, cmd_build_manifest, cmd_codegen] GROUPS = [git, hook] @@ -61,9 +70,7 @@ def _create_parser() -> argparse.ArgumentParser: def main() -> None: - from dotenv import load_dotenv - - load_dotenv() + sys.dont_write_bytecode = True parser = _create_parser() args = parser.parse_args() @@ -88,12 +95,22 @@ def main() -> None: try: root = common.find_root() + + env_file = root / '.env' + if env_file.exists(): + from dotenv import load_dotenv + + load_dotenv(env_file) + + python_command = [sys.executable, '-B'] + # Pre-subcommand: exec the manager's `.genvm-tool.py` so subcommands can # ask it for the test suite (`tests`) and the manager's commit hooks. ctx = common.Context( root=root, logger=logger, printer=printer, + python_command=python_command, project=common.load_project(root), ) func = args.func diff --git a/support/tools/genvm-tool/genvm_tool/cmd_codegen.py b/support/tools/genvm-tool/genvm_tool/cmd_codegen.py new file mode 100644 index 00000000..5ab8c0b9 --- /dev/null +++ b/support/tools/genvm-tool/genvm_tool/cmd_codegen.py @@ -0,0 +1,60 @@ +"""`genvm-tool codegen` β€” generate language bindings from genvm data JSON. + +Replaces the per-executor ruby templates (``rs.rb`` / ``py.rb`` / ``rst.rb`` and +the standalone go generator). One typed front-end parses the data JSON; the +``--lang`` backend renders it. The build invokes this for the rust/python/rst +outputs; ``--lang go`` is a manual/external entry point (the generated ``.go`` +lands in a separate Go module), with the package name set by ``--go-package``. +""" + +from pathlib import Path + +from . import common + +NAME = 'codegen' +HELP = 'generate language bindings (rust/python/rst/go) from genvm data JSON' + + +def configure(parser): + from . import codegen + + parser.add_argument( + '--lang', + required=True, + choices=codegen.LANGS, + help='target language backend', + ) + parser.add_argument( + '-i', + '--in', + dest='input', + required=True, + help='path to the codegen data JSON', + ) + parser.add_argument( + '-o', + '--out', + dest='output', + required=True, + help='path to write the generated file', + ) + parser.add_argument( + '--go-package', + default='genvm', + help="package name for the go backend (default: 'genvm')", + ) + + +def main(ctx: common.Context, args) -> int: + from . import codegen + + codegen.generate( + args.lang, + Path(args.input), + Path(args.output), + go_package=args.go_package, + ) + ctx.printer.put( + 'codegen', lang=args.lang, input=str(args.input), output=str(args.output) + ) + return 0 diff --git a/support/tools/genvm-tool/genvm_tool/cmd_configure.py b/support/tools/genvm-tool/genvm_tool/cmd_configure.py index 441ee162..07985369 100644 --- a/support/tools/genvm-tool/genvm_tool/cmd_configure.py +++ b/support/tools/genvm-tool/genvm_tool/cmd_configure.py @@ -1,9 +1,18 @@ """`genvm-tool configure` β€” generate build/build.ninja and build/info.json. Generates the ninja build graph that drives cargo (build / clippy / clippy-fix / -fmt), the ruby codegen templates, the nix runner build, and the install `cp` +fmt), the `genvm-tool codegen` outputs, the nix runner build, and the install `cp` steps. (This was historically the manager-root `configure.rb` Ruby script.) +The ninja DSL and the *per-line* build registrations live in the importable +``genvm_tool_plugins.ninja`` plugin. This command owns only the manager-global +graph (manager crates, shared codegen, the runner build, info.json/manifest); for +each active executor line it builds a :class:`~genvm_tool_plugins.ninja.LineContext` +and hands it to that line's ``.genvm-tool.py:configure(line)`` hook. The lines +already diverge on how they resolve their runner registry (a frozen line copies a +committed manifest, a live line derives it through nix), so each line owns its own +configuration rather than this command branching on the version. + Every path is resolved relative to the manager root (`ctx.root`), so the graph is identical regardless of the directory the tool is invoked from β€” the original script assumed it was run from the source root, which is the same thing once the @@ -11,8 +20,6 @@ """ import json -import os -import re import subprocess import sys from pathlib import Path @@ -26,6 +33,10 @@ # Umbrella manifest listing the active top-level version lines and the primary. MONOREPO_ROOT_FILE = '.genvm-monorepo-root' +# The importable plugins live here (mirrors `extra_python_paths`); configure adds +# it to `sys.path` so it (and each line's hook) can import `genvm_tool_plugins`. +PLUGINS_REL = Path('tests/runner') + class ExecVersion(NamedTuple): """One active executor line resolved to its on-disk locations. @@ -61,17 +72,6 @@ def _load_versions(source_dir: Path) -> tuple[str, list[ExecVersion]]: return cfg['version'], versions -# Build directory, manager-root-relative. Kept as a relative path because the -# generated `build_dir` ninja variable must read `build`, not an absolute path. -BUILD_DIR_REL = Path('build') - -# Cargo needs the rust toolchain's libs on LD_LIBRARY_PATH; `$$` escapes the `$` -# from ninja so the shell expands it, `:+` keeps an unset LD_LIBRARY_PATH empty. -CARGO_LD_LIBRARY_PATH = ( - 'LD_LIBRARY_PATH="$${CARGO_LD_LIBRARY_PATH}$${LD_LIBRARY_PATH:+:$$LD_LIBRARY_PATH}"' -) - - def configure(parser): parser.add_argument( '--test-always-fail-module', @@ -85,269 +85,6 @@ def configure(parser): ) -# --- ninja DSL ------------------------------------------------------------- - - -class RawStr(str): - """A string emitted verbatim into the ninja file (never shell-escaped).""" - - -AND = RawStr('&&') -VAR_IN = RawStr('$in') -VAR_OUT = RawStr('$out') - -# clippy/build runs are instrumented for coverage but discard the profile here. -COVERAGE_ENV = RawStr('RUSTFLAGS="-C instrument-coverage" LLVM_PROFILE_FILE=/dev/null') - -# Ruby's Shellwords.escape leaves this set unescaped; everything else (including -# spaces, quotes, `$`, `?`, `#`) gets a leading backslash. -_UNSAFE = re.compile(r'[^A-Za-z0-9_\-.,:+/@\n]') - - -def _shellescape(s: str) -> str: - if s == '': - return "''" - s = _UNSAFE.sub(lambda m: '\\' + m.group(0), s) - # A backslash before LF is a line continuation, so wrap newlines in quotes. - return s.replace('\n', "'\n'") - - -def _escape(s: str) -> str: - # configure.rb un-escapes `\=`, keeping `FOO=bar` env assignments readable. - return _shellescape(s).replace('\\=', '=') - - -def _glob(base: Path, pattern: str) -> list[Path]: - """Mirror Ruby's `Pathname#glob`: a depth-first, per-component sorted walk - that skips dotfile entries (no FNM_DOTMATCH). - - Sorting by path *components* (not the joined string) is what makes - `calldata/...` precede `calldata-derive/...`: the `/` separator dominates, - exactly as Ruby's recursive directory traversal does. - """ - out = [] - for p in base.glob(pattern): - rel = p.relative_to(base) - if any(part.startswith('.') for part in rel.parts): - continue - out.append(p) - return sorted(out, key=lambda p: p.parts) - - -def _relpath(target: Path, start: Path) -> str: - return os.path.relpath(str(target), str(start)) - - -def _is_subpath(path: Path, base: Path) -> bool: - return not _relpath(path, base).startswith('..') - - -class Build: - """One `build` edge; mirrors configure.rb's BuildBuilder.""" - - def __init__(self, ninja: 'Ninja', rule: str, outputs): - self.ninja = ninja - self.rule = rule - self.outputs = list(outputs) - self.implicit_outputs: list = [] - self.deps: list = [] - self.implicit_deps: list = [] - self.order_only_deps: list = [] - self.props: dict = {} - - @staticmethod - def _add(arr: list, item) -> None: - if isinstance(item, (list, tuple)): - arr.extend(item) - else: - arr.append(item) - - def add_output(self, output): - self._add(self.outputs, output) - return self - - def add_implicit_output(self, output): - self._add(self.implicit_outputs, output) - return self - - def add_dependency(self, dep): - self._add(self.deps, dep) - return self - - def add_implicit_dependency(self, dep): - self._add(self.implicit_deps, dep) - return self - - def add_order_only_dependency(self, dep): - self._add(self.order_only_deps, dep) - return self - - def var(self, name: str, value): - self.props[name] = value - return self - - def description(self, desc: str): - self.props['description'] = desc - return self - - def finish(self) -> None: - assert self.outputs, 'build edge must have at least one output' - n = self.ninja - n.buf.append('build') - for o in self.outputs: - n.buf.append(' ') - n._scalar(o) - if self.implicit_outputs: - n.buf.append(' |') - for o in self.implicit_outputs: - n.buf.append(' ') - n._scalar(o) - n.buf.append(': ') - n.buf.append(self.rule) - for d in self.deps: - n.buf.append(' ') - n._scalar(d) - if self.implicit_deps: - n.buf.append(' |') - for d in self.implicit_deps: - n.buf.append(' ') - n._scalar(d) - if self.order_only_deps: - n.buf.append(' ||') - for d in self.order_only_deps: - n.buf.append(' ') - n._scalar(d) - n.buf.append('\n') - for key, value in self.props.items(): - n.buf.append(f' {key} = ') - n._value(value) - n.buf.append('\n') - n.buf.append('\n') - - -class Ninja: - """Accumulates the build.ninja text; mirrors configure.rb's Ninja::File.""" - - def __init__(self, source_dir: Path, build_dir: Path): - self.source_dir = source_dir - self.build_dir = build_dir # absolute; this is ninja's working directory - self.buf: list[str] = [] - self.all_format: list[str] = [] - self.all_clippy: list[str] = [] - self.all_clippy_fix: list[str] = [] - self.rust_target = '' - self.rust_target_dir = build_dir # set by main - build_dir.mkdir(parents=True, exist_ok=True) - - def _resolve_path(self, value: Path) -> str: - """Render a Path as ninja sees it: relative to the build (working) dir.""" - if not value.is_absolute(): - # Relative paths are interpreted against the source root. - return _relpath(self.source_dir / value, self.build_dir) - if _is_subpath(value, self.build_dir) or _is_subpath(value, self.source_dir): - return _relpath(value, self.build_dir) - return str(value) - - def _scalar(self, value) -> None: - if isinstance(value, RawStr): - self.buf.append(str(value)) - elif isinstance(value, Path): - self.buf.append(_escape(self._resolve_path(value))) - else: - self.buf.append(_escape(str(value))) - - def _value(self, value) -> None: - if isinstance(value, (list, tuple)): - for i, v in enumerate(value): - if i: - self.buf.append(' ') - self._value(v) - else: - self._scalar(value) - - def comment(self, text: str) -> None: - for line in text.splitlines(): - self.buf.append(f'# {line.strip()}\n') - - def var(self, name: str, value) -> None: - self.buf.append(f'{name} = ') - self._value(value) - self.buf.append('\n\n') - - def rule(self, name: str, **props) -> None: - assert 'command' in props, f'rule {name} must have a command' - self.buf.append(f'rule {name}\n') - for key, value in props.items(): - self.buf.append(f' {key} = ') - self._value(value) - self.buf.append('\n') - self.buf.append('\n') - - def build(self, rule: str, *outputs) -> Build: - return Build(self, rule, outputs) - - # --- per-crate cargo edges -------------------------------------------- - - def register_cargo( - self, rel_path: str, extra_args=(), build_to: str | None = None - ) -> None: - extra_args = list(extra_args) - to = BUILD_DIR_REL / rel_path - (self.source_dir / to).mkdir(parents=True, exist_ok=True) - - crate_dir = Path(rel_path) - base = self.source_dir / rel_path - all_files = [crate_dir / p.relative_to(base) for p in _glob(base, '**/*.rs')] - all_files += [crate_dir / 'Cargo.toml', crate_dir / 'Cargo.lock'] - - files_trg = BUILD_DIR_REL / 'ya-build' / rel_path / 'files.trg' - (self.source_dir / files_trg).parent.mkdir(parents=True, exist_ok=True) - - self.build('phony_touch', files_trg).add_implicit_dependency(all_files).finish() - - clippy_lints = ['--', '-A', 'clippy::upper_case_acronyms', '-Dwarnings'] - - clippy = self.build('cargo', 'target/' + rel_path + '/clippy') - clippy.add_dependency(files_trg) - clippy.var('subcommand', 'clippy') - clippy.var('wd', crate_dir) - clippy.var('extra_args', extra_args + clippy_lints) - clippy.var('env', COVERAGE_ENV) - clippy.description('Run cargo clippy for ' + rel_path) - clippy.finish() - self.all_clippy.append('target/' + rel_path + '/clippy') - - fix = self.build('cargo', 'target/' + rel_path + '/clippy/fix') - fix.var('subcommand', 'clippy') - fix.var('wd', crate_dir) - fix.var( - 'extra_args', - extra_args + ['--fix', '--allow-dirty', '--allow-staged'] + clippy_lints, - ) - fix.var('env', COVERAGE_ENV) - fix.finish() - self.all_clippy_fix.append('target/' + rel_path + '/clippy/fix') - - fmt = self.build('CUSTOM_COMMAND', 'target/' + rel_path + '/fmt') - fmt.var('command', ['cd', crate_dir, AND, 'cargo', 'fmt']) - fmt.description('Run cargo fmt for ' + rel_path) - fmt.finish() - self.all_format.append('target/' + rel_path + '/fmt') - - if build_to is not None: - bin_name = str( - self.rust_target_dir / self.rust_target / 'debug' / build_to.split('/')[-1] - ) - build = self.build('cargo_build', bin_name) - build.add_dependency(files_trg) - build.var('wd', crate_dir) - build.var('extra_args', extra_args) - build.var('env', COVERAGE_ENV) - build.finish() - - self.build('cp', build_to).add_dependency(bin_name).finish() - - def _detect_rust_target() -> str: out = subprocess.run( ['rustc', '-vV'], capture_output=True, text=True, check=True @@ -358,24 +95,34 @@ def _detect_rust_target() -> str: raise common.ToolError("failed to detect rust target from 'rustc -vV' output") -# Nix expressions that evaluate a checkout's runner derivations to their -# {id: hash} (latest.json) and {id: [hash]} (all.json) maps. -def _runner_manifest_expr(version: str, kind: str) -> str: - """Nix expr for one executor's `latest`/`all` runner manifest. +def _line_configurator(ctx: common.Context, source_dir: Path, exec_rel: str): + """Resolve a line's `configure(line)` hook from its `.genvm-tool.py`. - The umbrella's ./runners owns this: `all` is every runner compatible up to - `version`, `latest` is that executor's own current runners. Evaluated with - cwd at the manager root (the nix_eval rule cds into ``$wd``). + Falls back to the plugin's default (committed-registry-or-nix) when the line + carries no hook, so a line without one still configures sensibly. """ - return ( - f'(import ./runners/manifest.nix ' - f'{{ executorVersion = "{version}"; host-system = builtins.currentSystem; }}).{kind}' - ) + from genvm_tool_plugins import ninja + + project = common.load_project(source_dir / exec_rel) + hook = getattr(project, 'configure', None) + if hook is None: + ctx.logger.debug('line has no configure hook, using default', line=exec_rel) + return ninja.configure_line_default + return hook def main(ctx: common.Context, args) -> int: source_dir = ctx.root - build_dir = source_dir / BUILD_DIR_REL + + # The ninja DSL + per-line helpers live in the importable plugin; put the + # plugin search path on sys.path before importing it (and before any line's + # `.genvm-tool.py:configure` hook, which may import it too). + plugins_path = str(source_dir / PLUGINS_REL) + if plugins_path not in sys.path: + sys.path.insert(0, plugins_path) + from genvm_tool_plugins import ninja + + build_dir = source_dir / ninja.BUILD_DIR_REL primary_key, versions = _load_versions(source_dir) # Manager-global generated files (test fixtures, docs) have a single output, @@ -387,23 +134,33 @@ def main(ctx: common.Context, args) -> int: rust_target_dir = build_dir / 'ya-build' / 'rust-target' rust_target_dir.mkdir(parents=True, exist_ok=True) - n = Ninja(source_dir, build_dir) + n = ninja.Ninja(source_dir, build_dir) n.rust_target = rust_target n.rust_target_dir = rust_target_dir n.comment('Generated by `genvm-tool configure`, DO NOT EDIT MANUALLY') n.var('ninja_required_version', '1.5') + # Default cargo target dir. Version-independent crates share it; each + # executor line overrides it (register_cargo `target_dir=`) so two lines + # building a `genvm` binary don't clobber the same `debug/genvm`. + n.var('target_dir', str(rust_target_dir)) n.rule( 'CLEAN', - command=['ninja', RawStr('$FILE_ARG'), '-t', 'clean', RawStr('$TARGETS')], + command=[ + 'ninja', + ninja.RawStr('$FILE_ARG'), + '-t', + 'clean', + ninja.RawStr('$TARGETS'), + ], description='Cleaning all built files...', ) n.rule( 'HELP', command=[ 'ninja', - RawStr('$FILE_ARG'), + ninja.RawStr('$FILE_ARG'), '-t', 'targets', 'rule', @@ -416,82 +173,108 @@ def main(ctx: common.Context, args) -> int: description='All primary targets available', ) - n.var('build_dir', str(BUILD_DIR_REL)) + n.var('build_dir', str(ninja.BUILD_DIR_REL)) n.build('CLEAN', 'clean').finish() n.build('HELP', 'help').finish() - n.rule('phony_touch', command=['touch', VAR_OUT]) + n.rule('phony_touch', command=['touch', ninja.VAR_OUT]) n.rule( 'CUSTOM_COMMAND', - command=['cd', RawStr('$CWD'), AND, RawStr('$ENV'), RawStr('$COMMAND')], + command=[ + 'cd', + ninja.RawStr('$CWD'), + ninja.AND, + ninja.RawStr('$ENV'), + ninja.RawStr('$COMMAND'), + ], description='Running custom command', ) - n.rule('cp', command=['cp', VAR_IN, VAR_OUT]) + n.rule('cp', command=['cp', ninja.VAR_IN, ninja.VAR_OUT]) + + # In-tree genvm-tool launcher that the generated build edges (regen + codegen) + # shell out to, instead of repeating the `PYTHONPATH=… python -m genvm_tool` + # incantation. Written directly here (not via a ninja edge) so it exists for the + # first build. It front-loads the in-tree source on PYTHONPATH β€” so an edit to + # cmd_configure.py or a codegen backend takes effect on the next build rather + # than running a stale installed copy β€” and pins this env's interpreter (there + # is no `genvm-tool` wrapper in the source tree, and PATH may not carry it under + # ninja). `${PYTHONPATH:-}` keeps an unset inherited path empty. + tool_src = source_dir / 'support' / 'tools' / 'genvm-tool' + genvm_tool_sh = build_dir / 'genvm_tool.sh' + build_dir.mkdir(parents=True, exist_ok=True) + genvm_tool_sh.write_text( + '#!/bin/sh\n' + '# Generated by `genvm-tool configure`, DO NOT EDIT MANUALLY.\n' + f'export PYTHONPATH="{tool_src}:${{PYTHONPATH:-}}"\n' + f'exec "{sys.executable}" -m genvm_tool "$@"\n' + ) + genvm_tool_sh.chmod(0o755) - # Regenerate build.ninja when this script changes (re-runs the tool). - # Re-invoke this exact program via its interpreter + module rather than a - # `genvm-tool` wrapper (there is none in the source tree, and PATH may not - # carry it under ninja). Preserve the configure flags so a regen keeps mode. - genvm_tool = [sys.executable, '-m', 'genvm_tool'] + # Regenerate build.ninja when this script changes (re-runs the tool via the + # launcher above). Preserve the configure flags so a regen keeps mode. regen_args = [] if args.test_always_fail_module: regen_args.append('--test-always-fail-module') if args.ci: regen_args.append('--ci') regen = n.build('CUSTOM_COMMAND', 'build.ninja') - regen.add_dependency( - source_dir / 'support' / 'tools' / 'genvm-tool' / 'genvm_tool' / 'cmd_configure.py' - ) + regen.add_dependency(tool_src / 'genvm_tool' / 'cmd_configure.py') + # The build graph is also shaped by the shared ninja plugin and by each line's + # own `configure` hook, so editing any of them must trigger a regen too. + regen.add_dependency(source_dir / PLUGINS_REL / 'genvm_tool_plugins' / 'ninja.py') # The active version set and each line's pinned version drive the graph too. regen.add_dependency(source_dir / MONOREPO_ROOT_FILE) for v in versions: regen.add_dependency(source_dir / v.exec_rel / 'manifest.json') - regen.var('COMMAND', genvm_tool + ['configure'] + regen_args) + regen.add_dependency(source_dir / v.exec_rel / common.PROJECT_FILE) + regen.var('COMMAND', [str(genvm_tool_sh), 'configure'] + regen_args) regen.var('CWD', str(source_dir)) regen.finish() - n.rule('codegen', command=['ruby', VAR_IN, VAR_OUT]) - - def codegen(out: Path, template: Path, data: Path) -> None: - b = n.build('codegen', out) - b.add_dependency(template) - b.add_dependency(data) - b.finish() + # Codegen shells out to the same launcher; `$lang` is per-edge, `$in` is the + # data JSON, `$out` the generated file. + n.rule( + 'codegen', + command=[ + str(genvm_tool_sh), + 'codegen', + '--lang', + ninja.RawStr('$lang'), + '--in', + ninja.VAR_IN, + '--out', + ninja.VAR_OUT, + ], + description='Codegen $out', + ) + # Regenerate when any codegen backend changes (the data JSON is a per-edge dep). + n.codegen_deps = ninja.glob(tool_src / 'genvm_tool' / 'codegen', '*.py') + [ + tool_src / 'genvm_tool' / 'cmd_codegen.py' + ] codegen_phony = n.build('phony', 'codegen') # Manager-global generated files (test fixtures + docs) have a single output, # so generate them from the primary line's codegen data. - p_tmpl = primary_exec_root / 'executor' / 'codegen' / 'templates' p_data = primary_exec_root / 'executor' / 'codegen' / 'data' host_fns_py = source_dir / 'tests/runner/origin/host_fns.py' public_abi_py = source_dir / 'tests/runner/origin/public_abi.py' constants_rst = source_dir / 'docs/website/src/spec/appendix/constants.rst' + # Pending public-abi constants (ADR-012): documented in their own appendix + # page so the spec can reference them, without feeding the runner-hashed + # `public_abi.py`. Sourced from the primary line's pending data file. + constants_pending_rst = ( + source_dir / 'docs/website/src/spec/appendix/constants-pending.rst' + ) - codegen(host_fns_py, p_tmpl / 'py.rb', p_data / 'host-fns.json') - codegen(public_abi_py, p_tmpl / 'py.rb', p_data / 'public-abi.json') - codegen(constants_rst, p_tmpl / 'rst.rb', p_data / 'public-abi.json') - for out in (host_fns_py, public_abi_py, constants_rst): + n.codegen(host_fns_py, 'python', p_data / 'host-fns.json') + n.codegen(public_abi_py, 'python', p_data / 'public-abi.json') + n.codegen(constants_rst, 'rst', p_data / 'public-abi.json') + n.codegen(constants_pending_rst, 'rst', p_data / 'public-abi-pending.json') + for out in (host_fns_py, public_abi_py, constants_rst, constants_pending_rst): codegen_phony.add_dependency(out) - # Per-line generated files live inside each executor/runner source tree. - for v in versions: - exec_root = source_dir / v.exec_rel - tmpl = exec_root / 'executor' / 'codegen' / 'templates' - data = exec_root / 'executor' / 'codegen' / 'data' - consts_rs = exec_root / 'executor/crates/sdk-rs/src/abi/consts.rs' - py_std_abi = exec_root / 'runners/genlayer-py-std/src/genlayer/vm/public_abi.py' - host_fns_rs = exec_root / 'executor/crates/common/src/host_fns.rs' - - codegen(consts_rs, tmpl / 'rs.rb', data / 'public-abi.json') - codegen(py_std_abi, tmpl / 'py.rb', data / 'public-abi.json') - codegen(host_fns_rs, tmpl / 'rs.rb', data / 'host-fns.json') - for out in (consts_rs, py_std_abi, host_fns_rs): - codegen_phony.add_dependency(out) - - codegen_phony.finish() - cargo_cmd = [common.command_to_executable('cargo')] # CI mode forbids touching Cargo.lock during the build. @@ -501,24 +284,24 @@ def codegen(out: Path, template: Path, data: Path) -> None: 'cargo', command=[ 'cd', - RawStr('$wd'), - AND, - RawStr(CARGO_LD_LIBRARY_PATH), - RawStr('$env'), + ninja.RawStr('$wd'), + ninja.AND, + ninja.RawStr(ninja.CARGO_LD_LIBRARY_PATH), + ninja.RawStr('$env'), *cargo_cmd, - RawStr('$subcommand'), + ninja.RawStr('$subcommand'), '--target', rust_target, '--target-dir', - str(rust_target_dir), + ninja.RawStr('$target_dir'), *locked, - RawStr('$extra_args'), - AND, + ninja.RawStr('$extra_args'), + ninja.AND, 'cd', str(build_dir), - AND, + ninja.AND, 'touch', - VAR_OUT, + ninja.VAR_OUT, ], description='Running cargo $subcommand', pool='console', @@ -527,21 +310,21 @@ def codegen(out: Path, template: Path, data: Path) -> None: 'cargo_build', command=[ 'cd', - RawStr('$wd'), - AND, - RawStr(CARGO_LD_LIBRARY_PATH), - RawStr('$env'), + ninja.RawStr('$wd'), + ninja.AND, + ninja.RawStr(ninja.CARGO_LD_LIBRARY_PATH), + ninja.RawStr('$env'), *cargo_cmd, 'build', '--target', rust_target, '--target-dir', - str(rust_target_dir), + ninja.RawStr('$target_dir'), *locked, - RawStr('$extra_args'), + ninja.RawStr('$extra_args'), ], description='Running cargo $subcommand', - depfile=RawStr('$out.d'), + depfile=ninja.RawStr('$out.d'), pool='console', ) @@ -556,11 +339,11 @@ def codegen(out: Path, template: Path, data: Path) -> None: n.rule( 'nix_eval', command=[ - RawStr('WD=$$(pwd)'), - AND, + ninja.RawStr('WD=$$(pwd)'), + ninja.AND, 'cd', - RawStr('$wd'), - AND, + ninja.RawStr('$wd'), + ninja.AND, 'nix', 'eval', '--verbose', @@ -569,9 +352,9 @@ def codegen(out: Path, template: Path, data: Path) -> None: '--show-trace', '--json', '--expr', - RawStr('$expr'), - RawStr('>'), - RawStr('$$WD/$out'), + ninja.RawStr('$expr'), + ninja.RawStr('>'), + ninja.RawStr('$$WD/$out'), ], pool='console', ) @@ -582,52 +365,25 @@ def codegen(out: Path, template: Path, data: Path) -> None: all_bin = n.build('phony', 'all/bin') all_bin.add_dependency('out/bin/genvm-modules') - def install(frm: str, to: str) -> None: - install_dir = source_dir / frm - for f in _glob(install_dir, '**/*'): - if f.is_dir(): - continue - out = to + '/' + str(f.relative_to(install_dir)) - all_bin.add_dependency(out) - n.build('cp', out).add_dependency(f).finish() - - # Per-line executor build, runner data, and install tree, each landing under - # its own `out/executor/` directory. + # Per-line executor build, runner data, codegen, and install tree, each + # landing under its own `out/executor/` directory. Each line's + # `.genvm-tool.py:configure(line)` hook owns its registrations (the lines + # already diverge β€” e.g. frozen runner registry vs nix-derived manifests). for v in versions: - exec_root = source_dir / v.exec_rel - out_exec = f'out/executor/{v.real}' - - n.register_cargo(f'{v.exec_rel}/executor', build_to=f'{out_exec}/bin/genvm') - n.register_cargo(f'{v.exec_rel}/executor/crates/calldata') - n.register_cargo(f'{v.exec_rel}/executor/crates/calldata-derive') - n.register_cargo(f'{v.exec_rel}/executor/crates/common') - n.register_cargo(f'{v.exec_rel}/executor/crates/sdk-rs') - - # Manifests are derived by the umbrella machinery (which imports every - # active line's runners), so both the line's and the umbrella's nix - # inputs matter. - runners_nix_inputs = _glob(exec_root / 'runners', '**/*.nix') + _glob( - source_dir / 'runners', '**/*.nix' + line = ninja.LineContext( + n=n, + source_dir=source_dir, + key=v.key, + exec_rel=v.exec_rel, + real=v.real, + codegen_phony=codegen_phony, + data_phony=data_phony, + all_bin=all_bin, ) + configurator = _line_configurator(ctx, source_dir, v.exec_rel) + configurator(line) - latest = n.build('nix_eval', f'{out_exec}/data/latest.json') - latest.var('expr', _runner_manifest_expr(v.real, 'latest')) - latest.var('wd', source_dir) - latest.add_implicit_dependency(runners_nix_inputs) - latest.finish() - - all_json = n.build('nix_eval', f'{out_exec}/data/all.json') - all_json.var('expr', _runner_manifest_expr(v.real, 'all')) - all_json.var('wd', source_dir) - all_json.add_implicit_dependency(runners_nix_inputs) - all_json.finish() - - data_phony.add_dependency(f'{out_exec}/data/latest.json') - data_phony.add_dependency(f'{out_exec}/data/all.json') - - install(f'{v.exec_rel}/executor/install', out_exec) - all_bin.add_dependency(f'{out_exec}/bin/genvm') - + codegen_phony.finish() data_phony.finish() def _runner_file(f: Path) -> bool: @@ -638,7 +394,10 @@ def _runner_file(f: Path) -> bool: # ./runners owns the accumulate-and-build machinery (runners-all). Watch both. umbrella_runners = source_dir / 'runners' runner_inputs = [ - f for d in (runners, umbrella_runners) for f in _glob(d, '**/*') if _runner_file(f) + f + for d in (runners, umbrella_runners) + for f in ninja.glob(d, '**/*') + if _runner_file(f) ] runners_build = n.build('CUSTOM_COMMAND', 'target/runners') @@ -651,22 +410,44 @@ def _runner_file(f: Path) -> bool: '-v', '-L', '-o', - BUILD_DIR_REL / 'runners-nix', + ninja.BUILD_DIR_REL / 'runners-nix', f'git+file:{source_dir}?submodules=1#runners-all', - AND, + ninja.AND, 'mkdir', '-p', './out/runners', - AND, + ninja.AND, 'cp', '-r', './runners-nix/.', './out/runners/.', - AND, + ninja.AND, 'chmod', '-R', '+w', './out/runners/.', + # Legacy lines (v0.2.x) keep their runners under their own executor + # root (out/executor//legacy-runners); the nix output is + # already laid out at that relative path, so overlay it onto ./out. + ninja.AND, + 'nix', + 'build', + '--keep-going', + '-v', + '-L', + '-o', + ninja.BUILD_DIR_REL / 'legacy-runners-nix', + f'git+file:{source_dir}?submodules=1#legacy-runners-all', + ninja.AND, + 'cp', + '-r', + './legacy-runners-nix/.', + './out/.', + ninja.AND, + 'chmod', + '-R', + '+w', + './out/executor', ], ) runners_build.add_dependency(source_dir / 'flake.nix') @@ -678,7 +459,7 @@ def _runner_file(f: Path) -> bool: n.build('phony', 'cargo/clippy').add_dependency(n.all_clippy).finish() n.build('phony', 'cargo/clippy/fix').add_dependency(n.all_clippy_fix).finish() - install('install', 'out') + n.install('install', 'out', all_bin) all_bin.finish() all_phony = n.build('phony', 'all') diff --git a/support/tools/genvm-tool/genvm_tool/codegen/__init__.py b/support/tools/genvm-tool/genvm_tool/codegen/__init__.py new file mode 100644 index 00000000..05aac43d --- /dev/null +++ b/support/tools/genvm-tool/genvm_tool/codegen/__init__.py @@ -0,0 +1,39 @@ +"""`genvm-tool codegen` language backends. + +A single front-end (:mod:`genvm_tool.codegen.model`) parses the codegen data JSON +into a typed model + shared string-trie; each language module renders that model. +Adding a language is just a new ``render(defs, **opts)`` registered in +:data:`BACKENDS`. +""" + +from __future__ import annotations + +from pathlib import Path + +from . import go, python, rst, rust +from .model import Definition, load + +BACKENDS = { + 'rust': rust.render, + 'python': python.render, + 'rst': rst.render, + 'go': go.render, +} + +LANGS = tuple(BACKENDS) + + +def render(lang: str, defs: list[Definition], **opts) -> str: + try: + backend = BACKENDS[lang] + except KeyError: + raise ValueError(f'unknown codegen language {lang!r}') from None + return backend(defs, **opts) + + +def generate(lang: str, input_path: Path, output_path: Path, **opts) -> None: + """Parse ``input_path``, render with ``lang``, write to ``output_path``.""" + Path(output_path).write_text(render(lang, load(input_path), **opts)) + + +__all__ = ['BACKENDS', 'LANGS', 'generate', 'load', 'render'] diff --git a/support/tools/genvm-tool/genvm_tool/codegen/go.py b/support/tools/genvm-tool/genvm_tool/codegen/go.py new file mode 100644 index 00000000..818e5a8d --- /dev/null +++ b/support/tools/genvm-tool/genvm_tool/codegen/go.py @@ -0,0 +1,105 @@ +"""Go backend for ``genvm-tool codegen``. + +Ports the standalone go generator (trie builders emit flat ``func``s rather than +structs). The package name is configurable via ``--go-package`` since the +generated file lands in an external Go module. +""" + +from __future__ import annotations + +import json + +from .model import Const, Consts, Definition, Enum, StrTrie, TrieNode, to_camel + +_GO_REPR = { + 'str': 'string', + 'u8': 'uint8', + 'u16': 'uint16', + 'u32': 'uint32', + 'i32': 'int32', +} + + +def _go_repr(s: str) -> str: + return _GO_REPR.get(s, s) + + +def _dump(v) -> str: + if isinstance(v, str): + return json.dumps(v) + return str(v) + + +def _has_param(node: TrieNode) -> bool: + for _head, child in node.methods: + if child.param is not None or _has_param(child): + return True + return False + + +def _trie_inner(node: TrieNode, root_camel: str, buf: list[str]) -> None: + # Walk children in source order: unlike the struct-based backends, go emits a + # flat function per terminal/param exactly where it appears in the JSON. + prefix = root_camel + node.suffix + for kind, head, payload in node.order: + if kind == 'leaf': + val = ' '.join(payload) + buf.append( + f'func {prefix}{to_camel(head)}() {root_camel} {{ return {_dump(val)} }}\n' + ) + continue + child = payload + if child.param is not None: + param_type, parts = child.param + fmt_str = ' '.join(parts) + ' %v' + buf.append( + f'func {root_camel}{child.suffix}{to_camel(param_type)}' + f'(v {_go_repr(param_type)}) {root_camel} {{ ' + f'return {root_camel}(fmt.Sprintf({_dump(fmt_str)}, v)) }}\n' + ) + else: + if child.terminal: + val = ' '.join(child.parts) + buf.append( + f'func {root_camel}{child.suffix}() {root_camel} {{ return {_dump(val)} }}\n' + ) + _trie_inner(child, root_camel, buf) + + +def render(defs: list[Definition], *, go_package: str = 'genvm', **_opts) -> str: + buf: list[str] = [] + buf.append('// Code generated from genvm data. DO NOT EDIT.\n') + buf.append('\n') + buf.append(f'package {go_package}\n') + + needs_fmt = any(isinstance(d, StrTrie) and _has_param(d.root) for d in defs) + if needs_fmt: + buf.append('\nimport "fmt"\n') + + for d in defs: + if isinstance(d, Enum): + name = to_camel(d.name) + buf.append('\n') + buf.append(f'type {name} {_go_repr(d.repr)}\n') + buf.append('\n') + buf.append('const (\n') + for k, v in d.values.items(): + buf.append(f'\t{name}{to_camel(k)} {name} = {_dump(v)}\n') + buf.append(')\n') + elif isinstance(d, Const): + buf.append(f'const {to_camel(d.name)} = {_dump(d.value)}\n') + elif isinstance(d, Consts): + buf.append('\nconst (\n') + for k, v in d.values.items(): + buf.append( + f'\t{to_camel(d.name)}{to_camel(k)} {_go_repr(d.repr)} = {_dump(v)}\n' + ) + buf.append(')\n') + elif isinstance(d, StrTrie): + root_camel = to_camel(d.name) + buf.append(f'\ntype {root_camel} string\n\n') + _trie_inner(d.root, root_camel, buf) + return ''.join(buf) + + +__all__ = ['render'] diff --git a/support/tools/genvm-tool/genvm_tool/codegen/model.py b/support/tools/genvm-tool/genvm_tool/codegen/model.py new file mode 100644 index 00000000..2286f3e6 --- /dev/null +++ b/support/tools/genvm-tool/genvm_tool/codegen/model.py @@ -0,0 +1,187 @@ +"""Shared front-end for the ``genvm-tool codegen`` language backends. + +Parses the codegen data JSON (a list of typed definitions) into a small typed +model and provides the shared string-trie machinery every backend walks. The +backends (``rust``, ``python``, ``rst``, ``go``) differ only in how they render +this model; all parsing, trie unfolding, and node classification live here, so a +new language is just a renderer. +""" + +from __future__ import annotations + +import dataclasses +import json +from pathlib import Path + + +def to_camel(s: str) -> str: + """``a_b_c`` -> ``ABC`` (each underscore-separated part title-cased).""" + return ''.join( + '' if len(part) == 0 else part[0].upper() + part[1:].lower() + for part in s.split('_') + ) + + +@dataclasses.dataclass +class Enum: + name: str + repr: str + values: dict[str, object] + + +@dataclasses.dataclass +class Const: + name: str + repr: str + value: object + + +@dataclasses.dataclass +class Consts: + name: str + repr: str + values: dict[str, object] + + +@dataclasses.dataclass +class TrieNode: + """One node of an unfolded string trie. + + ``suffix`` is the CamelCase concatenation of the path heads from the root to + this node (``''`` at the root); backends use it to name the generated + struct/function. ``parts`` is the space-joined path the node maps to. A node is + either a *param* node (``param`` set: consumes one typed argument) or a regular + node carrying ``leaves`` (terminal strings) and ``methods`` (child nodes), + mirroring the original ruby generators. + """ + + suffix: str + parts: list[str] + terminal: bool + param: tuple[str, list[str]] | None + leaves: list[tuple[str, list[str]]] + methods: list[tuple[str, 'TrieNode']] + # Children in original JSON order, tagged ``('leaf', head, parts)`` or + # ``('method', head, child)``. ``leaves``/``methods`` are the grouped views the + # struct-based backends (rust, python) want; the go backend, which emits flat + # functions in source order, walks this instead. + order: list[tuple] + + +@dataclasses.dataclass +class StrTrie: + name: str + root: TrieNode + entries: list # unfolded entries, kept for flat path enumeration (rst) + + +Definition = Enum | Const | Consts | StrTrie + + +def _unfold(entry): + """Normalize a trie entry to ``{'head', 'tail'}`` form (recursively). + + A bare string ``"x"`` becomes ``{'head': 'x', 'tail': []}``; a dict whose + ``tail`` is a list has each child unfolded; everything else (``$param`` tails) + is left as-is. + """ + if entry is None: + return None + if isinstance(entry, str): + return {'head': entry, 'tail': []} + if isinstance(entry, dict): + tail = entry.get('tail') + if isinstance(tail, list): + return {'head': entry['head'], 'tail': [_unfold(e) for e in tail]} + return entry + # Anything else (notably a bare `[]` placeholder) is a null/terminal marker, + # exactly like an explicit `null` β€” matches ruby's `unfold_trie_entry`. + return None + + +def _build(entries, prefix_parts: list[str], suffix: str, terminal_parts) -> TrieNode: + node = TrieNode( + suffix=suffix, + parts=terminal_parts if terminal_parts is not None else [], + terminal=terminal_parts is not None, + param=None, + leaves=[], + methods=[], + order=[], + ) + for entry in entries: + if entry is None: + continue + head = entry['head'] + tail = entry['tail'] + current = prefix_parts + [head] + child_suffix = suffix + to_camel(head) + if isinstance(tail, str) and tail.startswith('$'): + param_type = tail[1:] + child = TrieNode( + suffix=child_suffix, + parts=current, + terminal=False, + param=(param_type, current), + leaves=[], + methods=[], + order=[], + ) + node.methods.append((head, child)) + node.order.append(('method', head, child)) + elif isinstance(tail, list): + non_null = [e for e in tail if e is not None] + # A trie level is terminal if any sibling was a `null` placeholder (or it + # has no real children): the path itself is a valid leaf string. + is_terminal = (len(tail) != len(non_null)) or len(non_null) == 0 + if len(non_null) == 0: + node.leaves.append((head, current)) + node.order.append(('leaf', head, current)) + else: + child_terminal = current if is_terminal else None + child = _build(non_null, current, child_suffix, child_terminal) + node.methods.append((head, child)) + node.order.append(('method', head, child)) + return node + + +def enumerate_paths(entries, prefix: str = ''): + """Flatten a trie to ``(path, param_type | None)`` pairs (used by the rst docs).""" + result = [] + for entry in entries: + if entry is None: + continue + head = entry['head'] + tail = entry['tail'] + current = head if prefix == '' else f'{prefix} {head}' + if isinstance(tail, str) and tail.startswith('$'): + result.append((current, tail[1:])) + elif isinstance(tail, list): + non_null = [e for e in tail if e is not None] + is_terminal = (len(tail) != len(non_null)) or len(non_null) == 0 + if is_terminal: + result.append((current, None)) + result.extend(enumerate_paths(non_null, current)) + return result + + +def parse(data) -> list[Definition]: + defs: list[Definition] = [] + for t in data: + kind = t['type'] + if kind == 'enum': + defs.append(Enum(t['name'], t['repr'], t['values'])) + elif kind == 'const': + defs.append(Const(t['name'], t['repr'], t['value'])) + elif kind == 'consts': + defs.append(Consts(t['name'], t['repr'], t['values'])) + elif kind == 'str_trie': + entries = [_unfold(e) for e in t['values']] + defs.append(StrTrie(t['name'], _build(entries, [], '', None), entries)) + else: + raise ValueError(f'unknown codegen type {kind!r}') + return defs + + +def load(path: Path) -> list[Definition]: + return parse(json.loads(Path(path).read_text())) diff --git a/support/tools/genvm-tool/genvm_tool/codegen/python.py b/support/tools/genvm-tool/genvm_tool/codegen/python.py new file mode 100644 index 00000000..6c07c3a5 --- /dev/null +++ b/support/tools/genvm-tool/genvm_tool/codegen/python.py @@ -0,0 +1,120 @@ +"""Python backend for ``genvm-tool codegen`` (ports the old ``py.rb`` template). + +Output is byte-for-byte identical to the previous ruby generator (tab-indented, +single-quoted), so regeneration produces no diff. +""" + +from __future__ import annotations + +import re + +from .model import Const, Consts, Definition, Enum, StrTrie, TrieNode, to_camel + +_INT_REPR = re.compile(r'^(u|i)\d+$') + + +def _pydump(v) -> str: + if isinstance(v, str): + return f"'{v}'" + return str(v) + + +def _py_repr(s: str) -> str: + return 'int' if _INT_REPR.match(s) else s + + +def _emit_struct(node: TrieNode, root_name: str, buf: list[str]) -> None: + for _head, child in node.methods: + if child.param is None: + _emit_struct(child, root_name, buf) + buf.append(f'class _{root_name}{node.suffix}:\n') + if node.terminal: + val = ' '.join(node.parts) + buf.append('\t@staticmethod\n') + buf.append(f"\tdef val() -> '{root_name}':\n") + buf.append(f"\t\treturn {root_name}('{val}')\n") + for name, parts in node.leaves: + val = ' '.join(parts) + buf.append('\t@staticmethod\n') + buf.append(f"\tdef {name.lower()}() -> '{root_name}':\n") + buf.append(f"\t\treturn {root_name}('{val}')\n") + for head, child in node.methods: + buf.append('\t@staticmethod\n') + buf.append(f"\tdef {head.lower()}() -> '_{root_name}{child.suffix}':\n") + buf.append(f'\t\treturn _{root_name}{child.suffix}()\n') + buf.append('\n') + for _head, child in node.methods: + if child.param is not None: + _emit_param(child, root_name, buf) + + +def _emit_param(node: TrieNode, root_name: str, buf: list[str]) -> None: + param_type, parts = node.param + fmt = ' '.join(parts) + buf.append(f'class _{root_name}{node.suffix}:\n') + buf.append('\t@staticmethod\n') + buf.append(f"\tdef val_{param_type}(v: {_py_repr(param_type)}) -> '{root_name}':\n") + buf.append(f"\t\treturn {root_name}(f'{fmt} {{v}}')\n") + buf.append('\n') + + +def _trie(t: StrTrie, buf: list[str]) -> None: + root_name = to_camel(t.name) + root = t.root + for _head, child in root.methods: + if child.param is None: + _emit_struct(child, root_name, buf) + for _head, child in root.methods: + if child.param is not None: + _emit_param(child, root_name, buf) + + buf.append(f'class {root_name}:\n') + buf.append("\t__slots__ = ('value',)\n") + buf.append('\tdef __init__(self, value: str):\n') + buf.append('\t\tself.value = value\n') + buf.append('\tdef __str__(self) -> str:\n') + buf.append('\t\treturn self.value\n') + for name, parts in root.leaves: + val = ' '.join(parts) + buf.append('\t@staticmethod\n') + buf.append(f"\tdef {name.lower()}() -> '{root_name}':\n") + buf.append(f"\t\treturn {root_name}('{val}')\n") + for head, child in root.methods: + buf.append('\t@staticmethod\n') + buf.append(f"\tdef {head.lower()}() -> '_{root_name}{child.suffix}':\n") + buf.append(f'\t\treturn _{root_name}{child.suffix}()\n') + buf.append('\n') + + +def render(defs: list[Definition], **_opts) -> str: + buf: list[str] = [] + buf.append('# This file is auto-generated. Do not edit!\n') + buf.append('\n') + buf.append('# fmt: off\n') + buf.append('# ruff: noqa\n') + buf.append('\n') + buf.append('import typing\n') + buf.append('from enum import IntEnum, StrEnum\n') + for d in defs: + if isinstance(d, Enum): + base = 'StrEnum' if d.repr == 'str' else 'IntEnum' + buf.append('\n\n') + buf.append(f'class {to_camel(d.name)}({base}):\n') + for k, v in d.values.items(): + buf.append(f'\t{k.upper()} = {_pydump(v)}\n') + elif isinstance(d, Const): + buf.append( + f'\n\n{d.name.upper()}: typing.Final[{_py_repr(d.repr)}] = {_pydump(d.value)}\n' + ) + elif isinstance(d, Consts): + buf.append(f'\n\nclass _{to_camel(d.name)}(typing.NamedTuple):\n') + for k, v in d.values.items(): + buf.append(f'\t{k.upper()}: {_py_repr(d.repr)} = {_pydump(v)}\n') + buf.append(f'\n{d.name}: typing.Final = _{to_camel(d.name)}()\n') + elif isinstance(d, StrTrie): + buf.append('\n') + _trie(d, buf) + return ''.join(buf) + + +__all__ = ['render'] diff --git a/support/tools/genvm-tool/genvm_tool/codegen/rst.py b/support/tools/genvm-tool/genvm_tool/codegen/rst.py new file mode 100644 index 00000000..07b463c7 --- /dev/null +++ b/support/tools/genvm-tool/genvm_tool/codegen/rst.py @@ -0,0 +1,64 @@ +"""reStructuredText backend for ``genvm-tool codegen`` (ports ``rst.rb``). + +Generates the spec appendix listing every constant/enum/trie path with stable +``.. _gvm-def-*`` cross-reference anchors. +""" + +from __future__ import annotations + +from . import model +from .model import Const, Consts, Definition, Enum, StrTrie + + +def _dash(s: str) -> str: + return s.replace('_', '-') + + +def render(defs: list[Definition], **_opts) -> str: + buf: list[str] = [] + buf.append('Constants\n') + buf.append('=========\n') + buf.append('\n') + for d in defs: + if isinstance(d, Enum): + buf.append(f'.. _gvm-def-enum-{_dash(d.name)}:\n\n') + buf.append(f'{d.name}\n') + buf.append('-' * len(d.name) + '\n\n') + buf.append(f'Type: {d.repr}\n\n') + for k, v in d.values.items(): + buf.append(f'.. _gvm-def-enum-value-{_dash(d.name)}-{_dash(k)}:\n\n') + buf.append(f'{k}\n') + buf.append('~' * len(k) + '\n\n') + buf.append(f'Value: ``{v}``\n\n') + elif isinstance(d, Const): + buf.append(f'.. _gvm-def-const-{_dash(d.name)}:\n\n') + buf.append(f'{d.name}\n') + buf.append('-' * len(d.name) + '\n\n') + buf.append(f'Type: {d.repr}\n\n') + buf.append(f'Value: ``{d.value}``\n\n') + elif isinstance(d, Consts): + buf.append(f'.. _gvm-def-consts-{_dash(d.name)}:\n\n') + buf.append(f'{d.name}\n') + buf.append('-' * len(d.name) + '\n\n') + buf.append(f'Type: {d.repr}\n\n') + for k, v in d.values.items(): + buf.append(f'.. _gvm-def-consts-value-{_dash(d.name)}-{_dash(k)}:\n\n') + buf.append(f'{k}\n') + buf.append('~' * len(k) + '\n\n') + buf.append(f'Value: ``{v}``\n\n') + elif isinstance(d, StrTrie): + buf.append(f'.. _gvm-def-str-trie-{_dash(d.name)}:\n\n') + buf.append(f'{d.name}\n') + buf.append('-' * len(d.name) + '\n\n') + buf.append('Type: str_trie\n\n') + for path, param in model.enumerate_paths(d.entries): + rst_name = path.replace('_', '-').replace(' ', '-') + buf.append(f'.. _gvm-def-str-trie-value-{_dash(d.name)}-{rst_name}:\n\n') + buf.append(f'``{path}``\n') + buf.append('~' * (len(path) + 4) + '\n\n') + if param is not None: + buf.append(f'Param: {param}\n\n') + return ''.join(buf) + + +__all__ = ['render'] diff --git a/support/tools/genvm-tool/genvm_tool/codegen/rust.py b/support/tools/genvm-tool/genvm_tool/codegen/rust.py new file mode 100644 index 00000000..0d056d90 --- /dev/null +++ b/support/tools/genvm-tool/genvm_tool/codegen/rust.py @@ -0,0 +1,217 @@ +"""Rust backend for ``genvm-tool codegen`` (ports the old ``rs.rb`` template). + +Output is byte-for-byte identical to the previous ruby generator: the build +regenerates these files in-tree and a non-empty diff would surface as drift. +""" + +from __future__ import annotations + +import json + +from .model import Const, Consts, Definition, Enum, StrTrie, TrieNode, to_camel + + +def _dump(v) -> str: + if isinstance(v, str): + return json.dumps(v) + return str(v) + + +def _rust_repr(s: str) -> str: + return "&'static str" if s == 'str' else s + + +def _rust_repr_from(s: str) -> str: + return '&str' if s == 'str' else s + + +def _rust_param_type(s: str) -> str: + return '&str' if s == 'str' else s + + +def _enum(e: Enum) -> str: + buf: list[str] = [] + name = to_camel(e.name) + buf.append('#[derive(\n') + for d in ( + 'Debug', + 'PartialEq', + 'Clone', + 'Copy', + 'Serialize', + 'Deserialize', + '::genlayer_calldata::Encode', + '::genlayer_calldata::Decode', + ): + buf.append(f' {d},\n') + buf.append(')]\n') + if e.repr != 'str': + buf.append(f'#[repr({e.repr})]\n') + buf.append(f'pub enum {name} {{\n') + for k, v in e.values.items(): + buf.append(f' {to_camel(k)} = {_dump(v)},\n') + buf.append('}\n') + else: + buf.append(f'pub enum {name} {{\n') + for k in e.values: + buf.append(f' {to_camel(k)},\n') + buf.append('}\n') + buf.append('\n') + buf.append(f'impl {name} {{\n') + vals = list(e.values.values()) + if ( + e.repr != 'str' + and all(isinstance(v, int) for v in vals) + and sorted(vals) == list(range(len(vals))) + ): + buf.append(f' pub const SIZE: usize = {len(vals)};\n') + buf.append(f' pub fn value(self) -> {_rust_repr(e.repr)} {{\n') + buf.append(' match self {\n') + for k, v in e.values.items(): + buf.append(f' {name}::{to_camel(k)} => {_dump(v)},\n') + buf.append(' }\n') + buf.append(' }\n') + buf.append(" pub fn str_snake_case(self) -> &'static str {\n") + buf.append(' match self {\n') + for k in e.values: + buf.append(f' {name}::{to_camel(k)} => "{k}",\n') + buf.append(' }\n') + buf.append(' }\n') + buf.append('}\n') + buf.append('\n') + buf.append(f'impl TryFrom<{_rust_repr_from(e.repr)}> for {name} {{\n') + buf.append(' type Error = ();\n') + buf.append('\n') + buf.append( + f' fn try_from(value: {_rust_repr_from(e.repr)}) -> Result {{\n' + ) + buf.append(' match value {\n') + for k, v in e.values.items(): + buf.append(f' {_dump(v)} => Ok({name}::{to_camel(k)}),\n') + buf.append(' _ => Err(()),\n') + buf.append(' }\n') + buf.append(' }\n') + buf.append('}\n') + return ''.join(buf) + + +def _emit_struct(node: TrieNode, root_name: str, buf: list[str]) -> None: + """Emit one non-root trie struct: children first, then this struct, then its + param children (matching the original ruby buffer order).""" + for _head, child in node.methods: + if child.param is None: + _emit_struct(child, root_name, buf) + buf.append(f'pub struct {node.suffix};\n\n') + buf.append(f'impl {node.suffix} {{\n') + if node.terminal: + val = ' '.join(node.parts) + buf.append( + f' pub const fn val(&self) -> {root_name} {{ ' + f'{root_name}(Cow::Borrowed("{val}")) }}\n' + ) + for name, parts in node.leaves: + val = ' '.join(parts) + buf.append( + f' pub const fn {name.lower()}(&self) -> {root_name} {{ ' + f'{root_name}(Cow::Borrowed("{val}")) }}\n' + ) + for head, child in node.methods: + buf.append( + f' pub const fn {head.lower()}(&self) -> {child.suffix} {{ {child.suffix} }}\n' + ) + buf.append('}\n\n') + for _head, child in node.methods: + if child.param is not None: + _emit_param(child, root_name, buf) + + +def _emit_param(node: TrieNode, root_name: str, buf: list[str]) -> None: + param_type, parts = node.param + fmt_str = ' '.join(parts) + ' {v}' + buf.append(f'pub struct {node.suffix};\n\n') + buf.append(f'impl {node.suffix} {{\n') + buf.append( + f' pub fn val_{param_type}(&self, v: {_rust_param_type(param_type)}) ' + f'-> {root_name} {{\n' + ) + buf.append(f' {root_name}(Cow::Owned(format!("{fmt_str}")))\n') + buf.append(' }\n') + buf.append('}\n\n') + + +def _trie(t: StrTrie, buf: list[str]) -> None: + root_name = to_camel(t.name) + root = t.root + mod: list[str] = [] + for _head, child in root.methods: + if child.param is None: + _emit_struct(child, root_name, mod) + for _head, child in root.methods: + if child.param is not None: + _emit_param(child, root_name, mod) + mod_buf = ''.join(mod) + + mod_name = f'__{root_name}' + if mod_buf != '': + buf.append('#[allow(non_snake_case)]\n') + buf.append('#[rustfmt::skip]\n') + buf.append(f'pub mod {mod_name} {{\n') + buf.append(' use std::borrow::Cow;\n') + buf.append(f' use super::{root_name};\n\n') + for line in mod_buf.splitlines(keepends=True): + buf.append('\n' if line.strip() == '' else f' {line}') + buf.append('}\n\n') + + buf.append('#[derive(Debug, Clone, PartialEq, Eq, Serialize)]\n') + buf.append(f"pub struct {root_name}(pub Cow<'static, str>);\n\n") + buf.append(f'impl From<{root_name}> for String {{\n') + buf.append(f' fn from(val: {root_name}) -> String {{\n') + buf.append(' val.0.into()\n') + buf.append(' }\n') + buf.append('}\n') + buf.append('#[rustfmt::skip]\n') + buf.append(f'impl {root_name} {{\n') + for name, parts in root.leaves: + val = ' '.join(parts) + buf.append( + f' pub const fn {name.lower()}() -> Self {{ Self(Cow::Borrowed("{val}")) }}\n' + ) + for head, child in root.methods: + buf.append( + f' pub const fn {head.lower()}() -> {mod_name}::{child.suffix} {{ ' + f'{mod_name}::{child.suffix} }}\n' + ) + buf.append('}\n\n') + + +def render(defs: list[Definition], **_opts) -> str: + has_str_trie = any(isinstance(d, StrTrie) for d in defs) + # Only enums and str-tries derive serde; a const-only data file (e.g. + # public-abi-pending.json) must not emit an unused `use serde` import. + has_serde = any(isinstance(d, (Enum, StrTrie)) for d in defs) + buf: list[str] = [] + buf.append('// This file is auto-generated. Do not edit!\n\n') + buf.append('#![allow(dead_code, clippy::redundant_static_lifetimes)]\n') + buf.append('\n') + if has_serde: + buf.append('use serde::{Deserialize, Serialize};\n\n') + if has_str_trie: + buf.append('use std::borrow::Cow;\n\n') + for d in defs: + if isinstance(d, Enum): + buf.append(_enum(d)) + elif isinstance(d, Const): + buf.append( + f'pub const {d.name.upper()}: {_rust_repr(d.repr)} = {_dump(d.value)};\n' + ) + elif isinstance(d, Consts): + buf.append(f'pub mod {d.name} {{\n') + for k, v in d.values.items(): + buf.append(f' pub const {k.upper()}: {_rust_repr(d.repr)} = {_dump(v)};\n') + buf.append('}\n\n') + elif isinstance(d, StrTrie): + _trie(d, buf) + return ''.join(buf) + + +__all__ = ['render'] diff --git a/support/tools/genvm-tool/genvm_tool/common.py b/support/tools/genvm-tool/genvm_tool/common.py index 237fb555..f9f8bfc8 100644 --- a/support/tools/genvm-tool/genvm_tool/common.py +++ b/support/tools/genvm-tool/genvm_tool/common.py @@ -9,6 +9,7 @@ import functools import json import os +import re import subprocess import types from dataclasses import dataclass @@ -45,6 +46,7 @@ class Context: root: Path logger: formatter.Formatter printer: formatter.Sink + python_command: list[str] # The manager's loaded `.genvm-tool.py` module (or None if absent). Populated # once in `__main__` before dispatch; subcommands query its exported funcs. project: types.ModuleType | None = None @@ -103,6 +105,11 @@ def active_versions(root: Path) -> list[str]: return sorted(vers, key=_ver_key) +# Release/branch-model branches (`main`, a version branch `v0.3.x`, or a dev +# branch `v0.3-dev`) β€” already line-scoped, so they are never feature-namespaced. +_MODEL_BRANCH = re.compile(r'main|v[\d.]+\.x|v[\d.]+-dev') + + class Repo: """One git repo in the umbrella: the manager or an executor submodule.""" @@ -127,6 +134,30 @@ def prefix(self, rel_path: str) -> str: """Make a repo-relative path manager-root-relative.""" return f'{self.rel}/{rel_path}' if self.rel else rel_path + @property + def line(self) -> str | None: + """The version-line tag (e.g. `v0.3`) for an executor submodule, or None + for the manager.""" + m = re.fullmatch(r'executors/(v\d+\.\d+)\.x', self.name) + return m.group(1) if m else None + + def feature_branch(self, name: str) -> str: + """Physical branch name for the logical feature branch `name` in this repo. + + The manager is its own repo, so it carries `name` verbatim. Every executor + submodule, however, shares ONE remote (`genvm-executor`): a bare `name` + would collide between two active lines that branch off it at once. So an + executor feature branch is namespaced under `pr//` (e.g. + `pr/v0.3/`) β€” the `` segment is what keeps the two lines + distinct, matching how the release branches (`v0.3-dev`, `v0.3.x`) are + already line-scoped and thus never clash. Release-model branches and an + already-prefixed `name` pass through unchanged (idempotent). + """ + line = self.line + if line is None or name.startswith(f'pr/{line}/') or _MODEL_BRANCH.fullmatch(name): + return name + return f'pr/{line}/{name}' + def discover_repos(root: Path, which: str = 'all') -> list[Repo]: repos = [Repo('manager', root, '')] @@ -149,6 +180,32 @@ def executor_rels(root: Path) -> set[str]: # --- git ------------------------------------------------------------------- +# git scopes itself to one repo through these variables. A pre-commit / commit-msg +# hook runs with them set to the *parent* repo (GIT_INDEX_FILE points at the +# in-progress commit's index, GIT_DIR at the parent .git, ...). Because we operate +# on each repo β€” and its submodules β€” by explicit path (`git -C `), those +# inherited values must be dropped: otherwise a submodule `git` reads the parent's +# index/objects and dies (e.g. `git diff --cached` -> exit 128, "unable to read +# "). See `strip_inherited_git_env`. +_INHERITED_GIT_ENV_VARS = ( + 'GIT_DIR', + 'GIT_WORK_TREE', + 'GIT_INDEX_FILE', + 'GIT_PREFIX', + 'GIT_COMMON_DIR', + 'GIT_OBJECT_DIRECTORY', + 'GIT_ALTERNATE_OBJECT_DIRECTORIES', + 'GIT_NAMESPACE', +) + + +def strip_inherited_git_env() -> None: + """Drop the repo-scoping GIT_* variables inherited from a parent git hook, so + our per-repo `git -C ` calls discover each repo from its own path + instead of being pinned to the invoking repo.""" + for var in _INHERITED_GIT_ENV_VARS: + os.environ.pop(var, None) + def _git_z(repo: Repo, *args: str) -> list[str]: out = subprocess.run( diff --git a/support/tools/genvm-tool/genvm_tool/git/__init__.py b/support/tools/genvm-tool/genvm_tool/git/__init__.py index fa1f20da..ff6c3dda 100644 --- a/support/tools/genvm-tool/genvm_tool/git/__init__.py +++ b/support/tools/genvm-tool/genvm_tool/git/__init__.py @@ -1,7 +1,7 @@ """`git` command group: git helpers spanning the manager + executor submodules.""" -from . import list_repo, ls +from . import check_for_push, create_branches, list_repo, ls NAME = 'git' HELP = 'git helpers across the manager and executor submodules' -COMMANDS = [ls, list_repo] +COMMANDS = [ls, list_repo, create_branches, check_for_push] diff --git a/support/tools/genvm-tool/genvm_tool/git/check_for_push.py b/support/tools/genvm-tool/genvm_tool/git/check_for_push.py new file mode 100644 index 00000000..4ebcbc21 --- /dev/null +++ b/support/tools/genvm-tool/genvm_tool/git/check_for_push.py @@ -0,0 +1,327 @@ +"""`genvm-tool git check-for-push` β€” pre-push status across the sub-repos. + +Read-only survey of the manager and every executor submodule. For each repo it +reports, folds into a per-repo `ready` verdict, and suggests the next action: + + - dirty β€” uncommitted work in the tree + - on_branch β€” on a real branch, not a detached HEAD + - committed_in_manager β€” (submodule) the submodule's HEAD is the gitlink the + manager has committed, i.e. the branch advance is + recorded in the superproject + - in_sync_with_origin β€” the branch tip vs its branch on `origin` + - action β€” the next step in the add β†’ commit β†’ push pipeline + +By default the origin comparison is a LIVE `git ls-remote` query. The queries are +launched up front β€” one per distinct origin URL (the executor submodules share a +remote, so they batch into a single multi-ref call) and all in parallel β€” before +any local work or output, so their network latency overlaps everything else. + +`ls-remote` only yields the remote branch's existence + hash, so when the remote +holds commits we don't have locally the exact ahead/behind cannot be counted +without a fetch β€” surfaced as `fetch (cannot compare with origin)`. Pass +`--offline` to compare against the local `origin/` tracking ref (your +last fetch) with no network. + +`same_branch_as_manager` is shown for submodules as context (the line-namespaced +mirror convention: `feat/x` on the manager maps to `pr/v0.3/feat/x` in the +executor β€” see `common.Repo.feature_branch`) but does not gate `ready`. Exit +status is non-zero when any repo is not ready, so the command can gate a push. + +This NEVER fetches, resets, switches, or writes anything β€” `ls-remote` reads the +remote without updating any local ref. +""" + +import os +import subprocess + +from .. import common + +NAME = 'check-for-push' +HELP = 'read-only pre-push status + suggested action across manager and submodules' + +REMOTE = 'origin' +# `ls-remote` is a network round-trip; cap it so an unreachable remote can't hang +# the survey, and never let git pop an interactive auth prompt. +_LS_REMOTE_TIMEOUT = 20 +_NET_ENV = {'GIT_TERMINAL_PROMPT': '0'} + + +def configure(parser): + parser.add_argument( + '--repo', + default='all', + help="'all' (default), 'manager', or an executor name like executors/v0.3.x", + ) + parser.add_argument( + '--offline', + action='store_true', + help='compare against the local origin/ tracking ref (last fetch) ' + 'instead of querying the remote live', + ) + + +def _git(repo: common.Repo, *args: str) -> subprocess.CompletedProcess: + return subprocess.run( + ['git', '-C', str(repo.path), *args], + capture_output=True, + text=True, + ) + + +def _current_branch(repo: common.Repo) -> str: + """The repo's checked-out branch, or '' when detached.""" + return _git(repo, 'branch', '--show-current').stdout.strip() + + +def _origin_url(repo: common.Repo) -> str: + r = _git(repo, 'config', '--get', f'remote.{REMOTE}.url') + return r.stdout.strip() if r.returncode == 0 else '' + + +def _worktree_state( + repo: common.Repo, ignore_submodules: bool +) -> tuple[int, bool, bool]: + """Porcelain summary: (changed-line count, has-staged, has-unstaged/untracked). + + The manager passes ``--ignore-submodules=dirty`` so a submodule's own dirty + tree is left to that submodule's row, while a *moved gitlink* (new commits) + still counts β€” that genuinely needs an add + commit in the manager. + """ + args = ['status', '--porcelain'] + if ignore_submodules: + args.append('--ignore-submodules=dirty') + lines = [ln for ln in _git(repo, *args).stdout.splitlines() if len(ln) >= 2] + # Porcelain columns: X = index (staged), Y = worktree. '??' is untracked. + staged = any(ln[0] not in ' ?' for ln in lines) + unstaged = any(ln[1] != ' ' for ln in lines) + return len(lines), staged, unstaged + + +def _committed_in_manager( + manager: common.Repo, rel: str, sub: common.Repo +) -> tuple[str, bool]: + """Whether the submodule's HEAD is the gitlink committed in the manager.""" + recorded = _git(manager, 'rev-parse', f'HEAD:{rel}') + if recorded.returncode != 0: + return 'no (not committed in manager HEAD)', False + rec = recorded.stdout.strip() + head = _git(sub, 'rev-parse', 'HEAD').stdout.strip() + if rec == head: + return 'yes', True + return f'no (manager has {rec[:10]}, submodule at {head[:10]})', False + + +def _launch_origin_queries(targets: list[tuple[common.Repo, str]]): + """Start one parallel `ls-remote` per distinct origin URL. + + `targets` is (repo, branch) for the repos we want a live remote answer for. + Repos that share an origin URL (the executor submodules) are batched into a + single multi-ref `ls-remote`. Returns a handle for :func:`_collect_origin_queries`; + this only spawns the processes β€” it does not wait β€” so the network overlaps + whatever the caller does next. + """ + groups: dict[str, dict] = {} + # repo.name -> (hash, reachable); pre-seed repos that have no origin remote. + result: dict[str, tuple[str | None, bool]] = {} + for repo, branch in targets: + url = _origin_url(repo) + if not url: + result[repo.name] = (None, False) + continue + g = groups.setdefault(url, {'repo': repo, 'branches': set(), 'members': []}) + g['branches'].add(branch) + g['members'].append((repo, branch)) + + env = {**os.environ, **_NET_ENV} + procs = [] + for g in groups.values(): + refs = [f'refs/heads/{b}' for b in sorted(g['branches'])] + proc = subprocess.Popen( + ['git', '-C', str(g['repo'].path), 'ls-remote', REMOTE, *refs], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + env=env, + ) + procs.append((g, proc)) + return procs, result + + +def _collect_origin_queries(launched) -> dict[str, tuple[str | None, bool]]: + """Wait for the launched `ls-remote` processes and map each repo to its remote + branch hash: `{repo.name: (hash | None, reachable)}` (None hash = branch absent). + """ + procs, result = launched + for g, proc in procs: + try: + out, _ = proc.communicate(timeout=_LS_REMOTE_TIMEOUT) + ok = proc.returncode == 0 + except subprocess.TimeoutExpired: + proc.kill() + proc.communicate() + out, ok = '', False + hashes = {} + if ok: + for line in out.splitlines(): + sha, _, ref = line.partition('\t') + if ref.startswith('refs/heads/'): + hashes[ref[len('refs/heads/') :]] = sha + for repo, branch in g['members']: + result[repo.name] = (hashes.get(branch), True) if ok else (None, False) + return result + + +def _kind_from_counts(ahead: int, behind: int) -> str: + if ahead and behind: + return 'diverged' + if ahead: + return 'ahead' + if behind: + return 'behind' + return 'synced' + + +def _origin_state_remote( + repo: common.Repo, branch: str, lookup: dict[str, tuple[str | None, bool]] +) -> tuple[str, str]: + """Resolve origin status from the pre-fetched live `ls-remote` answer.""" + if not branch: + return 'no (detached)', 'detached' + rhash, reachable = lookup.get(repo.name, (None, False)) + if not reachable: + return 'unknown (origin unreachable)', 'unreachable' + if rhash is None: + return f'no (no origin/{branch})', 'absent' + head = _git(repo, 'rev-parse', 'HEAD').stdout.strip() + if rhash == head: + return 'yes', 'synced' + # Differ: count precisely only if the remote commit is already in our object + # store (e.g. the remote is behind us, so its tip is an ancestor of HEAD). + if _git(repo, 'cat-file', '-e', f'{rhash}^{{commit}}').returncode == 0: + ahead = int(_git(repo, 'rev-list', '--count', f'{rhash}..HEAD').stdout or 0) + behind = int(_git(repo, 'rev-list', '--count', f'HEAD..{rhash}').stdout or 0) + return f'no (ahead {ahead}, behind {behind})', _kind_from_counts(ahead, behind) + # Remote holds commits we lack β€” counts need a fetch first. + return f'no (differs; origin at {rhash[:10]}, fetch to compare)', 'differs' + + +def _origin_state_local(repo: common.Repo, branch: str) -> tuple[str, str]: + """Compare HEAD to the local `origin/` tracking ref (no network).""" + if not branch: + return 'no (detached)', 'detached' + ref = f'refs/remotes/origin/{branch}' + if _git(repo, 'rev-parse', '--verify', '--quiet', ref).returncode != 0: + return f'no (no local origin/{branch})', 'absent' + counts = _git( + repo, 'rev-list', '--left-right', '--count', f'{ref}...HEAD' + ).stdout.split() + if len(counts) != 2: + return 'no (cannot compare)', 'differs' + behind, ahead = int(counts[0]), int(counts[1]) + if ahead == 0 and behind == 0: + return 'yes', 'synced' + return f'no (ahead {ahead}, behind {behind})', _kind_from_counts(ahead, behind) + + +def _suggest(detached: bool, staged: bool, unstaged: bool, origin_kind: str) -> str: + """The next step in the add β†’ commit β†’ push pipeline for this repo.""" + if detached: + return 'switch to a branch (detached HEAD)' + if unstaged: + return 'add files' + if staged: + return 'commit files' + return { + 'absent': 'push (no origin branch yet)', + 'ahead': 'push', + 'behind': 'pull (behind origin)', + 'diverged': 'reconcile (diverged from origin)', + 'differs': 'fetch (cannot compare with origin)', + 'unreachable': 'fetch (origin unreachable)', + 'synced': 'none (ready to push)', + }.get(origin_kind, 'fetch (cannot compare with origin)') + + +def main(ctx: common.Context, args) -> int: + manager = common.discover_repos(ctx.root, 'manager')[0] + manager_branch = _current_branch(manager) + + # Cheap local pass: checked-out state + current branch (needed to know which + # refs to query on the remote). + entries = [] + for repo in common.discover_repos(ctx.root, args.repo): + checked_out = repo.is_checked_out() + branch = _current_branch(repo) if checked_out else '' + entries.append((repo, checked_out, branch)) + + # Fire the remote queries first (parallel, one per origin URL) so they run + # while we gather local status below and before anything is printed. + launched = None + if not args.offline: + launched = _launch_origin_queries( + [ + (repo, branch) + for repo, checked_out, branch in entries + if checked_out and branch + ] + ) + + # Local status, overlapping the in-flight network calls. + local: dict[str, dict] = {} + for repo, checked_out, branch in entries: + if not checked_out: + continue + is_sub = repo.rel != '' + changed, staged, unstaged = _worktree_state(repo, ignore_submodules=not is_sub) + row = {'changed': changed, 'staged': staged, 'unstaged': unstaged} + if is_sub: + row['committed_msg'], row['committed_ok'] = _committed_in_manager( + manager, repo.rel, repo + ) + local[repo.name] = row + + lookup = _collect_origin_queries(launched) if launched is not None else {} + + all_ready = True + for repo, checked_out, branch in entries: + if not checked_out: + # Not checked out: nothing local to push, and nothing we can verify. + ctx.printer.put(repo.name, checked_out='no', action='checkout submodule') + continue + + is_sub = repo.rel != '' + detached = not branch + row = local[repo.name] + if args.offline: + sync_msg, kind = _origin_state_local(repo, branch) + else: + sync_msg, kind = _origin_state_remote(repo, branch, lookup) + + ready = row['changed'] == 0 and not detached and kind == 'synced' + fields = { + 'branch': branch or 'DETACHED', + 'dirty': f'yes ({row["changed"]})' if row['changed'] else 'no', + 'on_branch': 'yes' if branch else 'no (detached)', + } + if is_sub: + # A submodule mirrors the manager branch under the line-namespaced + # convention (`feat/x` on the manager -> `pr/v0.3/feat/x` here), so + # compare against the manager branch mapped through feature_branch, + # not the literal string. + mirror = repo.feature_branch(manager_branch) if manager_branch else '' + same = bool(branch) and branch == mirror + fields['same_branch_as_manager'] = ( + 'yes' + if same + else f'no (manager on {manager_branch or "DETACHED"}, expected {mirror or "β€”"})' + ) + fields['committed_in_manager'] = row['committed_msg'] + ready = ready and row['committed_ok'] + + fields['in_sync_with_origin'] = sync_msg + fields['action'] = _suggest(detached, row['staged'], row['unstaged'], kind) + fields['ready'] = 'yes' if ready else 'NO' + all_ready = all_ready and ready + ctx.printer.put(repo.name, **fields) + + return 0 if all_ready else 1 diff --git a/support/tools/genvm-tool/genvm_tool/git/create_branches.py b/support/tools/genvm-tool/genvm_tool/git/create_branches.py new file mode 100644 index 00000000..d1a115fc --- /dev/null +++ b/support/tools/genvm-tool/genvm_tool/git/create_branches.py @@ -0,0 +1,182 @@ +"""`genvm-tool git create-branches` β€” branch the sub-repos that have new content. + +Surveys the manager and every checked-out executor submodule, keeps the ones +that carry uncommitted or unpushed work, lets you tick which of those to branch, +and creates the feature branch (`git checkout -b`) across all picked repos. + +The logical branch name is shared, but every executor submodule namespaces it +under `pr//` (e.g. `pr/v0.3/`) via :meth:`common.Repo.feature_branch`, +because all executor submodules push to ONE remote (`genvm-executor`) and a bare +name would collide between two lines branched at once β€” the `` segment keeps +them apart. The manager carries the bare name. This keeps the manager and its +executor submodule in lockstep while staying collision-free, matching the +line-scoped `v-dev` release branches CI already relies on. +""" + +import subprocess + +from .. import common + +NAME = 'create-branches' +HELP = 'create a feature branch across selected sub-repos that have new content' + + +def configure(parser): + parser.add_argument( + 'name', + nargs='?', + help='branch name to create across the selected repos (prompted if omitted); ' + 'must not contain spaces', + ) + parser.add_argument( + '--repo', + default='all', + help="limit candidates: 'all' (default), 'manager', or an executor name " + 'like executors/v0.3.x', + ) + + +def _git(repo: common.Repo, *args: str) -> subprocess.CompletedProcess: + return subprocess.run( + ['git', '-C', str(repo.path), *args], + capture_output=True, + text=True, + ) + + +def _new_content(repo: common.Repo) -> str | None: + """Describe a repo's pending work, or None if it has nothing to branch. + + "New content" is a dirty working tree (staged / unstaged / untracked) or + commits ahead of the upstream branch β€” anything a fresh branch would capture. + """ + changed = [ + ln for ln in _git(repo, 'status', '--porcelain').stdout.splitlines() if ln.strip() + ] + # `@{upstream}` errors out when the branch has no upstream; treat that as 0. + ahead = 0 + r = _git(repo, 'rev-list', '--count', '@{upstream}..HEAD') + if r.returncode == 0 and r.stdout.strip(): + ahead = int(r.stdout.strip()) + + if not changed and ahead == 0: + return None + parts = [] + if changed: + parts.append(f'{len(changed)} change(s)') + if ahead: + parts.append(f'{ahead} commit(s) ahead') + return ', '.join(parts) + + +def _validate_name(name: str) -> bool | str: + """questionary-style validator: True when ok, else the error message.""" + if not name: + return 'branch name must not be empty' + if ' ' in name: + return 'branch name must not contain spaces' + return True + + +def _branch_exists(repo: common.Repo, name: str) -> bool: + return ( + _git(repo, 'show-ref', '--verify', '--quiet', f'refs/heads/{name}').returncode == 0 + ) + + +def _current_branch(repo: common.Repo) -> str: + """The repo's checked-out branch, or '' when detached.""" + return _git(repo, 'branch', '--show-current').stdout.strip() + + +def _default_name(repo: common.Repo) -> str: + """The manager's current branch as a prompt default, unless it is a release/ + branch-model branch (main / version / dev) or detached β€” then no default. + """ + cur = _current_branch(repo) + if not cur or common._MODEL_BRANCH.fullmatch(cur): + return '' + return cur + + +def main(ctx: common.Context, args) -> int: + import questionary + + # Fail fast on a bad CLI name before bothering with the survey/prompt. + if args.name is not None: + verdict = _validate_name(args.name) + if verdict is not True: + raise common.ToolError(verdict) + + # Survey candidates: checked-out repos that carry something worth branching. + candidates = [] + for repo in common.discover_repos(ctx.root, args.repo): + if not repo.is_checked_out(): + ctx.logger.debug('skipping repo (not checked out)', repo=repo.name) + continue + summary = _new_content(repo) + if summary is None: + ctx.logger.debug('skipping repo (no new content)', repo=repo.name) + continue + candidates.append((repo, summary)) + + if not candidates: + ctx.printer.put('no sub-repos have new content to branch') + return 0 + + # Tick which candidates to branch (all pre-checked). + selected = questionary.checkbox( + 'Select sub-repos to branch', + choices=[ + questionary.Choice(f'{repo.name} β€” {summary}', value=repo, checked=True) + for repo, summary in candidates + ], + ).ask() + # `ask()` returns None on Ctrl-C/Esc; an empty list means everything unticked. + if not selected: + ctx.printer.put('nothing selected; no branches created') + return 0 + + # Branch name: the CLI value (already validated above) or an interactive prompt + # defaulted to the manager's current feature branch (e.g. while mid-work on it). + name = args.name + if name is None: + default = _default_name(common.discover_repos(ctx.root, 'manager')[0]) + name = questionary.text( + 'Branch name', default=default, validate=_validate_name + ).ask() + if not name: + ctx.printer.put('no branch name given; no branches created') + return 0 + + # Each repo gets its own physical branch name: the manager carries `name` + # verbatim; every executor submodule line-namespaces it (`v0.3/`) so two + # lines branching the shared executor remote at once never collide. + branch_of = {repo: repo.feature_branch(name) for repo in selected} + + # A repo already sitting on its branch needs nothing done β€” that is not a clash + # (e.g. the manager you started this branch on). Leave it exactly as it is. + on_branch = [repo for repo in selected if _current_branch(repo) == branch_of[repo]] + todo = [repo for repo in selected if repo not in on_branch] + + # Pre-flight: refuse if the branch already exists (as a ref) in any repo we + # would create it in, so we never half-create the set β€” and never reset/move an + # existing branch. Repos already on the branch are exempt (handled above). + clash = [repo.name for repo in todo if _branch_exists(repo, branch_of[repo])] + if clash: + raise common.ToolError(f'branch {name!r} already exists in: {", ".join(clash)}') + + for repo in on_branch: + ctx.printer.put('already on branch', repo=repo.name, branch=branch_of[repo]) + + for repo in todo: + # `checkout -b` only creates a new ref and moves HEAD onto the same commit; + # it never touches the working tree, so pending work is carried, not lost. + r = _git(repo, 'checkout', '-b', branch_of[repo]) + if r.returncode != 0: + raise common.ToolError( + f'failed to create branch {branch_of[repo]!r} in {repo.name}: ' + f'{r.stderr.strip()}' + ) + ctx.printer.put('created branch', repo=repo.name, branch=branch_of[repo]) + return 0 diff --git a/support/tools/genvm-tool/genvm_tool/hook/install.py b/support/tools/genvm-tool/genvm_tool/hook/install.py index 9b87d0be..2f3b9ecf 100644 --- a/support/tools/genvm-tool/genvm_tool/hook/install.py +++ b/support/tools/genvm-tool/genvm_tool/hook/install.py @@ -28,6 +28,11 @@ [ -f "$dir/.genvm-monorepo-root" ] || exit 0 # standalone checkout: skip tool="$dir/{cache}/{link}/bin/genvm-tool" [ -x "$tool" ] || {{ echo "genvm-tool not built (run 'genvm-tool hook install'); skipping {stage}" >&2; exit 0; }} +# The dev shell exports PYTHONPATH pointing at *its* genvm-tool build; left set, +# the hook's python would import genvm_tool from there instead of the tool we +# just resolved (running stale code). The nix wrapper bakes in every dep, so drop +# the inherited python path and let it use its own. +unset PYTHONPATH NIX_PYTHONPATH exec "$tool" hook run --repo {repo} --hook-stage {stage} -- "$@" """ diff --git a/support/tools/genvm-tool/genvm_tool/hook/run.py b/support/tools/genvm-tool/genvm_tool/hook/run.py index 36ef872c..b4358980 100644 --- a/support/tools/genvm-tool/genvm_tool/hook/run.py +++ b/support/tools/genvm-tool/genvm_tool/hook/run.py @@ -72,6 +72,11 @@ async def _one_repo(ctx, repo, stage, all_files, msg_file, fix): async def main(ctx: common.Context, args) -> int: + # When invoked as a git hook, git exports GIT_DIR / GIT_INDEX_FILE / ... for + # the *parent* repo. We fan out to every repo (and its submodules) by explicit + # path, so leaving those set would pin our `git -C ` calls to the + # parent's index/objects and crash them (exit 128). Strip them up front. + common.strip_inherited_git_env() repos = common.discover_repos(ctx.root, args.repo) msg_file = ( args.hook_args[0] if (args.hook_stage == 'commit-msg' and args.hook_args) else None diff --git a/support/tools/genvm-tool/genvm_tool/tests/cli.py b/support/tools/genvm-tool/genvm_tool/tests/cli.py index 7b8cb63b..03189e9a 100755 --- a/support/tools/genvm-tool/genvm_tool/tests/cli.py +++ b/support/tools/genvm-tool/genvm_tool/tests/cli.py @@ -55,6 +55,13 @@ def create_parser() -> _ParserResult: help='Stop execution after the first test failure', ) + run_parser.add_argument( + '--ci', + action='store_true', + default=False, + help='Emit full failure and service log context for CI diagnostics', + ) + default_cpu_count = os.cpu_count() or 1 run_parser.add_argument( diff --git a/support/tools/genvm-tool/pyproject.toml b/support/tools/genvm-tool/pyproject.toml index 35b08af6..d2afdf6d 100644 --- a/support/tools/genvm-tool/pyproject.toml +++ b/support/tools/genvm-tool/pyproject.toml @@ -1,13 +1,21 @@ [build-system] -requires = ["setuptools>=61"] build-backend = "setuptools.build_meta" +requires = ["setuptools>=61"] [project] -name = "genvm-tool" -version = "0.1.0" +dependencies = [ + "aiohttp", + "jsonnet", + "shtab", + "ruamel.yaml", + "cloudpickle", + "python-dotenv", + "questionary", +] description = "GenVM monorepo git helper (build, tests, hooks, ls)" +name = "genvm-tool" requires-python = ">=3.12" -dependencies = ["aiohttp", "jsonnet", "shtab", "ruamel.yaml", "cloudpickle"] +version = "0.1.0" [project.scripts] genvm-tool = "genvm_tool.__main__:main" diff --git a/support/tools/git-third-party/ruff.toml b/support/tools/git-third-party/ruff.toml index 18748aab..34b109c4 100644 --- a/support/tools/git-third-party/ruff.toml +++ b/support/tools/git-third-party/ruff.toml @@ -2,8 +2,8 @@ indent-width = 2 [format] indent-style = "tab" -quote-style = "single" line-ending = "lf" +quote-style = "single" -skip-magic-trailing-comma = false docstring-code-format = true +skip-magic-trailing-comma = false diff --git a/tests/runner/genvm_tool_plugins/docker.py b/tests/runner/genvm_tool_plugins/docker.py index fd6a2f1f..38a9b77a 100644 --- a/tests/runner/genvm_tool_plugins/docker.py +++ b/tests/runner/genvm_tool_plugins/docker.py @@ -122,8 +122,9 @@ def _stop_cleanup(container_id: str) -> functools.partial: class ContainerHandle(genvm_tool.tests.exec.service.Handle): - def __init__(self, container_id: str): + def __init__(self, container_id: str, *, ci: bool): self._container_id = container_id + self._ci = ci self._watchdog_token = local_ctx.shared.watchdog.register( _stop_cleanup(container_id) ) @@ -147,6 +148,7 @@ async def run( cwd: Path, env: dict[str, str] | None = None, log_context: dict | None = None, + ci: bool = False, ) -> 'ContainerHandle': if env is None: env = copy.deepcopy(DEFAULT_ENV) @@ -183,7 +185,7 @@ async def run( ) raise DockerRunError(f'Docker run failed with exit code {res.exit_code}') container_id = res.stdout.strip() - return ContainerHandle(container_id=container_id) + return ContainerHandle(container_id=container_id, ci=ci) async def healthy(self) -> bool: try: @@ -202,23 +204,25 @@ async def healthy(self) -> bool: ctx=local_ctx.shared, mode=genvm_tool.tests.exec.command.RunMode.SILENT ) if res.exit_code != 0: - logs = await self.get_logs() - local_ctx.shared.logger.warning( - 'Docker health check failed', - container_id=self._container_id, - returncode=res.exit_code, - logs=logs, - ) + if self._ci: + logs = await self.get_logs() + local_ctx.shared.logger.warning( + 'Docker health check failed', + container_id=self._container_id, + returncode=res.exit_code, + logs=logs, + ) return False status = res.stdout.strip() if status != 'healthy': - logs = await self.get_logs() - local_ctx.shared.logger.warning( - 'Container not healthy', - container_id=self._container_id, - status=status, - logs=logs, - ) + if self._ci: + logs = await self.get_logs() + local_ctx.shared.logger.warning( + 'Container not healthy', + container_id=self._container_id, + status=status, + logs=logs, + ) return status == 'healthy' except Exception as e: local_ctx.shared.logger.error( diff --git a/tests/runner/genvm_tool_plugins/genvm.py b/tests/runner/genvm_tool_plugins/genvm.py index 102c25a7..4965ff56 100644 --- a/tests/runner/genvm_tool_plugins/genvm.py +++ b/tests/runner/genvm_tool_plugins/genvm.py @@ -22,6 +22,8 @@ def get_manager_port(env: genvm_tool.tests.stage.configuration.Env) -> int: async def start_webdriver_service( env: genvm_tool.tests.stage.configuration.Env, + *, + ci: bool = False, ) -> genvm_tool.tests.exec.service.Handle: context_dir = local_ctx.shared.root_dir.joinpath('webdriver') @@ -36,7 +38,10 @@ async def start_webdriver_service( proc_env[k] = v return await docker.ContainerHandle.run( - ['-p', f'{port}:4444', image_sha], cwd=local_ctx.shared.root_dir, env=proc_env + ['-p', f'{port}:4444', image_sha], + cwd=local_ctx.shared.root_dir, + env=proc_env, + ci=ci, ) @@ -54,6 +59,8 @@ async def _read_log_pipe( stream: asyncio.StreamReader, log_file: typing.IO[str], logger: genvm_tool.tests.Formatter, + *, + ci: bool, ) -> None: while True: line_bytes = await stream.readline() @@ -65,13 +72,15 @@ async def _read_log_pipe( try: parsed = json.loads(line) level = parsed.get('level', '') - if _LOG_LEVEL_PRIORITY.get(level, -1) >= _LOG_LEVEL_PRIORITY['info']: + min_level = 'info' if ci else 'warning' + if _LOG_LEVEL_PRIORITY.get(level, -1) >= _LOG_LEVEL_PRIORITY[min_level]: message = parsed.get('message', line) extra = {k: v for k, v in parsed.items() if k not in ('level', 'message')} fmt_level = genvm_tool.tests.Formatter.Level.from_str(level) logger.log(fmt_level, f'[manager] {message}', **extra) except (json.JSONDecodeError, ValueError): - logger.warning(f'[manager] {line}') + if ci: + logger.warning(f'[manager] {line}') class ManagerHandle(genvm_tool.tests.exec.service.Handle): @@ -132,10 +141,12 @@ def __init__( env: genvm_tool.tests.stage.configuration.Env, bin_path: Path, log_path: Path | None = None, + ci: bool = False, ): self._bin_path = bin_path self._log_path = log_path self._env = env + self._ci = ci async def start(self) -> ManagerHandle: log_file = None @@ -162,8 +173,12 @@ async def start(self) -> ManagerHandle: if use_pipe: logger = local_ctx.shared.logger log_tasks = [ - asyncio.create_task(_read_log_pipe(process.stdout, log_file, logger)), - asyncio.create_task(_read_log_pipe(process.stderr, log_file, logger)), + asyncio.create_task( + _read_log_pipe(process.stdout, log_file, logger, ci=self._ci) + ), + asyncio.create_task( + _read_log_pipe(process.stderr, log_file, logger, ci=self._ci) + ), ] handle = ManagerHandle(port, process, log_file, log_tasks) diff --git a/tests/runner/genvm_tool_plugins/integration.py b/tests/runner/genvm_tool_plugins/integration.py index f727e335..7b1db4e9 100644 --- a/tests/runner/genvm_tool_plugins/integration.py +++ b/tests/runner/genvm_tool_plugins/integration.py @@ -24,6 +24,7 @@ import origin.base_host as base_host import origin.calldata as gvm_calldata import origin.fees as fees +import origin.log_asserts as log_asserts import origin.logger as origin_logger import origin.public_abi as public_abi from genvm_tool.tests.exec.command import Command, RunMode @@ -54,13 +55,12 @@ def _is_ignore_hash_enabled() -> bool: return '--ignore-hash' in sys.argv -# Integration cases live in the executor checkout (per-version, per-branch). -# The active executor version line is mounted as a submodule at executors/v0.3.x. -# TODO(multi-version): iterate over executors/* once several minors coexist. -# The test harness (templates, runner) is version-independent and lives in the -# manager root. -EXEC_SUBDIR = 'executors/v0.3.x' -CASES_DIR = local_ctx.shared.root_dir.joinpath(EXEC_SUBDIR, 'tests', 'integration') +# Integration cases live per-version in each executor checkout +# (executors/.x/tests/integration). `integration_test` iterates the active +# version lines from .genvm-monorepo-root and runs each line's suite against that +# line's own executor (see `integration_test_single_executor`, which resolves the +# line's cases dir and reroute target). The harness itself (templates, runner) is +# version-independent and lives in the manager root. TEMPLATES_DIR = local_ctx.shared.root_dir.joinpath('tests', 'templates') # Default environment for tests @@ -96,6 +96,73 @@ class _SavedLog(typing.TypedDict): kwargs: dict +_SHORT_TEXT_LIMIT = 2000 +_SHORT_LIST_LIMIT = 12 +_NOISY_CONTEXT_KEYS = frozenset( + { + 'exp', + 'got', + 'genvm_log', + 'log', + 'logs', + 'semantics', + 'stderr', + 'stdout', + } +) + + +def _shorten_text(value: str, limit: int = _SHORT_TEXT_LIMIT) -> str: + if len(value) <= limit: + return value + omitted = len(value) - limit + return f'\n{value[-limit:]}' + + +def _shorten_value(value: typing.Any) -> typing.Any: + if isinstance(value, str): + return _shorten_text(value) + if isinstance(value, bytes): + if len(value) <= _SHORT_TEXT_LIMIT: + return value + return value[:_SHORT_TEXT_LIMIT] + b'' + if isinstance(value, BaseException): + return { + 'type': value.__class__.__name__, + 'message': str(value), + } + if isinstance(value, list): + if len(value) <= _SHORT_LIST_LIMIT: + return value + return [ + *value[:_SHORT_LIST_LIMIT], + f'', + ] + return value + + +def _short_failure_context( + context: dict[str, typing.Any], artifacts_dir: Path | None = None, *, ci: bool +) -> dict[str, typing.Any]: + if ci: + return context + + short = { + k: _shorten_value(v) for k, v in context.items() if k not in _NOISY_CONTEXT_KEYS + } + for key in ('stderr', 'stdout'): + if key in context and context[key]: + short[f'{key}_tail'] = _shorten_value(context[key]) + if 'logs' in context: + short['log_count'] = len(context['logs']) + if 'genvm_log' in context: + short['genvm_log_count'] = len(context['genvm_log']) + if artifacts_dir is not None: + short['artifacts_dir'] = str(artifacts_dir) + short['hint'] = 'run with --ci for full inline failure context' + return short + + def _make_log_adapter(formatter: genvm_tool.tests.Formatter) -> 'origin_logger.Logger': class _FormatterLoggerAdapter(origin_logger.Logger): """Adapts genvm_tool.formatter.Formatter to base_host.Logger interface.""" @@ -167,6 +234,7 @@ class IntegrationPrepareCase(genvm_tool.tests.test.Case): jsonnet_path: Path top_level_conf: dict tmp_dir: Path + ci: bool hidden: bool = True @@ -180,6 +248,8 @@ class IntegrationSingleCase(genvm_tool.tests.test.Case): description: genvm_tool.tests.test.Description jsonnet_path: Path + cases_dir: Path + reroute_to: str manager_service: genvm_tool.tests.stage.collection.Service tree_path: str parent_tree_path: str | None @@ -187,7 +257,9 @@ class IntegrationSingleCase(genvm_tool.tests.test.Case): total_steps: int tmp_dir: Path max_attempts: int + ignore_hash: bool = False is_benchmark: bool = False + ci: bool = False async def into_steps(self) -> list[genvm_tool.tests.exec.step.Step]: step = IntegrationSingleStep(self) @@ -282,16 +354,21 @@ async def run( ) result = await cmd.run(local_ctx.shared, mode=RunMode.SILENT) if result.exit_code != 0: + context = _short_failure_context( + { + 'reason': 'prepare script failed', + 'exit_code': result.exit_code, + 'stdout': result.stdout, + 'stderr': result.stderr, + 'log': result.stderr, + }, + tmp_dir, + ci=self._case.ci, + ) raise genvm_tool.tests.test.FinishedEarlyException( result=genvm_tool.tests.test.Result( passed=False, - context={ - 'reason': 'prepare script failed', - 'exit_code': result.exit_code, - 'stdout': result.stdout, - 'stderr': result.stderr, - 'log': result.stderr, - }, + context=context, elapsed_seconds=0, ) ) @@ -384,6 +461,8 @@ def __init__(self, case: IntegrationSingleCase): self._total_steps = case.total_steps self._tmp_dir = case.tmp_dir self._max_attempts = case.max_attempts + self._ignore_hash = case.ignore_hash + self._ci = case.ci def to_str(self) -> str: return f'' @@ -407,6 +486,11 @@ async def run( # Raise FinishedEarlyException to stop subsequent steps context = result.get('context', {}) context['logs'] = sub_logger.saved_logs + context = _short_failure_context( + context, + self._tmp_dir.joinpath(self._tree_path), + ci=self._ci, + ) raise genvm_tool.tests.test.FinishedEarlyException( result=genvm_tool.tests.test.Result( passed=False, @@ -421,7 +505,11 @@ async def run( max_attempts=self._max_attempts, test_name=str(self._test_case.description.name), tree_path=self._tree_path, - context=result.get('context', {}), + context=_short_failure_context( + result.get('context', {}), + self._tmp_dir.joinpath(self._tree_path), + ci=self._ci, + ), ) # Should not reach here @@ -509,13 +597,16 @@ async def _run_single_step( single_conf['message']['origin_address'] = Address( single_conf['message']['origin_address'] ) + single_conf['message']['signer_address'] = Address( + single_conf['message']['signer_address'] + ) path_to_which_leader_puts_result = my_tmp_dir.parent.joinpath( my_tmp_dir.name[:-1] + '_leader_nondet.pickle' ) # Set up paths - rel_path = jsonnet_path.relative_to(CASES_DIR) + rel_path = jsonnet_path.relative_to(self._test_case.cases_dir) mock_sock_path = Path('/tmp', 'genvm-test', rel_path.with_suffix(f'.sock{suff}')) mock_sock_path.parent.mkdir(exist_ok=True, parents=True) @@ -547,6 +638,17 @@ async def _run_single_step( encoded_nondet.append( bytes([public_abi.ResultCode.VM_ERROR]) + res['value'].encode('utf-8') ) + elif res['kind'] == 'rollback': + # v0.2.x: a rollback is a user error carrying a plain UTF-8 + # string (v0.2 user errors are string-only). + encoded_nondet.append( + bytes([public_abi.ResultCode.USER_ERROR]) + res['value'].encode('utf-8') + ) + elif res['kind'] == 'contract_error': + # v0.2.x: a vm error carrying a plain UTF-8 string. + encoded_nondet.append( + bytes([public_abi.ResultCode.VM_ERROR]) + res['value'].encode('utf-8') + ) elif res['kind'] == 'raw': encoded_nondet.append(bytes(res['value'])) else: @@ -569,7 +671,7 @@ async def _run_single_step( # Get manager URI from the service manager_svc = self._test_case.manager_service port = manager_svc.meta['port'] - reroute_to = manager_svc.meta.get('reroute_to', '') + reroute_to = self._test_case.reroute_to manager_uri = f'http://localhost:{port}' # Run the test @@ -589,9 +691,7 @@ async def _run_single_step( if case_permissions is not None: request_extra['permissions'] = case_permissions dflt_bucket = 2**200 - bucket_totals: list[int] = single_conf.get( - 'bucket_totals', [dflt_bucket, dflt_bucket] - ) + bucket_totals: list[int] = single_conf.get('bucket_totals', [dflt_bucket] * 10) default_message_fee_allocation = [ fees.DEFAULT_EXTERNAL_MESSAGE_ALLOC, @@ -761,6 +861,32 @@ async def _run_single_step( # Create expected output file exp_semantics_path.write_text(semantics) + # ADR-012 load-action log-grep assertions. A case may declare + # `runner_load_asserts` on a step to assert on the executor's stable + # `runner load` log records (charge counts, per-runner sizes, cached vs + # charged); see origin.log_asserts for the schema. Gated on the main mode + # (semantics_components is non-empty only there) so a nondet leader's + # extra loads do not make validator/sandbox re-runs flaky. + runner_load_asserts = single_conf.get('runner_load_asserts') + if runner_load_asserts and semantics_components: + load_errors = log_asserts.check( + res.genvm_log, + runner_load_asserts, + code_len=(len(code) if code is not None else None), + ) + if load_errors: + return { + 'passed': False, + 'context': { + 'reason': 'runner load assertion failed', + 'tree_path': tree_path, + 'errors': load_errors, + 'runner_loads': log_asserts.summarize(res.genvm_log), + 'stderr': res.stderr, + 'genvm_log': res.genvm_log, + }, + } + my_tmp_dir.joinpath('hash').write_bytes(base64.b64encode(hash_data)) expected_hash_path = single_conf['expected_hash_path'] @@ -770,7 +896,7 @@ async def _run_single_step( res.result_kind == public_abi.ResultCode.VM_ERROR and res.result_data == 'timeout' ): check_expected_hash = False # Don't check hash for timeouts, as they can be flaky - if _is_ignore_hash_enabled(): + if _is_ignore_hash_enabled() or self._ignore_hash: check_expected_hash = False if check_expected_hash: @@ -813,6 +939,30 @@ def integration_test( manager_service: genvm_tool.tests.stage.collection.Service, modules_service: genvm_tool.tests.stage.collection.Service, webdriver_service: genvm_tool.tests.stage.collection.Service, + ci: bool, +) -> None: + active_versions: list[str] = json.loads( + local_ctx.shared.root_dir.joinpath('.genvm-monorepo-root').read_text() + )['active-versions'] + for ver in active_versions: + integration_test_single_executor( + ctx, + executor_version=ver, + manager_service=manager_service, + modules_service=modules_service, + webdriver_service=webdriver_service, + ci=ci, + ) + + +def integration_test_single_executor( + ctx: genvm_tool.tests.stage.collection.Context, + *, + executor_version: str, + manager_service: genvm_tool.tests.stage.collection.Service, + modules_service: genvm_tool.tests.stage.collection.Service, + webdriver_service: genvm_tool.tests.stage.collection.Service, + ci: bool, ) -> None: """ Collect integration tests from tests/integration/ directory. @@ -824,6 +974,25 @@ def integration_test( Steps depend on /prepare, and child steps depend on their parent step. """ + import genvm_tool.common + + EXEC_SUBDIR = os.environ.get( + 'GENVM_TEST_EXEC_SUBDIR', f'executors/{executor_version}.x' + ) + executor_root = local_ctx.shared.root_dir / EXEC_SUBDIR + CASES_DIR = executor_root / 'tests' / 'integration' + + # Run this line's cases against this line's own executor. The version key + # (e.g. "v0.2") maps to the concrete built version (e.g. "v0.2.16"). + reroute_to = build_info['executor_versions'].get(executor_version, executor_version) + + executor_project = genvm_tool.common.load_project(executor_root) + executor_integration_fn = getattr(executor_project, 'integration', None) + executor_integration_conf: dict = ( + executor_integration_fn() if executor_integration_fn else {} + ) + ignore_hash = executor_integration_conf.get('ignore-hash', False) + jsonnet_files = list(CASES_DIR.glob('**/*.jsonnet')) jsonnet_files.sort() @@ -837,9 +1006,13 @@ def integration_test( _single_integration_test, ctx, jsonnet_file, + cases_dir=CASES_DIR, + reroute_to=reroute_to, + ignore_hash=ignore_hash, manager_service=manager_service, modules_service=modules_service, webdriver_service=webdriver_service, + ci=ci, ): jsonnet_file for jsonnet_file in jsonnet_files } @@ -851,13 +1024,17 @@ def _single_integration_test( ctx: genvm_tool.tests.stage.collection.Context, jsonnet_file: Path, *, + cases_dir: Path, + reroute_to: str, + ignore_hash: bool, manager_service: genvm_tool.tests.stage.collection.Service, modules_service: genvm_tool.tests.stage.collection.Service, webdriver_service: genvm_tool.tests.stage.collection.Service, + ci: bool, ) -> None: import _jsonnet - rel_path = jsonnet_file.relative_to(CASES_DIR) + rel_path = jsonnet_file.relative_to(cases_dir) # Determine stability tag from path tags: set[str] = {'integration'} @@ -945,6 +1122,7 @@ def _single_integration_test( jsonnet_path=jsonnet_file, top_level_conf=top_level_conf, tmp_dir=tmp_dir, + ci=ci, ) ) @@ -977,6 +1155,8 @@ def _single_integration_test( IntegrationSingleCase( description=step_desc, jsonnet_path=jsonnet_file, + cases_dir=cases_dir, + reroute_to=reroute_to, manager_service=manager_service, tree_path=tree_path, parent_tree_path=single_conf.get('parent_tree_path'), @@ -984,6 +1164,8 @@ def _single_integration_test( total_steps=total_steps, tmp_dir=tmp_dir, max_attempts=max_attempts, + ignore_hash=ignore_hash, is_benchmark=is_benchmark, + ci=ci, ) ) diff --git a/tests/runner/genvm_tool_plugins/ninja.py b/tests/runner/genvm_tool_plugins/ninja.py new file mode 100644 index 00000000..3306b9f9 --- /dev/null +++ b/tests/runner/genvm_tool_plugins/ninja.py @@ -0,0 +1,477 @@ +"""Ninja build-graph DSL + per-line `configure` helpers (importable plugin). + +Extracted from ``genvm-tool configure`` so each executor line can own its own +build configuration. The shared machinery lives here β€” the ninja DSL +(:class:`Ninja` / :class:`Build`) and the *standard* per-line registrations +(cargo crates, `genvm-tool codegen`, install tree) bundled on :class:`LineContext`. + +The configure command builds one :class:`LineContext` per active line and hands +it to that line's ``.genvm-tool.py:configure(line)`` hook. A line's hook composes +the standard steps and picks how to resolve the runner registry β€” the one place +the lines already diverge: a frozen line copies its committed +``executor/registry`` verbatim (:meth:`LineContext.frozen_registry`), a live line +derives the manifests through the umbrella nix machinery +(:meth:`LineContext.nix_manifests`). New incompatibilities are added as line +hooks here, never as ``if version ==`` branches in the command. + +This is a plain importable library (per the plugin philosophy); it is pulled in +via ``extra_python_paths`` and used both by the command and by the executor +hooks. Stdlib-only, so it keeps ``configure`` free of the test dependency closure. +""" + +import os +import re +from dataclasses import dataclass +from pathlib import Path + +# Build directory, manager-root-relative. Kept as a relative path because the +# generated `build_dir` ninja variable must read `build`, not an absolute path. +BUILD_DIR_REL = Path('build') + +# Cargo needs the rust toolchain's libs on LD_LIBRARY_PATH; `$$` escapes the `$` +# from ninja so the shell expands it, `:+` keeps an unset LD_LIBRARY_PATH empty. +CARGO_LD_LIBRARY_PATH = ( + 'LD_LIBRARY_PATH="$${CARGO_LD_LIBRARY_PATH}$${LD_LIBRARY_PATH:+:$$LD_LIBRARY_PATH}"' +) + + +# --- ninja DSL ------------------------------------------------------------- + + +class RawStr(str): + """A string emitted verbatim into the ninja file (never shell-escaped).""" + + +AND = RawStr('&&') +VAR_IN = RawStr('$in') +VAR_OUT = RawStr('$out') + +# clippy/build runs are instrumented for coverage but discard the profile here. +COVERAGE_ENV = RawStr('RUSTFLAGS="-C instrument-coverage" LLVM_PROFILE_FILE=/dev/null') + +# Ruby's Shellwords.escape leaves this set unescaped; everything else (including +# spaces, quotes, `$`, `?`, `#`) gets a leading backslash. +_UNSAFE = re.compile(r'[^A-Za-z0-9_\-.,:+/@\n]') + + +def _shellescape(s: str) -> str: + if s == '': + return "''" + s = _UNSAFE.sub(lambda m: '\\' + m.group(0), s) + # A backslash before LF is a line continuation, so wrap newlines in quotes. + return s.replace('\n', "'\n'") + + +def _escape(s: str) -> str: + # configure.rb un-escapes `\=`, keeping `FOO=bar` env assignments readable. + return _shellescape(s).replace('\\=', '=') + + +def glob(base: Path, pattern: str) -> list[Path]: + """Mirror Ruby's `Pathname#glob`: a depth-first, per-component sorted walk + that skips dotfile entries (no FNM_DOTMATCH). + + Sorting by path *components* (not the joined string) is what makes + `calldata/...` precede `calldata-derive/...`: the `/` separator dominates, + exactly as Ruby's recursive directory traversal does. + """ + out = [] + for p in base.glob(pattern): + rel = p.relative_to(base) + if any(part.startswith('.') for part in rel.parts): + continue + out.append(p) + return sorted(out, key=lambda p: p.parts) + + +def relpath(target: Path, start: Path) -> str: + return os.path.relpath(str(target), str(start)) + + +def is_subpath(path: Path, base: Path) -> bool: + return not relpath(path, base).startswith('..') + + +# Nix expressions that evaluate a checkout's runner derivations to their +# {id: hash} (latest.json) and {id: [hash]} (all.json) maps. +def runner_manifest_expr(version: str, kind: str) -> str: + """Nix expr for one executor's `latest`/`all` runner manifest. + + The umbrella's ./runners owns this: `all` is every runner compatible up to + `version`, `latest` is that executor's own current runners. Evaluated with + cwd at the manager root (the nix_eval rule cds into ``$wd``). + """ + return ( + f'(import ./runners/manifest.nix ' + f'{{ executorVersion = "{version}"; host-system = builtins.currentSystem; }}).{kind}' + ) + + +class Build: + """One `build` edge; mirrors configure.rb's BuildBuilder.""" + + def __init__(self, ninja: 'Ninja', rule: str, outputs): + self.ninja = ninja + self.rule = rule + self.outputs = list(outputs) + self.implicit_outputs: list = [] + self.deps: list = [] + self.implicit_deps: list = [] + self.order_only_deps: list = [] + self.props: dict = {} + + @staticmethod + def _add(arr: list, item) -> None: + if isinstance(item, (list, tuple)): + arr.extend(item) + else: + arr.append(item) + + def add_output(self, output): + self._add(self.outputs, output) + return self + + def add_implicit_output(self, output): + self._add(self.implicit_outputs, output) + return self + + def add_dependency(self, dep): + self._add(self.deps, dep) + return self + + def add_implicit_dependency(self, dep): + self._add(self.implicit_deps, dep) + return self + + def add_order_only_dependency(self, dep): + self._add(self.order_only_deps, dep) + return self + + def var(self, name: str, value): + self.props[name] = value + return self + + def description(self, desc: str): + self.props['description'] = desc + return self + + def finish(self) -> None: + assert self.outputs, 'build edge must have at least one output' + n = self.ninja + n.buf.append('build') + for o in self.outputs: + n.buf.append(' ') + n._scalar(o) + if self.implicit_outputs: + n.buf.append(' |') + for o in self.implicit_outputs: + n.buf.append(' ') + n._scalar(o) + n.buf.append(': ') + n.buf.append(self.rule) + for d in self.deps: + n.buf.append(' ') + n._scalar(d) + if self.implicit_deps: + n.buf.append(' |') + for d in self.implicit_deps: + n.buf.append(' ') + n._scalar(d) + if self.order_only_deps: + n.buf.append(' ||') + for d in self.order_only_deps: + n.buf.append(' ') + n._scalar(d) + n.buf.append('\n') + for key, value in self.props.items(): + n.buf.append(f' {key} = ') + n._value(value) + n.buf.append('\n') + n.buf.append('\n') + + +class Ninja: + """Accumulates the build.ninja text; mirrors configure.rb's Ninja::File.""" + + def __init__(self, source_dir: Path, build_dir: Path): + self.source_dir = source_dir + self.build_dir = build_dir # absolute; this is ninja's working directory + self.buf: list[str] = [] + self.all_format: list[str] = [] + self.all_clippy: list[str] = [] + self.all_clippy_fix: list[str] = [] + self.rust_target = '' + self.rust_target_dir = build_dir # set by main + build_dir.mkdir(parents=True, exist_ok=True) + + def _resolve_path(self, value: Path) -> str: + """Render a Path as ninja sees it: relative to the build (working) dir.""" + if not value.is_absolute(): + # Relative paths are interpreted against the source root. + return relpath(self.source_dir / value, self.build_dir) + if is_subpath(value, self.build_dir) or is_subpath(value, self.source_dir): + return relpath(value, self.build_dir) + return str(value) + + def _scalar(self, value) -> None: + if isinstance(value, RawStr): + self.buf.append(str(value)) + elif isinstance(value, Path): + self.buf.append(_escape(self._resolve_path(value))) + else: + self.buf.append(_escape(str(value))) + + def _value(self, value) -> None: + if isinstance(value, (list, tuple)): + for i, v in enumerate(value): + if i: + self.buf.append(' ') + self._value(v) + else: + self._scalar(value) + + def comment(self, text: str) -> None: + for line in text.splitlines(): + self.buf.append(f'# {line.strip()}\n') + + def var(self, name: str, value) -> None: + self.buf.append(f'{name} = ') + self._value(value) + self.buf.append('\n\n') + + def rule(self, name: str, **props) -> None: + assert 'command' in props, f'rule {name} must have a command' + self.buf.append(f'rule {name}\n') + for key, value in props.items(): + self.buf.append(f' {key} = ') + self._value(value) + self.buf.append('\n') + self.buf.append('\n') + + def build(self, rule: str, *outputs) -> Build: + return Build(self, rule, outputs) + + def codegen(self, out: Path, lang: str, data: Path) -> None: + """Emit one `codegen` edge (`genvm-tool codegen --lang `). + + Requires the `codegen` rule. The generated file depends on its data JSON and + on the codegen backend sources (``codegen_deps``, set by the command) so an + edit to either regenerates it. + """ + b = self.build('codegen', out) + b.add_dependency(data) + b.add_implicit_dependency(getattr(self, 'codegen_deps', [])) + b.var('lang', lang) + b.finish() + + def install(self, frm: str, to: str, phony: Build) -> None: + """`cp` every file under `frm` into `to`, registering each on `phony`.""" + install_dir = self.source_dir / frm + for f in glob(install_dir, '**/*'): + if f.is_dir(): + continue + out = to + '/' + str(f.relative_to(install_dir)) + phony.add_dependency(out) + self.build('cp', out).add_dependency(f).finish() + + # --- per-crate cargo edges -------------------------------------------- + + def register_cargo( + self, + rel_path: str, + extra_args=(), + build_to: str | None = None, + target_dir: Path | None = None, + ) -> None: + extra_args = list(extra_args) + # Per-line cargo target dir; falls back to the shared default. Executor + # lines pass their own so two `genvm` binaries don't share `debug/genvm`. + td = target_dir if target_dir is not None else self.rust_target_dir + to = BUILD_DIR_REL / rel_path + (self.source_dir / to).mkdir(parents=True, exist_ok=True) + + crate_dir = Path(rel_path) + base = self.source_dir / rel_path + all_files = [crate_dir / p.relative_to(base) for p in glob(base, '**/*.rs')] + all_files += [crate_dir / 'Cargo.toml', crate_dir / 'Cargo.lock'] + + files_trg = BUILD_DIR_REL / 'ya-build' / rel_path / 'files.trg' + (self.source_dir / files_trg).parent.mkdir(parents=True, exist_ok=True) + + self.build('phony_touch', files_trg).add_implicit_dependency(all_files).finish() + + clippy_lints = ['--', '-A', 'clippy::upper_case_acronyms', '-Dwarnings'] + + clippy = self.build('cargo', 'target/' + rel_path + '/clippy') + clippy.add_dependency(files_trg) + clippy.var('subcommand', 'clippy') + clippy.var('wd', crate_dir) + clippy.var('extra_args', extra_args + clippy_lints) + clippy.var('env', COVERAGE_ENV) + if target_dir is not None: + clippy.var('target_dir', str(td)) + clippy.description('Run cargo clippy for ' + rel_path) + clippy.finish() + self.all_clippy.append('target/' + rel_path + '/clippy') + + fix = self.build('cargo', 'target/' + rel_path + '/clippy/fix') + fix.var('subcommand', 'clippy') + fix.var('wd', crate_dir) + fix.var( + 'extra_args', + extra_args + ['--fix', '--allow-dirty', '--allow-staged'] + clippy_lints, + ) + fix.var('env', COVERAGE_ENV) + if target_dir is not None: + fix.var('target_dir', str(td)) + fix.finish() + self.all_clippy_fix.append('target/' + rel_path + '/clippy/fix') + + fmt = self.build('CUSTOM_COMMAND', 'target/' + rel_path + '/fmt') + fmt.var('command', ['cd', crate_dir, AND, 'cargo', 'fmt']) + fmt.description('Run cargo fmt for ' + rel_path) + fmt.finish() + self.all_format.append('target/' + rel_path + '/fmt') + + if build_to is not None: + bin_name = str(td / self.rust_target / 'debug' / build_to.split('/')[-1]) + build = self.build('cargo_build', bin_name) + build.add_dependency(files_trg) + build.var('wd', crate_dir) + build.var('extra_args', extra_args) + build.var('env', COVERAGE_ENV) + if target_dir is not None: + build.var('target_dir', str(td)) + build.finish() + + self.build('cp', build_to).add_dependency(bin_name).finish() + + +# --- per-executor-line configuration -------------------------------------- + + +@dataclass +class LineContext: + """Everything one executor line's ``configure(line)`` hook needs. + + Built by the configure command per active line. The hook composes the + ``register_standard_*`` / ``install_tree`` steps and chooses a runner-registry + resolver (:meth:`frozen_registry` vs :meth:`nix_manifests`). The phony nodes + are shared, command-owned aggregates the hook appends to; the command finishes + them after every line's hook has run. + """ + + n: Ninja + source_dir: Path + key: str # version line, e.g. 'v0.3' + exec_rel: str # 'executors/v0.3.x' + real: str # concrete pinned version, e.g. 'v0.3.0-rc7' + codegen_phony: Build + data_phony: Build + all_bin: Build + + @property + def exec_root(self) -> Path: + return self.source_dir / self.exec_rel + + @property + def out_exec(self) -> str: + """Install prefix for this line: ``out/executor/``.""" + return f'out/executor/{self.real}' + + @property + def line_target_dir(self) -> Path: + """Per-line cargo target dir; lines share crate names so each needs its own.""" + return self.n.rust_target_dir / self.key + + def codegen(self, out: Path, lang: str, data: Path) -> None: + """Emit a codegen edge and register its output on the codegen phony.""" + self.n.codegen(out, lang, data) + self.codegen_phony.add_dependency(out) + + def register_standard_codegen(self) -> None: + """Per-line generated files inside this line's executor/runner source tree.""" + data = self.exec_root / 'executor' / 'codegen' / 'data' + self.codegen( + self.exec_root / 'executor/crates/sdk-rs/src/abi/consts.rs', + 'rust', + data / 'public-abi.json', + ) + self.codegen( + self.exec_root / 'runners/genlayer-py-std/src/genlayer/vm/public_abi.py', + 'python', + data / 'public-abi.json', + ) + self.codegen( + self.exec_root / 'executor/crates/common/src/host_fns.rs', + 'rust', + data / 'host-fns.json', + ) + + def register_standard_crates(self) -> None: + """Build this line's `genvm` binary plus its `common`/`sdk-rs` crates. + + calldata + calldata-derive are shared, manager-root crates (registered + once by the command); they are no longer per-line. + """ + td = self.line_target_dir + self.n.register_cargo( + f'{self.exec_rel}/executor', + build_to=f'{self.out_exec}/bin/genvm', + target_dir=td, + ) + self.n.register_cargo(f'{self.exec_rel}/executor/crates/common', target_dir=td) + self.n.register_cargo(f'{self.exec_rel}/executor/crates/sdk-rs', target_dir=td) + + def frozen_registry(self) -> None: + """Copy this line's committed ``executor/registry`` runner manifests verbatim. + + Frozen legacy lines ship the exact ``latest.json``/``all.json`` from their + release, which the nix build copies verbatim (see that line's + ``executor/default.nix``). The debug build must use the same files, not + recompute via the umbrella machinery, or the two builds disagree on runner + hashes. + """ + registry_dir = self.exec_root / 'executor' / 'registry' + for name in ('latest.json', 'all.json'): + out = f'{self.out_exec}/data/{name}' + self.n.build('cp', out).add_dependency(registry_dir / name).finish() + self.data_phony.add_dependency(out) + + def nix_manifests(self) -> None: + """Derive this line's `latest`/`all` runner manifests via the nix machinery. + + Manifests are derived by the umbrella machinery (which imports every active + line's runners), so both the line's and the umbrella's nix inputs matter. + Requires the `nix_eval` rule. + """ + runners_nix_inputs = glob(self.exec_root / 'runners', '**/*.nix') + glob( + self.source_dir / 'runners', '**/*.nix' + ) + for kind in ('latest', 'all'): + out = f'{self.out_exec}/data/{kind}.json' + edge = self.n.build('nix_eval', out) + edge.var('expr', runner_manifest_expr(self.real, kind)) + edge.var('wd', self.source_dir) + edge.add_implicit_dependency(runners_nix_inputs) + edge.finish() + self.data_phony.add_dependency(out) + + def install_tree(self) -> None: + """`cp` this line's install tree into `out_exec` and register its binary.""" + self.n.install(f'{self.exec_rel}/executor/install', self.out_exec, self.all_bin) + self.all_bin.add_dependency(f'{self.out_exec}/bin/genvm') + + +def configure_line_default(line: LineContext) -> None: + """Fallback per-line configuration for a line whose `.genvm-tool.py` has no + `configure` hook: the standard codegen + crates + install, resolving the + runner registry from a committed ``executor/registry`` when present, else nix. + """ + line.register_standard_codegen() + line.register_standard_crates() + if (line.exec_root / 'executor' / 'registry' / 'all.json').is_file(): + line.frozen_registry() + else: + line.nix_manifests() + line.install_tree() diff --git a/tests/runner/gvm_extra/mock_host.py b/tests/runner/gvm_extra/mock_host.py index 08081797..db2f3adc 100644 --- a/tests/runner/gvm_extra/mock_host.py +++ b/tests/runner/gvm_extra/mock_host.py @@ -114,7 +114,7 @@ async def remaining_fuel_as_gen(self) -> int: return 2**32 async def eth_call(self, account: bytes, calldata: bytes, /) -> bytes: - raise HostException(host_fns.Errors.FORBIDDEN) + raise HostException(host_fns.Errors.EVM_REVERTED) async def consume_gas(self, gas: int): pass diff --git a/tests/runner/origin/base_host.py b/tests/runner/origin/base_host.py index d053b474..9082ffea 100644 --- a/tests/runner/origin/base_host.py +++ b/tests/runner/origin/base_host.py @@ -23,6 +23,15 @@ ACCOUNT_ADDR_SIZE = 20 SLOT_ID_SIZE = 32 +# Mirrors the executor's `DebugMode` enum (crates/common/src/debug_mode.rs). +DebugMode = typing.Literal[ + 'disabled', + 'safe', + 'safe-unbounded', + 'unsafe', + 'unsafe-tracing', +] + # Default host-provided `node` fee constants (see fees.expr_prelude in # install/config/genvm.yaml). Values are strings (gas_data is Map) # and are kept minimal/deterministic for tests. `validatorsPerRound` is @@ -36,6 +45,10 @@ 'fixedProposeReceiptGas': '0', 'fixedMessageRevealGas': '0', 'genPerTimeUnit': '0', + # 0 = no per-phase timeunit floor, so default-allocation tests are unaffected. + 'minTimeUnitsPerPhase': '0', + # 0 = no per-round execution-budget floor for balance-funded messages. + 'messageBudgetFloor': '0', } @@ -94,6 +107,7 @@ class Message(typing.TypedDict): contract_address: Address sender_address: Address origin_address: Address + signer_address: Address chain_id: int value: typing.NotRequired[int] is_init: bool @@ -131,6 +145,8 @@ class PostMessageInner(typing.TypedDict): fee_params: fees.InternalMessageParams # ABI-encoded allocation subtree carried in the receipt under commitment modes. subtree: bytes + # Chain `useBalance`: fee funded from the emitting contract's balance. + use_balance: bool class DeployContractInner(typing.TypedDict): @@ -145,6 +161,8 @@ class DeployContractInner(typing.TypedDict): fee_params: fees.InternalMessageParams # ABI-encoded allocation subtree carried in the receipt under commitment modes. subtree: bytes + # Chain `useBalance`: fee funded from the emitting contract's balance. + use_balance: bool class EmitEventInner(typing.TypedDict): @@ -434,7 +452,7 @@ async def run_genvm( manager_uri: str = 'http://127.0.0.1:3999', ctx: Context, is_sync: bool, - debug_mode: str = 'disabled', + debug_mode: DebugMode = 'disabled', message: Message, host_data: str = '', gas_data: dict[str, str] | None = None, diff --git a/tests/runner/origin/host_fns.py b/tests/runner/origin/host_fns.py index 1b94961b..24024ae2 100644 --- a/tests/runner/origin/host_fns.py +++ b/tests/runner/origin/host_fns.py @@ -1,9 +1,10 @@ # This file is auto-generated. Do not edit! # fmt: off +# ruff: noqa import typing -from enum import IntEnum +from enum import IntEnum, StrEnum class Methods(IntEnum): @@ -19,9 +20,22 @@ class Methods(IntEnum): class Errors(IntEnum): OK = 0 - ABSENT = 1 + EVM_REVERTED = 1 FORBIDDEN = 2 - OUT_OF_STORAGE_GAS = 3 + +class VmErrorDetail: + __slots__ = ('value',) + def __init__(self, value: str): + self.value = value + def __str__(self) -> str: + return self.value + @staticmethod + def internal() -> 'VmErrorDetail': + return VmErrorDetail('internal') + @staticmethod + def external() -> 'VmErrorDetail': + return VmErrorDetail('external') + CURRENT_MAJOR: typing.Final[int] = 0 diff --git a/tests/runner/origin/log_asserts.py b/tests/runner/origin/log_asserts.py new file mode 100644 index 00000000..828a8d5a --- /dev/null +++ b/tests/runner/origin/log_asserts.py @@ -0,0 +1,118 @@ +"""Assertions over the executor's structured log records. + +The manager captures every executor-emitted log line and surfaces them as +``RunHostAndProgramRes.genvm_log`` β€” a list of JSON objects shaped like +``{"message": ..., "target": ..., ...}``. Capture is *unbounded* +under ``debug_mode >= safe-unbounded`` (integration tests run ``unsafe``), so no +record is evicted regardless of how chatty the run is. + +The ADR-012 load action emits one stable ``"runner load"`` record per load, +carrying: + +- ``runner`` β€” the canonical runner id (``chain:``/``custom:``/``name:hash``); +- ``runner_load_cost`` β€” the flat per-load constant + (``public_abi::memory_limiter_consts::RUNNER_LOAD_COST``); +- ``size`` β€” the charged content size (archive ``total_size``); +- ``status`` β€” ``"charged"`` (first load in this VM) or ``"cached"`` + (already in the VM's loaded set β€” free). + +This module is deliberately message-agnostic: a matcher's ``message`` defaults +to ``"runner load"`` but may name any message, so the same machinery serves +future charge-related log lines. + +Numeric fields (``size``, ``runner_load_cost``) are emitted by the executor as JSON +*strings* (the logger's default ``Display`` capture), so every comparison here +is done on the string form β€” an assertion may write ``4096`` or ``'4096'`` +interchangeably. + +Assertion schema entries may include ``match`` for subset matching over record +fields, ``runner_prefix`` to isolate runner ids such as ``custom:``, count bounds +(``count``/``min``/``max``), and ``size_is_code_len`` for init steps whose load +size must equal the raw contract code length. + +`check` returns a list of human-readable failure strings (empty == all passed). +`extract` is a convenience returning the matching records for eyeballing. +""" + +import typing + +DEFAULT_MESSAGE = 'runner load' + + +def _record_matches(record: dict, crit: dict, runner_prefix: str | None) -> bool: + for k, v in crit.items(): + if str(record.get(k)) != str(v): + return False + if runner_prefix is not None: + runner = record.get('runner') + if not isinstance(runner, str) or not runner.startswith(runner_prefix): + return False + return True + + +def _select(genvm_log: list[dict], crit: dict, runner_prefix: str | None) -> list[dict]: + crit = dict(crit) + crit.setdefault('message', DEFAULT_MESSAGE) + return [r for r in genvm_log if _record_matches(r, crit, runner_prefix)] + + +def extract( + genvm_log: list[dict], + *, + match: dict | None = None, + runner_prefix: str | None = None, +) -> list[dict]: + """Return the ``genvm_log`` records matching ``match`` (defaulting to the + ``runner load`` message) and an optional ``runner`` prefix.""" + return _select(genvm_log, match or {}, runner_prefix) + + +def check( + genvm_log: list[dict], + asserts: list[dict], + *, + code_len: int | None = None, +) -> list[str]: + """Evaluate ``asserts`` against ``genvm_log``; return a list of failures.""" + errors: list[str] = [] + for i, a in enumerate(asserts): + crit = a.get('match', {}) + runner_prefix = a.get('runner_prefix') + matched = _select(genvm_log, crit, runner_prefix) + n = len(matched) + + label = f'assert[{i}] match={crit}' + if runner_prefix is not None: + label += f' runner_prefix={runner_prefix!r}' + + if 'count' in a and n != a['count']: + errors.append(f'{label}: expected count {a["count"]}, got {n}') + if 'min' in a and n < a['min']: + errors.append(f'{label}: expected at least {a["min"]}, got {n}') + if 'max' in a and n > a['max']: + errors.append(f'{label}: expected at most {a["max"]}, got {n}') + + if a.get('size_is_code_len'): + if code_len is None: + errors.append(f'{label}: size_is_code_len set but this step ships no code') + else: + bad = [r for r in matched if str(r.get('size')) != str(code_len)] + if bad: + got = [r.get('size') for r in bad] + errors.append(f'{label}: expected size == code_len {code_len}, got {got}') + return errors + + +def summarize(genvm_log: list[dict]) -> list[dict]: + """Compact view of every ``runner load`` record, for failure context.""" + out = [] + for r in _select(genvm_log, {}, None): + out.append( + { + 'runner': r.get('runner'), + 'runner_load_cost': r.get('runner_load_cost'), + 'size': r.get('size'), + 'status': r.get('status'), + } + ) + return typing.cast(list[dict], out) diff --git a/tests/runner/origin/public_abi.py b/tests/runner/origin/public_abi.py index 65b3c07c..cc3445bb 100644 --- a/tests/runner/origin/public_abi.py +++ b/tests/runner/origin/public_abi.py @@ -1,6 +1,7 @@ # This file is auto-generated. Do not edit! # fmt: off +# ruff: noqa import typing from enum import IntEnum, StrEnum @@ -25,10 +26,16 @@ class EntryKind(IntEnum): CONSENSUS_STAGE = 2 +class Permissions(IntEnum): + CAN_USE_BALANCE_FOR_MESSAGE_FEES = 1 + + class _MemoryLimiterConsts(typing.NamedTuple): TABLE_ENTRY: int = 64 FILE_MAPPING: int = 256 FD_ALLOCATION: int = 96 + RUNNER_LOAD_COST: int = 4096 + VM_SPAWN_COST: int = 134217728 memory_limiter_consts: typing.Final = _MemoryLimiterConsts() @@ -40,6 +47,7 @@ class _RootOffsets(typing.NamedTuple): LOCKED_SLOTS: int = 3 UPGRADERS: int = 4 CODE_SLOT: int = 5 + PERMISSIONS: int = 37 root_offsets: typing.Final = _RootOffsets() @@ -52,6 +60,8 @@ class _TopLimits(typing.NamedTuple): WEB_REQUEST_MIN_SPACE: int = 65536 WEB_RENDER_MIN_SPACE: int = 134217728 MAX_FDS: int = 1024 + WASM_CALL_DEPTH: int = 1024 + WASM_STACK_VALUE_SLOTS: int = 65535 top_limits: typing.Final = _TopLimits() @@ -60,57 +70,130 @@ class SpecialMethod(StrEnum): GET_SCHEMA = '#get-schema' ERRORED_MESSAGE = '#error' -class _VmErrorOomRam: +class _VmErrorWasmTrap: @staticmethod def val() -> 'VmError': - return VmError('OOM RAM') + return VmError('wasm_trap') + @staticmethod + def unreachable() -> 'VmError': + return VmError('wasm_trap unreachable') + @staticmethod + def stack_overflow() -> 'VmError': + return VmError('wasm_trap stack_overflow') + @staticmethod + def memory_out_of_bounds() -> 'VmError': + return VmError('wasm_trap memory_out_of_bounds') + @staticmethod + def table_out_of_bounds() -> 'VmError': + return VmError('wasm_trap table_out_of_bounds') + @staticmethod + def indirect_call_to_null() -> 'VmError': + return VmError('wasm_trap indirect_call_to_null') + @staticmethod + def bad_signature() -> 'VmError': + return VmError('wasm_trap bad_signature') + @staticmethod + def integer_overflow() -> 'VmError': + return VmError('wasm_trap integer_overflow') @staticmethod - def table() -> 'VmError': - return VmError('OOM RAM table') + def integer_divide_by_zero() -> 'VmError': + return VmError('wasm_trap integer_divide_by_zero') @staticmethod - def memory() -> 'VmError': - return VmError('OOM RAM memory') + def bad_conversion_to_integer() -> 'VmError': + return VmError('wasm_trap bad_conversion_to_integer') @staticmethod - def limit() -> 'VmError': - return VmError('OOM RAM limit') + def heap_misaligned() -> 'VmError': + return VmError('wasm_trap heap_misaligned') + @staticmethod + def atomic_wait_non_shared_memory() -> 'VmError': + return VmError('wasm_trap atomic_wait_non_shared_memory') + @staticmethod + def out_of_fuel() -> 'VmError': + return VmError('wasm_trap out_of_fuel') + @staticmethod + def interrupt() -> 'VmError': + return VmError('wasm_trap interrupt') + @staticmethod + def nondet_instruction() -> 'VmError': + return VmError('wasm_trap nondet_instruction') + @staticmethod + def fault() -> 'VmError': + return VmError('wasm_trap fault') -class _VmErrorOomReceiptMessage: +class _VmErrorOutOfMemory: @staticmethod - def internal() -> 'VmError': - return VmError('OOM receipt message internal') + def val() -> 'VmError': + return VmError('out_of memory') + @staticmethod + def wasm_memory() -> 'VmError': + return VmError('out_of memory wasm_memory') @staticmethod - def external() -> 'VmError': - return VmError('OOM receipt message external') + def wasm_table() -> 'VmError': + return VmError('out_of memory wasm_table') -class _VmErrorOomReceipt: +class _VmErrorOutOfReceipt: @staticmethod def nondet_output() -> 'VmError': - return VmError('OOM receipt nondet_output') + return VmError('out_of receipt nondet_output') @staticmethod - def message() -> '_VmErrorOomReceiptMessage': - return _VmErrorOomReceiptMessage() + def message() -> 'VmError': + return VmError('out_of receipt message') + @staticmethod + def event() -> 'VmError': + return VmError('out_of receipt event') -class _VmErrorOomFees: +class _VmErrorOutOfMessageFee: @staticmethod - def internal() -> 'VmError': - return VmError('OOM fees internal') + def total() -> 'VmError': + return VmError('out_of message_fee total') @staticmethod - def external() -> 'VmError': - return VmError('OOM fees external') + def node() -> 'VmError': + return VmError('out_of message_fee node') -class _VmErrorOom: +class _VmErrorOutOf: @staticmethod def storage() -> 'VmError': - return VmError('OOM storage') + return VmError('out_of storage') + @staticmethod + def vm_recursion() -> 'VmError': + return VmError('out_of vm_recursion') @staticmethod - def ram() -> '_VmErrorOomRam': - return _VmErrorOomRam() + def nondet_blocks() -> 'VmError': + return VmError('out_of nondet_blocks') @staticmethod - def receipt() -> '_VmErrorOomReceipt': - return _VmErrorOomReceipt() + def locked_slots() -> 'VmError': + return VmError('out_of locked_slots') @staticmethod - def fees() -> '_VmErrorOomFees': - return _VmErrorOomFees() + def upgraders() -> 'VmError': + return VmError('out_of upgraders') + @staticmethod + def fds() -> 'VmError': + return VmError('out_of fds') + @staticmethod + def memory() -> '_VmErrorOutOfMemory': + return _VmErrorOutOfMemory() + @staticmethod + def receipt() -> '_VmErrorOutOfReceipt': + return _VmErrorOutOfReceipt() + @staticmethod + def message_fee() -> '_VmErrorOutOfMessageFee': + return _VmErrorOutOfMessageFee() + +class _VmErrorFee: + @staticmethod + def no_matching_node() -> 'VmError': + return VmError('fee no_matching_node') + @staticmethod + def below_minimum() -> 'VmError': + return VmError('fee below_minimum') + @staticmethod + def too_many_rounds() -> 'VmError': + return VmError('fee too_many_rounds') + +class _VmErrorEvm: + @staticmethod + def reverted() -> 'VmError': + return VmError('evm reverted') class _VmErrorInvalidContractWasm: @staticmethod @@ -148,16 +231,6 @@ class _VmErrorExitCode: def val_i32(v: int) -> 'VmError': return VmError(f'exit_code {v}') -class _VmErrorWasmTrap: - @staticmethod - def val_str(v: str) -> 'VmError': - return VmError(f'wasm_trap {v}') - -class _VmErrorHost: - @staticmethod - def val_str(v: str) -> 'VmError': - return VmError(f'host {v}') - class VmError: __slots__ = ('value',) def __init__(self, value: str): @@ -171,20 +244,26 @@ def timeout() -> 'VmError': def absent_leader_nondet_output() -> 'VmError': return VmError('absent_leader_nondet_output') @staticmethod + def host_forbidden() -> 'VmError': + return VmError('host_forbidden') + @staticmethod def exit_code() -> '_VmErrorExitCode': return _VmErrorExitCode() @staticmethod def wasm_trap() -> '_VmErrorWasmTrap': return _VmErrorWasmTrap() @staticmethod - def oom() -> '_VmErrorOom': - return _VmErrorOom() + def out_of() -> '_VmErrorOutOf': + return _VmErrorOutOf() + @staticmethod + def fee() -> '_VmErrorFee': + return _VmErrorFee() + @staticmethod + def evm() -> '_VmErrorEvm': + return _VmErrorEvm() @staticmethod def invalid_contract() -> '_VmErrorInvalidContract': return _VmErrorInvalidContract() - @staticmethod - def host() -> '_VmErrorHost': - return _VmErrorHost() diff --git a/tests/system/permits/test.py b/tests/system/permits/test.py index 8605ffa5..23c18b7d 100644 --- a/tests/system/permits/test.py +++ b/tests/system/permits/test.py @@ -26,6 +26,7 @@ 'contract_address': Address('0x' + '00' * 20), 'sender_address': Address('0x' + '01' * 20), 'origin_address': Address('0x' + '01' * 20), + 'signer_address': Address('0x' + '01' * 20), 'chain_id': 0, 'is_init': True, } diff --git a/tests/templates/message.json b/tests/templates/message.json index 17cedda9..08ef0755 100644 --- a/tests/templates/message.json +++ b/tests/templates/message.json @@ -2,6 +2,7 @@ "contract_address": "AQAAAAAAAAAAAAAAAAAAAAAAAAA=", "sender_address": "AgAAAAAAAAAAAAAAAAAAAAAAAAA=", "origin_address": "AgAAAAAAAAAAAAAAAAAAAAAAAAA=", + "signer_address": "AgAAAAAAAAAAAAAAAAAAAAAAAAA=", "chain_id": 0, "value": 0, "is_init": false From 12f85fc02dd7f56c5808f21187e1ed94a90caf25 Mon Sep 17 00:00:00 2001 From: kp2pml30 Date: Fri, 17 Jul 2026 00:37:11 +0900 Subject: [PATCH 03/45] chore: CI rework MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: revive release CI * fix(test): discover executor crates by recursing submodules πŸ› collect_rust finds Cargo.toml/fuzz targets by scanning Context.git_files, which ran a plain 'git ls-files'. Since the monorepo split each executors/.x is a submodule, so plain ls-files stops at the boundary and only the two manager crates are seen β€” every executor rust crate (calldata and its fuzz targets included) silently went untested, and executors carry no tests() hook of their own. Recurse into submodules (uninitialized ones are skipped) to restore pre-split discovery: 2 -> 16 crates. * fix(build): resolve a Build passed as a dependency to its output πŸ› A Build used as an add_dependency/output reference β€” e.g. all/bin depending on the all/manager and all/executor/ phony aggregators β€” fell through _scalar's type switch to str(build), emitting the object's default repr ('') as a literal ninja path, so 'ninja all/bin' failed with 'missing and no known rule to make it'. Handle Build in _scalar by referencing its primary output (enough to force a multi-output edge), so passing a Build anywhere a target reference is expected just works. * fix(ci): retry transient gh reads in the executor-PR precondition πŸ”’ check-executor-prs went red because 'gh api repos/.../pulls/4' came back with an empty body, which gh surfaces as 'unexpected end of JSON input'; manager_info cannot tolerate a miss, so that single hiccup failed the whole landability gate (and cascaded into gate-check and validate-end, skipping the test matrix). Every gh call here is an idempotent read, so wrap the gh helper in a bounded retry with linear backoff for transient signals (empty body, 5xx, rate-limit, network). A terminal 4xx β€” including the 404 that allow_missing callers expect for an absent executor line β€” is not retried, so that path stays fast, and an exhausted retry still raises exactly as before. * fix(ci): stop the PR-diff fetch from recursing submodules πŸ› ci-changes.py reads gitlink SHAs out of the superproject trees via git ls-tree and never touches a submodule object, but its fetch inherited git's default fetch.recurseSubmodules=on-demand once get-all-git.py had populated the submodules. On-demand recursion then tried to fetch every executor gitlink the PR range touches; a historical commit in the range pins an executor commit that was never pushed to its remote, so the whole fetch aborted with 'upload-pack: not our ref'. Pass --no-recurse-submodules so only the superproject refs are fetched. * chore(ci): run full tests as a planned matrix of reusable cells ♻️ Replace the fixed module-test-* jobs with plan-queue-matrix (emits the cell matrix, marker-gated) fanned out through a reusable test_cell.yaml; each cell is declared once as DefaultStepInfo (genvm-tool test filters + per-cell runner setup) and executed by test-cell. Supporting granularity: split ninja `all/bin` into `all/manager` + `all/executor/` and add an `all/runners` phony so cells build only what they need; a `minimal` devshell for `genvm-tool configure`; tag the genlayer-py-std suite so `--filter-tag` selects it. * feat(ci): executor mirror PR provisioning and a manager-PR landability gate ✨ Adds `pr-branches`, a CI tool over a manager PR and every executor line it ships: - pr_branches_info: per-repo base/head sha, ahead/behind, mirror-PR url and rebased/synced/moved flags, all read through the gh API (no checkout) - subcommands show-info, provision (force-push each moved line's mirror branch to its pinned commit and open/reuse its PR) and check - queue gate check-executor-prs (every repo rebased, every executor's pinned commit present so it can land), aggregated by gate-check before the heavy jobs - a "Provision executor PRs" panel checkbox that dispatches branch_provision_executor_prs.yaml Panel and merge now run their scripts from the PR branch under a ci-safe guard, so a PR exercises its own support/ci. * chore: split docs, rework ci * fix(ci): harden the merge squash against review findings πŸ”’ A review of the squash-on-merge change surfaced several ways it could corrupt a protected branch: - The re-tick provenance lookup grepped the whole commit message, and the body is the raw PR body. A PR quoting `GenVM-Squashed-From: ` (a revert quoting what it reverts) could make a squash record bogus provenance, so a later PR pinning that sha would push nothing and repoint the manager gitlink at the wrong commit. Strip that trailer from the body and normalize CRLF while there (commit-tree does no cleanup, unlike git commit, so the executor and manager messages for one PR would otherwise differ byte-for-byte). - The executor squash message referenced the manager PR as a bare `#N`, which resolves to an unrelated issue in the executor repo; qualify it as owner/repo#N. - coauthor_trailers claimed to filter AI/tool attribution but did not, so a bot commit in the range could land a Co-authored-by the commit-message hook rejects β€” and nothing re-validates a generated squash message. Filter it. - Nothing stopped the manager gitlink moving backward: the 0-behind gate is on commits, not the pointer, so a PR current on commits but stale on the gitlink could silently revert another PR`s executor work. Refuse a landed commit that is an ancestor of the base`s current gitlink. - A hand-merged executor PR (right tree, new sha, no trailer) used to wedge the line as diverged; recognise it by tree equality and mirror the tip. Also preserve the author date (was becoming job time), parse author identity NUL-separated (names with `<`/spaces), and correct the now-stale strategy comment in branch_merge_into_dev.yaml. * fix(docs): build the docs site again after the repo split πŸ“š The docs pipeline had been broken since the split, in three places. generate.py read flake-config.json, which no longer exists, and passed commitToTagStr / build-config-str to runners/docs.nix, which no longer takes arguments β€” each runner now carries the version of the line it came from, so docs.nix groups by that. It also resolved its own directory through Path("__file__") β€” the literal string, not the file β€” so it only ever worked from one cwd. docs.nix assumed every runner has a `hash`. Only the forward-rolling lines do; the legacy v0.2 line exposes just `uid`, so adding that line made the eval fail. The hash the executor puts in the on-disk path is in `uid` either way. The runner pages are built from git: tags, tag messages, and the log between two runner versions. A runner version is an EXECUTOR version, tagged in the executor repo, with its sources under that repo`s runners/ β€” but generate.py asked the manager, whose history contains no executor commits and whose tags are manager versions. It never matched, so the changelog was silently empty. Ask the executor checkout instead. It stays empty until the executor repo has tags, which the reworked release now creates. Finally, the gen-docs shell needs `?submodules=1` like every other nix invocation here, since the manager flake imports the executor lines. * feat(ci): squash executor commits on merge and repoint the manager gitlink πŸ”€ Merge now leaves ONE commit per merged PR in every repo. Each executor line had its commits fast-forwarded onto its dev branch, so the executor history kept every intermediate commit while the manager was squashed. Both sides are squashed now, and since a squashed line lands under a NEW sha, the manager commit is rewritten to gitlink that commit rather than the one the PR pinned, which never lands. A line that is unchanged, or already a single commit on its dev tip, is left alone and keeps its sha. Squashing collapses several authors into one, so Co-authored-by trailers are added for the other human authors in the range. No tool/AI attribution β€” the commit-message hook rejects it. The squashed executor commit carries a GenVM-Squashed-From trailer naming the commit whose content it holds. Without it, re-ticking Merge after a rejected manager push (executors are pushed first, so they are already on the dev branch) would find a tip that is neither an ancestor nor a descendant of the pinned sha β€” squashing keeps only the tree β€” and would block the PR as diverged, permanently. The trailer is how the second run recognises its own work, reuses that commit and pushes nothing. * feat(install): download executors, add --use-patchelf, rename to genvm-post-install πŸ“₯ The manager bundle no longer ships the executors, so post-install can no longer assume they are on disk. It now downloads each line the manifest lists from that line`s own release, keyed by its executor-version, and unpacks it at the install root (the tarball is already laid out as executor//...). The URL template is a manifest field, next to the runner one. A failed download is fatal only when a missing executor is (--error-on-missing-executor). --use-patchelf sets the ELF interpreter with patchelf instead of lief. lief is a heavy binary wheel pulled in for that one job: with the flag it is neither imported nor installed into the venv, since the wrapper drops it from the requirements (the venv is keyed by their hash, so the two variants get separate venvs). Default stays lief. patchelf missing from PATH is a hard error; non-ELF and static binaries are skipped, as with lief. The entry point is now bin/genvm-post-install, matching bin/genvm-manager and bin/genvm-modules. * fix(ci): make the executor release honest about what it publishes πŸ”’ Three bugs a review of the release split turned up: Per-line executor targets prefetched no dependencies at all. The registry keys deps by platform (`executor-amd64-linux`), so `executor-v0.3-amd64-linux` matched nothing and the prefetch was a silent no-op β€” masked only because the manager target, whose dep set happens to be identical, is built first in the same job. Match the per-line target against the platform key. A line whose release already existed was skipped without checking WHERE that release sits. An executor commit that moves the gitlink without bumping executor-version would leave the manager pinning commit X while the published executor for that version was built from commit Y. Skip only when the existing release is on the pinned commit, and hard-fail otherwise. The legacy line was published as a full release, so GitHub marked v0.2.17 "Latest" ahead of the v0.3 line. Both lines publish identically named assets, so /releases/latest/download/... served the legacy executor. Only the head line claims Latest now. Also drop `actions: write` from the reusable build workflow: a called workflow cannot exceed the caller (`actions: read`), and uploading an artifact does not need it. * ci(release): publish executors to the executor repo, runners as -universal πŸ“¦ Split the release along repo lines now that the monorepo is split. The manager release keeps the three platform bundles and gains genvm-universal (the runners, which are platform-independent); it no longer carries the executors. Each active executor line is released into the executor repo instead, at its own executor-version and at the commit the manager pins as its gitlink. A line that has not moved keeps its existing release. Everything is still BUILT here: the executor is deliberately not a build entry point (the umbrella flake feeds it compiled-libs/deps/support), so the built assets are pushed across with EXECUTOR_RELEASE_TOKEN β€” GITHUB_TOKEN cannot write to another repo. The manager is tagged only after the executor releases land. Platforms are now named as nix names them (amd64-linux, not linux-amd64) everywhere: targets, jobs and published assets. incl_build_trg takes a list of targets rather than a primary/secondary pair, and uploads them as one artifact per job; the targets of a platform share a nix store, so building the manager and every executor line together is no more expensive than separate jobs. * ci(hooks): check commit messages across manager and executors πŸ“ get-src now emits a `changes` output: a JSON object mapping each repo (`.` for the manager, path for each executor submodule) to its base_commit, branch_commit and has_changes. A submodule has no history the manager can see, so its endpoints are the gitlinks recorded on either side of the PR. Outside a pull request base == branch and has_changes is false. Use it to lint commit messages in incl_initial: `check-commit-message` is a commit-msg-stage hook, so the CI pre-commit run (--hook-stage pre-commit) never fired it and no commit message was checked on CI at all. Every commit a PR adds is now checked, in the manager and in each executor that moved, each against its own copy of the script. * chore(ci): match all dev base branches in coderabbit 🐰 CodeRabbit reads .coderabbit.yaml from the PR head branch, and declaring base_branches overrides the org default. The `v\d+\.\d+(-dev)?` pattern did not match `v0.6-dev`, so PRs to the dev branch were auto-skipped (the review on #4 was skipped, while #3 branched before the pattern landed and was reviewed). Spell the suffixes out as separate alternatives instead of an optional group, and do the same for the executors, whose dev branches were unmatched too. * ci(tests): emit GitHub Actions error annotations * chore(ci): keep submodules referenced to cache πŸ”¨ * ci(workflows): use pipeline-local artifacts * chore(ci): cache submodule checkouts πŸ”¨ * ci: group pipeline logs and make nix artifacts symlink trees πŸ”— Wrap each CI pipeline stage in GitHub Actions ::group::/::endgroup:: sections for collapsible logs (never wrapping a nix develop that re-emits its own groups, since Actions groups cannot nest). Make the top-level combining derivations (runner trees, executor/manager bundles) symlink trees into the store via cp -rsf instead of copying, keeping their store paths as runtime deps; materialize real files only at the upload tar (find -L plus tar --dereference) and symlink the built runners under build/out/runners in the rust-test flow. * chore: replace in-house commit-hook engine with git-hooks.nix πŸͺ Each repo (manager + every executor submodule) now defines its own git-hooks.nix config in its flake, pinned to that repo's toolchain, replacing the bespoke genvm-tool hook engine and the per-repo support/nix/precommit tool flakes. Expose a formatter output so nix fmt runs the same generated hooks in the working tree. CI runs support/ci/pipelines/commit-hooks.sh, which runs pre-commit in-tree per repo β€” the executor crates reach the manager's shared crates via relative paths that only resolve in the nested checkout, so a sandboxed nix flake check cannot run their cargo-fmt. Remove the hook package, the hooks() project functions and now-dead helpers; bump the executor gitlinks. * fix(web): exempt webdriver connection from SSRF dns filter πŸ› * fix(post-install): reject wrong-OS executor before trusting it πŸ”’οΈ A wrong-platform binary shipped in a tarball was silently accepted and only surfaced as an opaque `Exec format error` at runtime. Sniff the ELF/Mach-O header and error on a clear mismatch with the target OS. --- .claude/skills/commit-style/SKILL.md | 8 +- .claude/skills/macos/SKILL.md | 2 +- .coderabbit.yaml | 9 +- .genvm-monorepo-root | 2 +- .genvm-tool.py | 201 +-- .github/actions/get-src/action.yaml | 43 +- .github/actions/matrix-cell-step/action.yaml | 22 + .github/actions/postmortem/action.yaml | 132 ++ .github/workflows/branch_executor_prs.yaml | 4 +- .github/workflows/branch_forward.yaml | 2 +- .github/workflows/branch_merge_into_dev.yaml | 55 +- .github/workflows/branch_pr_actions.yaml | 21 +- .github/workflows/branch_pr_checklist.yaml | 2 +- .github/workflows/branch_provision.yaml | 4 +- .../branch_provision_executor_prs.yaml | 64 + .github/workflows/branch_retarget.yaml | 2 +- .github/workflows/incl_build_trg.yaml | 118 -- .github/workflows/incl_initial.yaml | 32 +- .../workflows/incl_release_build_test.yaml | 128 ++ .../incl_release_build_test_cell_build.yaml | 72 + .../incl_release_build_test_cell_test.yaml | 114 ++ .github/workflows/queue.yaml | 181 +-- .github/workflows/queue_test_cell.yaml | 117 ++ .github/workflows/release.yaml | 325 ++--- .gitignore | 1 + crates/modules-interfaces/Cargo.lock | 2 +- crates/modules-interfaces/Cargo.toml | 2 +- docs/contributing/howto/building/docs.md | 41 +- docs/contributing/howto/building/runners.md | 4 +- .../howto/committing/submodules.md | 36 +- docs/contributing/howto/genvm-tool.md | 1 - .../howto/releasing/release-build.md | 30 +- docs/contributing/howto/setup.md | 26 +- docs/website/.gitignore | 4 +- docs/website/generate.py | 171 ++- docs/website/merge-txts.rb | 19 - docs/website/merge_txts.py | 26 + docs/website/poetry.lock | 1222 ----------------- docs/website/pyproject.toml | 20 - docs/website/src/conf.py | 161 +-- .../impl-spec/02-vm/02-version-management.rst | 5 +- .../src/impl-spec/03-greyboxing/01-lua-api.md | 480 ------- .../src/impl-spec/03-greyboxing/index.rst | 2 - docs/website/src/impl-spec/appendix/index.rst | 1 - .../src/impl-spec/appendix/manager-api.rst | 5 +- docs/website/src/index.rst | 1 - docs/website/src/overview/index.rst | 8 + .../src/python-sdk/changelog-notes/v0.1.3.rst | 113 -- .../src/python-sdk/changelog-notes/v0.1.8.rst | 19 - .../src/python-sdk/changelog-notes/v0.3.rst | 348 ----- .../src/python-sdk/guides/boot_process.rst | 73 - .../src/python-sdk/guides/floating_point.rst | 14 - docs/website/src/python-sdk/guides/index.rst | 7 - docs/website/src/python-sdk/index.rst | 14 - docs/website/src/python-sdk/introduction.rst | 15 - .../src/python-sdk/reference/genlayer.rst | 127 -- .../reference/genlayer_embeddings.rst | 8 - .../src/python-sdk/reference/index.rst | 7 - docs/website/src/python-sdk/runners/index.rst | 38 - .../03-wasi_genlayer_sdk/02-gl_call.rst | 2 +- docs/website/yabuild.rb | 11 - executors/v0.2.x | 2 +- executors/v0.3.x | 2 +- flake.lock | 38 + flake.nix | 375 ++++- implementation/Cargo.lock | 5 +- implementation/Cargo.toml | 4 +- implementation/tests/web_render_webdriver.rs | 196 +++ install/bin/genvm-post-install | 102 ++ install/bin/post-install.py | 75 +- install/config/genvm-web-default.lua | 1 + install/lib/python/post-install/__main__.py | 177 ++- libs/deps/dependency-urls.json | 14 +- runners/docs.nix | 9 +- support/ci/README.md | 10 + support/ci/__main__.py | 60 + support/ci/branch-versions.py | 60 - support/ci/cargo-clippy.sh | 25 - support/ci/check-versions.py | 205 --- support/ci/ci_lib.py | 260 ++++ support/ci/genvm-merge-into-dev.py | 367 ----- support/ci/pipelines/README.md | 9 - support/ci/pipelines/_common.sh | 7 - support/ci/pipelines/build.py | 100 ++ support/ci/pipelines/build.sh | 114 -- support/ci/pipelines/checks.py | 168 +++ support/ci/pipelines/docs.py | 120 ++ support/ci/pipelines/docs.sh | 11 - support/ci/pipelines/release.py | 299 ++++ support/ci/pipelines/src/docs.sh | 18 - support/ci/pipelines/src/test-python.sh | 13 - support/ci/pipelines/src/test-rust-fuzz.sh | 9 - support/ci/pipelines/src/test-rust.sh | 15 - support/ci/pipelines/test-python.sh | 6 - support/ci/pipelines/test-rust-fuzz.sh | 9 - support/ci/pipelines/test-rust.sh | 38 - support/ci/pipelines/tests.py | 444 ++++++ support/ci/pr_branches_info.py | 357 +++++ support/ci/provision-branches.sh | 81 -- support/ci/run.sh | 7 + support/ci/tools/__init__.py | 0 support/ci/tools/branches.py | 118 ++ support/ci/tools/genvm_merge_into_dev.py | 585 ++++++++ .../make_release_notes.py} | 15 +- .../open_executor_prs.py} | 15 +- .../pr_action_panel.py} | 23 + support/ci/tools/pr_branches.py | 243 ++++ support/ci/tools/script_adapters.py | 44 + support/ci/tools/versions.py | 188 +++ support/manifest-base.yaml | 5 + support/nix/precommit/flake.lock | 61 - support/nix/precommit/flake.nix | 47 - support/runner-script.py | 23 + support/scripts/ci-changes.py | 154 +++ support/scripts/get-all-git.py | 273 ++++ .../mdlinks.py => scripts/md-local-links.py} | 28 +- support/tools/genvm-tool/default.nix | 2 +- .../tools/genvm-tool/genvm_tool/__main__.py | 26 +- .../genvm-tool/genvm_tool/cmd_configure.py | 45 +- .../tools/genvm-tool/genvm_tool/cmd_test.py | 4 +- .../genvm-tool/genvm_tool/codegen/rust.py | 2 + support/tools/genvm-tool/genvm_tool/common.py | 52 +- .../genvm_tool/git/check_for_push.py | 38 +- .../genvm-tool/genvm_tool/hook/__init__.py | 7 - .../genvm-tool/genvm_tool/hook/engine.py | 300 ---- .../genvm-tool/genvm_tool/hook/install.py | 109 -- .../tools/genvm-tool/genvm_tool/hook/run.py | 98 -- .../genvm-tool/genvm_tool/tests/__init__.py | 10 +- .../tools/genvm-tool/genvm_tool/tests/cli.py | 3 + .../genvm_tool/tests/stage/execution.py | 7 + support/tools/git-third-party/git-third-party | 269 +++- tests/runner/genvm_tool_plugins/cargo.py | 1 + .../runner/genvm_tool_plugins/integration.py | 21 + tests/runner/genvm_tool_plugins/ninja.py | 9 +- 134 files changed, 5875 insertions(+), 5486 deletions(-) create mode 100644 .github/actions/matrix-cell-step/action.yaml create mode 100644 .github/actions/postmortem/action.yaml create mode 100644 .github/workflows/branch_provision_executor_prs.yaml delete mode 100644 .github/workflows/incl_build_trg.yaml create mode 100644 .github/workflows/incl_release_build_test.yaml create mode 100644 .github/workflows/incl_release_build_test_cell_build.yaml create mode 100644 .github/workflows/incl_release_build_test_cell_test.yaml create mode 100644 .github/workflows/queue_test_cell.yaml delete mode 100755 docs/website/merge-txts.rb create mode 100644 docs/website/merge_txts.py delete mode 100644 docs/website/poetry.lock delete mode 100644 docs/website/pyproject.toml delete mode 100644 docs/website/src/impl-spec/03-greyboxing/01-lua-api.md delete mode 100644 docs/website/src/python-sdk/changelog-notes/v0.1.3.rst delete mode 100644 docs/website/src/python-sdk/changelog-notes/v0.1.8.rst delete mode 100644 docs/website/src/python-sdk/changelog-notes/v0.3.rst delete mode 100644 docs/website/src/python-sdk/guides/boot_process.rst delete mode 100644 docs/website/src/python-sdk/guides/floating_point.rst delete mode 100644 docs/website/src/python-sdk/guides/index.rst delete mode 100644 docs/website/src/python-sdk/index.rst delete mode 100644 docs/website/src/python-sdk/introduction.rst delete mode 100644 docs/website/src/python-sdk/reference/genlayer.rst delete mode 100644 docs/website/src/python-sdk/reference/genlayer_embeddings.rst delete mode 100644 docs/website/src/python-sdk/reference/index.rst delete mode 100644 docs/website/src/python-sdk/runners/index.rst delete mode 100644 docs/website/yabuild.rb create mode 100644 implementation/tests/web_render_webdriver.rs create mode 100755 install/bin/genvm-post-install mode change 100755 => 120000 install/bin/post-install.py create mode 100644 support/ci/README.md create mode 100644 support/ci/__main__.py delete mode 100755 support/ci/branch-versions.py delete mode 100755 support/ci/cargo-clippy.sh delete mode 100755 support/ci/check-versions.py create mode 100644 support/ci/ci_lib.py delete mode 100755 support/ci/genvm-merge-into-dev.py delete mode 100644 support/ci/pipelines/README.md delete mode 100755 support/ci/pipelines/_common.sh create mode 100644 support/ci/pipelines/build.py delete mode 100755 support/ci/pipelines/build.sh create mode 100644 support/ci/pipelines/checks.py create mode 100644 support/ci/pipelines/docs.py delete mode 100755 support/ci/pipelines/docs.sh create mode 100644 support/ci/pipelines/release.py delete mode 100644 support/ci/pipelines/src/docs.sh delete mode 100755 support/ci/pipelines/src/test-python.sh delete mode 100755 support/ci/pipelines/src/test-rust-fuzz.sh delete mode 100755 support/ci/pipelines/src/test-rust.sh delete mode 100755 support/ci/pipelines/test-python.sh delete mode 100755 support/ci/pipelines/test-rust-fuzz.sh delete mode 100755 support/ci/pipelines/test-rust.sh create mode 100644 support/ci/pipelines/tests.py create mode 100644 support/ci/pr_branches_info.py delete mode 100755 support/ci/provision-branches.sh create mode 100755 support/ci/run.sh create mode 100644 support/ci/tools/__init__.py create mode 100644 support/ci/tools/branches.py create mode 100755 support/ci/tools/genvm_merge_into_dev.py rename support/ci/{make-relase-notes.py => tools/make_release_notes.py} (93%) rename support/ci/{open-executor-prs.py => tools/open_executor_prs.py} (93%) rename support/ci/{pr-action-panel.py => tools/pr_action_panel.py} (86%) create mode 100644 support/ci/tools/pr_branches.py create mode 100644 support/ci/tools/script_adapters.py create mode 100644 support/ci/tools/versions.py delete mode 100644 support/nix/precommit/flake.lock delete mode 100644 support/nix/precommit/flake.nix create mode 100755 support/scripts/ci-changes.py create mode 100755 support/scripts/get-all-git.py rename support/{tools/genvm-tool/genvm_tool/hook/mdlinks.py => scripts/md-local-links.py} (74%) delete mode 100644 support/tools/genvm-tool/genvm_tool/hook/__init__.py delete mode 100644 support/tools/genvm-tool/genvm_tool/hook/engine.py delete mode 100644 support/tools/genvm-tool/genvm_tool/hook/install.py delete mode 100644 support/tools/genvm-tool/genvm_tool/hook/run.py diff --git a/.claude/skills/commit-style/SKILL.md b/.claude/skills/commit-style/SKILL.md index ceae8050..6857ba4c 100644 --- a/.claude/skills/commit-style/SKILL.md +++ b/.claude/skills/commit-style/SKILL.md @@ -136,7 +136,7 @@ the diff and not a bullet list of squashed sub-commits. A change that touches an executor submodule spans repos: commit inside the submodule first, then bump the manager's gitlink (`git add executors/.x`) in a manager commit. Keep each commit's files coherent and don't let a -submodule's linter reformat leak an unrelated file into a commit. The cross-repo -pre-commit hook (`genvm-tool hook run`) fans out to all repos; use `--no-verify` -for pure gitlink bumps. Full workflow (order, pushing, `--force-with-lease` after -a rebase): `/submodules`. +submodule's linter reformat leak an unrelated file into a commit. Each repo's +per-repo pre-commit hook (git-hooks.nix, in its flake) runs on the repo you +commit in. Do not `--no-verify` (even for gitlink bumps β€” let the hooks run). +Full workflow (order, pushing, `--force-with-lease` after a rebase): `/submodules`. diff --git a/.claude/skills/macos/SKILL.md b/.claude/skills/macos/SKILL.md index 45a83eb3..b465bc46 100644 --- a/.claude/skills/macos/SKILL.md +++ b/.claude/skills/macos/SKILL.md @@ -91,7 +91,7 @@ You usually do not need to build runners at all. Download them instead (works on any platform): ```bash -nix develop '.?submodules=1#full' --command python3 build/out/bin/post-install.py \ +nix develop '.?submodules=1#full' --command python3 build/out/bin/genvm-post-install \ --create-venv false --default-step false \ --runners-download true --error-on-missing-executor false ``` diff --git a/.coderabbit.yaml b/.coderabbit.yaml index 960bc334..7c65a682 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -1,9 +1,12 @@ reviews: - # Review PRs targeting the long-lived version branches (e.g. v0.6, v0.6-dev) - # and main; feature/topic branches are reviewed via their target base. + # Auto-review PRs targeting any long-lived branch: main, a release branch + # (v0.6) or its dev branch (v0.6-dev). Feature/topic branches are reviewed + # through the base they target. Patterns are regexes; keep them free of + # optional groups, an `(-dev)?` suffix here did not match `v0.6-dev`. base_branches: - "main" - - "v\\d+\\.\\d+(-dev)?" + - "v\\d+\\.\\d+" + - "v\\d+\\.\\d+-dev" path_filters: - "!**/*.onnx" - "!**/*.txt" diff --git a/.genvm-monorepo-root b/.genvm-monorepo-root index e97bae45..f691af1d 100644 --- a/.genvm-monorepo-root +++ b/.genvm-monorepo-root @@ -1,5 +1,5 @@ { - "version": "v0.6.0-rc0", + "version": "v0.6.0-rc1", "active-versions": ["v0.3", "v0.2"], "artifacts_dir": "build/test-artifacts", "extra_python_paths": ["tests/runner"] diff --git a/.genvm-tool.py b/.genvm-tool.py index 5ac477fd..9d82383c 100644 --- a/.genvm-tool.py +++ b/.genvm-tool.py @@ -3,9 +3,6 @@ Loaded once by genvm-tool (``common.load_project``) before any subcommand runs; subcommands ask it for what they need: -- ``hooks(ctx)`` β€” the manager's commit-hook definitions (was - ``support/nix/precommit/hooks.toml``), returned in the same shape the hook - engine already understands. - ``tests(ctx)`` β€” the umbrella test suite (was the top-level ``.ya-test.py``): imports the plugins it needs and registers collectors + CLI args on the runner's configuration ``Context``. This is the plugins' "initial run" hook; @@ -17,189 +14,10 @@ applies before calling us. General runner config (``artifacts_dir`` / ``extra_python_paths``) lives in -``.genvm-monorepo-root``. Each executor carries its own ``.genvm-tool.py`` with -its own ``hooks``. +``.genvm-monorepo-root``. Each executor carries its own ``.genvm-tool.py``. +Commit hooks live in each repo's flake (git-hooks.nix), not here. """ -import genvm_tool - - -def hooks(ctx: 'genvm_tool.common.Context'): - """Manager commit-hook definitions (was support/nix/precommit/hooks.toml). - - Tools resolve from the sibling flake's buildEnv (``nix = ""``); - ``local`` hooks run a repo-owned script and ``builtin`` hooks run logic baked - into genvm-tool. A hook's ``args`` are its check-mode invocation; an optional - ``fix_args`` is the rewrite-in-place invocation used when ``hook run`` fixes - (the default β€” CI passes ``--check``). ``files``/``exclude`` are repo-relative - regexes. - """ - return [ - # --- generic checks (mirror the executor's) ------------------------ - { - 'id': 'trailing-whitespace', - 'nix': 'pre-commit-hooks', - 'entry': 'trailing-whitespace-fixer', - 'types_or': ['text'], - 'exclude': r'^\.git-third-party|/fuzz/', - }, - { - 'id': 'end-of-file-fixer', - 'nix': 'pre-commit-hooks', - 'entry': 'end-of-file-fixer', - 'types_or': ['text'], - 'exclude': r'^\.git-third-party|/fuzz/', - }, - { - 'id': 'check-added-large-files', - 'nix': 'pre-commit-hooks', - 'entry': 'check-added-large-files', - }, - { - 'id': 'check-json', - 'nix': 'pre-commit-hooks', - 'entry': 'check-json', - 'types_or': ['json'], - 'exclude': r'(^\.git-third-party)|(/tsconfig\.json$)', - }, - { - 'id': 'check-yaml', - 'nix': 'pre-commit-hooks', - 'entry': 'check-yaml', - 'types_or': ['yaml'], - }, - { - 'id': 'check-toml', - 'nix': 'pre-commit-hooks', - 'entry': 'check-toml', - 'types_or': ['toml'], - }, - { - 'id': 'toml-format', - 'nix': 'taplo', - 'entry': 'taplo', - 'args': ['format', '--check'], - 'fix_args': ['format'], - 'types_or': ['toml'], - 'pass_filenames': True, - }, - { - 'id': 'check-merge-conflict', - 'nix': 'pre-commit-hooks', - 'entry': 'check-merge-conflict', - 'types_or': ['text'], - }, - { - # Matches both the hidden vendored `.git-third-party` trees and the - # `support/tools/git-third-party` tool dir (vendored LICENSE etc.). - 'id': 'editorconfig-checker', - 'nix': 'editorconfig-checker', - 'entry': 'editorconfig-checker', - # text-only: the engine's `text` pseudo-type filters out binaries - # (e.g. model/onnx blobs) that editorconfig-checker should not see. - 'types_or': ['text'], - 'exclude': r'git-third-party|/fuzz/', - }, - # --- python / lua / ts (manager-owned languages) ------------------- - { - 'id': 'ruff-format', - 'nix': 'ruff', - 'entry': 'ruff', - 'args': ['format', '--check'], - 'fix_args': ['format'], - 'types_or': ['python'], - }, - { - 'id': 'ruff-check', - 'nix': 'ruff', - 'entry': 'ruff', - 'args': ['check'], - 'fix_args': ['check', '--fix'], - 'types_or': ['python'], - }, - { - 'id': 'stylua', - 'nix': 'stylua', - 'entry': 'stylua', - 'args': ['--check'], - 'fix_args': [], - 'files': r'\.lua$', - }, - { - 'id': 'prettier', - 'nix': 'prettier', - 'entry': 'prettier', - 'args': ['--check'], - 'fix_args': ['--write'], - 'types_or': ['ts', 'tsx'], - 'exclude': r'^\.git-third-party', - }, - { - 'id': 'nixfmt', - 'nix': 'nixfmt', - 'entry': 'nixfmt', - 'args': ['--check'], - 'fix_args': [], - 'files': r'\.nix$', - }, - # --- github workflow/action schemas ------------------------------- - { - 'id': 'check-github-workflows', - 'nix': 'check-jsonschema', - 'entry': 'check-jsonschema', - 'args': ['--builtin-schema', 'vendor.github-workflows'], - 'files': r'^\.github/workflows/.*\.ya?ml$', - }, - { - 'id': 'check-github-actions', - 'nix': 'check-jsonschema', - 'entry': 'check-jsonschema', - 'args': ['--builtin-schema', 'vendor.github-actions'], - 'files': r'^\.github/(actions/.+/)?action\.ya?ml$', - }, - # --- local scripts ------------------------------------------------- - { - 'id': 'cargo-fmt', - 'nix': 'cargo', - 'builtin': 'cargo-fmt', - 'files': r'\.rs$', - 'pass_filenames': False, - }, - { - # Keep the manager crate's [package] version in lockstep with - # .genvm-monorepo-root. The executor submodule is versioned - # independently (its own repo/hooks own that); we never touch it. - 'id': 'check-cargo-versions', - 'local': True, - 'entry': ctx.python_command[0], - 'args': [ - *ctx.python_command[1:], - str(ctx.root / 'support/ci/check-versions.py'), - 'sync', - ], - 'pass_filenames': False, - 'files': r'^(implementation/Cargo\.toml|\.genvm-monorepo-root)$', - }, - { - 'id': 'markdown-local-links', - 'builtin': 'md-local-links', - 'types_or': ['markdown'], - }, - # --- commit-msg ---------------------------------------------------- - { - 'id': 'check-commit-message', - 'local': True, - 'entry': ctx.python_command[0], - 'args': [ - *ctx.python_command[1:], - str(ctx.root / 'support/scripts/check-commit-message.py'), - '--message-file', - ], - 'stages': ['commit-msg'], - 'pass_filenames': False, - }, - ] - def tests(ctx): """Umbrella test suite (was the top-level .ya-test.py). @@ -211,6 +29,7 @@ def tests(ctx): import sys from pathlib import Path + import genvm_tool.cmd_configure import genvm_tool.tests _info_path = ctx.shared.root_dir / 'build' / 'info.json' @@ -224,6 +43,11 @@ def tests(ctx): 'coverage_dir': str(_build_dir / 'cov'), 'build_dir': str(_build_dir), 'rust_target_dir': str(_build_dir / 'ya-build' / 'rust-target'), + # executor_versions / primary_executor_version: resolved from + # source (.genvm-monorepo-root + committed manifests), so a + # build-less test run still knows which out/executor/ dir + # to reroute to. configure emits the same keys. + **genvm_tool.cmd_configure.build_independent_info(ctx.shared.root_dir), }, indent=2, ) @@ -290,7 +114,9 @@ def collect_pytest(ctx: genvm_tool.tests.stage.collection.Context): pytest.pytest( ctx, genvm_tool.tests.test.Description( - 'runners/genlayer-py-std/test', + str(p.relative_to(ctx.shared.root_dir)), + console_pool=True, + tags=frozenset({'unit'}), ), project_root_dir=p, ) @@ -349,11 +175,12 @@ def collect_integration(ctx: genvm_tool.tests.stage.collection.Context): # per-request only under debug_mode >= safe (tests run with unsafe). reroute_to = ( getattr(ctx.configuration.args, 'genvm_reroute_to', '') - or build_info['primary_executor_version'] + or build_info.get('primary_executor_version') + or ctx.shared.config['active-versions'][0] ) no_manager = getattr(ctx.configuration.args, 'no_manager', False) no_webdriver = getattr(ctx.configuration.args, 'no_webdriver', False) - ci = getattr(ctx.configuration.args, 'ci', False) + ci = ctx.shared.ci manager_port = genvm.get_manager_port(ctx.configuration) diff --git a/.github/actions/get-src/action.yaml b/.github/actions/get-src/action.yaml index d386d441..e0ae6c74 100644 --- a/.github/actions/get-src/action.yaml +++ b/.github/actions/get-src/action.yaml @@ -5,10 +5,6 @@ inputs: description: if should setup nix required: false default: "false" - load_submodules: - description: if should update submodules - required: false - default: "true" third_party: description: third-party modules to install required: false @@ -16,6 +12,13 @@ inputs: github_token: description: GitHub token for nix setup required: false +outputs: + changes: + description: >- + JSON object mapping each repo (`.` for the manager, submodule path for + each executor) to {base_commit, branch_commit, has_changes}. Outside a + pull request base_commit == branch_commit and has_changes is false. + value: ${{ steps.changes.outputs.changes }} runs: using: composite steps: @@ -24,26 +27,16 @@ runs: cd "$GITHUB_WORKSPACE" git config --global user.email "ci@genlayerlabs.com" git config --global user.name "CI worker" - if [ "${{ inputs.load_submodules }}" == "true" ] - then - git submodule update --init --recursive --depth 1 - if [ "${{ inputs.third_party }}" != "none" ] - then - source env.sh - # Vendored third-party trees are registered per executor submodule: - # `.git-third-party/config.json` lives inside each executor, which is - # its own git repo. `git third-party` resolves its config from the - # current git toplevel, so it must run *inside* each executor β€” - # running it once at the manager root finds no config and - # materializes nothing (which then breaks the cargo build). - for cfg in executors/*/.git-third-party/config.json - do - [ -e "$cfg" ] || continue - ( cd "$(dirname "$(dirname "$cfg")")" && git third-party update ${{ inputs.third_party }} ) - done - fi - fi + python3 support/scripts/get-all-git.py "--third-party=${{ inputs.third_party }}" shell: bash -ex {0} + - name: compute per-repo changes + id: changes + shell: bash + env: + GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + cd "$GITHUB_WORKSPACE" + python3 support/scripts/ci-changes.py --github-output - name: setup nix if: ${{ inputs.with_nix == 'true' }} uses: cachix/install-nix-action@v31.6.2 @@ -52,5 +45,9 @@ runs: experimental-features = nix-command flakes recursive-nix substituters = https://cache.nixos.org/ https://nix-community.cachix.org https://cache.nix.kp2pml30.moe trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs= cache.nix.kp2pml30.moe:9izVvysdKza7+bjw2gI/etkDVfEQy+kne2aU/TZpA5o= + download-attempts = 10 + connect-timeout = 15 + stalled-download-timeout = 60 + fallback = true github_access_token: ${{ inputs.github_token }} install_url: https://releases.nixos.org/nix/nix-2.24.14/install diff --git a/.github/actions/matrix-cell-step/action.yaml b/.github/actions/matrix-cell-step/action.yaml new file mode 100644 index 00000000..3f3f3429 --- /dev/null +++ b/.github/actions/matrix-cell-step/action.yaml @@ -0,0 +1,22 @@ +name: GenVM run matrix cell step +description: >- + Replay one key of a matrix cell's step mapping (from a plan-*-matrix pipeline). + Does nothing if that key holds no steps. +inputs: + job_json: + description: "JSON {key: step list} mapping, as passed to test_cell.yaml" + required: true + key: + description: which key of job_json to replay + required: true +runs: + using: composite + steps: + # Skip when the key holds no steps: indexing an empty list yields null, and + # null != null is false. The list is passed via env (not interpolated into + # the shell) so its quotes/brackets survive. + - if: ${{ fromJSON(inputs.job_json)[inputs.key][0] != null }} + shell: bash + env: + JOB_JSON: ${{ toJSON(fromJSON(inputs.job_json)[inputs.key]) }} + run: ./support/ci/run.sh pipeline test-cell --job-json "$JOB_JSON" diff --git a/.github/actions/postmortem/action.yaml b/.github/actions/postmortem/action.yaml new file mode 100644 index 00000000..661bc1b7 --- /dev/null +++ b/.github/actions/postmortem/action.yaml @@ -0,0 +1,132 @@ +name: GenVM postmortem +description: >- + Report runner state after a cell's steps have run: disk, inodes, space by + path, memory, surviving processes, kernel OOM/IO errors, and nix substituter + config. Diagnoses failures that leave no trace in the failing step's own + output β€” disk/inode exhaustion, the OOM killer, and substituter errors all + surface as an unrelated-looking non-zero exit. + + A composite action, not an `incl_*` reusable workflow, because a reusable + workflow is a separate job on a fresh runner: it would report a pristine + machine instead of the one that just failed. + + Callers should add `if: ${{ !cancelled() }}` (the grace window on a cancel is + too short to spend here) and `continue-on-error: true`. Linux-only, matching + every cell that calls it. +runs: + using: composite + steps: + # Never fails the job: `set +e` keeps one probe's failure from skipping the + # rest, and the trailing `exit 0` swallows the last probe's status. + - shell: bash + run: | + set +e + + # /dev/shm, not /tmp: scratch must not live on the filesystem being + # diagnosed, or inode/space exhaustion silently blanks every probe + # that redirects through it. + scratch="$(mktemp -d -p /dev/shm 2>/dev/null || mktemp -d)" + if [ -z "${scratch}" ] || [ ! -d "${scratch}" ]; then + echo "::warning::postmortem: no writable scratch dir; probes degraded" + scratch="" + fi + + echo "::group::disk free (df -h)" + df -h + echo "::endgroup::" + + echo "::group::inodes (df -i)" + df -i + echo "::endgroup::" + + # /nix/store is deliberately absent: `du -s` emits a total only once an + # argument completes, so a store walk would eat the whole budget and + # starve every path after it. `df` above already covers that + # filesystem. `-k 10` because `du` wedged in D-state on a failing disk + # ignores SIGTERM, and this step is built for failing disks. + echo "::group::space by path (du, 60s budget)" + timeout -k 10 60 du -shx \ + "${GITHUB_WORKSPACE}" \ + "${RUNNER_TEMP}" \ + "${HOME}/.cargo" \ + "${HOME}/.rustup" \ + >"${scratch:-/dev/null}/du" 2>/dev/null + du_rc=$? + [ "${du_rc}" -eq 124 ] && echo "(du timed out; totals below are partial)" + if [ -s "${scratch:-/nonexistent}/du" ]; then + sort -h "${scratch}/du" + else + echo "(du produced nothing; rc=${du_rc})" + fi + echo "::endgroup::" + + # free/ps run after the failing step exited, so an OOM-killed hog is + # already gone and these render a healthy box. They catch leftover + # daemons and give a baseline; `dmesg` below is what sees the kill. + echo "::group::memory (post-step; see dmesg for OOM)" + free -h + swapon --show + echo "::endgroup::" + + echo "::group::load" + uptime + nproc + echo "::endgroup::" + + echo "::group::surviving processes by memory (post-step)" + # `-rss`, not `-%rss` β€” procps rejects the latter as an unknown sort + # specifier and prints only a usage message. + ps -eo pid,ppid,rss,pcpu,etime,comm --sort=-rss 2>/dev/null | head -n 15 + echo "::endgroup::" + + # The OOM killer reports here and nowhere else: a build that "fails" + # with a bare signal 9 and no diagnostic is almost always this. + # `sudo -n` so a runner without passwordless sudo fails fast instead of + # blocking on a password prompt. + echo "::group::kernel log (OOM / IO errors)" + if [ -z "${scratch}" ]; then + echo "(skipped: no scratch dir)" + elif ! sudo -n dmesg -T >"${scratch}/dmesg" 2>"${scratch}/dmesg.err"; then + # Distinguished, because "dmesg refused" and "the disk we are + # diagnosing ate the redirect" want opposite next steps. + echo "(dmesg unavailable: $(tr -d '\n' <"${scratch}/dmesg.err"))" + else + # Match into a file, then test it: `grep | tail` reports tail's + # status, so a no-match branch behind it could never fire. + grep -iE 'oom|killed process|out of memory|i/o error|ext4-fs error' \ + "${scratch}/dmesg" >"${scratch}/oom" + if [ -s "${scratch}/oom" ]; then + tail -n 30 "${scratch}/oom" + else + echo "(nothing matched)" + fi + echo "::endgroup::" + + echo "::group::kernel log (tail)" + tail -n 40 "${scratch}/dmesg" + fi + echo "::endgroup::" + + # Aimed at substituter failures (a truncated NAR fails the job with no + # hint of which cache served it): the effective substituter list and + # retry knobs are what identify the culprit afterwards. Store *content* + # is not probed β€” nix builds under TMPDIR, not in the store, so a + # failed build leaves nothing there to find. + echo "::group::nix" + if command -v nix >/dev/null 2>&1; then + nix --version + nix config show 2>/dev/null \ + | grep -E '^(substituters|extra-substituters|fallback|download-attempts|connect-timeout|stalled-download-timeout) ' + if [ -d /nix/store ]; then + find /nix/store -maxdepth 1 -mindepth 1 -printf . 2>/dev/null \ + | wc -c | sed 's/^/store entries: /' + else + echo "store entries: (no /nix/store)" + fi + else + echo "(no nix on PATH)" + fi + echo "::endgroup::" + + [ -n "${scratch}" ] && rm -rf "${scratch}" + exit 0 diff --git a/.github/workflows/branch_executor_prs.yaml b/.github/workflows/branch_executor_prs.yaml index 30d3fdee..b613d2d8 100644 --- a/.github/workflows/branch_executor_prs.yaml +++ b/.github/workflows/branch_executor_prs.yaml @@ -6,7 +6,7 @@ name: branch / open linked executor PRs # line (.genvm-monorepo-root): for each, `pr//` -> -dev. Each # is listed on the manager PR as an `executor: ` line. The executor work lives # on line-namespaced mirror branches the developer pushed via -# `genvm-tool git create-branches`. See support/ci/open-executor-prs.py. +# `genvm-tool git create-branches`. See `support/ci/run.sh tool open-executor-prs`. # # pull_request_target gives a manager-scoped write token for the comment; the # cross-repo executor PR needs a PAT with genvm-executor access @@ -38,4 +38,4 @@ jobs: MANAGER_REPO: ${{ github.repository }} PR_NUMBER: ${{ github.event.pull_request.number }} HEAD_REF: ${{ github.event.pull_request.head.ref }} - run: python3 support/ci/open-executor-prs.py + run: ./support/ci/run.sh tool open-executor-prs diff --git a/.github/workflows/branch_forward.yaml b/.github/workflows/branch_forward.yaml index 975a6e78..fddaf7b7 100644 --- a/.github/workflows/branch_forward.yaml +++ b/.github/workflows/branch_forward.yaml @@ -51,7 +51,7 @@ jobs: # branch carries a stale active-versions and would otherwise try # to drag main backwards once a newer train exists. git show origin/main:.genvm-monorepo-root > /tmp/main-monorepo-root.json - LATEST="$(MONOREPO_ROOT=/tmp/main-monorepo-root.json python3 support/ci/branch-versions.py latest)" + LATEST="$(MONOREPO_ROOT=/tmp/main-monorepo-root.json ./support/ci/run.sh tool branch-versions latest)" if [ "$BRANCH" != "v${LATEST}-dev" ]; then echo "Pushed branch ${BRANCH} is not the latest dev branch (v${LATEST}-dev); nothing to do" exit 0 diff --git a/.github/workflows/branch_merge_into_dev.yaml b/.github/workflows/branch_merge_into_dev.yaml index e58fe231..4f74d4fa 100644 --- a/.github/workflows/branch_merge_into_dev.yaml +++ b/.github/workflows/branch_merge_into_dev.yaml @@ -14,25 +14,29 @@ name: branch / merge PR into dev # 4. the cross-repo E2E check concluded success on the head commit # 5. the PR is 0 commits behind base (head already contains base tip) # -# Merge strategy: -# - 1 commit -> fast-forward the original commit (SHA preserved) -# - more than 1 commit -> squash into a single commit on top of base, -# then fast-forward -# Either way the PR is closed afterwards. +# Merge strategy: one commit per merged PR in every repo. A manager PR that is +# a single commit and rewrites no gitlink is fast-forwarded as-is (SHA +# preserved); otherwise its commits are squashed onto base. Each executor line +# is squashed onto its own -dev, which gives it a NEW commit, so the +# manager commit is rewritten to gitlink that squashed commit. The PR is closed +# afterwards. # -# Executor mirror: the landed manager commit gitlinks into the executor -# submodule; the script fast-forwards the executor's SAME-NAMED branch to -# that gitlink commit. It fast-forward-checks both repos up front, then -# pushes the executor first (the dependency) and the manager second. +# Executor mirror: the landed manager commit gitlinks into each active executor +# line; the script squashes each line onto its SAME-NAMED -dev branch. It +# plans and fast-forward-checks every side up front, then pushes the executors +# first (the dependencies) and the manager second. # # The dev branches are protected; the default GITHUB_TOKEN cannot push to # them. We push over SSH with the GENVM_CI_PRIVATE_KEY deploy key (on the # dev-branch ruleset bypass list), and to the executor repo with a separate # GENVM_EXECUTOR_CI_PRIVATE_KEY deploy key (on the executor's bypass list). # Pushes are non-force, so a base that advanced between the checks and the -# push is safely rejected. This job never runs PR code β€” only git plumbing -# and API reads. The caller is responsible for restricting who can trigger a -# merge (maintainers only). +# push is safely rejected. The merge scripts are checked out from the PR head, +# so a PR that changes support/ci/ merges with its own version β€” which means +# this job runs PR-authored code WITH the deploy keys in scope. It is gated on +# the `ci-safe` label: a guard step verifies it (via the plain GITHUB_TOKEN, +# before any PR code is fetched or run) and fails otherwise. The caller must +# still restrict who can trigger a merge (maintainers only). on: workflow_call: @@ -62,12 +66,27 @@ jobs: merge: runs-on: ubuntu-latest steps: - # Checkout with the manager deploy key so the (protected) dev branch - # push succeeds. Submodules are initialised in the next step with a - # SEPARATE executor key β€” the manager key cannot read/write the - # executor repo. The script never runs PR code. + # Gate: this job checks out and runs the PR's own CI scripts while the + # deploy keys are in scope, so refuse unless the PR is `ci-safe`. Uses the + # plain GITHUB_TOKEN and runs BEFORE any PR code is fetched. + - name: Require ci-safe label + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ inputs.pr_number }} + run: | + if ! gh api "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/labels" \ + --jq '.[].name' | grep -qxF 'ci-safe'; then + echo "::error::PR #$PR_NUMBER lacks the 'ci-safe' label; refusing to merge (this job runs PR code)." + exit 1 + fi + + # Check out the PR head (so the merge runs the PR's own scripts) with the + # manager deploy key so the (protected) dev branch push succeeds. + # Submodules are initialised in the next step with a SEPARATE executor key + # β€” the manager key cannot read/write the executor repo. - uses: actions/checkout@v4 with: + ref: refs/pull/${{ inputs.pr_number }}/head ssh-key: ${{ secrets.GENVM_CI_PRIVATE_KEY }} fetch-depth: 0 submodules: false @@ -90,7 +109,7 @@ jobs: # all, binding the executor key per-submodule (core.sshCommand) so the # later mirror pushes use it while the manager push keeps the checkout # credentials. - for v in $(python3 support/ci/branch-versions.py list); do + for v in $(./support/ci/run.sh tool branch-versions list); do sub="executors/v${v}.x" git -c core.sshCommand="$ssh" submodule update --init "$sub" git -C "$sub" config core.sshCommand "$ssh" @@ -100,4 +119,4 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} PR_NUMBER: ${{ inputs.pr_number }} - run: python3 support/ci/genvm-merge-into-dev.py + run: ./support/ci/run.sh tool genvm-merge-into-dev diff --git a/.github/workflows/branch_pr_actions.yaml b/.github/workflows/branch_pr_actions.yaml index 4d11a25c..8e05db5d 100644 --- a/.github/workflows/branch_pr_actions.yaml +++ b/.github/workflows/branch_pr_actions.yaml @@ -6,6 +6,12 @@ name: branch / PR action panel handler # tests), unticks the boxes, and reports whether Merge was requested. If so # the `merge` job calls the reusable merge workflow. # +# The handler runs the CI scripts checked out from the PR head branch, not +# from the default branch, so a PR that changes support/ci/ exercises its own +# version. That means this workflow DOES run PR-authored code, so it is gated +# on the `ci-safe` label (added only for vetted / write-access PRs) β€” the +# `dispatch` job does not even start without it. +# # Resetting the panel re-fires issue_comment:edited, but that re-run finds # no ticked box (and is sent by the bot), so it is a no-op β€” no loop. @@ -30,17 +36,22 @@ defaults: jobs: dispatch: - # Only the panel comment, only on a PR. + # Only the panel comment, only on a PR, only when the PR is `ci-safe` + # (we check out and run its scripts, so an unvetted PR must not reach here). if: > github.event.issue.pull_request && - contains(github.event.comment.body, '') + contains(github.event.comment.body, '') && + contains(github.event.issue.labels.*.name, 'ci-safe') runs-on: ubuntu-latest outputs: merge: ${{ steps.act.outputs.merge }} steps: - # Checkout of the default branch only β€” to run the handler script. No - # PR code is executed. + # Check out the PR head so the handler runs the PR's own CI scripts. + # This runs PR-authored code β€” the `ci-safe` gate on the job above is + # what makes that safe. - uses: actions/checkout@v4 + with: + ref: refs/pull/${{ github.event.issue.number }}/head - name: Handle ticked boxes id: act env: @@ -48,7 +59,7 @@ jobs: PR_NUMBER: ${{ github.event.issue.number }} COMMENT_ID: ${{ github.event.comment.id }} SENDER: ${{ github.event.sender.login }} - run: python3 support/ci/pr-action-panel.py + run: ./support/ci/run.sh tool pr-action-panel merge: needs: dispatch diff --git a/.github/workflows/branch_pr_checklist.yaml b/.github/workflows/branch_pr_checklist.yaml index 8440487a..657acb19 100644 --- a/.github/workflows/branch_pr_checklist.yaml +++ b/.github/workflows/branch_pr_checklist.yaml @@ -1,7 +1,6 @@ name: branch / post PR action panel # When a PR is opened against a dev branch (v-dev) we: -# - make sure the action labels exist; # - post the action panel (checklist comment) that replaces the old # /genvm-* slash commands; # - auto-add `ci-safe` if the PR author has write access, so their PR can @@ -52,6 +51,7 @@ jobs: - [ ] Force run full tests - [ ] Rerun full tests + - [ ] Provision executor PRs - [ ] Merge into dev Full GenVM CI runs only when **`rtm`** or **`run-full-tests`** is set β€” "Force run full tests" is a sticky toggle for `run-full-tests`. Adding **`rtm`** marks the PR ready-to-merge and also runs full tests. **Merge** requires: `rtm`, green full tests, green E2E, and the branch 0 commits behind. diff --git a/.github/workflows/branch_provision.yaml b/.github/workflows/branch_provision.yaml index d45f56b2..efe29277 100644 --- a/.github/workflows/branch_provision.yaml +++ b/.github/workflows/branch_provision.yaml @@ -3,7 +3,7 @@ name: branch / provision dev branches and standing PRs # Materialize the branch model for every active version train listed in # .genvm-monorepo-root. For each version X it ensures the version branch # v.x, the dev branch v-dev, and a standing release-gate PR -# v-dev -> v.x all exist (see support/ci/provision-branches.sh). +# v-dev -> v.x all exist (see support/ci/run.sh tool provision-branches). # # Idempotent β€” safe to re-run. Fires automatically when active-versions # changes, and on demand via the Actions tab. @@ -37,4 +37,4 @@ jobs: - name: Provision branches and standing PRs env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: support/ci/provision-branches.sh + run: ./support/ci/run.sh tool provision-branches diff --git a/.github/workflows/branch_provision_executor_prs.yaml b/.github/workflows/branch_provision_executor_prs.yaml new file mode 100644 index 00000000..5dd2ecae --- /dev/null +++ b/.github/workflows/branch_provision_executor_prs.yaml @@ -0,0 +1,64 @@ +name: branch / provision executor mirror PRs + +# Triggered by the "Provision executor PRs" box on the PR action panel +# (pr_action_panel.py fires a workflow_dispatch with the manager PR number). For +# each executor line the manager PR moved, force-push its mirror branch +# `pr//` to the pinned gitlink commit and open (or reuse) its PR into +# `-dev`. See `support/ci/run.sh tool pr-branches provision`. +# +# The provision scripts are checked out from the PR head, so a PR iterating on +# them runs its own version β€” which means this job runs PR-authored code WITH the +# executor write PAT (GENVM_EXECUTOR_PR_TOKEN) in scope. It is gated on the +# `ci-safe` label: a guard step verifies it (via the plain GITHUB_TOKEN, before +# any PR code runs) and fails otherwise, exactly like branch_merge_into_dev.yaml. + +on: + workflow_dispatch: + inputs: + pr: + description: manager PR number to provision executor mirror PRs for + type: string + required: true + +run-name: 'branch / provision executor mirror PRs (PR #${{ inputs.pr }})' + +permissions: + contents: read + # For the GITHUB_TOKEN only: read the ci-safe label and the manager PR title. + # Writes to the executor repo go through GENVM_EXECUTOR_PR_TOKEN, not this token. + pull-requests: read + issues: read + +defaults: + run: + shell: bash -x {0} + +jobs: + provision: + runs-on: ubuntu-latest + steps: + # Gate: this job checks out and runs the PR's scripts with the executor + # write PAT in scope, so refuse unless the PR is `ci-safe`. Uses the plain + # GITHUB_TOKEN and runs BEFORE any PR code is fetched. + - name: Require ci-safe label + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ inputs.pr }} + run: | + if ! gh api "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/labels" \ + --jq '.[].name' | grep -qxF 'ci-safe'; then + echo "::error::PR #$PR_NUMBER lacks the 'ci-safe' label; refusing to provision (this job runs PR code with the executor token)." + exit 1 + fi + + - uses: actions/checkout@v4 + with: + ref: refs/pull/${{ inputs.pr }}/head + + - name: Provision executor mirror PRs + env: + PR_NUMBER: ${{ inputs.pr }} + MANAGER_REPO: ${{ github.repository }} + MANAGER_TOKEN: ${{ secrets.GITHUB_TOKEN }} + EXECUTOR_TOKEN: ${{ secrets.GENVM_EXECUTOR_PR_TOKEN }} + run: ./support/ci/run.sh tool pr-branches provision diff --git a/.github/workflows/branch_retarget.yaml b/.github/workflows/branch_retarget.yaml index a7f45715..7e2bea60 100644 --- a/.github/workflows/branch_retarget.yaml +++ b/.github/workflows/branch_retarget.yaml @@ -35,7 +35,7 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - LATEST="$(python3 support/ci/branch-versions.py latest)" + LATEST="$(./support/ci/run.sh tool branch-versions latest)" DEV="v${LATEST}-dev" PR="${{ github.event.pull_request.number }}" diff --git a/.github/workflows/incl_build_trg.yaml b/.github/workflows/incl_build_trg.yaml deleted file mode 100644 index c12df64e..00000000 --- a/.github/workflows/incl_build_trg.yaml +++ /dev/null @@ -1,118 +0,0 @@ -name: Build Release For Target - -on: - workflow_call: - inputs: - target: - description: 'Target to build' - required: true - type: string - secondary_target: - description: 'Secondary target to build' - required: false - type: string - default: '' - executor_version: - description: 'Version name of the executor' - required: true - type: string - secrets: - GCP_SA_KEY: - description: 'Google Cloud Service Account Key' - required: true - outputs: - artifact_url: - description: "URL to download the built artifact" - value: ${{ jobs.build.outputs.artifact_url }} - secondary_artifact_url: - description: "URL to download the secondary built artifact" - value: ${{ jobs.build.outputs.secondary_artifact_url }} - -env: - GCS_BUCKET: "gh-af" - -jobs: - build: - runs-on: ubuntu-latest - permissions: - contents: read - actions: read - outputs: - artifact_url: ${{ steps.upload.outputs.gcs_url }} - secondary_artifact_url: ${{ steps.upload.outputs.secondary_gcs_url }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - uses: insightsengineering/disk-space-reclaimer@v1 - with: - tools-cache: false - swap-storage: false - docker-images: false - - - name: Get source - uses: ./.github/actions/get-src - with: - # The executor lines are submodules; nix builds run on - # `.?submodules=1` and need them checked out. - load_submodules: "true" - with_nix: "true" - github_token: ${{ secrets.GITHUB_TOKEN }} - - - name: Build - run: | - ./support/ci/pipelines/build.sh \ - --target ${{ inputs.target }} \ - --executor-version ${{ inputs.executor_version }} - - - name: Build Secondary - if: ${{ inputs.secondary_target != '' }} - run: | - ./support/ci/pipelines/build.sh \ - --target ${{ inputs.secondary_target }} \ - --executor-version ${{ inputs.executor_version }} - - - name: Authenticate to Google Cloud - uses: google-github-actions/auth@v2 - with: - credentials_json: ${{ secrets.GCP_SA_KEY }} - - - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v2 - - - name: Check gcloud authentication - run: gcloud auth list - - - name: Generate upload url - id: upload - run: | - TIMESTAMP=$(date +%Y%m%d_%H%M%S) - DIR_NAME="genvm_executor_${GITHUB_SHA}_${TIMESTAMP}" - echo "dirname=$DIR_NAME" >> $GITHUB_OUTPUT - BASE_NAME="genvm-${{ inputs.target }}.tar.xz" - echo "basename=$BASE_NAME" >> $GITHUB_OUTPUT - echo "gcs_url=https://storage.googleapis.com/$GCS_BUCKET/$DIR_NAME/$BASE_NAME" >> $GITHUB_OUTPUT - - if [ "${{ inputs.secondary_target }}" != "" ]; then - SECONDARY_BASE_NAME="genvm-${{ inputs.secondary_target }}.tar.xz" - echo "secondary_basename=$SECONDARY_BASE_NAME" >> $GITHUB_OUTPUT - echo "secondary_gcs_url=https://storage.googleapis.com/$GCS_BUCKET/$DIR_NAME/$SECONDARY_BASE_NAME" >> $GITHUB_OUTPUT - else - echo "secondary_basename=" >> $GITHUB_OUTPUT - echo "secondary_gcs_url=" >> $GITHUB_OUTPUT - fi - - - name: Upload to GCS - uses: google-github-actions/upload-cloud-storage@v2 - with: - path: build/${{ steps.upload.outputs.basename }} - destination: ${{ env.GCS_BUCKET }}/${{ steps.upload.outputs.dirname }} - parent: false - - - name: Upload Secondary to GCS - uses: google-github-actions/upload-cloud-storage@v2 - if: ${{ inputs.secondary_target != '' }} - with: - path: build/${{ steps.upload.outputs.secondary_basename }} - destination: ${{ env.GCS_BUCKET }}/${{ steps.upload.outputs.dirname }} - parent: false diff --git a/.github/workflows/incl_initial.yaml b/.github/workflows/incl_initial.yaml index d16fed5c..6aae6c66 100644 --- a/.github/workflows/incl_initial.yaml +++ b/.github/workflows/incl_initial.yaml @@ -15,13 +15,33 @@ jobs: with: with_nix: "true" github_token: ${{ secrets.GITHUB_TOKEN }} - # genvm-tool runs every repo's hooks with tools from each repo's pinned - # support/nix/precommit flake (rustfmt included), so no separate toolchain - # setup is needed. - # --check: CI verifies formatting and fails on drift instead of fixing - # in place (fixing is the default for local/git-hook runs). + # Each repo (manager + every executor submodule) supplies its own + # pre-commit binary + generated config from its flake (git-hooks.nix), + # with tools pinned per repo β€” so no separate toolchain setup is needed. + # The formatters fix in place and the run fails if anything changed, so + # unformatted code turns the build red (the diff is shown on failure). - name: Run hooks - run: nix develop '.?submodules=1#initial-check' --command genvm-tool hook run --all-files --check + run: ./support/ci/run.sh pipeline commit-hooks + + # `check-commit-message` is a commit-msg-stage hook, so the pre-commit run + # above never fires it. Check every commit the PR adds, in the manager and in + # each executor submodule that moved. + commit-messages: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Get source + id: src + uses: ./.github/actions/get-src + with: + with_nix: "false" + - name: Check commit messages + env: + CHANGES: ${{ steps.src.outputs.changes }} + run: ./support/ci/run.sh pipeline commit-messages # The Merge action only fast-forwards, so the PR head must already # contain the base tip (0 commits behind). We assert it here too (as part diff --git a/.github/workflows/incl_release_build_test.yaml b/.github/workflows/incl_release_build_test.yaml new file mode 100644 index 00000000..a705d155 --- /dev/null +++ b/.github/workflows/incl_release_build_test.yaml @@ -0,0 +1,128 @@ +name: Build and test release artifacts + +# The whole release pipeline except publishing: plan -> build every artifact -> +# test every platform against what was built. +# +# Callers see four artifacts, a tag and the executor lines; the matrices that +# produce them are an implementation detail. `release.yaml` is this plus a +# publisher; `queue.yaml` runs it on a PR (label `test-release-pipeline`) to +# exercise the pipeline itself without releasing anything. + +on: + workflow_call: + inputs: + version_override: + description: "Release tag. Defaults to `version` from .genvm-monorepo-root" + required: false + type: string + default: "" + outputs: + tag: + description: the tag the artifacts were planned at + value: ${{ jobs.plan.outputs.tag }} + artifact_universal: + description: artifact holding the platform-independent runners + value: ${{ jobs.plan.outputs.artifact_universal }} + artifact_amd64_linux: + description: artifact holding amd64-linux's manager + executor tarballs + value: ${{ jobs.plan.outputs.artifact_amd64_linux }} + artifact_arm64_linux: + description: artifact holding arm64-linux's manager + executor tarballs + value: ${{ jobs.plan.outputs.artifact_arm64_linux }} + artifact_arm64_macos: + description: artifact holding arm64-macos's manager + executor tarballs + value: ${{ jobs.plan.outputs.artifact_arm64_macos }} + artifact_notes: + description: artifact holding the generated release notes + value: ${{ jobs.plan.outputs.artifact_notes }} + notes_file: + description: the notes file's name inside artifact_notes + value: ${{ jobs.plan.outputs.notes_file }} + +jobs: + plan: + runs-on: ubuntu-latest + outputs: + tag: ${{ steps.plan.outputs.tag }} + build_matrix: ${{ steps.plan.outputs.build_matrix }} + test_matrix: ${{ steps.plan.outputs.test_matrix }} + artifact_universal: ${{ steps.plan.outputs.artifact_universal }} + artifact_amd64_linux: ${{ steps.plan.outputs.artifact_amd64_linux }} + artifact_arm64_linux: ${{ steps.plan.outputs.artifact_arm64_linux }} + artifact_arm64_macos: ${{ steps.plan.outputs.artifact_arm64_macos }} + artifact_notes: ${{ steps.plan.outputs.artifact_notes }} + notes_file: ${{ steps.plan.outputs.notes_file }} + steps: + - uses: actions/checkout@v4 + with: + # Each line's manifest.json (its executor-version) lives in the + # submodule, so the gitlinks must be materialized. + submodules: true + - name: Plan release matrix + id: plan + env: + VERSION_OVERRIDE: ${{ inputs.version_override }} + run: ./support/ci/run.sh pipeline plan-release-matrix --version-override "$VERSION_OVERRIDE" + + # Generated here rather than in the publisher so a PR-label run produces them + # too: they are worth reading before the release, not after. The range ends at + # HEAD, not at the tag β€” the tag will point here, but does not exist yet. + notes: + needs: plan + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + # make-release-notes diffs against the previous tag via `git describe`. + fetch-depth: 0 + - name: Generate release notes + env: + TAG: ${{ needs.plan.outputs.tag }} + NOTES_FILE: ${{ needs.plan.outputs.notes_file }} + run: | + set -euo pipefail + prev_tag=$(git describe --tags --abbrev=0 HEAD~1 2>/dev/null || echo "") + if [ -n "$prev_tag" ]; then + echo "notes for $TAG: ${prev_tag}..HEAD" + ./support/ci/run.sh tool make-release-notes "${prev_tag}..HEAD" > "$NOTES_FILE" + else + echo "Initial release" > "$NOTES_FILE" + fi + # Echoed so a debug run needs no artifact download. + cat "$NOTES_FILE" + - uses: actions/upload-artifact@v4 + with: + name: ${{ needs.plan.outputs.artifact_notes }} + path: ${{ needs.plan.outputs.notes_file }} + if-no-files-found: error + + build: + needs: plan + name: ${{ matrix.name }} + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.plan.outputs.build_matrix) }} + uses: ./.github/workflows/incl_release_build_test_cell_build.yaml + with: + name: ${{ matrix.name }} + runs_on: ${{ matrix.runs_on }} + artifact_name: ${{ matrix.artifact_name }} + job_json: ${{ matrix.job_json }} + secrets: inherit + + test: + needs: [plan, build] + name: ${{ matrix.name }} + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.plan.outputs.test_matrix) }} + uses: ./.github/workflows/incl_release_build_test_cell_test.yaml + with: + name: ${{ matrix.name }} + runs_on: ${{ matrix.runs_on }} + platform_artifact: ${{ matrix.platform_artifact }} + runners_artifact: ${{ matrix.runners_artifact }} + with_nix: ${{ matrix.with_nix }} + setup_python: ${{ matrix.setup_python }} + job_json: ${{ matrix.job_json }} + secrets: inherit diff --git a/.github/workflows/incl_release_build_test_cell_build.yaml b/.github/workflows/incl_release_build_test_cell_build.yaml new file mode 100644 index 00000000..ed227565 --- /dev/null +++ b/.github/workflows/incl_release_build_test_cell_build.yaml @@ -0,0 +1,72 @@ +name: Build one release artifact cell + +# One cell of plan-release-matrix's build matrix: replays the cell's `build` +# steps (one build-and-pack per target) and uploads everything they packed under +# the plan-assigned artifact name. +# +# The name comes from the plan rather than being minted here: every leg of a +# matrix job writes the same `outputs` key, so a per-leg name cannot be handed +# back to the test matrix as an output. + +on: + workflow_call: + inputs: + name: + required: true + type: string + runs_on: + required: true + type: string + artifact_name: + description: artifact to upload the cell's tarballs as + required: true + type: string + job_json: + # JSON {key: step list} from plan-release-matrix; matrix-cell-step + # slices out the list for its key. + required: true + type: string + +jobs: + build: + name: build / ${{ inputs.name }} + runs-on: ${{ inputs.runs_on }} + steps: + - uses: actions/checkout@v4 + + # Guards the "Job:" step below: every key must have a step, every step a key. + - name: Check job keys + env: + JOB_JSON: ${{ inputs.job_json }} + run: ./support/ci/run.sh pipeline check-cell-keys --job-json "$JOB_JSON" build + + - uses: insightsengineering/disk-space-reclaimer@v1 + with: + tools-cache: false + swap-storage: false + docker-images: false + + - name: Get source + uses: ./.github/actions/get-src + with: + # The executor lines are submodules; nix builds run on + # `.?submodules=1` and need them checked out. + third_party: --all + with_nix: "true" + github_token: ${{ secrets.GITHUB_TOKEN }} + + - name: "Job: build" + uses: ./.github/actions/matrix-cell-step + with: + job_json: ${{ inputs.job_json }} + key: build + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.artifact_name }} + # upload-artifact strips the common prefix, so entries land flat as + # genvm-.tar.xz β€” the names the test cells and the release + # uploader index by. + path: build/artifacts/*.tar.xz + if-no-files-found: error diff --git a/.github/workflows/incl_release_build_test_cell_test.yaml b/.github/workflows/incl_release_build_test_cell_test.yaml new file mode 100644 index 00000000..7cda7f53 --- /dev/null +++ b/.github/workflows/incl_release_build_test_cell_test.yaml @@ -0,0 +1,114 @@ +name: Test one release platform cell + +# One cell of plan-release-matrix's test matrix. Materializes the built tarballs +# into build/out and replays the cell's `test` steps against them: `genvm-tool +# configure` writes build/info.json pointing at that tree, so the runner +# exercises the downloaded artifacts and never builds. + +on: + workflow_call: + inputs: + name: + required: true + type: string + runs_on: + required: true + type: string + platform_artifact: + description: artifact holding this platform's manager + executor tarballs + required: true + type: string + runners_artifact: + description: artifact holding the universal runners; empty to skip + required: false + type: string + default: "" + with_nix: + required: false + type: boolean + default: true + setup_python: + required: false + type: boolean + default: false + job_json: + required: true + type: string + +jobs: + test: + name: release-test / ${{ inputs.name }} + runs-on: ${{ inputs.runs_on }} + steps: + - uses: actions/checkout@v4 + + - name: Check job keys + env: + JOB_JSON: ${{ inputs.job_json }} + run: ./support/ci/run.sh pipeline check-cell-keys --job-json "$JOB_JSON" test + + - name: Get source + uses: ./.github/actions/get-src + with: + third_party: --all + with_nix: ${{ inputs.with_nix && 'true' || 'false' }} + github_token: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup Python + if: ${{ inputs.setup_python }} + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Download platform artifacts + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.platform_artifact }} + path: ${{ runner.temp }}/release-artifacts + + # Skipped for cells that cannot use them: a foreign-arch tree has nothing + # to run the runners with. + - name: Download runners artifact + if: ${{ inputs.runners_artifact != '' }} + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.runners_artifact }} + path: ${{ runner.temp }}/release-artifacts + + - name: Materialize artifacts + env: + ARTIFACTS_DIR: ${{ runner.temp }}/release-artifacts + run: | + set -euo pipefail + # Every tarball β€” manager, each executor line, and the runners β€” + # overlays onto the same install root. + mkdir -p build/out + shopt -s nullglob + archives=("$ARTIFACTS_DIR"/*.tar.xz) + if [ "${#archives[@]}" -eq 0 ]; then + echo "::error::no tarballs in $ARTIFACTS_DIR" + exit 1 + fi + printf 'extracting:\n' + printf ' %s\n' "${archives[@]}" + for archive in "${archives[@]}"; do + tar -xf "$archive" -C build/out --no-same-owner + done + + - name: "Job: test" + uses: ./.github/actions/matrix-cell-step + with: + job_json: ${{ inputs.job_json }} + key: test + + # Postmortem on the same runner the cell just used; see the action for + # what it probes and why it is not an `incl_*` reusable workflow. + # `failure()` covers any earlier step failing and is already false on a + # cancel, so it needs no !cancelled() guard. + # `continue-on-error` is belt-and-braces: the action already exits 0. + # Probes are Linux-only, so skipped on this workflow's macOS cells. + - name: Postmortem + if: ${{ failure() && !startsWith(inputs.runs_on, 'macos') }} + continue-on-error: true + timeout-minutes: 5 + uses: ./.github/actions/postmortem diff --git a/.github/workflows/queue.yaml b/.github/workflows/queue.yaml index 7153c198..11b5ea9a 100644 --- a/.github/workflows/queue.yaml +++ b/.github/workflows/queue.yaml @@ -6,10 +6,13 @@ name: GenVM full # fast-forwards. # # For every push to a PR targeting a dev branch (v-dev): -# - `initial` (cheap pre-commit + 0-behind check) ALWAYS runs; -# - the heavy test jobs run only when a run-full-tests marker is set β€” -# the `rtm` (ready-to-merge) label or the `run-full-tests` label (the -# "Force run full tests" checkbox sets the latter); +# - `initial` (cheap pre-commit + 0-behind) and `check-executor-prs` (the +# executor landability precondition) ALWAYS run, in parallel; +# - `gate-check` aggregates them and plans the test matrix (plan-queue-matrix); +# the heavy `matrix-tests` cells run only when a run-full-tests marker is set +# β€” the `rtm` (ready-to-merge) label or the `run-full-tests` label ("Force run +# full tests" sets the latter). Without a marker the plan is empty, so +# matrix-tests is skipped; # - without a marker `validate-end` fails, so the CI check stays red # until a full run has happened. # @@ -45,105 +48,107 @@ jobs: uses: ./.github/workflows/incl_initial.yaml secrets: inherit - module-test-python: - if: ${{ github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'rtm') || contains(github.event.pull_request.labels.*.name, 'run-full-tests') }} - needs: [initial] + # Precondition (run.sh tool pr-branches check): the PR must be landable via the + # panel (Provision then Merge), which lands the manager and its executors almost- + # atomically. So every repo (manager + each executor line) must be rebased on its + # base, and every executor line present at base must have its pinned commit in the + # executor repo (ahead-of-base is fine β€” that work lands with the PR; a commit + # missing from the executor repo is not). Read-only (gh API, no submodule checkout). + # + # Runs on same-repo PRs AND on workflow_dispatch with a PR number (the panel's + # rerun path, whose run the Merge gate consumes β€” so the precondition must hold + # there too; the PR number then comes from inputs.pr). Fork PRs are skipped: they + # get no secrets (the executor PAT would be empty) and cannot push executor mirror + # branches, so there is nothing to gate. It checks out the PR being validated + # (refs/pull//head) so the scripts and .genvm-monorepo-root match that PR β€” on + # dispatch the base branch is not in the event context, and its default-branch tree + # is the wrong commit. (Checking out PR code is not an extra exposure: for + # pull_request GitHub already runs this workflow file from the PR, so a same-repo + # author controls it regardless.) + check-executor-prs: + if: > + (github.event_name == 'workflow_dispatch' && inputs.pr != '') || + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository) runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: - lfs: true - - name: Get source - uses: ./.github/actions/get-src - with: - load_submodules: "true" - with_nix: "true" - github_token: ${{ secrets.GITHUB_TOKEN }} - - run: ./support/ci/pipelines/test-python.sh - - module-test-rust-fuzz: - if: ${{ github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'rtm') || contains(github.event.pull_request.labels.*.name, 'run-full-tests') }} - needs: [initial] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - lfs: true - - uses: insightsengineering/disk-space-reclaimer@v1 - with: - tools-cache: false - swap-storage: false - docker-images: false - - name: Get source - uses: ./.github/actions/get-src - with: - load_submodules: "true" - with_nix: "true" - github_token: ${{ secrets.GITHUB_TOKEN }} - - run: | - echo core | sudo tee /proc/sys/kernel/core_pattern - echo performance | sudo tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor - echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope - ./support/ci/pipelines/test-rust-fuzz.sh && ./support/ci/cargo-clippy.sh + ref: refs/pull/${{ github.event.pull_request.number || inputs.pr }}/head + - name: Executor mirror PRs are landed env: - OPENAIKEY: ${{ secrets.OPENAIKEY }} - HEURISTKEY: ${{ secrets.HEURISTKEY }} - ANTHROPICKEY: ${{ secrets.ANTHROPICKEY }} - XAIKEY: ${{ secrets.XAIKEY }} - GEMINIKEY: ${{ secrets.GEMINIKEY }} + PR_NUMBER: ${{ github.event.pull_request.number || inputs.pr }} + MANAGER_REPO: ${{ github.repository }} + MANAGER_TOKEN: ${{ secrets.GITHUB_TOKEN }} + EXECUTOR_TOKEN: ${{ secrets.GENVM_EXECUTOR_PR_TOKEN }} + run: ./support/ci/run.sh tool pr-branches check - module-test-rust: - if: ${{ github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'rtm') || contains(github.event.pull_request.labels.*.name, 'run-full-tests') }} - needs: [initial] + gate-check: + needs: [initial, check-executor-prs] runs-on: ubuntu-latest - env: - # Only used to upload freshly-built runners to the GCS cache; this is - # best-effort (see the `|| true` upload in test-rust.sh). Absent on fork - # PRs, so the GCP steps below are skipped and runners come from the public - # bucket instead of failing the whole job. - GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} + if: ${{ always() }} + outputs: + matrix: ${{ steps.plan.outputs.matrix }} steps: + - name: Check gate results + run: | + if [ "${{ needs.initial.result }}" != "success" ]; then + echo "::error::initial checks failed" + exit 1 + fi + # Executor precondition: fail on failure/cancelled. Skipped (fork PR or + # manual dispatch, where check-executor-prs does not run) is acceptable. + case "${{ needs.check-executor-prs.result }}" in + failure | cancelled) + echo "::error::executor mirror PR precondition failed" + exit 1 + ;; + esac - uses: actions/checkout@v4 - with: - lfs: true - - uses: insightsengineering/disk-space-reclaimer@v1 - with: - tools-cache: false - swap-storage: false - docker-images: false - - name: Get source - uses: ./.github/actions/get-src - with: - load_submodules: "true" - with_nix: "true" - github_token: ${{ secrets.GITHUB_TOKEN }} - - name: Authenticate to Google Cloud - if: ${{ env.GCP_SA_KEY != '' }} - uses: google-github-actions/auth@v2 - with: - credentials_json: ${{ secrets.GCP_SA_KEY }} - - name: Set up Cloud SDK # needed for gcloud to upload runners to GCS for caching - if: ${{ env.GCP_SA_KEY != '' }} - uses: google-github-actions/setup-gcloud@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - run: | - ./support/ci/pipelines/test-rust.sh + - name: Plan test matrix + id: plan env: - OPENAIKEY: ${{ secrets.OPENAIKEY }} - HEURISTKEY: ${{ secrets.HEURISTKEY }} - ANTHROPICKEY: ${{ secrets.ANTHROPICKEY }} - XAIKEY: ${{ secrets.XAIKEY }} - GEMINIKEY: ${{ secrets.GEMINIKEY }} + # Heavy cells run only with a marker; without one the plan is empty and + # matrix-tests is skipped. + RUN_FULL_TESTS: ${{ github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'rtm') || contains(github.event.pull_request.labels.*.name, 'run-full-tests') }} + run: ./support/ci/run.sh pipeline plan-queue-matrix + + matrix-tests: + needs: gate-check + name: ${{ matrix.name }} + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.gate-check.outputs.matrix) }} + uses: ./.github/workflows/queue_test_cell.yaml + with: + name: ${{ matrix.name }} + job_json: ${{ matrix.job_json }} + runs_on: ${{ matrix.runs_on }} + disk_reclaim: ${{ matrix.disk_reclaim }} + fuzz_host: ${{ matrix.fuzz_host }} + buildx: ${{ matrix.buildx }} + gcp: ${{ matrix.gcp }} + secrets: inherit + + + # Opt-in: exercises the release pipeline itself (plan -> build every artifact + # -> test every platform) without publishing anything. Not part of + # validate-end's gate β€” it is skipped by default, and `skipped` there is a + # failure. + release-pipeline-test: + needs: [initial, gate-check] + if: ${{ contains(github.event.pull_request.labels.*.name, 'test-release-pipeline') }} + uses: ./.github/workflows/incl_release_build_test.yaml + secrets: inherit + validate-end: runs-on: ubuntu-latest if: ${{ always() }} needs: - initial - - module-test-python - - module-test-rust - - module-test-rust-fuzz + - gate-check + - matrix-tests env: HAS_MARKER: ${{ github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'rtm') || contains(github.event.pull_request.labels.*.name, 'run-full-tests') }} steps: @@ -160,7 +165,7 @@ jobs: echo "::error::full tests have not run β€” add the 'rtm' label or check 'Force run full tests' on the action panel" exit 1 fi - results="${{ needs.module-test-python.result }} ${{ needs.module-test-rust.result }} ${{ needs.module-test-rust-fuzz.result }}" + results="${{ needs.matrix-tests.result }}" echo "$results" if echo "$results" | grep -qiE '(failure|cancelled|skipped)'; then echo "::error::one or more full-test jobs failed/cancelled/skipped" diff --git a/.github/workflows/queue_test_cell.yaml b/.github/workflows/queue_test_cell.yaml new file mode 100644 index 00000000..8cbda019 --- /dev/null +++ b/.github/workflows/queue_test_cell.yaml @@ -0,0 +1,117 @@ +on: + workflow_call: + inputs: + name: + required: true + type: string + job_json: + # JSON {key: step list} from plan-queue-matrix, one key per "Job:" step + # below; matrix-cell-step slices out the list for its key. + required: true + type: string + runs_on: + required: true + type: string + disk_reclaim: + required: false + type: boolean + default: false + fuzz_host: + required: false + type: boolean + default: false + buildx: + required: false + type: boolean + default: false + gcp: + required: false + type: boolean + default: false + +jobs: + test: + name: test / ${{ inputs.name }} + runs-on: ${{ inputs.runs_on }} + env: + GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} + OPENAIKEY: ${{ secrets.OPENAIKEY }} + HEURISTKEY: ${{ secrets.HEURISTKEY }} + ANTHROPICKEY: ${{ secrets.ANTHROPICKEY }} + XAIKEY: ${{ secrets.XAIKEY }} + GEMINIKEY: ${{ secrets.GEMINIKEY }} + + steps: + - uses: actions/checkout@v4 + with: + lfs: true + + # Guards the "Job:" steps below: every key must have a step, every step a key. + - name: Check job keys + env: + JOB_JSON: ${{ inputs.job_json }} + run: ./support/ci/run.sh pipeline check-cell-keys --job-json "$JOB_JSON" configure build tool + + - uses: insightsengineering/disk-space-reclaimer@v1 + if: ${{ inputs.disk_reclaim }} + with: + tools-cache: false + swap-storage: false + docker-images: false + + - name: Get source + uses: ./.github/actions/get-src + with: + third_party: --all + with_nix: "true" + github_token: ${{ secrets.GITHUB_TOKEN }} + + - name: Configure fuzzing host + if: ${{ inputs.fuzz_host }} + run: | + echo core | sudo tee /proc/sys/kernel/core_pattern || true + echo performance | sudo tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor || true + echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope || true + + - name: Authenticate to Google Cloud + if: ${{ inputs.gcp && env.GCP_SA_KEY != '' }} + uses: google-github-actions/auth@v2 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + + - name: Set up Cloud SDK + if: ${{ inputs.gcp && env.GCP_SA_KEY != '' }} + uses: google-github-actions/setup-gcloud@v2 + + - name: Set up Docker Buildx + if: ${{ inputs.buildx }} + uses: docker/setup-buildx-action@v3 + + - name: "Job: configure" + uses: ./.github/actions/matrix-cell-step + with: + job_json: ${{ inputs.job_json }} + key: configure + + - name: "Job: build" + uses: ./.github/actions/matrix-cell-step + with: + job_json: ${{ inputs.job_json }} + key: build + + - name: "Job: tool" + uses: ./.github/actions/matrix-cell-step + with: + job_json: ${{ inputs.job_json }} + key: tool + + # Postmortem on the same runner the cell just used; see the action for + # what it probes and why it is not an `incl_*` reusable workflow. + # `failure()` covers any earlier step failing and is already false on a + # cancel, so it needs no !cancelled() guard. + # `continue-on-error` is belt-and-braces: the action already exits 0. + - name: Postmortem + if: ${{ failure() }} + continue-on-error: true + timeout-minutes: 5 + uses: ./.github/actions/postmortem diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 59164965..5a83b27d 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,16 +1,21 @@ name: GenVM release -# Builds and publishes the release artifacts consumed by other projects -# (node's download_genvm.sh and friends): -# genvm--.tar.xz β€” manager bundle (bin/, lib/, config/, data/) -# genvm---executor.tar.xz β€” every active executor line (executor//...) -# genvm-runners-all.tar.xz β€” shared runners under runners/ + legacy -# lines' runners at executor//legacy-runners -# All three extract at the same install root. +# incl_release_build_test.yaml plus a publisher: it plans, builds every artifact +# and tests every platform; this workflow attaches what it produced to a +# genvm-manager release tagged with `version` from .genvm-monorepo-root. +# `queue.yaml` runs the same pipeline on a PR (label `test-release-pipeline`), +# so everything below is the only release-only part. # -# The release tag is the manager version from .genvm-monorepo-root (also baked -# into the binaries as GENVM_PROFILE), so tagging is derived from the source -# tree instead of bumping the previous git tag. +# genvm--.tar.xz β€” manager + every active executor line for that +# platform (bin/, lib/, config/, data/, executor/) +# genvm-universal.tar.xz β€” shared runners under runners/ + legacy lines' +# runners at executor//legacy-runners +# +# All four extract at the same install root: a full install is one platform +# asset plus genvm-universal. +# +# Platforms are named the way nix names them (amd64-linux, arm64-linux, +# arm64-macos) everywhere: flake attrs, cells and published assets. on: workflow_dispatch: @@ -28,219 +33,38 @@ permissions: actions: read contents: read -env: - GCS_BUCKET: "gh-af" - jobs: - gen-tag: + # Publishing is the only thing that cares whether the tag is free, so the + # check lives here rather than in the shared pipeline (which a PR runs at the + # same tag on every push). Guards the build so a doomed release fails in + # seconds instead of after three platform builds. + check-tag: runs-on: ubuntu-latest - outputs: - tag: ${{ steps.determine-version.outputs.new_version }} steps: - uses: actions/checkout@v4 - - - name: Determine final version - id: determine-version + - name: Ensure tag is free + env: + VERSION_OVERRIDE: ${{ inputs.version_override }} run: | set -euo pipefail - if [ -n "${{ github.event.inputs.version_override }}" ]; then - tag="${{ github.event.inputs.version_override }}" - echo "Using override version: $tag" - else - tag=$(jq -r .version .genvm-monorepo-root) - echo "Using .genvm-monorepo-root version: $tag" - fi - echo "$tag" | grep -qE '^v[0-9]+\.[0-9]+\.[0-9]+' || { - echo "::error::invalid version '$tag'" - exit 1 - } + tag="${VERSION_OVERRIDE:-$(jq -r .version .genvm-monorepo-root)}" if git ls-remote --exit-code origin "refs/tags/$tag" > /dev/null; then echo "::error::tag $tag already exists on origin" exit 1 fi - echo "new_version=$tag" >> $GITHUB_OUTPUT - - build-runners-all: - needs: [gen-tag] - uses: ./.github/workflows/incl_build_trg.yaml - with: - target: runners-all-dist - executor_version: ${{ needs.gen-tag.outputs.tag }} - secrets: - GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} - - build-linux-amd64: - needs: - - gen-tag - uses: ./.github/workflows/incl_build_trg.yaml - with: - target: manager-amd64-linux - secondary_target: executor-amd64-linux - executor_version: ${{ needs.gen-tag.outputs.tag }} - secrets: - GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} + echo "tag $tag is free" - build-linux-arm64: - needs: - - gen-tag - uses: ./.github/workflows/incl_build_trg.yaml + # Plan -> build every artifact -> test every platform. Its matrices are an + # implementation detail; all we consume are the four artifact names. + build-and-test: + needs: [check-tag] + uses: ./.github/workflows/incl_release_build_test.yaml with: - target: manager-arm64-linux - secondary_target: executor-arm64-linux - executor_version: ${{ needs.gen-tag.outputs.tag }} - secrets: - GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} - - build-macos-arm64: - needs: - - gen-tag - uses: ./.github/workflows/incl_build_trg.yaml - with: - target: manager-arm64-macos - secondary_target: executor-arm64-macos - executor_version: ${{ needs.gen-tag.outputs.tag }} - secrets: - GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} - - test-linux-arm64: - needs: - - build-linux-arm64 - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Get source - uses: ./.github/actions/get-src - with: - # Even devShell evaluation reads the executor submodules' manifests. - load_submodules: "true" - with_nix: "true" - github_token: ${{ secrets.GITHUB_TOKEN }} - - name: Download artifact - run: | - set -euo pipefail - mkdir -p build/out - pushd build/out - curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-linux-arm64.tar.xz ${{ needs.build-linux-arm64.outputs.artifact_url }} - tar -xf genvm-linux-arm64.tar.xz --no-same-owner - curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-linux-arm64-executor.tar.xz ${{ needs.build-linux-arm64.outputs.secondary_artifact_url }} - tar -xf genvm-linux-arm64-executor.tar.xz --no-same-owner - popd - - name: Run post-install - run: | - ./build/out/bin/post-install.py \ - --error-on-missing-executor=false \ - --default-download=false || true - nix develop '.?submodules=1#check-qemu' --command bash -c 'qemu-aarch64 ./build/out/bin/genvm-modules --version' - - name: Smoke test modules startup - run: | - nix develop '.?submodules=1#check-qemu' --command bash -c ' - timeout 5 qemu-aarch64 ./build/out/bin/genvm-modules llm --allow-empty-backends --config ./build/out/config/genvm-module-llm.yaml || test $? -eq 124 - ' - - test-linux-amd64: - needs: - - build-linux-amd64 - - build-runners-all - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Get source - uses: ./.github/actions/get-src - with: - load_submodules: "true" - with_nix: "true" - github_token: ${{ secrets.GITHUB_TOKEN }} - - name: Download artifact - run: | - set -euo pipefail - mkdir -p build/out - pushd build/out - curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-linux-amd64.tar.xz ${{ needs.build-linux-amd64.outputs.artifact_url }} - curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-runners-all.tar.xz ${{ needs.build-runners-all.outputs.artifact_url }} - curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-linux-amd64-executor.tar.xz ${{ needs.build-linux-amd64.outputs.secondary_artifact_url }} - tar -xf genvm-linux-amd64.tar.xz --no-same-owner - tar -xf genvm-runners-all.tar.xz --no-same-owner - tar -xf genvm-linux-amd64-executor.tar.xz --no-same-owner - popd - - name: Run post-install - run: | - ./build/out/bin/post-install.py \ - --default-download=false - - name: Smoke test modules startup - run: | - timeout 5 ./build/out/bin/genvm-modules llm --allow-empty-backends --config ./build/out/config/genvm-module-llm.yaml || test $? -eq 124 - - name: Run stable tests - run: | - # `configure` only writes build/info.json + the ninja files; the test - # runner itself never builds, so it exercises the downloaded - # artifacts in build/out. Rerouting each line to its concrete built - # version comes from build/info.json (no CLI flag anymore). - nix develop '.?submodules=1#mock-tests' --command bash -c ' - set -euo pipefail - genvm-tool configure - genvm-tool test run --ci --filter-tag "$(cat tests/presets/release.txt)" - ' - - test-macos-arm64: - needs: - - build-macos-arm64 - - build-runners-all - runs-on: macos-latest - steps: - - uses: actions/checkout@v4 - - name: Get source - uses: ./.github/actions/get-src - with: - load_submodules: "true" - with_nix: "false" - - name: Download artifact - run: | - set -euo pipefail - mkdir -p build/out - pushd build/out - curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-macos-arm64.tar.xz ${{ needs.build-macos-arm64.outputs.artifact_url }} - curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-runners-all.tar.xz ${{ needs.build-runners-all.outputs.artifact_url }} - curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-macos-arm64-executor.tar.xz ${{ needs.build-macos-arm64.outputs.secondary_artifact_url }} - tar -xf genvm-macos-arm64.tar.xz --no-same-owner - tar -xf genvm-runners-all.tar.xz --no-same-owner - tar -xf genvm-macos-arm64-executor.tar.xz --no-same-owner - popd - - name: Get wabt - run: | - brew install wabt coreutils - - name: Setup Rust - run: | - rustup target add wasm32-wasip1 - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - name: Run post-install - run: | - ./build/out/bin/post-install.py \ - --default-download=false - - name: Smoke test modules startup - run: | - gtimeout 5 ./build/out/bin/genvm-modules llm --allow-empty-backends --config ./build/out/config/genvm-module-llm.yaml || test $? -eq 124 - - name: Run stable tests - run: | - set -euo pipefail - python3 -m venv .venv - source .venv/bin/activate - pip install ./support/tools/genvm-tool - genvm-tool configure - genvm-tool test run --ci --filter-tag "$(cat tests/presets/release.txt)" + version_override: ${{ inputs.version_override }} + secrets: inherit release-publish: - needs: - - gen-tag - - build-linux-amd64 - - build-linux-arm64 - - build-macos-arm64 - - build-runners-all - - test-linux-amd64 - - test-linux-arm64 - - test-macos-arm64 + needs: [build-and-test] runs-on: ubuntu-latest permissions: contents: write # Needed for creating releases @@ -250,45 +74,60 @@ jobs: # Full history + tags: the release-notes step diffs against the # previous tag via `git describe`. fetch-depth: 0 - - run: | - set -euo pipefail - curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-runners-all.tar.xz ${{ needs.build-runners-all.outputs.artifact_url }} - curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-linux-amd64.tar.xz ${{ needs.build-linux-amd64.outputs.artifact_url }} - curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-linux-arm64.tar.xz ${{ needs.build-linux-arm64.outputs.artifact_url }} - curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-macos-arm64.tar.xz ${{ needs.build-macos-arm64.outputs.artifact_url }} - curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-linux-amd64-executor.tar.xz ${{ needs.build-linux-amd64.outputs.secondary_artifact_url }} - curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-linux-arm64-executor.tar.xz ${{ needs.build-linux-arm64.outputs.secondary_artifact_url }} - curl -L --fail-with-body -H 'Accept: application/octet-stream' -o genvm-macos-arm64-executor.tar.xz ${{ needs.build-macos-arm64.outputs.secondary_artifact_url }} - - run: | - git tag ${{ needs.gen-tag.outputs.tag }} && \ - git push origin ${{ needs.gen-tag.outputs.tag }} + - name: Download release notes + uses: actions/download-artifact@v4 + with: + name: ${{ needs.build-and-test.outputs.artifact_notes }} + path: artifacts/notes + # Each artifact holds exactly one tarball, already named as the asset it + # is published as, so they all land in one dir and need no renaming. + - name: Download universal artifact + uses: actions/download-artifact@v4 + with: + name: ${{ needs.build-and-test.outputs.artifact_universal }} + path: release-assets + - name: Download amd64-linux artifact + uses: actions/download-artifact@v4 + with: + name: ${{ needs.build-and-test.outputs.artifact_amd64_linux }} + path: release-assets + - name: Download arm64-linux artifact + uses: actions/download-artifact@v4 + with: + name: ${{ needs.build-and-test.outputs.artifact_arm64_linux }} + path: release-assets + - name: Download arm64-macos artifact + uses: actions/download-artifact@v4 + with: + name: ${{ needs.build-and-test.outputs.artifact_arm64_macos }} + path: release-assets - - name: Generate release notes - id: release_notes + - name: Check release assets run: | - prev_tag=$(git describe --tags --abbrev=0 HEAD~1 2>/dev/null || echo "") - new_tag="${{ needs.gen-tag.outputs.tag }}" - if [ -n "$prev_tag" ]; then - python3 support/ci/make-relase-notes.py "${prev_tag}..${new_tag}" > release-notes.md - else - echo "Initial release" > release-notes.md - fi + set -euo pipefail + ls -l release-assets + for asset in genvm-universal genvm-amd64-linux genvm-arm64-linux genvm-arm64-macos; do + test -f "release-assets/$asset.tar.xz" || { + echo "::error::missing release asset $asset.tar.xz" + exit 1 + } + done + + - name: Tag the release + env: + TAG: ${{ needs.build-and-test.outputs.tag }} + run: | + set -euo pipefail + git tag "$TAG" + git push origin "$TAG" - name: Create Release - id: create_release uses: softprops/action-gh-release@v2 with: - files: | - genvm-runners-all.tar.xz - genvm-linux-amd64.tar.xz - genvm-linux-arm64.tar.xz - genvm-macos-arm64.tar.xz - genvm-linux-amd64-executor.tar.xz - genvm-linux-arm64-executor.tar.xz - genvm-macos-arm64-executor.tar.xz - name: Release ${{ needs.gen-tag.outputs.tag }} - tag_name: ${{ needs.gen-tag.outputs.tag }} - body_path: release-notes.md + files: release-assets/*.tar.xz + name: Release ${{ needs.build-and-test.outputs.tag }} + tag_name: ${{ needs.build-and-test.outputs.tag }} + body_path: artifacts/notes/${{ needs.build-and-test.outputs.notes_file }} draft: false - prerelease: ${{ contains(needs.gen-tag.outputs.tag, '-') }} + prerelease: ${{ contains(needs.build-and-test.outputs.tag, '-') }} diff --git a/.gitignore b/.gitignore index b6a017df..eab50f70 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ !**/.vscode/extensions.json !**/.vscode/settings.example.json +/.pre-commit-config.yaml .claude/hooks .idea .envrc diff --git a/crates/modules-interfaces/Cargo.lock b/crates/modules-interfaces/Cargo.lock index b139de89..66beea9d 100644 --- a/crates/modules-interfaces/Cargo.lock +++ b/crates/modules-interfaces/Cargo.lock @@ -277,7 +277,7 @@ dependencies = [ ] [[package]] -name = "genvm-modules-interfaces" +name = "genvm_modules_interfaces" version = "0.1.0" dependencies = [ "anyhow", diff --git a/crates/modules-interfaces/Cargo.toml b/crates/modules-interfaces/Cargo.toml index 224522e5..bbc07fdd 100644 --- a/crates/modules-interfaces/Cargo.toml +++ b/crates/modules-interfaces/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "genvm-modules-interfaces" +name = "genvm_modules_interfaces" version = "0.1.0" edition = "2021" diff --git a/docs/contributing/howto/building/docs.md b/docs/contributing/howto/building/docs.md index aac6e0f0..49280c0d 100644 --- a/docs/contributing/howto/building/docs.md +++ b/docs/contributing/howto/building/docs.md @@ -2,16 +2,27 @@ ## Website (`docs/website`) -Sphinx site with a poetry env in that dir: +Sphinx site. The toolchain (sphinx + extensions) comes from the `gen-docs` Nix +shell β€” no poetry/venv. Build everything with: ```bash -cd docs/website -poetry install --no-root -poetry run sphinx-build -q -b dirhtml src +./support/ci/run.sh pipeline docs ``` -CI builds `-b html` and `-b text` β€” see `support/ci/pipelines/src/docs.sh` -(also merges the text output into `_static/ai/*.txt`). +That regenerates the derived pages, then enters `.#gen-docs` and runs +`sphinx-build` for `-b html` and `-b text` (merging the text output into +`_static/ai/*.txt`). Finally, for every active executor line it enters that +line's own `.#gen-docs` shell (`executors/.x/flake.nix`) and builds the +line's standalone docs sub-site into `build/doc/html/executors//`; the +manager's generated `executors.rst` page links to them. To iterate on just the +sphinx build: + +```bash +nix develop '.?submodules=1#gen-docs' --command \ + sphinx-build -q -b dirhtml docs/website/src +``` + +See `support/ci/pipelines/docs.py` for the full pipeline. Source layout under `docs/website/src/`: @@ -19,10 +30,20 @@ Source layout under `docs/website/src/`: `spec/appendix/constants.rst` is **generated** from codegen data β€” do not edit by hand, see [genvm-tool.md](../genvm-tool.md). - `impl-spec/` β€” the implementation specification (how this codebase does it). -- `python-sdk/`, `overview/` β€” SDK API docs and intro material. - -`docs/website/generate.py` regenerates derived pages (runner/version tables) -from the repo state; wired into the build via `docs/website/yabuild.rb`. +- `overview/` β€” intro material. + +The Python SDK reference and the per-line runner-version pages are **not** in the +manager tree β€” they live in the executor line that ships them, under +`executors/.x/docs/website/src/python-sdk/` (see that line's `conf.py`), +and build as sub-sites under `build/doc/html/executors//`. Cross-linking is +one-way: the sub-sites reference the manager via intersphinx (the `genvm` +inventory); the manager only links out to them from the generated +`executors.rst` index page. + +`docs/website/generate.py` regenerates derived pages β€” the runner-version tables +(`available-runners.rst` + `runners-versions.json`) and the SDK `changelog.rst` +into the primary line's sub-site, and the `executors.rst` index into the manager +β€” from the repo state; the `docs` pipeline runs it before the sphinx build. ## ADRs diff --git a/docs/contributing/howto/building/runners.md b/docs/contributing/howto/building/runners.md index abec0b6f..d326fe0f 100644 --- a/docs/contributing/howto/building/runners.md +++ b/docs/contributing/howto/building/runners.md @@ -4,10 +4,10 @@ Runners build only on x86_64 Linux (part of the `all` ninja target / `#runners-all` nix attr). On other hosts, download them: ```bash -python3 build/out/bin/post-install.py --create-venv false --default-step false --runners-download true --error-on-missing-executor false +python3 build/out/bin/genvm-post-install --create-venv false --default-step false --runners-download true --error-on-missing-executor false ``` -or fetch `genvm-runners-all.tar.xz` from a +or fetch `genvm-universal.tar.xz` from a [genvm-manager release](https://github.com/genlayerlabs/genvm-manager/releases) and extract it over `build/out`. diff --git a/docs/contributing/howto/committing/submodules.md b/docs/contributing/howto/committing/submodules.md index 751ef380..405eacb0 100644 --- a/docs/contributing/howto/committing/submodules.md +++ b/docs/contributing/howto/committing/submodules.md @@ -37,24 +37,32 @@ git -C executors/v0.3.x rev-parse HEAD # must match To reshape unpushed history: `git reset --mixed `, re-stage per commit, re-bump the gitlink to the new submodule HEAD. -## Pre-commit hook (cross-repo) - -`git commit` triggers `genvm-tool hook run`, which fans linters/formatters out -to the manager **and** submodules. - -- Everything at once: `genvm-tool hook run --all-files` (default `--fix` - rewrites; `--check` only verifies β€” what CI uses). -- `git commit --no-verify` skips the hook β€” fine for pure gitlink bumps or - already-formatted content. -- After changing genvm-tool itself: `genvm-tool hook install` rebuilds the tool - and rewrites the hook stubs. A hook crash referencing an old - `/nix/store/...genvm-tool` means a stale dev-shell `PYTHONPATH` β€” refresh the - dev env or re-run `hook install` with `PYTHONPATH` unset. +## Pre-commit hook (per repo) + +Each repo β€” the manager and every executor submodule β€” carries its own +[git-hooks.nix](https://github.com/cachix/git-hooks.nix) config in its +`flake.nix`, pinned to that repo's own toolchain. Entering the dev shell +(`.envrc` β†’ `nix develop .#full` for the manager; `nix develop` for a +standalone executor checkout) installs the `pre-commit` / `commit-msg` stubs +into that repo's `.git/hooks`, so `git commit` runs the hooks for the repo you +are committing in. + +- Run everything by hand: `nix develop -c pre-commit run --all-files` in the + repo (or `bash support/ci/pipelines/commit-hooks.sh` from the manager root to + check the manager + every executor at once, as CI does). +- Do not `git commit --no-verify` β€” let the hooks run, even for a pure gitlink + bump (they are cheap when nothing needs formatting). +- The executor crates reach the manager's shared `crates/` via relative paths, + so executor hooks must run against the **nested working tree** (submodule + checked out under the manager), not an isolated flake sandbox β€” this is why + CI uses `commit-hooks.sh` (in-tree) instead of `nix flake check`. ## Pushing Check first: `genvm-tool git check-for-push` (`--offline` compares against the -last fetch). Push **submodules before the manager** so gitlinks resolve: +last fetch). After the repo rows it prints one aggregated +`suggested_push_command` block; when anything is not ready, that block is +`none`. Push **submodules before the manager** so gitlinks resolve: ```bash git -C executors/v0.2.x push origin pr/v0.2/feat/ diff --git a/docs/contributing/howto/genvm-tool.md b/docs/contributing/howto/genvm-tool.md index 681cc365..5520cfe7 100644 --- a/docs/contributing/howto/genvm-tool.md +++ b/docs/contributing/howto/genvm-tool.md @@ -12,7 +12,6 @@ outside it, call `support/tools/genvm-tool/genvm-tool`). | `codegen` | generate language bindings from data JSON (below) | | `build-manifest` | generate the manager's `data/manifest.yaml` | | `git ls` / `list-repo` / `create-branches` / `check-for-push` | git helpers across manager + submodules β€” [submodules.md](committing/submodules.md) | -| `hook run` / `hook install` | cross-repo pre-commit fan-out / stub installer β€” [submodules.md](committing/submodules.md) | Command registration: `genvm_tool/__main__.py` (`TOPLEVEL`, `GROUPS`); each command module has `NAME`/`HELP` β€” look there for details. diff --git a/docs/contributing/howto/releasing/release-build.md b/docs/contributing/howto/releasing/release-build.md index b5008f58..49588219 100644 --- a/docs/contributing/howto/releasing/release-build.md +++ b/docs/contributing/howto/releasing/release-build.md @@ -12,18 +12,34 @@ Platform suffixes: none (native), `-amd64-linux`, `-arm64-linux`, `-arm64-macos` | `manager[-]` | manager bundle (`bin/`, `lib/`, `config/`, `data/`) | | `executor[-]` | all active executor lines merged (`executor//…`) | | `executor-[-]` | one executor line | -| `runners-all-dist` | platform-independent runners (`runners/` prefix + legacy-line overlay) | +| `artifact-prepack-genvm-` | a platform's release asset: manager + every active executor line | +| `artifact-prepack-genvm-universal` | the universal release asset: runners (`runners/` prefix + legacy-line overlay) | | `genvm-tool` | the tool itself | ```bash nix build -v -L '.?submodules=1#genvm-amd64-linux' ``` -Release assets (`.github/workflows/release.yaml`; tag = `version` from -`.genvm-monorepo-root`, see [versioning.md](versioning.md)): +## Release assets -- `genvm--.tar.xz` ← `manager-` -- `genvm---executor.tar.xz` ← `executor-` -- `genvm-runners-all.tar.xz` ← `runners-all-dist` +`.github/workflows/release.yaml` is `incl_release_build_test.yaml` (plan β†’ build +every asset β†’ test every platform) plus a publisher. Everything is built here β€” +the executor is not a standalone build entry point. -All three extract at the same install root. +Assets are named with the same platform token as the nix targets +(`amd64-linux`, …), not a reversed one. Each is one prepacked tree, and the +tarball is already named as the asset, so nothing renames it on the way out: + +| Asset | Package | +|---|---| +| `genvm-.tar.xz` | `artifact-prepack-genvm-` | +| `genvm-universal.tar.xz` | `artifact-prepack-genvm-universal` | + +All four go onto one **genvm-manager** release, tagged with `version` from +`.genvm-monorepo-root` (see [versioning.md](versioning.md)). They overlay onto a +single install root, so a full install is one platform asset + `genvm-universal`. + +To exercise the whole pipeline on a PR without releasing anything, add the +`test-release-pipeline` label: `queue.yaml` then runs the same +`incl_release_build_test.yaml`. It also generates the release notes and prints +them, so they can be checked before the release. diff --git a/docs/contributing/howto/setup.md b/docs/contributing/howto/setup.md index 40f26389..1be964e4 100644 --- a/docs/contributing/howto/setup.md +++ b/docs/contributing/howto/setup.md @@ -1,11 +1,17 @@ # First-time setup -1. Init the executor submodules **before** entering the dev shell (the nix flake reads them): +1. Get the executor submodules and vendored third-party trees **before** entering + the dev shell (the nix flake reads them): ```bash - git submodule update --init --recursive --depth 1 + python3 support/scripts/get-all-git.py ``` + The default options are sufficient for normal setup: the command resolves + duplicate submodule remotes, fetches each unique remote once into a local + cache, initializes submodules from that cache, and materializes all + third-party trees. + 2. Enter the dev shell: ```bash @@ -17,19 +23,9 @@ direnv (`.envrc` is `use flake '.?submodules=1#full'`). 3. `source env.sh` β€” adds `support/tools/git-third-party` to PATH and sources - `.env` if present. Only needed outside the dev shell. - -4. Materialize the vendored third-party trees (wasmtime, wasm-tools). These are - registered per executor submodule, and `git third-party` reads its config - from the current git toplevel β€” so run it *inside each executor*, not at the - manager root (which has no config and would materialize nothing): - - ```bash - for cfg in executors/*/.git-third-party/config.json; do - ( cd "$(dirname "$(dirname "$cfg")")" && git third-party update --all ) - done - ``` + `.env` if present. Only needed outside the dev shell for manual + `git third-party` use. - See [git-third-party.md](committing/git-third-party.md) for how vendoring works. +See [git-third-party.md](committing/git-third-party.md) for how vendoring works. Next: [build.md](building/build.md). diff --git a/docs/website/.gitignore b/docs/website/.gitignore index d5063148..97f10dee 100644 --- a/docs/website/.gitignore +++ b/docs/website/.gitignore @@ -1,5 +1,3 @@ /src/spec/appendix/constants.rst -/src/impl-spec/appendix/runners-versions.json /src/_static/versions.json -/src/python-sdk/changelog.rst -/src/impl-spec/appendix/available-runners.rst +/src/overview/executor-lines_generated.rst diff --git a/docs/website/generate.py b/docs/website/generate.py index f0cdb3ec..99c52dc9 100755 --- a/docs/website/generate.py +++ b/docs/website/generate.py @@ -1,34 +1,42 @@ #!/usr/bin/env python3 import json -import os import re import subprocess import sys from pathlib import Path -script_dir = Path('__file__').parent.absolute() +script_dir = Path(__file__).resolve().parent genvm_root_dir = script_dir while not genvm_root_dir.joinpath('.genvm-monorepo-root').exists(): genvm_root_dir = genvm_root_dir.parent docs_src_root_dir = genvm_root_dir.joinpath('docs', 'website', 'src') -proc = subprocess.run( - ['git', 'remote', 'get-url', 'origin'], - check=True, - capture_output=True, - text=True, +# Runners belong to the EXECUTOR, not the manager: a runner's version is an +# executor-version, tagged in the executor repo, and its sources live under that +# repo's runners/. So every git lookup behind this page β€” tags, tag messages, the +# log between two runner versions β€” has to run against an executor checkout, not +# the manager's. All active lines are branches of ONE repo, so any of them sees +# every executor tag; take the first. +monorepo = json.loads(genvm_root_dir.joinpath('.genvm-monorepo-root').read_text()) +executor_dir = genvm_root_dir.joinpath( + 'executors', f'{monorepo["active-versions"][0]}.x' ) -origin_url = proc.stdout.strip() -proc = subprocess.run( - ['git', 'ls-remote', '--tags', origin_url], - check=True, - capture_output=True, - text=True, -) +def executor_git(*args: str, check: bool = True) -> subprocess.CompletedProcess: + return subprocess.run( + ['git', '-C', str(executor_dir), *args], + check=check, + capture_output=True, + text=True, + ) + + +origin_url = executor_git('remote', 'get-url', 'origin').stdout.strip() + +proc = executor_git('ls-remote', '--tags', origin_url) commit2tag = {} @@ -45,18 +53,11 @@ commit2tag[commit] = bad_id.sub('_', rev) -cwd = Path(os.getcwd()) - -# Ensure tags are available locally (CI may use shallow checkout) -subprocess.run(['git', 'fetch', '--tags'], capture_output=True, check=True, text=True) +# Ensure tags are available locally (CI may use a shallow checkout) +executor_git('fetch', '--tags', check=False) # Collect tag messages (one-line) for descriptions -proc_tags = subprocess.run( - ['git', 'tag', '-l', '-n1'], - check=True, - capture_output=True, - text=True, -) +proc_tags = executor_git('tag', '-l', '-n1') tag_messages = {} for line in proc_tags.stdout.splitlines(): parts = line.split(None, 1) @@ -65,24 +66,10 @@ elif len(parts) == 1: tag_messages[parts[0]] = '' -current_commit = subprocess.run( - ['git', 'rev-parse', 'HEAD'], - check=True, - capture_output=True, - text=True, -) - -current_commit = current_commit.stdout.strip() - eval_file = genvm_root_dir.joinpath('runners', 'docs.nix') -build_config = json.loads(genvm_root_dir.joinpath('flake-config.json').read_text()) -build_config['head-revision'] = current_commit -commit2tag[current_commit] = build_config['executor-version'] - -print(commit2tag) -print(build_config) - +# Each runner carries the version of the executor line it came from, so docs.nix +# groups them itself and needs no commit-to-tag map (it takes no arguments). proc = subprocess.run( [ 'nix', @@ -93,13 +80,9 @@ '--show-trace', '--json', '--file', - str(eval_file.relative_to(cwd)), + str(eval_file), '--apply', - 'f: f { commitToTagStr = "' - + json.dumps(commit2tag).replace('"', '\\"') - + '"; build-config-str = "' - + json.dumps(build_config).replace('"', '\\"') - + '"; }', + 'f: f { }', ], check=True, capture_output=True, @@ -108,6 +91,13 @@ res = json.loads(proc.stdout) +# docs.nix accumulates the runners of EVERY active line, but this page is +# per-line (it renders into active-versions[0]'s sub-site). Post-split the lines +# are independent β€” v0.2 is not "before" v0.3 β€” so scope to this line's own +# versions; otherwise the other lines' hashes leak in as bogus history. +_line = monorepo['active-versions'][0] +res = {v: r for v, r in res.items() if v.startswith(_line + '.')} + # --- Post-process: sort by semver, enrich with descriptions, diffs, commits --- SEMVER_RE = re.compile(r'^v?(\d+)\.(\d+)\.(\d+)(.*)$') @@ -170,27 +160,26 @@ def is_semver(version: str) -> bool: prev_commit_hash = tag2commit.get(prev_ver) curr_commit_hash = tag2commit.get(ver) if prev_commit_hash and curr_commit_hash: - try: - log_proc = subprocess.run( - [ - 'git', - 'log', - '--oneline', - f'{prev_commit_hash}..{curr_commit_hash}', - '--', - 'executors/v0.3.x/runners/', - ], - capture_output=True, - text=True, - check=True, - ) - commits_map[ver] = [ + # In the executor repo, and over ITS runners/ β€” the manager's history has + # no executor commits (they sit behind a gitlink), so asking it for the + # log between two runner versions silently yields nothing. + log_proc = executor_git( + 'log', + '--oneline', + f'{prev_commit_hash}..{curr_commit_hash}', + '--', + 'runners/', + check=False, + ) + commits_map[ver] = ( + [ line.split(None, 1)[1] if len(line.split(None, 1)) > 1 else line for line in log_proc.stdout.strip().splitlines() if line.strip() ] - except subprocess.CalledProcessError: - commits_map[ver] = [] + if log_proc.returncode == 0 + else [] + ) else: commits_map[ver] = [] @@ -372,17 +361,15 @@ def render_runner_section(rid, rst_parts): rst_parts.append(f'Current hash: ``{current_hash}``') rst_parts.append('') + # Non-latest hashes: link each to the version it first appeared in + # (start_v of its run), not a range \u2014 a small "when did this hash ship" map. prev_groups = format_version_range(rid) if prev_groups: - for h, start_v, end_v in prev_groups: - if start_v == end_v: - anchor = version_to_anchor(start_v) - rst_parts.append(f'- ``{h}`` \u2014 `{start_v} `_') - else: - anchor = version_to_anchor(end_v) - rst_parts.append( - f'- ``{h}`` \u2014 {start_v} through `{end_v} `_' - ) + rst_parts.append('Earlier hashes (first appeared in):') + rst_parts.append('') + for h, start_v, _end_v in prev_groups: + anchor = version_to_anchor(start_v) + rst_parts.append(f'- ``{h}`` \u2014 `{start_v} `_') rst_parts.append('') @@ -428,12 +415,13 @@ def render_runner_section(rid, rst_parts): # --- Generate changelog --- -# Sphinx resolves include paths relative to the top-level source file, -changelog_notes_dir = docs_src_root_dir / 'python-sdk' / 'changelog-notes' -changelog_rst_path = docs_src_root_dir / 'python-sdk' / 'changelog.rst' -changelog_notes_include_prefix = changelog_notes_dir.relative_to( - changelog_rst_path.parent -).as_posix() +# The Python SDK docs (incl. this changelog) now live in the executor line that +# ships the SDK, not the manager tree. The changelog is still generated here β€” +# it is driven by runner-version data the manager owns β€” but written into that +# line's docs sub-site. +sdk_src_root_dir = executor_dir / 'docs' / 'website' / 'src' + +changelog_rst_path = sdk_src_root_dir / 'python-sdk' / 'changelog.rst' changelog_rst = [] @@ -465,18 +453,6 @@ def render_runner_section(rid, rst_parts): changelog_rst.append(f'*{desc}*') changelog_rst.append('') - # Check for hand-written notes β€” try exact version and base version (e.g. v0.1.8 for v0.1.8-runner-hash) - notes_file = changelog_notes_dir / f'{ver}.rst' - if not notes_file.exists(): - base_ver = SEMVER_RE.match(ver) - if base_ver: - base_name = f'v{base_ver.group(1)}.{base_ver.group(2)}.{base_ver.group(3)}' - notes_file = changelog_notes_dir / f'{base_name}.rst' - if notes_file.exists(): - rel_notes = f'{changelog_notes_include_prefix}/{notes_file.name}' - changelog_rst.append(f'.. include:: {rel_notes}') - changelog_rst.append('') - if changes: n = len(changes) s = 's' if n != 1 else '' @@ -517,3 +493,18 @@ def render_runner_section(rid, rst_parts): changelog_rst.append('') changelog_rst_path.write_text('\n'.join(changelog_rst) + '\n') + +# --- Generate the executor sub-site links (included by the overview page) --- +# Each active executor line ships a standalone docs sub-site, built separately by +# support/ci/pipelines/docs.py under build/doc/html/executors//. The +# overview page lists a link to each; this is generated as an include (kept out +# of the toctree by the *_generated.rst exclude) so overview/index.rst owns the +# heading/prose and this file is just the per-line bullets. Links are relative to +# the overview page (one level under the html root), hence `../executors/`. +executor_links = [ + f'- `{line} <../executors/{line}/index.html>`_' + for line in monorepo['active-versions'] +] +(docs_src_root_dir / 'overview' / 'executor-lines_generated.rst').write_text( + '\n'.join(executor_links) + '\n' +) diff --git a/docs/website/merge-txts.rb b/docs/website/merge-txts.rb deleted file mode 100755 index 565b9156..00000000 --- a/docs/website/merge-txts.rb +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env ruby - -require 'pathname' - -src_dir, dst_file = ARGV - -buf = String.new - -src_dir = Pathname.new(src_dir) - -src_dir.glob('**/*.txt').sort.each do |file| - name = file.relative_path_from(src_dir).to_s - buf << "### #{name}\n\n" - buf << file.read - buf << "\n\n" -end - -Pathname.new(dst_file).dirname.mkpath -File.write(dst_file, buf) diff --git a/docs/website/merge_txts.py b/docs/website/merge_txts.py new file mode 100644 index 00000000..a805bc4d --- /dev/null +++ b/docs/website/merge_txts.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +"""Concatenate every ``*.txt`` under a directory into one bundle. + +Used to fold a sphinx ``-b text`` build of one section into a single +``_static/ai/
.txt`` file (the "docs for LLMs" bundles). Each entry is +prefixed with its path relative to the source dir, files sorted for a stable +output. +""" + +import sys +from pathlib import Path + + +def merge(src_dir: Path, dst_file: Path) -> None: + buf: list[str] = [] + for file in sorted(src_dir.glob('**/*.txt')): + name = file.relative_to(src_dir).as_posix() + buf.append(f'### {name}\n\n') + buf.append(file.read_text()) + buf.append('\n\n') + dst_file.parent.mkdir(parents=True, exist_ok=True) + dst_file.write_text(''.join(buf)) + + +if __name__ == '__main__': + merge(Path(sys.argv[1]), Path(sys.argv[2])) diff --git a/docs/website/poetry.lock b/docs/website/poetry.lock deleted file mode 100644 index accb5e6f..00000000 --- a/docs/website/poetry.lock +++ /dev/null @@ -1,1222 +0,0 @@ -# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. - -[[package]] -name = "accessible-pygments" -version = "0.0.5" -description = "A collection of accessible pygments styles" -optional = false -python-versions = ">=3.9" -files = [ - {file = "accessible_pygments-0.0.5-py3-none-any.whl", hash = "sha256:88ae3211e68a1d0b011504b2ffc1691feafce124b845bd072ab6f9f66f34d4b7"}, - {file = "accessible_pygments-0.0.5.tar.gz", hash = "sha256:40918d3e6a2b619ad424cb91e556bd3bd8865443d9f22f1dcdf79e33c8046872"}, -] - -[package.dependencies] -pygments = ">=1.5" - -[package.extras] -dev = ["pillow", "pkginfo (>=1.10)", "playwright", "pre-commit", "setuptools", "twine (>=5.0)"] -tests = ["hypothesis", "pytest"] - -[[package]] -name = "alabaster" -version = "1.0.0" -description = "A light, configurable Sphinx theme" -optional = false -python-versions = ">=3.10" -files = [ - {file = "alabaster-1.0.0-py3-none-any.whl", hash = "sha256:fc6786402dc3fcb2de3cabd5fe455a2db534b371124f1f21de8731783dec828b"}, - {file = "alabaster-1.0.0.tar.gz", hash = "sha256:c00dca57bca26fa62a6d7d0a9fcce65f3e026e9bfe33e9c538fd3fbb2144fd9e"}, -] - -[[package]] -name = "attrs" -version = "25.4.0" -description = "Classes Without Boilerplate" -optional = false -python-versions = ">=3.9" -files = [ - {file = "attrs-25.4.0-py3-none-any.whl", hash = "sha256:adcf7e2a1fb3b36ac48d97835bb6d8ade15b8dcce26aba8bf1d14847b57a3373"}, - {file = "attrs-25.4.0.tar.gz", hash = "sha256:16d5969b87f0859ef33a48b35d55ac1be6e42ae49d5e853b597db70c35c57e11"}, -] - -[[package]] -name = "babel" -version = "2.17.0" -description = "Internationalization utilities" -optional = false -python-versions = ">=3.8" -files = [ - {file = "babel-2.17.0-py3-none-any.whl", hash = "sha256:4d0b53093fdfb4b21c92b5213dba5a1b23885afa8383709427046b21c366e5f2"}, - {file = "babel-2.17.0.tar.gz", hash = "sha256:0c54cffb19f690cdcc52a3b50bcbf71e07a808d1c80d549f2459b9d2cf0afb9d"}, -] - -[package.extras] -dev = ["backports.zoneinfo", "freezegun (>=1.0,<2.0)", "jinja2 (>=3.0)", "pytest (>=6.0)", "pytest-cov", "pytz", "setuptools", "tzdata"] - -[[package]] -name = "beautifulsoup4" -version = "4.14.3" -description = "Screen-scraping library" -optional = false -python-versions = ">=3.7.0" -files = [ - {file = "beautifulsoup4-4.14.3-py3-none-any.whl", hash = "sha256:0918bfe44902e6ad8d57732ba310582e98da931428d231a5ecb9e7c703a735bb"}, - {file = "beautifulsoup4-4.14.3.tar.gz", hash = "sha256:6292b1c5186d356bba669ef9f7f051757099565ad9ada5dd630bd9de5fa7fb86"}, -] - -[package.dependencies] -soupsieve = ">=1.6.1" -typing-extensions = ">=4.0.0" - -[package.extras] -cchardet = ["cchardet"] -chardet = ["chardet"] -charset-normalizer = ["charset-normalizer"] -html5lib = ["html5lib"] -lxml = ["lxml"] - -[[package]] -name = "certifi" -version = "2026.1.4" -description = "Python package for providing Mozilla's CA Bundle." -optional = false -python-versions = ">=3.7" -files = [ - {file = "certifi-2026.1.4-py3-none-any.whl", hash = "sha256:9943707519e4add1115f44c2bc244f782c0249876bf51b6599fee1ffbedd685c"}, - {file = "certifi-2026.1.4.tar.gz", hash = "sha256:ac726dd470482006e014ad384921ed6438c457018f4b3d204aea4281258b2120"}, -] - -[[package]] -name = "charset-normalizer" -version = "3.4.4" -description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -optional = false -python-versions = ">=3.7" -files = [ - {file = "charset_normalizer-3.4.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e824f1492727fa856dd6eda4f7cee25f8518a12f3c4a56a74e8095695089cf6d"}, - {file = "charset_normalizer-3.4.4-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4bd5d4137d500351a30687c2d3971758aac9a19208fc110ccb9d7188fbe709e8"}, - {file = "charset_normalizer-3.4.4-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:027f6de494925c0ab2a55eab46ae5129951638a49a34d87f4c3eda90f696b4ad"}, - {file = "charset_normalizer-3.4.4-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f820802628d2694cb7e56db99213f930856014862f3fd943d290ea8438d07ca8"}, - {file = "charset_normalizer-3.4.4-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:798d75d81754988d2565bff1b97ba5a44411867c0cf32b77a7e8f8d84796b10d"}, - {file = "charset_normalizer-3.4.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d1bb833febdff5c8927f922386db610b49db6e0d4f4ee29601d71e7c2694313"}, - {file = "charset_normalizer-3.4.4-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9cd98cdc06614a2f768d2b7286d66805f94c48cde050acdbbb7db2600ab3197e"}, - {file = "charset_normalizer-3.4.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:077fbb858e903c73f6c9db43374fd213b0b6a778106bc7032446a8e8b5b38b93"}, - {file = "charset_normalizer-3.4.4-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:244bfb999c71b35de57821b8ea746b24e863398194a4014e4c76adc2bbdfeff0"}, - {file = "charset_normalizer-3.4.4-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:64b55f9dce520635f018f907ff1b0df1fdc31f2795a922fb49dd14fbcdf48c84"}, - {file = "charset_normalizer-3.4.4-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:faa3a41b2b66b6e50f84ae4a68c64fcd0c44355741c6374813a800cd6695db9e"}, - {file = "charset_normalizer-3.4.4-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:6515f3182dbe4ea06ced2d9e8666d97b46ef4c75e326b79bb624110f122551db"}, - {file = "charset_normalizer-3.4.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:cc00f04ed596e9dc0da42ed17ac5e596c6ccba999ba6bd92b0e0aef2f170f2d6"}, - {file = "charset_normalizer-3.4.4-cp310-cp310-win32.whl", hash = "sha256:f34be2938726fc13801220747472850852fe6b1ea75869a048d6f896838c896f"}, - {file = "charset_normalizer-3.4.4-cp310-cp310-win_amd64.whl", hash = "sha256:a61900df84c667873b292c3de315a786dd8dac506704dea57bc957bd31e22c7d"}, - {file = "charset_normalizer-3.4.4-cp310-cp310-win_arm64.whl", hash = "sha256:cead0978fc57397645f12578bfd2d5ea9138ea0fac82b2f63f7f7c6877986a69"}, - {file = "charset_normalizer-3.4.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6e1fcf0720908f200cd21aa4e6750a48ff6ce4afe7ff5a79a90d5ed8a08296f8"}, - {file = "charset_normalizer-3.4.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f819d5fe9234f9f82d75bdfa9aef3a3d72c4d24a6e57aeaebba32a704553aa0"}, - {file = "charset_normalizer-3.4.4-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a59cb51917aa591b1c4e6a43c132f0cdc3c76dbad6155df4e28ee626cc77a0a3"}, - {file = "charset_normalizer-3.4.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8ef3c867360f88ac904fd3f5e1f902f13307af9052646963ee08ff4f131adafc"}, - {file = "charset_normalizer-3.4.4-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d9e45d7faa48ee908174d8fe84854479ef838fc6a705c9315372eacbc2f02897"}, - {file = "charset_normalizer-3.4.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:840c25fb618a231545cbab0564a799f101b63b9901f2569faecd6b222ac72381"}, - {file = "charset_normalizer-3.4.4-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ca5862d5b3928c4940729dacc329aa9102900382fea192fc5e52eb69d6093815"}, - {file = "charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d9c7f57c3d666a53421049053eaacdd14bbd0a528e2186fcb2e672effd053bb0"}, - {file = "charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:277e970e750505ed74c832b4bf75dac7476262ee2a013f5574dd49075879e161"}, - {file = "charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:31fd66405eaf47bb62e8cd575dc621c56c668f27d46a61d975a249930dd5e2a4"}, - {file = "charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:0d3d8f15c07f86e9ff82319b3d9ef6f4bf907608f53fe9d92b28ea9ae3d1fd89"}, - {file = "charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:9f7fcd74d410a36883701fafa2482a6af2ff5ba96b9a620e9e0721e28ead5569"}, - {file = "charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ebf3e58c7ec8a8bed6d66a75d7fb37b55e5015b03ceae72a8e7c74495551e224"}, - {file = "charset_normalizer-3.4.4-cp311-cp311-win32.whl", hash = "sha256:eecbc200c7fd5ddb9a7f16c7decb07b566c29fa2161a16cf67b8d068bd21690a"}, - {file = "charset_normalizer-3.4.4-cp311-cp311-win_amd64.whl", hash = "sha256:5ae497466c7901d54b639cf42d5b8c1b6a4fead55215500d2f486d34db48d016"}, - {file = "charset_normalizer-3.4.4-cp311-cp311-win_arm64.whl", hash = "sha256:65e2befcd84bc6f37095f5961e68a6f077bf44946771354a28ad434c2cce0ae1"}, - {file = "charset_normalizer-3.4.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0a98e6759f854bd25a58a73fa88833fba3b7c491169f86ce1180c948ab3fd394"}, - {file = "charset_normalizer-3.4.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b5b290ccc2a263e8d185130284f8501e3e36c5e02750fc6b6bdeb2e9e96f1e25"}, - {file = "charset_normalizer-3.4.4-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:74bb723680f9f7a6234dcf67aea57e708ec1fbdf5699fb91dfd6f511b0a320ef"}, - {file = "charset_normalizer-3.4.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f1e34719c6ed0b92f418c7c780480b26b5d9c50349e9a9af7d76bf757530350d"}, - {file = "charset_normalizer-3.4.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2437418e20515acec67d86e12bf70056a33abdacb5cb1655042f6538d6b085a8"}, - {file = "charset_normalizer-3.4.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:11d694519d7f29d6cd09f6ac70028dba10f92f6cdd059096db198c283794ac86"}, - {file = "charset_normalizer-3.4.4-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ac1c4a689edcc530fc9d9aa11f5774b9e2f33f9a0c6a57864e90908f5208d30a"}, - {file = "charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:21d142cc6c0ec30d2efee5068ca36c128a30b0f2c53c1c07bd78cb6bc1d3be5f"}, - {file = "charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5dbe56a36425d26d6cfb40ce79c314a2e4dd6211d51d6d2191c00bed34f354cc"}, - {file = "charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:5bfbb1b9acf3334612667b61bd3002196fe2a1eb4dd74d247e0f2a4d50ec9bbf"}, - {file = "charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:d055ec1e26e441f6187acf818b73564e6e6282709e9bcb5b63f5b23068356a15"}, - {file = "charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:af2d8c67d8e573d6de5bc30cdb27e9b95e49115cd9baad5ddbd1a6207aaa82a9"}, - {file = "charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:780236ac706e66881f3b7f2f32dfe90507a09e67d1d454c762cf642e6e1586e0"}, - {file = "charset_normalizer-3.4.4-cp312-cp312-win32.whl", hash = "sha256:5833d2c39d8896e4e19b689ffc198f08ea58116bee26dea51e362ecc7cd3ed26"}, - {file = "charset_normalizer-3.4.4-cp312-cp312-win_amd64.whl", hash = "sha256:a79cfe37875f822425b89a82333404539ae63dbdddf97f84dcbc3d339aae9525"}, - {file = "charset_normalizer-3.4.4-cp312-cp312-win_arm64.whl", hash = "sha256:376bec83a63b8021bb5c8ea75e21c4ccb86e7e45ca4eb81146091b56599b80c3"}, - {file = "charset_normalizer-3.4.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:e1f185f86a6f3403aa2420e815904c67b2f9ebc443f045edd0de921108345794"}, - {file = "charset_normalizer-3.4.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b39f987ae8ccdf0d2642338faf2abb1862340facc796048b604ef14919e55ed"}, - {file = "charset_normalizer-3.4.4-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3162d5d8ce1bb98dd51af660f2121c55d0fa541b46dff7bb9b9f86ea1d87de72"}, - {file = "charset_normalizer-3.4.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:81d5eb2a312700f4ecaa977a8235b634ce853200e828fbadf3a9c50bab278328"}, - {file = "charset_normalizer-3.4.4-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5bd2293095d766545ec1a8f612559f6b40abc0eb18bb2f5d1171872d34036ede"}, - {file = "charset_normalizer-3.4.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a8a8b89589086a25749f471e6a900d3f662d1d3b6e2e59dcecf787b1cc3a1894"}, - {file = "charset_normalizer-3.4.4-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bc7637e2f80d8530ee4a78e878bce464f70087ce73cf7c1caf142416923b98f1"}, - {file = "charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f8bf04158c6b607d747e93949aa60618b61312fe647a6369f88ce2ff16043490"}, - {file = "charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:554af85e960429cf30784dd47447d5125aaa3b99a6f0683589dbd27e2f45da44"}, - {file = "charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:74018750915ee7ad843a774364e13a3db91682f26142baddf775342c3f5b1133"}, - {file = "charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c0463276121fdee9c49b98908b3a89c39be45d86d1dbaa22957e38f6321d4ce3"}, - {file = "charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:362d61fd13843997c1c446760ef36f240cf81d3ebf74ac62652aebaf7838561e"}, - {file = "charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9a26f18905b8dd5d685d6d07b0cdf98a79f3c7a918906af7cc143ea2e164c8bc"}, - {file = "charset_normalizer-3.4.4-cp313-cp313-win32.whl", hash = "sha256:9b35f4c90079ff2e2edc5b26c0c77925e5d2d255c42c74fdb70fb49b172726ac"}, - {file = "charset_normalizer-3.4.4-cp313-cp313-win_amd64.whl", hash = "sha256:b435cba5f4f750aa6c0a0d92c541fb79f69a387c91e61f1795227e4ed9cece14"}, - {file = "charset_normalizer-3.4.4-cp313-cp313-win_arm64.whl", hash = "sha256:542d2cee80be6f80247095cc36c418f7bddd14f4a6de45af91dfad36d817bba2"}, - {file = "charset_normalizer-3.4.4-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:da3326d9e65ef63a817ecbcc0df6e94463713b754fe293eaa03da99befb9a5bd"}, - {file = "charset_normalizer-3.4.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8af65f14dc14a79b924524b1e7fffe304517b2bff5a58bf64f30b98bbc5079eb"}, - {file = "charset_normalizer-3.4.4-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:74664978bb272435107de04e36db5a9735e78232b85b77d45cfb38f758efd33e"}, - {file = "charset_normalizer-3.4.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:752944c7ffbfdd10c074dc58ec2d5a8a4cd9493b314d367c14d24c17684ddd14"}, - {file = "charset_normalizer-3.4.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d1f13550535ad8cff21b8d757a3257963e951d96e20ec82ab44bc64aeb62a191"}, - {file = "charset_normalizer-3.4.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ecaae4149d99b1c9e7b88bb03e3221956f68fd6d50be2ef061b2381b61d20838"}, - {file = "charset_normalizer-3.4.4-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cb6254dc36b47a990e59e1068afacdcd02958bdcce30bb50cc1700a8b9d624a6"}, - {file = "charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c8ae8a0f02f57a6e61203a31428fa1d677cbe50c93622b4149d5c0f319c1d19e"}, - {file = "charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:47cc91b2f4dd2833fddaedd2893006b0106129d4b94fdb6af1f4ce5a9965577c"}, - {file = "charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:82004af6c302b5d3ab2cfc4cc5f29db16123b1a8417f2e25f9066f91d4411090"}, - {file = "charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7d8f6c26245217bd2ad053761201e9f9680f8ce52f0fcd8d0755aeae5b2152"}, - {file = "charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:799a7a5e4fb2d5898c60b640fd4981d6a25f1c11790935a44ce38c54e985f828"}, - {file = "charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:99ae2cffebb06e6c22bdc25801d7b30f503cc87dbd283479e7b606f70aff57ec"}, - {file = "charset_normalizer-3.4.4-cp314-cp314-win32.whl", hash = "sha256:f9d332f8c2a2fcbffe1378594431458ddbef721c1769d78e2cbc06280d8155f9"}, - {file = "charset_normalizer-3.4.4-cp314-cp314-win_amd64.whl", hash = "sha256:8a6562c3700cce886c5be75ade4a5db4214fda19fede41d9792d100288d8f94c"}, - {file = "charset_normalizer-3.4.4-cp314-cp314-win_arm64.whl", hash = "sha256:de00632ca48df9daf77a2c65a484531649261ec9f25489917f09e455cb09ddb2"}, - {file = "charset_normalizer-3.4.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:ce8a0633f41a967713a59c4139d29110c07e826d131a316b50ce11b1d79b4f84"}, - {file = "charset_normalizer-3.4.4-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eaabd426fe94daf8fd157c32e571c85cb12e66692f15516a83a03264b08d06c3"}, - {file = "charset_normalizer-3.4.4-cp38-cp38-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4ef880e27901b6cc782f1b95f82da9313c0eb95c3af699103088fa0ac3ce9ac"}, - {file = "charset_normalizer-3.4.4-cp38-cp38-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2aaba3b0819274cc41757a1da876f810a3e4d7b6eb25699253a4effef9e8e4af"}, - {file = "charset_normalizer-3.4.4-cp38-cp38-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:778d2e08eda00f4256d7f672ca9fef386071c9202f5e4607920b86d7803387f2"}, - {file = "charset_normalizer-3.4.4-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f155a433c2ec037d4e8df17d18922c3a0d9b3232a396690f17175d2946f0218d"}, - {file = "charset_normalizer-3.4.4-cp38-cp38-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a8bf8d0f749c5757af2142fe7903a9df1d2e8aa3841559b2bad34b08d0e2bcf3"}, - {file = "charset_normalizer-3.4.4-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:194f08cbb32dc406d6e1aea671a68be0823673db2832b38405deba2fb0d88f63"}, - {file = "charset_normalizer-3.4.4-cp38-cp38-musllinux_1_2_armv7l.whl", hash = "sha256:6aee717dcfead04c6eb1ce3bd29ac1e22663cdea57f943c87d1eab9a025438d7"}, - {file = "charset_normalizer-3.4.4-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:cd4b7ca9984e5e7985c12bc60a6f173f3c958eae74f3ef6624bb6b26e2abbae4"}, - {file = "charset_normalizer-3.4.4-cp38-cp38-musllinux_1_2_riscv64.whl", hash = "sha256:b7cf1017d601aa35e6bb650b6ad28652c9cd78ee6caff19f3c28d03e1c80acbf"}, - {file = "charset_normalizer-3.4.4-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:e912091979546adf63357d7e2ccff9b44f026c075aeaf25a52d0e95ad2281074"}, - {file = "charset_normalizer-3.4.4-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:5cb4d72eea50c8868f5288b7f7f33ed276118325c1dfd3957089f6b519e1382a"}, - {file = "charset_normalizer-3.4.4-cp38-cp38-win32.whl", hash = "sha256:837c2ce8c5a65a2035be9b3569c684358dfbf109fd3b6969630a87535495ceaa"}, - {file = "charset_normalizer-3.4.4-cp38-cp38-win_amd64.whl", hash = "sha256:44c2a8734b333e0578090c4cd6b16f275e07aa6614ca8715e6c038e865e70576"}, - {file = "charset_normalizer-3.4.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:a9768c477b9d7bd54bc0c86dbaebdec6f03306675526c9927c0e8a04e8f94af9"}, - {file = "charset_normalizer-3.4.4-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1bee1e43c28aa63cb16e5c14e582580546b08e535299b8b6158a7c9c768a1f3d"}, - {file = "charset_normalizer-3.4.4-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fd44c878ea55ba351104cb93cc85e74916eb8fa440ca7903e57575e97394f608"}, - {file = "charset_normalizer-3.4.4-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f04b14ffe5fdc8c4933862d8306109a2c51e0704acfa35d51598eb45a1e89fc"}, - {file = "charset_normalizer-3.4.4-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cd09d08005f958f370f539f186d10aec3377d55b9eeb0d796025d4886119d76e"}, - {file = "charset_normalizer-3.4.4-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4fe7859a4e3e8457458e2ff592f15ccb02f3da787fcd31e0183879c3ad4692a1"}, - {file = "charset_normalizer-3.4.4-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fa09f53c465e532f4d3db095e0c55b615f010ad81803d383195b6b5ca6cbf5f3"}, - {file = "charset_normalizer-3.4.4-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:7fa17817dc5625de8a027cb8b26d9fefa3ea28c8253929b8d6649e705d2835b6"}, - {file = "charset_normalizer-3.4.4-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:5947809c8a2417be3267efc979c47d76a079758166f7d43ef5ae8e9f92751f88"}, - {file = "charset_normalizer-3.4.4-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:4902828217069c3c5c71094537a8e623f5d097858ac6ca8252f7b4d10b7560f1"}, - {file = "charset_normalizer-3.4.4-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:7c308f7e26e4363d79df40ca5b2be1c6ba9f02bdbccfed5abddb7859a6ce72cf"}, - {file = "charset_normalizer-3.4.4-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:2c9d3c380143a1fedbff95a312aa798578371eb29da42106a29019368a475318"}, - {file = "charset_normalizer-3.4.4-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:cb01158d8b88ee68f15949894ccc6712278243d95f344770fa7593fa2d94410c"}, - {file = "charset_normalizer-3.4.4-cp39-cp39-win32.whl", hash = "sha256:2677acec1a2f8ef614c6888b5b4ae4060cc184174a938ed4e8ef690e15d3e505"}, - {file = "charset_normalizer-3.4.4-cp39-cp39-win_amd64.whl", hash = "sha256:f8e160feb2aed042cd657a72acc0b481212ed28b1b9a95c0cee1621b524e1966"}, - {file = "charset_normalizer-3.4.4-cp39-cp39-win_arm64.whl", hash = "sha256:b5d84d37db046c5ca74ee7bb47dd6cbc13f80665fdde3e8040bdd3fb015ecb50"}, - {file = "charset_normalizer-3.4.4-py3-none-any.whl", hash = "sha256:7a32c560861a02ff789ad905a2fe94e3f840803362c84fecf1851cb4cf3dc37f"}, - {file = "charset_normalizer-3.4.4.tar.gz", hash = "sha256:94537985111c35f28720e43603b8e7b43a6ecfb2ce1d3058bbe955b73404e21a"}, -] - -[[package]] -name = "colorama" -version = "0.4.6" -description = "Cross-platform colored terminal text." -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -files = [ - {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, - {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, -] - -[[package]] -name = "deepmerge" -version = "2.0" -description = "A toolset for deeply merging Python dictionaries." -optional = false -python-versions = ">=3.8" -files = [ - {file = "deepmerge-2.0-py3-none-any.whl", hash = "sha256:6de9ce507115cff0bed95ff0ce9ecc31088ef50cbdf09bc90a09349a318b3d00"}, - {file = "deepmerge-2.0.tar.gz", hash = "sha256:5c3d86081fbebd04dd5de03626a0607b809a98fb6ccba5770b62466fe940ff20"}, -] - -[package.extras] -dev = ["black", "build", "mypy", "pytest", "pyupgrade", "twine", "validate-pyproject[all]"] - -[[package]] -name = "docutils" -version = "0.21.2" -description = "Docutils -- Python Documentation Utilities" -optional = false -python-versions = ">=3.9" -files = [ - {file = "docutils-0.21.2-py3-none-any.whl", hash = "sha256:dafca5b9e384f0e419294eb4d2ff9fa826435bf15f15b7bd45723e8ad76811b2"}, - {file = "docutils-0.21.2.tar.gz", hash = "sha256:3a6b18732edf182daa3cd12775bbb338cf5691468f91eeeb109deff6ebfa986f"}, -] - -[[package]] -name = "idna" -version = "3.11" -description = "Internationalized Domain Names in Applications (IDNA)" -optional = false -python-versions = ">=3.8" -files = [ - {file = "idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea"}, - {file = "idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902"}, -] - -[package.extras] -all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] - -[[package]] -name = "imagesize" -version = "1.4.1" -description = "Getting image size from png/jpeg/jpeg2000/gif file" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -files = [ - {file = "imagesize-1.4.1-py2.py3-none-any.whl", hash = "sha256:0d8d18d08f840c19d0ee7ca1fd82490fdc3729b7ac93f49870406ddde8ef8d8b"}, - {file = "imagesize-1.4.1.tar.gz", hash = "sha256:69150444affb9cb0d5cc5a92b3676f0b2fb7cd9ae39e947a5e11a36b4497cd4a"}, -] - -[[package]] -name = "jinja2" -version = "3.1.6" -description = "A very fast and expressive template engine." -optional = false -python-versions = ">=3.7" -files = [ - {file = "jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67"}, - {file = "jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d"}, -] - -[package.dependencies] -MarkupSafe = ">=2.0" - -[package.extras] -i18n = ["Babel (>=2.7)"] - -[[package]] -name = "jsonschema" -version = "4.26.0" -description = "An implementation of JSON Schema validation for Python" -optional = false -python-versions = ">=3.10" -files = [ - {file = "jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce"}, - {file = "jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326"}, -] - -[package.dependencies] -attrs = ">=22.2.0" -jsonschema-specifications = ">=2023.03.6" -referencing = ">=0.28.4" -rpds-py = ">=0.25.0" - -[package.extras] -format = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3987", "uri-template", "webcolors (>=1.11)"] -format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "rfc3987-syntax (>=1.1.0)", "uri-template", "webcolors (>=24.6.0)"] - -[[package]] -name = "jsonschema-specifications" -version = "2025.9.1" -description = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry" -optional = false -python-versions = ">=3.9" -files = [ - {file = "jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe"}, - {file = "jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d"}, -] - -[package.dependencies] -referencing = ">=0.31.0" - -[[package]] -name = "markdown-it-py" -version = "4.0.0" -description = "Python port of markdown-it. Markdown parsing, done right!" -optional = false -python-versions = ">=3.10" -files = [ - {file = "markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147"}, - {file = "markdown_it_py-4.0.0.tar.gz", hash = "sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3"}, -] - -[package.dependencies] -mdurl = ">=0.1,<1.0" - -[package.extras] -benchmarking = ["psutil", "pytest", "pytest-benchmark"] -compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "markdown-it-pyrs", "mistletoe (>=1.0,<2.0)", "mistune (>=3.0,<4.0)", "panflute (>=2.3,<3.0)"] -linkify = ["linkify-it-py (>=1,<3)"] -plugins = ["mdit-py-plugins (>=0.5.0)"] -profiling = ["gprof2dot"] -rtd = ["ipykernel", "jupyter_sphinx", "mdit-py-plugins (>=0.5.0)", "myst-parser", "pyyaml", "sphinx", "sphinx-book-theme (>=1.0,<2.0)", "sphinx-copybutton", "sphinx-design"] -testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions", "requests"] - -[[package]] -name = "markupsafe" -version = "3.0.3" -description = "Safely add untrusted strings to HTML/XML markup." -optional = false -python-versions = ">=3.9" -files = [ - {file = "markupsafe-3.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559"}, - {file = "markupsafe-3.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419"}, - {file = "markupsafe-3.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ba88449deb3de88bd40044603fafffb7bc2b055d626a330323a9ed736661695"}, - {file = "markupsafe-3.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f42d0984e947b8adf7dd6dde396e720934d12c506ce84eea8476409563607591"}, - {file = "markupsafe-3.0.3-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c0c0b3ade1c0b13b936d7970b1d37a57acde9199dc2aecc4c336773e1d86049c"}, - {file = "markupsafe-3.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0303439a41979d9e74d18ff5e2dd8c43ed6c6001fd40e5bf2e43f7bd9bbc523f"}, - {file = "markupsafe-3.0.3-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:d2ee202e79d8ed691ceebae8e0486bd9a2cd4794cec4824e1c99b6f5009502f6"}, - {file = "markupsafe-3.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:177b5253b2834fe3678cb4a5f0059808258584c559193998be2601324fdeafb1"}, - {file = "markupsafe-3.0.3-cp310-cp310-win32.whl", hash = "sha256:2a15a08b17dd94c53a1da0438822d70ebcd13f8c3a95abe3a9ef9f11a94830aa"}, - {file = "markupsafe-3.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:c4ffb7ebf07cfe8931028e3e4c85f0357459a3f9f9490886198848f4fa002ec8"}, - {file = "markupsafe-3.0.3-cp310-cp310-win_arm64.whl", hash = "sha256:e2103a929dfa2fcaf9bb4e7c091983a49c9ac3b19c9061b6d5427dd7d14d81a1"}, - {file = "markupsafe-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad"}, - {file = "markupsafe-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a"}, - {file = "markupsafe-3.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50"}, - {file = "markupsafe-3.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf"}, - {file = "markupsafe-3.0.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f"}, - {file = "markupsafe-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a"}, - {file = "markupsafe-3.0.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115"}, - {file = "markupsafe-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a"}, - {file = "markupsafe-3.0.3-cp311-cp311-win32.whl", hash = "sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19"}, - {file = "markupsafe-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01"}, - {file = "markupsafe-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c"}, - {file = "markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e"}, - {file = "markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce"}, - {file = "markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d"}, - {file = "markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d"}, - {file = "markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a"}, - {file = "markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b"}, - {file = "markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f"}, - {file = "markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b"}, - {file = "markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d"}, - {file = "markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c"}, - {file = "markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f"}, - {file = "markupsafe-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795"}, - {file = "markupsafe-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219"}, - {file = "markupsafe-3.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6"}, - {file = "markupsafe-3.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676"}, - {file = "markupsafe-3.0.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9"}, - {file = "markupsafe-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1"}, - {file = "markupsafe-3.0.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc"}, - {file = "markupsafe-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12"}, - {file = "markupsafe-3.0.3-cp313-cp313-win32.whl", hash = "sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed"}, - {file = "markupsafe-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5"}, - {file = "markupsafe-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485"}, - {file = "markupsafe-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73"}, - {file = "markupsafe-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37"}, - {file = "markupsafe-3.0.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19"}, - {file = "markupsafe-3.0.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025"}, - {file = "markupsafe-3.0.3-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6"}, - {file = "markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f"}, - {file = "markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb"}, - {file = "markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009"}, - {file = "markupsafe-3.0.3-cp313-cp313t-win32.whl", hash = "sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354"}, - {file = "markupsafe-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218"}, - {file = "markupsafe-3.0.3-cp313-cp313t-win_arm64.whl", hash = "sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287"}, - {file = "markupsafe-3.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe"}, - {file = "markupsafe-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026"}, - {file = "markupsafe-3.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737"}, - {file = "markupsafe-3.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97"}, - {file = "markupsafe-3.0.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d"}, - {file = "markupsafe-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda"}, - {file = "markupsafe-3.0.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf"}, - {file = "markupsafe-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe"}, - {file = "markupsafe-3.0.3-cp314-cp314-win32.whl", hash = "sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9"}, - {file = "markupsafe-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581"}, - {file = "markupsafe-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4"}, - {file = "markupsafe-3.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab"}, - {file = "markupsafe-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175"}, - {file = "markupsafe-3.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634"}, - {file = "markupsafe-3.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50"}, - {file = "markupsafe-3.0.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e"}, - {file = "markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5"}, - {file = "markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523"}, - {file = "markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc"}, - {file = "markupsafe-3.0.3-cp314-cp314t-win32.whl", hash = "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d"}, - {file = "markupsafe-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9"}, - {file = "markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa"}, - {file = "markupsafe-3.0.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:15d939a21d546304880945ca1ecb8a039db6b4dc49b2c5a400387cdae6a62e26"}, - {file = "markupsafe-3.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f71a396b3bf33ecaa1626c255855702aca4d3d9fea5e051b41ac59a9c1c41edc"}, - {file = "markupsafe-3.0.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0f4b68347f8c5eab4a13419215bdfd7f8c9b19f2b25520968adfad23eb0ce60c"}, - {file = "markupsafe-3.0.3-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8fc20152abba6b83724d7ff268c249fa196d8259ff481f3b1476383f8f24e42"}, - {file = "markupsafe-3.0.3-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:949b8d66bc381ee8b007cd945914c721d9aba8e27f71959d750a46f7c282b20b"}, - {file = "markupsafe-3.0.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:3537e01efc9d4dccdf77221fb1cb3b8e1a38d5428920e0657ce299b20324d758"}, - {file = "markupsafe-3.0.3-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:591ae9f2a647529ca990bc681daebdd52c8791ff06c2bfa05b65163e28102ef2"}, - {file = "markupsafe-3.0.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a320721ab5a1aba0a233739394eb907f8c8da5c98c9181d1161e77a0c8e36f2d"}, - {file = "markupsafe-3.0.3-cp39-cp39-win32.whl", hash = "sha256:df2449253ef108a379b8b5d6b43f4b1a8e81a061d6537becd5582fba5f9196d7"}, - {file = "markupsafe-3.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:7c3fb7d25180895632e5d3148dbdc29ea38ccb7fd210aa27acbd1201a1902c6e"}, - {file = "markupsafe-3.0.3-cp39-cp39-win_arm64.whl", hash = "sha256:38664109c14ffc9e7437e86b4dceb442b0096dfe3541d7864d9cbe1da4cf36c8"}, - {file = "markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698"}, -] - -[[package]] -name = "mdit-py-plugins" -version = "0.5.0" -description = "Collection of plugins for markdown-it-py" -optional = false -python-versions = ">=3.10" -files = [ - {file = "mdit_py_plugins-0.5.0-py3-none-any.whl", hash = "sha256:07a08422fc1936a5d26d146759e9155ea466e842f5ab2f7d2266dd084c8dab1f"}, - {file = "mdit_py_plugins-0.5.0.tar.gz", hash = "sha256:f4918cb50119f50446560513a8e311d574ff6aaed72606ddae6d35716fe809c6"}, -] - -[package.dependencies] -markdown-it-py = ">=2.0.0,<5.0.0" - -[package.extras] -code-style = ["pre-commit"] -rtd = ["myst-parser", "sphinx-book-theme"] -testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] - -[[package]] -name = "mdurl" -version = "0.1.2" -description = "Markdown URL utilities" -optional = false -python-versions = ">=3.7" -files = [ - {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, - {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, -] - -[[package]] -name = "mistune" -version = "3.2.0" -description = "A sane and fast Markdown parser with useful plugins and renderers" -optional = false -python-versions = ">=3.8" -files = [ - {file = "mistune-3.2.0-py3-none-any.whl", hash = "sha256:febdc629a3c78616b94393c6580551e0e34cc289987ec6c35ed3f4be42d0eee1"}, - {file = "mistune-3.2.0.tar.gz", hash = "sha256:708487c8a8cdd99c9d90eb3ed4c3ed961246ff78ac82f03418f5183ab70e398a"}, -] - -[[package]] -name = "myst-parser" -version = "5.0.0" -description = "An extended [CommonMark](https://spec.commonmark.org/) compliant parser," -optional = false -python-versions = ">=3.11" -files = [ - {file = "myst_parser-5.0.0-py3-none-any.whl", hash = "sha256:ab31e516024918296e169139072b81592336f2fef55b8986aa31c9f04b5f7211"}, - {file = "myst_parser-5.0.0.tar.gz", hash = "sha256:f6f231452c56e8baa662cc352c548158f6a16fcbd6e3800fc594978002b94f3a"}, -] - -[package.dependencies] -docutils = ">=0.20,<0.23" -jinja2 = "*" -markdown-it-py = ">=4.0,<5.0" -mdit-py-plugins = ">=0.5,<1.0" -pyyaml = "*" -sphinx = ">=8,<10" - -[package.extras] -code-style = ["pre-commit (>=4.0,<5.0)"] -linkify = ["linkify-it-py (>=2.0,<3.0)"] -rtd = ["ipython", "sphinx (>=8)", "sphinx-autodoc2 (>=0.5.0,<0.6.0)", "sphinx-book-theme (>=1.1,<2.0)", "sphinx-copybutton", "sphinx-design", "sphinx-pyscript", "sphinx-tippy (>=0.4.3)", "sphinx-togglebutton", "sphinxext-opengraph (>=0.13.0,<0.14.0)", "sphinxext-rediraffe (>=0.3.0,<0.4.0)"] -testing = ["beautifulsoup4", "coverage[toml]", "defusedxml", "pygments (<2.20)", "pytest (>=9,<10)", "pytest-cov", "pytest-param-files (>=0.6.0,<0.7.0)", "pytest-regressions", "sphinx-pytest (>=0.3.0,<0.4.0)"] -testing-docutils = ["pygments", "pytest (>=9,<10)", "pytest-param-files (>=0.6.0,<0.7.0)"] - -[[package]] -name = "nodeenv" -version = "1.10.0" -description = "Node.js virtual environment builder" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -files = [ - {file = "nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827"}, - {file = "nodeenv-1.10.0.tar.gz", hash = "sha256:996c191ad80897d076bdfba80a41994c2b47c68e224c542b48feba42ba00f8bb"}, -] - -[[package]] -name = "numpy" -version = "2.4.1" -description = "Fundamental package for array computing in Python" -optional = false -python-versions = ">=3.11" -files = [ - {file = "numpy-2.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0cce2a669e3c8ba02ee563c7835f92c153cf02edff1ae05e1823f1dde21b16a5"}, - {file = "numpy-2.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:899d2c18024984814ac7e83f8f49d8e8180e2fbe1b2e252f2e7f1d06bea92425"}, - {file = "numpy-2.4.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:09aa8a87e45b55a1c2c205d42e2808849ece5c484b2aab11fecabec3841cafba"}, - {file = "numpy-2.4.1-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:edee228f76ee2dab4579fad6f51f6a305de09d444280109e0f75df247ff21501"}, - {file = "numpy-2.4.1-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a92f227dbcdc9e4c3e193add1a189a9909947d4f8504c576f4a732fd0b54240a"}, - {file = "numpy-2.4.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:538bf4ec353709c765ff75ae616c34d3c3dca1a68312727e8f2676ea644f8509"}, - {file = "numpy-2.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ac08c63cb7779b85e9d5318e6c3518b424bc1f364ac4cb2c6136f12e5ff2dccc"}, - {file = "numpy-2.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4f9c360ecef085e5841c539a9a12b883dff005fbd7ce46722f5e9cef52634d82"}, - {file = "numpy-2.4.1-cp311-cp311-win32.whl", hash = "sha256:0f118ce6b972080ba0758c6087c3617b5ba243d806268623dc34216d69099ba0"}, - {file = "numpy-2.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:18e14c4d09d55eef39a6ab5b08406e84bc6869c1e34eef45564804f90b7e0574"}, - {file = "numpy-2.4.1-cp311-cp311-win_arm64.whl", hash = "sha256:6461de5113088b399d655d45c3897fa188766415d0f568f175ab071c8873bd73"}, - {file = "numpy-2.4.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d3703409aac693fa82c0aee023a1ae06a6e9d065dba10f5e8e80f642f1e9d0a2"}, - {file = "numpy-2.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7211b95ca365519d3596a1d8688a95874cc94219d417504d9ecb2df99fa7bfa8"}, - {file = "numpy-2.4.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:5adf01965456a664fc727ed69cc71848f28d063217c63e1a0e200a118d5eec9a"}, - {file = "numpy-2.4.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:26f0bcd9c79a00e339565b303badc74d3ea2bd6d52191eeca5f95936cad107d0"}, - {file = "numpy-2.4.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0093e85df2960d7e4049664b26afc58b03236e967fb942354deef3208857a04c"}, - {file = "numpy-2.4.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7ad270f438cbdd402c364980317fb6b117d9ec5e226fff5b4148dd9aa9fc6e02"}, - {file = "numpy-2.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:297c72b1b98100c2e8f873d5d35fb551fce7040ade83d67dd51d38c8d42a2162"}, - {file = "numpy-2.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:cf6470d91d34bf669f61d515499859fa7a4c2f7c36434afb70e82df7217933f9"}, - {file = "numpy-2.4.1-cp312-cp312-win32.whl", hash = "sha256:b6bcf39112e956594b3331316d90c90c90fb961e39696bda97b89462f5f3943f"}, - {file = "numpy-2.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:e1a27bb1b2dee45a2a53f5ca6ff2d1a7f135287883a1689e930d44d1ff296c87"}, - {file = "numpy-2.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:0e6e8f9d9ecf95399982019c01223dc130542960a12edfa8edd1122dfa66a8a8"}, - {file = "numpy-2.4.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d797454e37570cfd61143b73b8debd623c3c0952959adb817dd310a483d58a1b"}, - {file = "numpy-2.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:82c55962006156aeef1629b953fd359064aa47e4d82cfc8e67f0918f7da3344f"}, - {file = "numpy-2.4.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:71abbea030f2cfc3092a0ff9f8c8fdefdc5e0bf7d9d9c99663538bb0ecdac0b9"}, - {file = "numpy-2.4.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:5b55aa56165b17aaf15520beb9cbd33c9039810e0d9643dd4379e44294c7303e"}, - {file = "numpy-2.4.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c0faba4a331195bfa96f93dd9dfaa10b2c7aa8cda3a02b7fd635e588fe821bf5"}, - {file = "numpy-2.4.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d3e3087f53e2b4428766b54932644d148613c5a595150533ae7f00dab2f319a8"}, - {file = "numpy-2.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:49e792ec351315e16da54b543db06ca8a86985ab682602d90c60ef4ff4db2a9c"}, - {file = "numpy-2.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:79e9e06c4c2379db47f3f6fc7a8652e7498251789bf8ff5bd43bf478ef314ca2"}, - {file = "numpy-2.4.1-cp313-cp313-win32.whl", hash = "sha256:3d1a100e48cb266090a031397863ff8a30050ceefd798f686ff92c67a486753d"}, - {file = "numpy-2.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:92a0e65272fd60bfa0d9278e0484c2f52fe03b97aedc02b357f33fe752c52ffb"}, - {file = "numpy-2.4.1-cp313-cp313-win_arm64.whl", hash = "sha256:20d4649c773f66cc2fc36f663e091f57c3b7655f936a4c681b4250855d1da8f5"}, - {file = "numpy-2.4.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f93bc6892fe7b0663e5ffa83b61aab510aacffd58c16e012bb9352d489d90cb7"}, - {file = "numpy-2.4.1-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:178de8f87948163d98a4c9ab5bee4ce6519ca918926ec8df195af582de28544d"}, - {file = "numpy-2.4.1-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:98b35775e03ab7f868908b524fc0a84d38932d8daf7b7e1c3c3a1b6c7a2c9f15"}, - {file = "numpy-2.4.1-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:941c2a93313d030f219f3a71fd3d91a728b82979a5e8034eb2e60d394a2b83f9"}, - {file = "numpy-2.4.1-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:529050522e983e00a6c1c6b67411083630de8b57f65e853d7b03d9281b8694d2"}, - {file = "numpy-2.4.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:2302dc0224c1cbc49bb94f7064f3f923a971bfae45c33870dcbff63a2a550505"}, - {file = "numpy-2.4.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:9171a42fcad32dcf3fa86f0a4faa5e9f8facefdb276f54b8b390d90447cff4e2"}, - {file = "numpy-2.4.1-cp313-cp313t-win32.whl", hash = "sha256:382ad67d99ef49024f11d1ce5dcb5ad8432446e4246a4b014418ba3a1175a1f4"}, - {file = "numpy-2.4.1-cp313-cp313t-win_amd64.whl", hash = "sha256:62fea415f83ad8fdb6c20840578e5fbaf5ddd65e0ec6c3c47eda0f69da172510"}, - {file = "numpy-2.4.1-cp313-cp313t-win_arm64.whl", hash = "sha256:a7870e8c5fc11aef57d6fea4b4085e537a3a60ad2cdd14322ed531fdca68d261"}, - {file = "numpy-2.4.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3869ea1ee1a1edc16c29bbe3a2f2a4e515cc3a44d43903ad41e0cacdbaf733dc"}, - {file = "numpy-2.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e867df947d427cdd7a60e3e271729090b0f0df80f5f10ab7dd436f40811699c3"}, - {file = "numpy-2.4.1-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:e3bd2cb07841166420d2fa7146c96ce00cb3410664cbc1a6be028e456c4ee220"}, - {file = "numpy-2.4.1-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:f0a90aba7d521e6954670550e561a4cb925713bd944445dbe9e729b71f6cabee"}, - {file = "numpy-2.4.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5d558123217a83b2d1ba316b986e9248a1ed1971ad495963d555ccd75dcb1556"}, - {file = "numpy-2.4.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2f44de05659b67d20499cbc96d49f2650769afcb398b79b324bb6e297bfe3844"}, - {file = "numpy-2.4.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:69e7419c9012c4aaf695109564e3387f1259f001b4326dfa55907b098af082d3"}, - {file = "numpy-2.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ffd257026eb1b34352e749d7cc1678b5eeec3e329ad8c9965a797e08ccba205"}, - {file = "numpy-2.4.1-cp314-cp314-win32.whl", hash = "sha256:727c6c3275ddefa0dc078524a85e064c057b4f4e71ca5ca29a19163c607be745"}, - {file = "numpy-2.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:7d5d7999df434a038d75a748275cd6c0094b0ecdb0837342b332a82defc4dc4d"}, - {file = "numpy-2.4.1-cp314-cp314-win_arm64.whl", hash = "sha256:ce9ce141a505053b3c7bce3216071f3bf5c182b8b28930f14cd24d43932cd2df"}, - {file = "numpy-2.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:4e53170557d37ae404bf8d542ca5b7c629d6efa1117dac6a83e394142ea0a43f"}, - {file = "numpy-2.4.1-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:a73044b752f5d34d4232f25f18160a1cc418ea4507f5f11e299d8ac36875f8a0"}, - {file = "numpy-2.4.1-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:fb1461c99de4d040666ca0444057b06541e5642f800b71c56e6ea92d6a853a0c"}, - {file = "numpy-2.4.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:423797bdab2eeefbe608d7c1ec7b2b4fd3c58d51460f1ee26c7500a1d9c9ee93"}, - {file = "numpy-2.4.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:52b5f61bdb323b566b528899cc7db2ba5d1015bda7ea811a8bcf3c89c331fa42"}, - {file = "numpy-2.4.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:42d7dd5fa36d16d52a84f821eb96031836fd405ee6955dd732f2023724d0aa01"}, - {file = "numpy-2.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e7b6b5e28bbd47b7532698e5db2fe1db693d84b58c254e4389d99a27bb9b8f6b"}, - {file = "numpy-2.4.1-cp314-cp314t-win32.whl", hash = "sha256:5de60946f14ebe15e713a6f22850c2372fa72f4ff9a432ab44aa90edcadaa65a"}, - {file = "numpy-2.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:8f085da926c0d491ffff3096f91078cc97ea67e7e6b65e490bc8dcda65663be2"}, - {file = "numpy-2.4.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6436cffb4f2bf26c974344439439c95e152c9a527013f26b3577be6c2ca64295"}, - {file = "numpy-2.4.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:8ad35f20be147a204e28b6a0575fbf3540c5e5f802634d4258d55b1ff5facce1"}, - {file = "numpy-2.4.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:8097529164c0f3e32bb89412a0905d9100bf434d9692d9fc275e18dcf53c9344"}, - {file = "numpy-2.4.1-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:ea66d2b41ca4a1630aae5507ee0a71647d3124d1741980138aa8f28f44dac36e"}, - {file = "numpy-2.4.1-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:d3f8f0df9f4b8be57b3bf74a1d087fec68f927a2fab68231fdb442bf2c12e426"}, - {file = "numpy-2.4.1-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2023ef86243690c2791fd6353e5b4848eedaa88ca8a2d129f462049f6d484696"}, - {file = "numpy-2.4.1-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8361ea4220d763e54cff2fbe7d8c93526b744f7cd9ddab47afeff7e14e8503be"}, - {file = "numpy-2.4.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:4f1b68ff47680c2925f8063402a693ede215f0257f02596b1318ecdfb1d79e33"}, - {file = "numpy-2.4.1.tar.gz", hash = "sha256:a1ceafc5042451a858231588a104093474c6a5c57dcc724841f5c888d237d690"}, -] - -[[package]] -name = "packaging" -version = "26.0" -description = "Core utilities for Python packages" -optional = false -python-versions = ">=3.8" -files = [ - {file = "packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529"}, - {file = "packaging-26.0.tar.gz", hash = "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4"}, -] - -[[package]] -name = "picobox" -version = "4.0.0" -description = "Dependency injection framework designed with Python in mind." -optional = false -python-versions = ">=3.8" -files = [ - {file = "picobox-4.0.0-py3-none-any.whl", hash = "sha256:4c27eb689fe45dabd9e64c382e04418147d0b746d155b4e80057dbb7ff82027e"}, - {file = "picobox-4.0.0.tar.gz", hash = "sha256:114da1b5606b2f615e8b0eb68d04198ad9de75af5adbcf5b36fe4f664ab927b6"}, -] - -[[package]] -name = "pydata-sphinx-theme" -version = "0.16.1" -description = "Bootstrap-based Sphinx theme from the PyData community" -optional = false -python-versions = ">=3.9" -files = [ - {file = "pydata_sphinx_theme-0.16.1-py3-none-any.whl", hash = "sha256:225331e8ac4b32682c18fcac5a57a6f717c4e632cea5dd0e247b55155faeccde"}, - {file = "pydata_sphinx_theme-0.16.1.tar.gz", hash = "sha256:a08b7f0b7f70387219dc659bff0893a7554d5eb39b59d3b8ef37b8401b7642d7"}, -] - -[package.dependencies] -accessible-pygments = "*" -Babel = "*" -beautifulsoup4 = "*" -docutils = "!=0.17.0" -pygments = ">=2.7" -sphinx = ">=6.1" -typing-extensions = "*" - -[package.extras] -a11y = ["pytest-playwright"] -dev = ["pandoc", "pre-commit", "pydata-sphinx-theme[doc,test]", "pyyaml", "sphinx-theme-builder[cli]", "tox"] -doc = ["ablog (>=0.11.8)", "colorama", "graphviz", "ipykernel", "ipyleaflet", "ipywidgets", "jupyter_sphinx", "jupyterlite-sphinx", "linkify-it-py", "matplotlib", "myst-parser", "nbsphinx", "numpy", "numpydoc", "pandas", "plotly", "rich", "sphinx-autoapi (>=3.0.0)", "sphinx-copybutton", "sphinx-design", "sphinx-favicon (>=1.0.1)", "sphinx-sitemap", "sphinx-togglebutton", "sphinxcontrib-youtube (>=1.4.1)", "sphinxext-rediraffe", "xarray"] -i18n = ["Babel", "jinja2"] -test = ["pytest", "pytest-cov", "pytest-regressions", "sphinx[test]"] - -[[package]] -name = "pygments" -version = "2.19.2" -description = "Pygments is a syntax highlighting package written in Python." -optional = false -python-versions = ">=3.8" -files = [ - {file = "pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b"}, - {file = "pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887"}, -] - -[package.extras] -windows-terminal = ["colorama (>=0.4.6)"] - -[[package]] -name = "pyright" -version = "1.1.408" -description = "Command line wrapper for pyright" -optional = false -python-versions = ">=3.7" -files = [ - {file = "pyright-1.1.408-py3-none-any.whl", hash = "sha256:090b32865f4fdb1e0e6cd82bf5618480d48eecd2eb2e70f960982a3d9a4c17c1"}, - {file = "pyright-1.1.408.tar.gz", hash = "sha256:f28f2321f96852fa50b5829ea492f6adb0e6954568d1caa3f3af3a5f555eb684"}, -] - -[package.dependencies] -nodeenv = ">=1.6.0" -typing-extensions = ">=4.1" - -[package.extras] -all = ["nodejs-wheel-binaries", "twine (>=3.4.1)"] -dev = ["twine (>=3.4.1)"] -nodejs = ["nodejs-wheel-binaries"] - -[[package]] -name = "pyyaml" -version = "6.0.3" -description = "YAML parser and emitter for Python" -optional = false -python-versions = ">=3.8" -files = [ - {file = "PyYAML-6.0.3-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3"}, - {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6"}, - {file = "PyYAML-6.0.3-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369"}, - {file = "PyYAML-6.0.3-cp38-cp38-win32.whl", hash = "sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295"}, - {file = "PyYAML-6.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b"}, - {file = "pyyaml-6.0.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b"}, - {file = "pyyaml-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956"}, - {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8"}, - {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198"}, - {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b"}, - {file = "pyyaml-6.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0"}, - {file = "pyyaml-6.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69"}, - {file = "pyyaml-6.0.3-cp310-cp310-win32.whl", hash = "sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e"}, - {file = "pyyaml-6.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c"}, - {file = "pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e"}, - {file = "pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824"}, - {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c"}, - {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00"}, - {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d"}, - {file = "pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a"}, - {file = "pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4"}, - {file = "pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b"}, - {file = "pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf"}, - {file = "pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196"}, - {file = "pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0"}, - {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28"}, - {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c"}, - {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc"}, - {file = "pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e"}, - {file = "pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea"}, - {file = "pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5"}, - {file = "pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b"}, - {file = "pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd"}, - {file = "pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8"}, - {file = "pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1"}, - {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c"}, - {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5"}, - {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6"}, - {file = "pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6"}, - {file = "pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be"}, - {file = "pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26"}, - {file = "pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c"}, - {file = "pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb"}, - {file = "pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac"}, - {file = "pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310"}, - {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7"}, - {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788"}, - {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5"}, - {file = "pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764"}, - {file = "pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35"}, - {file = "pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac"}, - {file = "pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3"}, - {file = "pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3"}, - {file = "pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba"}, - {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c"}, - {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702"}, - {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c"}, - {file = "pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065"}, - {file = "pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65"}, - {file = "pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9"}, - {file = "pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b"}, - {file = "pyyaml-6.0.3-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da"}, - {file = "pyyaml-6.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917"}, - {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9"}, - {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5"}, - {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a"}, - {file = "pyyaml-6.0.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926"}, - {file = "pyyaml-6.0.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7"}, - {file = "pyyaml-6.0.3-cp39-cp39-win32.whl", hash = "sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0"}, - {file = "pyyaml-6.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007"}, - {file = "pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f"}, -] - -[[package]] -name = "referencing" -version = "0.37.0" -description = "JSON Referencing + Python" -optional = false -python-versions = ">=3.10" -files = [ - {file = "referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231"}, - {file = "referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8"}, -] - -[package.dependencies] -attrs = ">=22.2.0" -rpds-py = ">=0.7.0" -typing-extensions = {version = ">=4.4.0", markers = "python_version < \"3.13\""} - -[[package]] -name = "requests" -version = "2.32.5" -description = "Python HTTP for Humans." -optional = false -python-versions = ">=3.9" -files = [ - {file = "requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6"}, - {file = "requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf"}, -] - -[package.dependencies] -certifi = ">=2017.4.17" -charset_normalizer = ">=2,<4" -idna = ">=2.5,<4" -urllib3 = ">=1.21.1,<3" - -[package.extras] -socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] - -[[package]] -name = "roman-numerals" -version = "4.1.0" -description = "Manipulate well-formed Roman numerals" -optional = false -python-versions = ">=3.10" -files = [ - {file = "roman_numerals-4.1.0-py3-none-any.whl", hash = "sha256:647ba99caddc2cc1e55a51e4360689115551bf4476d90e8162cf8c345fe233c7"}, - {file = "roman_numerals-4.1.0.tar.gz", hash = "sha256:1af8b147eb1405d5839e78aeb93131690495fe9da5c91856cb33ad55a7f1e5b2"}, -] - -[[package]] -name = "roman-numerals-py" -version = "4.1.0" -description = "This package is deprecated, switch to roman-numerals." -optional = false -python-versions = ">=3.10" -files = [ - {file = "roman_numerals_py-4.1.0-py3-none-any.whl", hash = "sha256:553114c1167141c1283a51743759723ecd05604a1b6b507225e91dc1a6df0780"}, - {file = "roman_numerals_py-4.1.0.tar.gz", hash = "sha256:f5d7b2b4ca52dd855ef7ab8eb3590f428c0b1ea480736ce32b01fef2a5f8daf9"}, -] - -[package.dependencies] -roman-numerals = "4.1.0" - -[[package]] -name = "rpds-py" -version = "0.30.0" -description = "Python bindings to Rust's persistent data structures (rpds)" -optional = false -python-versions = ">=3.10" -files = [ - {file = "rpds_py-0.30.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:679ae98e00c0e8d68a7fda324e16b90fd5260945b45d3b824c892cec9eea3288"}, - {file = "rpds_py-0.30.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4cc2206b76b4f576934f0ed374b10d7ca5f457858b157ca52064bdfc26b9fc00"}, - {file = "rpds_py-0.30.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:389a2d49eded1896c3d48b0136ead37c48e221b391c052fba3f4055c367f60a6"}, - {file = "rpds_py-0.30.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:32c8528634e1bf7121f3de08fa85b138f4e0dc47657866630611b03967f041d7"}, - {file = "rpds_py-0.30.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f207f69853edd6f6700b86efb84999651baf3789e78a466431df1331608e5324"}, - {file = "rpds_py-0.30.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:67b02ec25ba7a9e8fa74c63b6ca44cf5707f2fbfadae3ee8e7494297d56aa9df"}, - {file = "rpds_py-0.30.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c0e95f6819a19965ff420f65578bacb0b00f251fefe2c8b23347c37174271f3"}, - {file = "rpds_py-0.30.0-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:a452763cc5198f2f98898eb98f7569649fe5da666c2dc6b5ddb10fde5a574221"}, - {file = "rpds_py-0.30.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e0b65193a413ccc930671c55153a03ee57cecb49e6227204b04fae512eb657a7"}, - {file = "rpds_py-0.30.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:858738e9c32147f78b3ac24dc0edb6610000e56dc0f700fd5f651d0a0f0eb9ff"}, - {file = "rpds_py-0.30.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:da279aa314f00acbb803da1e76fa18666778e8a8f83484fba94526da5de2cba7"}, - {file = "rpds_py-0.30.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:7c64d38fb49b6cdeda16ab49e35fe0da2e1e9b34bc38bd78386530f218b37139"}, - {file = "rpds_py-0.30.0-cp310-cp310-win32.whl", hash = "sha256:6de2a32a1665b93233cde140ff8b3467bdb9e2af2b91079f0333a0974d12d464"}, - {file = "rpds_py-0.30.0-cp310-cp310-win_amd64.whl", hash = "sha256:1726859cd0de969f88dc8673bdd954185b9104e05806be64bcd87badbe313169"}, - {file = "rpds_py-0.30.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a2bffea6a4ca9f01b3f8e548302470306689684e61602aa3d141e34da06cf425"}, - {file = "rpds_py-0.30.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:dc4f992dfe1e2bc3ebc7444f6c7051b4bc13cd8e33e43511e8ffd13bf407010d"}, - {file = "rpds_py-0.30.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:422c3cb9856d80b09d30d2eb255d0754b23e090034e1deb4083f8004bd0761e4"}, - {file = "rpds_py-0.30.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:07ae8a593e1c3c6b82ca3292efbe73c30b61332fd612e05abee07c79359f292f"}, - {file = "rpds_py-0.30.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:12f90dd7557b6bd57f40abe7747e81e0c0b119bef015ea7726e69fe550e394a4"}, - {file = "rpds_py-0.30.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:99b47d6ad9a6da00bec6aabe5a6279ecd3c06a329d4aa4771034a21e335c3a97"}, - {file = "rpds_py-0.30.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33f559f3104504506a44bb666b93a33f5d33133765b0c216a5bf2f1e1503af89"}, - {file = "rpds_py-0.30.0-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:946fe926af6e44f3697abbc305ea168c2c31d3e3ef1058cf68f379bf0335a78d"}, - {file = "rpds_py-0.30.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:495aeca4b93d465efde585977365187149e75383ad2684f81519f504f5c13038"}, - {file = "rpds_py-0.30.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d9a0ca5da0386dee0655b4ccdf46119df60e0f10da268d04fe7cc87886872ba7"}, - {file = "rpds_py-0.30.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8d6d1cc13664ec13c1b84241204ff3b12f9bb82464b8ad6e7a5d3486975c2eed"}, - {file = "rpds_py-0.30.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3896fa1be39912cf0757753826bc8bdc8ca331a28a7c4ae46b7a21280b06bb85"}, - {file = "rpds_py-0.30.0-cp311-cp311-win32.whl", hash = "sha256:55f66022632205940f1827effeff17c4fa7ae1953d2b74a8581baaefb7d16f8c"}, - {file = "rpds_py-0.30.0-cp311-cp311-win_amd64.whl", hash = "sha256:a51033ff701fca756439d641c0ad09a41d9242fa69121c7d8769604a0a629825"}, - {file = "rpds_py-0.30.0-cp311-cp311-win_arm64.whl", hash = "sha256:47b0ef6231c58f506ef0b74d44e330405caa8428e770fec25329ed2cb971a229"}, - {file = "rpds_py-0.30.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a161f20d9a43006833cd7068375a94d035714d73a172b681d8881820600abfad"}, - {file = "rpds_py-0.30.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6abc8880d9d036ecaafe709079969f56e876fcf107f7a8e9920ba6d5a3878d05"}, - {file = "rpds_py-0.30.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca28829ae5f5d569bb62a79512c842a03a12576375d5ece7d2cadf8abe96ec28"}, - {file = "rpds_py-0.30.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a1010ed9524c73b94d15919ca4d41d8780980e1765babf85f9a2f90d247153dd"}, - {file = "rpds_py-0.30.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f8d1736cfb49381ba528cd5baa46f82fdc65c06e843dab24dd70b63d09121b3f"}, - {file = "rpds_py-0.30.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d948b135c4693daff7bc2dcfc4ec57237a29bd37e60c2fabf5aff2bbacf3e2f1"}, - {file = "rpds_py-0.30.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47f236970bccb2233267d89173d3ad2703cd36a0e2a6e92d0560d333871a3d23"}, - {file = "rpds_py-0.30.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:2e6ecb5a5bcacf59c3f912155044479af1d0b6681280048b338b28e364aca1f6"}, - {file = "rpds_py-0.30.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a8fa71a2e078c527c3e9dc9fc5a98c9db40bcc8a92b4e8858e36d329f8684b51"}, - {file = "rpds_py-0.30.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:73c67f2db7bc334e518d097c6d1e6fed021bbc9b7d678d6cc433478365d1d5f5"}, - {file = "rpds_py-0.30.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:5ba103fb455be00f3b1c2076c9d4264bfcb037c976167a6047ed82f23153f02e"}, - {file = "rpds_py-0.30.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7cee9c752c0364588353e627da8a7e808a66873672bcb5f52890c33fd965b394"}, - {file = "rpds_py-0.30.0-cp312-cp312-win32.whl", hash = "sha256:1ab5b83dbcf55acc8b08fc62b796ef672c457b17dbd7820a11d6c52c06839bdf"}, - {file = "rpds_py-0.30.0-cp312-cp312-win_amd64.whl", hash = "sha256:a090322ca841abd453d43456ac34db46e8b05fd9b3b4ac0c78bcde8b089f959b"}, - {file = "rpds_py-0.30.0-cp312-cp312-win_arm64.whl", hash = "sha256:669b1805bd639dd2989b281be2cfd951c6121b65e729d9b843e9639ef1fd555e"}, - {file = "rpds_py-0.30.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:f83424d738204d9770830d35290ff3273fbb02b41f919870479fab14b9d303b2"}, - {file = "rpds_py-0.30.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e7536cd91353c5273434b4e003cbda89034d67e7710eab8761fd918ec6c69cf8"}, - {file = "rpds_py-0.30.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2771c6c15973347f50fece41fc447c054b7ac2ae0502388ce3b6738cd366e3d4"}, - {file = "rpds_py-0.30.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0a59119fc6e3f460315fe9d08149f8102aa322299deaa5cab5b40092345c2136"}, - {file = "rpds_py-0.30.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:76fec018282b4ead0364022e3c54b60bf368b9d926877957a8624b58419169b7"}, - {file = "rpds_py-0.30.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:692bef75a5525db97318e8cd061542b5a79812d711ea03dbc1f6f8dbb0c5f0d2"}, - {file = "rpds_py-0.30.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9027da1ce107104c50c81383cae773ef5c24d296dd11c99e2629dbd7967a20c6"}, - {file = "rpds_py-0.30.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:9cf69cdda1f5968a30a359aba2f7f9aa648a9ce4b580d6826437f2b291cfc86e"}, - {file = "rpds_py-0.30.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a4796a717bf12b9da9d3ad002519a86063dcac8988b030e405704ef7d74d2d9d"}, - {file = "rpds_py-0.30.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5d4c2aa7c50ad4728a094ebd5eb46c452e9cb7edbfdb18f9e1221f597a73e1e7"}, - {file = "rpds_py-0.30.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ba81a9203d07805435eb06f536d95a266c21e5b2dfbf6517748ca40c98d19e31"}, - {file = "rpds_py-0.30.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:945dccface01af02675628334f7cf49c2af4c1c904748efc5cf7bbdf0b579f95"}, - {file = "rpds_py-0.30.0-cp313-cp313-win32.whl", hash = "sha256:b40fb160a2db369a194cb27943582b38f79fc4887291417685f3ad693c5a1d5d"}, - {file = "rpds_py-0.30.0-cp313-cp313-win_amd64.whl", hash = "sha256:806f36b1b605e2d6a72716f321f20036b9489d29c51c91f4dd29a3e3afb73b15"}, - {file = "rpds_py-0.30.0-cp313-cp313-win_arm64.whl", hash = "sha256:d96c2086587c7c30d44f31f42eae4eac89b60dabbac18c7669be3700f13c3ce1"}, - {file = "rpds_py-0.30.0-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:eb0b93f2e5c2189ee831ee43f156ed34e2a89a78a66b98cadad955972548be5a"}, - {file = "rpds_py-0.30.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:922e10f31f303c7c920da8981051ff6d8c1a56207dbdf330d9047f6d30b70e5e"}, - {file = "rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cdc62c8286ba9bf7f47befdcea13ea0e26bf294bda99758fd90535cbaf408000"}, - {file = "rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:47f9a91efc418b54fb8190a6b4aa7813a23fb79c51f4bb84e418f5476c38b8db"}, - {file = "rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1f3587eb9b17f3789ad50824084fa6f81921bbf9a795826570bda82cb3ed91f2"}, - {file = "rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:39c02563fc592411c2c61d26b6c5fe1e51eaa44a75aa2c8735ca88b0d9599daa"}, - {file = "rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51a1234d8febafdfd33a42d97da7a43f5dcb120c1060e352a3fbc0c6d36e2083"}, - {file = "rpds_py-0.30.0-cp313-cp313t-manylinux_2_31_riscv64.whl", hash = "sha256:eb2c4071ab598733724c08221091e8d80e89064cd472819285a9ab0f24bcedb9"}, - {file = "rpds_py-0.30.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6bdfdb946967d816e6adf9a3d8201bfad269c67efe6cefd7093ef959683c8de0"}, - {file = "rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c77afbd5f5250bf27bf516c7c4a016813eb2d3e116139aed0096940c5982da94"}, - {file = "rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:61046904275472a76c8c90c9ccee9013d70a6d0f73eecefd38c1ae7c39045a08"}, - {file = "rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:4c5f36a861bc4b7da6516dbdf302c55313afa09b81931e8280361a4f6c9a2d27"}, - {file = "rpds_py-0.30.0-cp313-cp313t-win32.whl", hash = "sha256:3d4a69de7a3e50ffc214ae16d79d8fbb0922972da0356dcf4d0fdca2878559c6"}, - {file = "rpds_py-0.30.0-cp313-cp313t-win_amd64.whl", hash = "sha256:f14fc5df50a716f7ece6a80b6c78bb35ea2ca47c499e422aa4463455dd96d56d"}, - {file = "rpds_py-0.30.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:68f19c879420aa08f61203801423f6cd5ac5f0ac4ac82a2368a9fcd6a9a075e0"}, - {file = "rpds_py-0.30.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ec7c4490c672c1a0389d319b3a9cfcd098dcdc4783991553c332a15acf7249be"}, - {file = "rpds_py-0.30.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f251c812357a3fed308d684a5079ddfb9d933860fc6de89f2b7ab00da481e65f"}, - {file = "rpds_py-0.30.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ac98b175585ecf4c0348fd7b29c3864bda53b805c773cbf7bfdaffc8070c976f"}, - {file = "rpds_py-0.30.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3e62880792319dbeb7eb866547f2e35973289e7d5696c6e295476448f5b63c87"}, - {file = "rpds_py-0.30.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4e7fc54e0900ab35d041b0601431b0a0eb495f0851a0639b6ef90f7741b39a18"}, - {file = "rpds_py-0.30.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47e77dc9822d3ad616c3d5759ea5631a75e5809d5a28707744ef79d7a1bcfcad"}, - {file = "rpds_py-0.30.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:b4dc1a6ff022ff85ecafef7979a2c6eb423430e05f1165d6688234e62ba99a07"}, - {file = "rpds_py-0.30.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4559c972db3a360808309e06a74628b95eaccbf961c335c8fe0d590cf587456f"}, - {file = "rpds_py-0.30.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:0ed177ed9bded28f8deb6ab40c183cd1192aa0de40c12f38be4d59cd33cb5c65"}, - {file = "rpds_py-0.30.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ad1fa8db769b76ea911cb4e10f049d80bf518c104f15b3edb2371cc65375c46f"}, - {file = "rpds_py-0.30.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:46e83c697b1f1c72b50e5ee5adb4353eef7406fb3f2043d64c33f20ad1c2fc53"}, - {file = "rpds_py-0.30.0-cp314-cp314-win32.whl", hash = "sha256:ee454b2a007d57363c2dfd5b6ca4a5d7e2c518938f8ed3b706e37e5d470801ed"}, - {file = "rpds_py-0.30.0-cp314-cp314-win_amd64.whl", hash = "sha256:95f0802447ac2d10bcc69f6dc28fe95fdf17940367b21d34e34c737870758950"}, - {file = "rpds_py-0.30.0-cp314-cp314-win_arm64.whl", hash = "sha256:613aa4771c99f03346e54c3f038e4cc574ac09a3ddfb0e8878487335e96dead6"}, - {file = "rpds_py-0.30.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:7e6ecfcb62edfd632e56983964e6884851786443739dbfe3582947e87274f7cb"}, - {file = "rpds_py-0.30.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a1d0bc22a7cdc173fedebb73ef81e07faef93692b8c1ad3733b67e31e1b6e1b8"}, - {file = "rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d08f00679177226c4cb8c5265012eea897c8ca3b93f429e546600c971bcbae7"}, - {file = "rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5965af57d5848192c13534f90f9dd16464f3c37aaf166cc1da1cae1fd5a34898"}, - {file = "rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9a4e86e34e9ab6b667c27f3211ca48f73dba7cd3d90f8d5b11be56e5dbc3fb4e"}, - {file = "rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e5d3e6b26f2c785d65cc25ef1e5267ccbe1b069c5c21b8cc724efee290554419"}, - {file = "rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:626a7433c34566535b6e56a1b39a7b17ba961e97ce3b80ec62e6f1312c025551"}, - {file = "rpds_py-0.30.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:acd7eb3f4471577b9b5a41baf02a978e8bdeb08b4b355273994f8b87032000a8"}, - {file = "rpds_py-0.30.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fe5fa731a1fa8a0a56b0977413f8cacac1768dad38d16b3a296712709476fbd5"}, - {file = "rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:74a3243a411126362712ee1524dfc90c650a503502f135d54d1b352bd01f2404"}, - {file = "rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:3e8eeb0544f2eb0d2581774be4c3410356eba189529a6b3e36bbbf9696175856"}, - {file = "rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:dbd936cde57abfee19ab3213cf9c26be06d60750e60a8e4dd85d1ab12c8b1f40"}, - {file = "rpds_py-0.30.0-cp314-cp314t-win32.whl", hash = "sha256:dc824125c72246d924f7f796b4f63c1e9dc810c7d9e2355864b3c3a73d59ade0"}, - {file = "rpds_py-0.30.0-cp314-cp314t-win_amd64.whl", hash = "sha256:27f4b0e92de5bfbc6f86e43959e6edd1425c33b5e69aab0984a72047f2bcf1e3"}, - {file = "rpds_py-0.30.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c2262bdba0ad4fc6fb5545660673925c2d2a5d9e2e0fb603aad545427be0fc58"}, - {file = "rpds_py-0.30.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:ee6af14263f25eedc3bb918a3c04245106a42dfd4f5c2285ea6f997b1fc3f89a"}, - {file = "rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3adbb8179ce342d235c31ab8ec511e66c73faa27a47e076ccc92421add53e2bb"}, - {file = "rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:250fa00e9543ac9b97ac258bd37367ff5256666122c2d0f2bc97577c60a1818c"}, - {file = "rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9854cf4f488b3d57b9aaeb105f06d78e5529d3145b1e4a41750167e8c213c6d3"}, - {file = "rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:993914b8e560023bc0a8bf742c5f303551992dcb85e247b1e5c7f4a7d145bda5"}, - {file = "rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58edca431fb9b29950807e301826586e5bbf24163677732429770a697ffe6738"}, - {file = "rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:dea5b552272a944763b34394d04577cf0f9bd013207bc32323b5a89a53cf9c2f"}, - {file = "rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ba3af48635eb83d03f6c9735dfb21785303e73d22ad03d489e88adae6eab8877"}, - {file = "rpds_py-0.30.0-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:dff13836529b921e22f15cb099751209a60009731a68519630a24d61f0b1b30a"}, - {file = "rpds_py-0.30.0-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:1b151685b23929ab7beec71080a8889d4d6d9fa9a983d213f07121205d48e2c4"}, - {file = "rpds_py-0.30.0-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:ac37f9f516c51e5753f27dfdef11a88330f04de2d564be3991384b2f3535d02e"}, - {file = "rpds_py-0.30.0.tar.gz", hash = "sha256:dd8ff7cf90014af0c0f787eea34794ebf6415242ee1d6fa91eaba725cc441e84"}, -] - -[[package]] -name = "six" -version = "1.17.0" -description = "Python 2 and 3 compatibility utilities" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -files = [ - {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, - {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, -] - -[[package]] -name = "snowballstemmer" -version = "3.0.1" -description = "This package provides 32 stemmers for 30 languages generated from Snowball algorithms." -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*" -files = [ - {file = "snowballstemmer-3.0.1-py3-none-any.whl", hash = "sha256:6cd7b3897da8d6c9ffb968a6781fa6532dce9c3618a4b127d920dab764a19064"}, - {file = "snowballstemmer-3.0.1.tar.gz", hash = "sha256:6d5eeeec8e9f84d4d56b847692bacf79bc2c8e90c7f80ca4444ff8b6f2e52895"}, -] - -[[package]] -name = "soupsieve" -version = "2.8.3" -description = "A modern CSS selector implementation for Beautiful Soup." -optional = false -python-versions = ">=3.9" -files = [ - {file = "soupsieve-2.8.3-py3-none-any.whl", hash = "sha256:ed64f2ba4eebeab06cc4962affce381647455978ffc1e36bb79a545b91f45a95"}, - {file = "soupsieve-2.8.3.tar.gz", hash = "sha256:3267f1eeea4251fb42728b6dfb746edc9acaffc4a45b27e19450b676586e8349"}, -] - -[[package]] -name = "sphinx" -version = "8.2.3" -description = "Python documentation generator" -optional = false -python-versions = ">=3.11" -files = [ - {file = "sphinx-8.2.3-py3-none-any.whl", hash = "sha256:4405915165f13521d875a8c29c8970800a0141c14cc5416a38feca4ea5d9b9c3"}, - {file = "sphinx-8.2.3.tar.gz", hash = "sha256:398ad29dee7f63a75888314e9424d40f52ce5a6a87ae88e7071e80af296ec348"}, -] - -[package.dependencies] -alabaster = ">=0.7.14" -babel = ">=2.13" -colorama = {version = ">=0.4.6", markers = "sys_platform == \"win32\""} -docutils = ">=0.20,<0.22" -imagesize = ">=1.3" -Jinja2 = ">=3.1" -packaging = ">=23.0" -Pygments = ">=2.17" -requests = ">=2.30.0" -roman-numerals-py = ">=1.0.0" -snowballstemmer = ">=2.2" -sphinxcontrib-applehelp = ">=1.0.7" -sphinxcontrib-devhelp = ">=1.0.6" -sphinxcontrib-htmlhelp = ">=2.0.6" -sphinxcontrib-jsmath = ">=1.0.1" -sphinxcontrib-qthelp = ">=1.0.6" -sphinxcontrib-serializinghtml = ">=1.1.9" - -[package.extras] -docs = ["sphinxcontrib-websupport"] -lint = ["betterproto (==2.0.0b6)", "mypy (==1.15.0)", "pypi-attestations (==0.0.21)", "pyright (==1.1.395)", "pytest (>=8.0)", "ruff (==0.9.9)", "sphinx-lint (>=0.9)", "types-Pillow (==10.2.0.20240822)", "types-Pygments (==2.19.0.20250219)", "types-colorama (==0.4.15.20240311)", "types-defusedxml (==0.7.0.20240218)", "types-docutils (==0.21.0.20241128)", "types-requests (==2.32.0.20241016)", "types-urllib3 (==1.26.25.14)"] -test = ["cython (>=3.0)", "defusedxml (>=0.7.1)", "pytest (>=8.0)", "pytest-xdist[psutil] (>=3.4)", "setuptools (>=70.0)", "typing_extensions (>=4.9)"] - -[[package]] -name = "sphinx-mdinclude" -version = "0.6.2" -description = "Markdown extension for Sphinx" -optional = false -python-versions = ">=3.8" -files = [ - {file = "sphinx_mdinclude-0.6.2-py3-none-any.whl", hash = "sha256:648e78edb067c0e4bffc22943278d49d54a0714494743592032fa3ad82a86984"}, - {file = "sphinx_mdinclude-0.6.2.tar.gz", hash = "sha256:447462e82cb8be61404a2204227f920769eb923d2f57608e3325f3bb88286b4c"}, -] - -[package.dependencies] -docutils = ">=0.19,<1.0" -mistune = ">=3.0,<4.0" -pygments = ">=2.8" -sphinx = ">=6" - -[package.extras] -dev = ["attribution (==1.7.1)", "black (==24.4.2)", "coverage (==7.5.1)", "docutils (==0.20.1)", "docutils (==0.21.2)", "flake8 (==7.0.0)", "flit (==3.9.0)", "mistune (==3.0.2)", "mypy (==1.10.0)", "sphinx (==7.1.2)", "sphinx (==7.3.7)", "ufmt (==2.5.1)", "usort (==1.0.8.post1)"] - -[[package]] -name = "sphinxcontrib-applehelp" -version = "2.0.0" -description = "sphinxcontrib-applehelp is a Sphinx extension which outputs Apple help books" -optional = false -python-versions = ">=3.9" -files = [ - {file = "sphinxcontrib_applehelp-2.0.0-py3-none-any.whl", hash = "sha256:4cd3f0ec4ac5dd9c17ec65e9ab272c9b867ea77425228e68ecf08d6b28ddbdb5"}, - {file = "sphinxcontrib_applehelp-2.0.0.tar.gz", hash = "sha256:2f29ef331735ce958efa4734873f084941970894c6090408b079c61b2e1c06d1"}, -] - -[package.extras] -lint = ["mypy", "ruff (==0.5.5)", "types-docutils"] -standalone = ["Sphinx (>=5)"] -test = ["pytest"] - -[[package]] -name = "sphinxcontrib-devhelp" -version = "2.0.0" -description = "sphinxcontrib-devhelp is a sphinx extension which outputs Devhelp documents" -optional = false -python-versions = ">=3.9" -files = [ - {file = "sphinxcontrib_devhelp-2.0.0-py3-none-any.whl", hash = "sha256:aefb8b83854e4b0998877524d1029fd3e6879210422ee3780459e28a1f03a8a2"}, - {file = "sphinxcontrib_devhelp-2.0.0.tar.gz", hash = "sha256:411f5d96d445d1d73bb5d52133377b4248ec79db5c793ce7dbe59e074b4dd1ad"}, -] - -[package.extras] -lint = ["mypy", "ruff (==0.5.5)", "types-docutils"] -standalone = ["Sphinx (>=5)"] -test = ["pytest"] - -[[package]] -name = "sphinxcontrib-htmlhelp" -version = "2.1.0" -description = "sphinxcontrib-htmlhelp is a sphinx extension which renders HTML help files" -optional = false -python-versions = ">=3.9" -files = [ - {file = "sphinxcontrib_htmlhelp-2.1.0-py3-none-any.whl", hash = "sha256:166759820b47002d22914d64a075ce08f4c46818e17cfc9470a9786b759b19f8"}, - {file = "sphinxcontrib_htmlhelp-2.1.0.tar.gz", hash = "sha256:c9e2916ace8aad64cc13a0d233ee22317f2b9025b9cf3295249fa985cc7082e9"}, -] - -[package.extras] -lint = ["mypy", "ruff (==0.5.5)", "types-docutils"] -standalone = ["Sphinx (>=5)"] -test = ["html5lib", "pytest"] - -[[package]] -name = "sphinxcontrib-httpdomain" -version = "1.8.1" -description = "Sphinx domain for documenting HTTP APIs" -optional = false -python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*" -files = [ - {file = "sphinxcontrib-httpdomain-1.8.1.tar.gz", hash = "sha256:6c2dfe6ca282d75f66df333869bb0ce7331c01b475db6809ff9d107b7cdfe04b"}, - {file = "sphinxcontrib_httpdomain-1.8.1-py2.py3-none-any.whl", hash = "sha256:21eefe1270e4d9de8d717cc89ee92cc4871b8736774393bafc5e38a6bb77b1d5"}, -] - -[package.dependencies] -six = "*" -Sphinx = ">=1.6" - -[[package]] -name = "sphinxcontrib-jsmath" -version = "1.0.1" -description = "A sphinx extension which renders display math in HTML via JavaScript" -optional = false -python-versions = ">=3.5" -files = [ - {file = "sphinxcontrib-jsmath-1.0.1.tar.gz", hash = "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8"}, - {file = "sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl", hash = "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178"}, -] - -[package.extras] -test = ["flake8", "mypy", "pytest"] - -[[package]] -name = "sphinxcontrib-mermaid" -version = "1.2.3" -description = "Mermaid diagrams in yours Sphinx powered docs" -optional = false -python-versions = ">=3.8" -files = [ - {file = "sphinxcontrib_mermaid-1.2.3-py3-none-any.whl", hash = "sha256:5be782b27026bef97bfb15ccb2f7868b674a1afc0982b54cb149702cfc25aa02"}, - {file = "sphinxcontrib_mermaid-1.2.3.tar.gz", hash = "sha256:358699d0ec924ef679b41873d9edd97d0773446daf9760c75e18dc0adfd91371"}, -] - -[package.dependencies] -pyyaml = "*" -sphinx = "*" - -[package.extras] -test = ["defusedxml", "myst-parser", "pytest", "ruff", "sphinx"] - -[[package]] -name = "sphinxcontrib-openapi" -version = "0.8.4" -description = "OpenAPI (fka Swagger) spec renderer for Sphinx" -optional = false -python-versions = ">=3.7" -files = [ - {file = "sphinxcontrib-openapi-0.8.4.tar.gz", hash = "sha256:df883808a5b5e4b4113ad697185c43a3f42df3dce70453af78ba7076907e9a20"}, - {file = "sphinxcontrib_openapi-0.8.4-py3-none-any.whl", hash = "sha256:50911c18d452d9390ee3a384ef8dc8bde6135f542ba55691f81e1fbc0b71014e"}, -] - -[package.dependencies] -deepmerge = ">=0.1" -jsonschema = ">=2.5.1" -picobox = ">=2.2" -PyYAML = ">=3.12" -sphinx = ">=2.0" -sphinx-mdinclude = ">=0.5.2" -sphinxcontrib-httpdomain = ">=1.5.0" - -[[package]] -name = "sphinxcontrib-qthelp" -version = "2.0.0" -description = "sphinxcontrib-qthelp is a sphinx extension which outputs QtHelp documents" -optional = false -python-versions = ">=3.9" -files = [ - {file = "sphinxcontrib_qthelp-2.0.0-py3-none-any.whl", hash = "sha256:b18a828cdba941ccd6ee8445dbe72ffa3ef8cbe7505d8cd1fa0d42d3f2d5f3eb"}, - {file = "sphinxcontrib_qthelp-2.0.0.tar.gz", hash = "sha256:4fe7d0ac8fc171045be623aba3e2a8f613f8682731f9153bb2e40ece16b9bbab"}, -] - -[package.extras] -lint = ["mypy", "ruff (==0.5.5)", "types-docutils"] -standalone = ["Sphinx (>=5)"] -test = ["defusedxml (>=0.7.1)", "pytest"] - -[[package]] -name = "sphinxcontrib-serializinghtml" -version = "2.0.0" -description = "sphinxcontrib-serializinghtml is a sphinx extension which outputs \"serialized\" HTML files (json and pickle)" -optional = false -python-versions = ">=3.9" -files = [ - {file = "sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl", hash = "sha256:6e2cb0eef194e10c27ec0023bfeb25badbbb5868244cf5bc5bdc04e4464bf331"}, - {file = "sphinxcontrib_serializinghtml-2.0.0.tar.gz", hash = "sha256:e9d912827f872c029017a53f0ef2180b327c3f7fd23c87229f7a8e8b70031d4d"}, -] - -[package.extras] -lint = ["mypy", "ruff (==0.5.5)", "types-docutils"] -standalone = ["Sphinx (>=5)"] -test = ["pytest"] - -[[package]] -name = "typing-extensions" -version = "4.15.0" -description = "Backported and Experimental Type Hints for Python 3.9+" -optional = false -python-versions = ">=3.9" -files = [ - {file = "typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548"}, - {file = "typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466"}, -] - -[[package]] -name = "urllib3" -version = "2.6.3" -description = "HTTP library with thread-safe connection pooling, file post, and more." -optional = false -python-versions = ">=3.9" -files = [ - {file = "urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4"}, - {file = "urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed"}, -] - -[package.extras] -brotli = ["brotli (>=1.2.0)", "brotlicffi (>=1.2.0.0)"] -h2 = ["h2 (>=4,<5)"] -socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] -zstd = ["backports-zstd (>=1.0.0)"] - -[metadata] -lock-version = "2.0" -python-versions = "^3.12" -content-hash = "d862f1d3f58e021e87551ff5c25128e997c1364600892ae0f54813818d3e6b68" diff --git a/docs/website/pyproject.toml b/docs/website/pyproject.toml deleted file mode 100644 index 3824d1a7..00000000 --- a/docs/website/pyproject.toml +++ /dev/null @@ -1,20 +0,0 @@ -[tool.poetry] -authors = ["GenLayer"] -description = "" -name = "genvm-docstypesgen" -version = "0.0.1" - -[tool.poetry.dependencies] -myst-parser = "^5.0.0" -numpy = "^2.1.3" -pydata-sphinx-theme = "^0.16.0" -pyright = "^1.1.390" -python = "^3.12" -sphinx = "^8.1.3" -sphinxcontrib-mermaid = "^1.0.0" -sphinxcontrib-openapi = "^0.8.4" - - -[build-system] -build-backend = "poetry.core.masonry.api" -requires = ["poetry-core"] diff --git a/docs/website/src/conf.py b/docs/website/src/conf.py index 6f0324b8..da41883f 100644 --- a/docs/website/src/conf.py +++ b/docs/website/src/conf.py @@ -1,21 +1,15 @@ import datetime -import enum import json import os import re as _re -import sys -import typing import urllib.request from pathlib import Path -from types import ModuleType - -import sphinx.util.inspect as _sphinx_inspect - -# Patch stringify_annotation to replace Annotated[int, StaticIntMeta(...)] with u8..u256/i8..i256 -_orig_stringify_annotation = _sphinx_inspect.stringify_annotation - -_annotated_type_aliases: dict[typing.Any, str] = {} +# The Python SDK reference (autodoc of the genlayer package) lives in the +# executor line that ships it β€” see executors/.x/docs/website. This +# manager site documents the spec / impl-spec / overview only, so it carries no +# autodoc machinery. Cross-linking is one-way: the executor sub-sites reference +# this manager site via intersphinx; the manager does NOT reference them back. project = 'GenVM SDK' @@ -30,10 +24,6 @@ version = release extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.viewcode', - 'sphinx.ext.todo', - 'sphinx.ext.intersphinx', 'sphinxcontrib.mermaid', 'sphinxcontrib.openapi', 'myst_parser', @@ -152,145 +142,4 @@ def _semver_sort_key(entry): html_js_files = ['favicon-swap.js'] html_favicon = f'/{release}/_static/favicon.png' -todo_include_todos = True - -autodoc_mock_imports = ['_genlayer_wasi', 'google', 'onnx', 'word_piece_tokenizer'] - -fake_genlayer_wasi = ModuleType( - '_genlayer_wasi', -) -fake_genlayer_wasi.__dict__['FAKE_VM'] = True -sys.modules['_genlayer_wasi'] = fake_genlayer_wasi - -MONO_REPO_ROOT_FILE = '.genvm-monorepo-root' -script_dir = Path(__file__).parent -root_dir = script_dir -while not root_dir.joinpath(MONO_REPO_ROOT_FILE).exists(): - root_dir = root_dir.parent -MONOREPO_CONF = json.loads(root_dir.joinpath(MONO_REPO_ROOT_FILE).read_text()) -active_versions = MONOREPO_CONF['active-versions'] -primary_executor = root_dir / 'executors' / f'{active_versions[0]}.x' -py_std = primary_executor / 'runners' / 'genlayer-py-std' / 'src' -py_emb = primary_executor / 'runners' / 'genlayer-py-std' / 'src-emb' - -sys.path.append(str(py_std)) -sys.path.append(str(py_emb)) - -os.environ['GENERATING_DOCS'] = 'true' - master_doc = 'index' -intersphinx_mapping = { - 'python': ('https://docs.python.org/3.12', None), - 'numpy': ('https://numpy.org/doc/stable/', None), -} - -ignored_special = [ - '__dict__', - '__abstractmethods__', - '__annotations__', - '__class_getitem__', - '__init_subclass__', - '__module__', - '__orig_bases__', - '__parameters__', - '__slots__', - '__subclasshook__', - '__type_params__', - '__weakref__', - '__reversed__', - '__protocol_attrs__', - '__dataclass_fields__', - '__match_args__', - '__dataclass_params__', -] - -autodoc_default_options: dict[str, str | bool] = { - 'inherited-members': True, - 'private-members': False, - 'special-members': True, - 'exclude-members': ','.join(ignored_special + ['gl']), -} - -autoapi_python_class_content = 'class' -autodoc_class_signature = 'separated' -autodoc_typehints = 'both' -autodoc_typehints_description_target = 'documented_params' -autodoc_inherit_docstrings = True -autodoc_typehints_format = 'short' -# autodoc_typehints_format = 'fully-qualified' -autodoc_preserve_defaults = True - -autodoc_signature_line_length = 1 - - -class PsuedoAll: - def __contains__(self, _x): - return True - - def append(self, _x): - pass - - def __iter__(self): - return - yield - - -def setup(app): - def handle_bases(app, name, obj, options, bases: list): - idx = 0 - for i in range(len(bases)): - cur = bases[i] - cur_name = cur if isinstance(cur, str) else cur.__qualname__ - if cur_name.startswith('_'): - pass - else: - bases[idx] = cur - idx += 1 - bases[idx:] = [] - if len(bases) == 0: - bases.append(object) - - def handle_skip_member(app, what, name, obj, skip, options): - import types - - if 'what' == 'class' and isinstance(obj, types.MethodType): - if obj.__self__.__class__ is typing.NewType: - return True - if what == 'module' and isinstance(obj, type): - if any(base in obj.mro() for base in [dict, tuple, bytes, enum.Enum]): - options['special-members'] = [] - options['inherited-members'] = False - return - if what == 'module': - if isinstance(obj, typing.NewType): - options['special-members'] = [] - options['inherited-members'] = False - return - # options['special-members'] = PsuedoAll() - # options['inherited-members'] = PsuedoAll() - - # Build u8..u256, i8..i256 display aliases for Annotated[int, StaticIntMeta(...)] - _type_display_aliases = {} - for _prefix, _signed in [('u', False), ('i', True)]: - for _sz in range(1, 33): - _bits = _sz * 8 - _type_display_aliases[ - f'typing.Annotated[int, StaticIntMeta(size={_sz}, signed={_signed})]' - ] = f'genlayer.types.{_prefix}{_bits}' - _type_display_aliases[ - f'Annotated[int, StaticIntMeta(size={_sz}, signed={_signed})]' - ] = f'genlayer.types.{_prefix}{_bits}' - - def autodoc_process_signature( - app, what, name, obj, options, signature, return_annotation - ): - for old, new in _type_display_aliases.items(): - if signature: - signature = signature.replace(old, new) - if return_annotation: - return_annotation = return_annotation.replace(old, new) - return (signature, return_annotation) - - app.connect('autodoc-process-bases', handle_bases) - app.connect('autodoc-skip-member', handle_skip_member) - app.connect('autodoc-process-signature', autodoc_process_signature) diff --git a/docs/website/src/impl-spec/02-vm/02-version-management.rst b/docs/website/src/impl-spec/02-vm/02-version-management.rst index 070f6b8f..06ce487d 100644 --- a/docs/website/src/impl-spec/02-vm/02-version-management.rst +++ b/docs/website/src/impl-spec/02-vm/02-version-management.rst @@ -45,8 +45,9 @@ GenVM could silently mis-interpret an older contract's calldata or storage. Runner manifest --------------- -The manager maintains an executor version manifest (the JSON document published as -:doc:`available-runners` and shipped at ``doc/website/src/impl-spec/appendix/runners-versions.json``). +The manager maintains an executor version manifest (the per-line available-runners +listing, generated as ``runners-versions.json`` and published on each executor +line's docs sub-site). For every supported runner version it lists the content hashes of every runtime artifact (``cpython``, ``py-genlayer``, ``py-lib-genlayer-std``, ``softfloat``, ...). When a contract selects a runner version (via its ``runner.json`` or the bundled `StartWasm` action) the diff --git a/docs/website/src/impl-spec/03-greyboxing/01-lua-api.md b/docs/website/src/impl-spec/03-greyboxing/01-lua-api.md deleted file mode 100644 index e0e0ed75..00000000 --- a/docs/website/src/impl-spec/03-greyboxing/01-lua-api.md +++ /dev/null @@ -1,480 +0,0 @@ -# Lua API docs - -## Format - - ---- - -## LLM - -### exec_prompt_in_provider - - -```lua -fun(ctx: any, data: { prompt: Prompt, format: "bool"|"json"|"text", model: string, provider: string }):any -``` - -### providers - - -```lua -table -``` - -### templates - - -```lua -{ eq_comparative: any, eq_non_comparative_leader: any, eq_non_comparative_validator: any } -``` - - ---- - -## LLMExecPromptPayload - -### images - - -```lua -userdata[] -``` - -### prompt - - -```lua -string -``` - -### response_format - - -```lua -"json"|"text" -``` - - ---- - -## LLMExecPromptTemplatePayload - -### [string] - - -```lua -string -``` - -### template - - -```lua -"EqComparative"|"EqNonComparativeLeader"|"EqNonComparativeValidator" -``` - - ---- - -## MappedPrompt - -### format - - -```lua -"bool"|"json"|"text" -``` - -### prompt - - -```lua -Prompt -``` - - ---- - -## MergeStrategy - - ---- - -## ModelConfig - -### enabled - - -```lua -boolean -``` - -### meta - - -```lua -any -``` - -### supports_image - - -```lua -boolean -``` - -### supports_json - - -```lua -boolean -``` - -### use_max_completion_tokens - - -```lua -boolean -``` - - ---- - -## ModuleError - -### causes - - -```lua -string[] -``` - -### ctx - - -```lua -table -``` - -### fatal - - -```lua -boolean -``` - - ---- - -## Prompt - -### extra - - -```lua -table|nil -``` - -### extra_merge_strategy - - -```lua -"merge_left"|"merge_right"|"none"|"replace"|table>...(+1) -``` - -### images - - -```lua -userdata[] -``` - -### max_tokens - - -```lua -integer -``` - -### seed - - -```lua -integer|nil -``` - -### system_message - - -```lua -string|nil -``` - -### temperature - - -```lua -number -``` - -### use_max_completion_tokens - - -```lua -boolean -``` - -### user_message - - -```lua -string -``` - - ---- - -## ProviderEntry - -### models - - -```lua -table -``` - - ---- - -## ProvidersDB - - ---- - -## RS - -### as_user_error - - -```lua -fun(val: any):ModuleError|nil -``` - -### base64_decode - - -```lua -fun(val: string):string -``` - -### base64_encode - - -```lua -fun(val: string):string -``` - -### filter_image - - -```lua -fun(image: string, filters: any[]):string -``` - -### filter_text - - -```lua -fun(text: string, filters: string[]):string -``` - -### json_parse - - -```lua -fun(val: string):any -``` - -### json_stringify - - -```lua -fun(val: any):string -``` - -### log_json - - -```lua -fun(val: any):nil -``` - -### random_bytes - - -```lua -fun(length: integer):string -``` - -### random_float - - -```lua -fun():number -``` - -### request - - -```lua -fun(ctx: any, req: { body: string|nil, url: string, headers: table, method: string, error_on_status: boolean|nil, json: false|nil, response_body_max_size: integer|n...(too long)... string> } -``` - -### sleep_seconds - - -```lua -fun(duration: number):nil -``` - -### split_url - - -```lua -fun(url: string):{ schema: string, port: number|nil, host: string }|nil -``` - -### url_encode - - -```lua -fun(url: string):string -``` - -### user_error - - -```lua -fun(val: ModuleError):nil -``` - - ---- - -## WEB - -### allowed_tld - - -```lua -{ [string]: boolean } -``` - -### config - - -```lua -table -``` - -### get_webdriver_session - - -```lua -fun(ctx: any):string -``` - - ---- - -## WebRenderPayload - -### mode - - -```lua -"html"|"screenshot"|"text" -``` - -### size_limit - - -```lua -integer? -``` - -### url - - -```lua -string -``` - -### wait_after_loaded - - -```lua -number -``` - - ---- - -## WebRequestPayload - -### body - - -```lua -string? -``` - -### headers - - -```lua -table -``` - -### method - - -```lua -"DELETE"|"GET"|"HEAD"|"OPTIONS"|"PATCH"...(+1) -``` - -### sign - - -```lua -boolean? -``` - -### size_limit - - -```lua -integer? -``` - -### url - - -```lua -string -``` - - ---- - -## my_data - - -```lua -table -``` - - -```lua -table -``` diff --git a/docs/website/src/impl-spec/03-greyboxing/index.rst b/docs/website/src/impl-spec/03-greyboxing/index.rst index 8fe1e836..c20928fd 100644 --- a/docs/website/src/impl-spec/03-greyboxing/index.rst +++ b/docs/website/src/impl-spec/03-greyboxing/index.rst @@ -4,8 +4,6 @@ Greyboxing Documentation .. toctree:: :maxdepth: 2 - 01-lua-api - It refers to the technique of preventing attacks on LLMs. Implementing it is a responsibility of every node, as bundled presets can be attacked. diff --git a/docs/website/src/impl-spec/appendix/index.rst b/docs/website/src/impl-spec/appendix/index.rst index 7083fbba..54dede99 100644 --- a/docs/website/src/impl-spec/appendix/index.rst +++ b/docs/website/src/impl-spec/appendix/index.rst @@ -6,4 +6,3 @@ Appendix host-loop manager-api - available-runners diff --git a/docs/website/src/impl-spec/appendix/manager-api.rst b/docs/website/src/impl-spec/appendix/manager-api.rst index ee7665f9..4edba3b5 100644 --- a/docs/website/src/impl-spec/appendix/manager-api.rst +++ b/docs/website/src/impl-spec/appendix/manager-api.rst @@ -3,8 +3,9 @@ Manager API The GenVM Manager is an HTTP server that provides an API for managing GenVM instances, modules, and related operations. -Two endpoints below are tightly coupled with the runner manifest published as -:doc:`available-runners` (generated from ``runners-versions.json``): +Two endpoints below are tightly coupled with the runner manifest (the per-line +available-runners listing, generated from ``runners-versions.json``, published on +each executor line's docs sub-site): - ``POST /contract/detect-version`` returns the public-ABI ``specified_major`` that the node MUST persist into the contract's root-slot ``major`` field (see diff --git a/docs/website/src/index.rst b/docs/website/src/index.rst index b77d98e9..57c8b050 100644 --- a/docs/website/src/index.rst +++ b/docs/website/src/index.rst @@ -11,6 +11,5 @@ For tutorials and getting started guides, see the :maxdepth: 2 overview/index - python-sdk/index spec/index impl-spec/index diff --git a/docs/website/src/overview/index.rst b/docs/website/src/overview/index.rst index b9722673..685387eb 100644 --- a/docs/website/src/overview/index.rst +++ b/docs/website/src/overview/index.rst @@ -6,3 +6,11 @@ Overview what-is-genvm key-concepts glossary + +Executor lines +-------------- + +Each executor line ships its own docs (the Python SDK reference and runner +versions) as a standalone sub-site: + +.. include:: executor-lines_generated.rst diff --git a/docs/website/src/python-sdk/changelog-notes/v0.1.3.rst b/docs/website/src/python-sdk/changelog-notes/v0.1.3.rst deleted file mode 100644 index 137890a1..00000000 --- a/docs/website/src/python-sdk/changelog-notes/v0.1.3.rst +++ /dev/null @@ -1,113 +0,0 @@ -Migration Guide -~~~~~~~~~~~~~~~ - -This section provides code examples for migrating from v0.1.0 to v0.1.3: - -Contract Interfaces -^^^^^^^^^^^^^^^^^^^ - -.. code-block:: python - - # v0.1.0 - contract = gl.ContractAt(address) - contract.view().some_method() - contract.emit().some_method() - - # v0.1.3 - contract = gl.get_contract_at(address) - contract.view().some_method() - contract.emit().some_method() - -Error handling -^^^^^^^^^^^^^^ - -.. code-block:: python - - # v0.1.0 - from genlayer import Rollback - gl.advanced.rollback_immediate("error message") - - # v0.1.3 - from genlayer.gl.vm import UserError # or use gl.vm.UserError - gl.advanced.user_error_immediate("error message") - -VM operations -^^^^^^^^^^^^^ - -.. code-block:: python - - # v0.1.0 - gl.advanced.run_nondet(leader_fn, validator_fn) - - # v0.1.3 - gl.vm.run_nondet(leader_fn, validator_fn) - -EVM contracts -^^^^^^^^^^^^^ - -.. code-block:: python - - # v0.1.0 - @gl.eth_contract - class MyEthContract: - # contract definition - - # v0.1.3 - @gl.evm.contract_interface - class MyEthContract: - # contract definition - -Non-deterministic functions -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: python - - # v0.1.0 - result = gl.get_webpage(url, mode='text') - response = gl.exec_prompt("prompt text") - - # v0.1.3 - result = gl.nondet.web.render(url, mode='text') - response = gl.nondet.exec_prompt("prompt text") - -Equivalence principles -^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: python - - # v0.1.0 - result = gl.eq_principle_strict_eq(fn) - result = gl.eq_principle_prompt_comparative(fn, principle) - - # v0.1.3 - result = gl.eq_principle.strict_eq(fn) - result = gl.eq_principle.prompt_comparative(fn, principle) - -Breaking Changes -~~~~~~~~~~~~~~~~ - -#. Removed ``Rollback`` exception from top-level imports - use ``genlayer.gl.vm.UserError`` instead -#. Error handling methods now use ``#error`` and ``#get-schema`` special method names from ``genlayer.py.public_abi.SpecialMethod`` -#. Contract interface changes: ``eth_contract`` renamed to ``evm_contract_interface`` in EVM module - -New Features -~~~~~~~~~~~~ - -#. **Module reorganization**: Core functionality moved to ``genlayer.gl`` namespace with lazy loading for better performance -#. **Contract interface system**: New ``@contract_interface`` decorator for type-safe contract interactions -#. **Enhanced contract deployment**: ``deploy_contract`` function with deterministic addressing via salt nonce -#. **Contract proxy system**: ``get_contract_at`` function returns proxy objects with ``view()`` and ``emit()`` methods -#. **Advanced event system**: ``Event`` class with proper topic generation and indexed field support -#. **VM operations**: New ``genlayer.gl.vm`` module with ``run_nondet``, ``spawn_sandbox`` functions -#. **EVM integration**: Enhanced ``genlayer.py.evm`` module with contract generation capabilities -#. **Equivalence principles**: New ``genlayer.gl.eq_principle`` module with ``strict_eq``, ``prompt_comparative``, ``prompt_non_comparative`` -#. **Advanced utilities**: ``genlayer.gl.advanced`` module with ``user_error_immediate`` and ``emit_raw_event`` - -API Improvements -~~~~~~~~~~~~~~~~ - -#. Storage system improvements with better slot management -#. Type annotations added for WASI bindings (``_genlayer_wasi.pyi``) -#. Enhanced error handling with ``UserError`` replacing ``Rollback`` -#. Better lazy loading system for improved import performance -#. Documentation generation support with ``GENERATING_DOCS`` environment variable diff --git a/docs/website/src/python-sdk/changelog-notes/v0.1.8.rst b/docs/website/src/python-sdk/changelog-notes/v0.1.8.rst deleted file mode 100644 index fd42f8d0..00000000 --- a/docs/website/src/python-sdk/changelog-notes/v0.1.8.rst +++ /dev/null @@ -1,19 +0,0 @@ -Breaking Changes -~~~~~~~~~~~~~~~~ - -#. Calldata encoding behavior changed - dataclasses now encoded automatically without requiring custom ``default`` parameter -#. ``genlayer.py.public_abi`` file marked as auto-generated - manual edits will be overwritten - -New Features -~~~~~~~~~~~~ - -#. **Enhanced calldata encoding**: Automatic dataclass encoding support without custom default functions -#. **Standardized error types**: New ``VmError`` enum for consistent error handling across the platform -#. **Extended public ABI**: Additional constants for internal VM operations (``CODE_SLOT_OFFSET``) - -API Improvements -~~~~~~~~~~~~~~~~ - -#. Simplified dataclass serialization - no longer requires explicit ``default`` parameter in ``calldata.encode()`` -#. Better type safety with standardized error enums -#. Cleaner API for encoding complex data structures diff --git a/docs/website/src/python-sdk/changelog-notes/v0.3.rst b/docs/website/src/python-sdk/changelog-notes/v0.3.rst deleted file mode 100644 index 3bdaab32..00000000 --- a/docs/website/src/python-sdk/changelog-notes/v0.3.rst +++ /dev/null @@ -1,348 +0,0 @@ -v0.3.0 -====== - -Migration Guide ---------------- - -v0.3.0 introduces a major restructuring of the standard library. The ``genlayer.gl`` and ``genlayer.py`` intermediate packages are removed. All public API is now accessible directly under the ``genlayer`` namespace. - -Import Pattern -~~~~~~~~~~~~~~~ - -The recommended import pattern has changed: - -.. code-block:: python - - # v0.2.x - from genlayer import * - - # v0.3.0 - import genlayer as gl - from genlayer.types import * - -The ``from genlayer import *`` star-import previously brought ``gl`` (a lazy proxy to ``genlayer.gl``), all types, and storage names into scope. Now ``import genlayer as gl`` gives you direct access to submodules (``gl.contract``, ``gl.vm``, ``gl.message``, ``gl.chain``, ``gl.storage``, etc.) and decorators (``gl.public``, ``gl.private``). The ``from genlayer.types import *`` import brings the type aliases (``u8``..``u256``, ``i8``..``i256``, ``Address``, ``bigint``, etc.) into local scope. - -Module Path Changes -~~~~~~~~~~~~~~~~~~~~~ - -All ``genlayer.py.*`` and ``genlayer.gl.*`` paths are removed. Here is the mapping: - -.. list-table:: - :header-rows: 1 - :widths: 50 50 - - * - v0.2.x - - v0.3.0 - * - ``genlayer.py.types`` - - ``genlayer.types`` - * - ``genlayer.py.keccak`` - - ``genlayer.types.keccak`` - * - ``genlayer.py.calldata`` - - ``genlayer.calldata`` - * - ``genlayer.py.storage`` - - ``genlayer.storage`` - * - ``genlayer.py.evm`` - - ``genlayer.evm`` - * - ``genlayer.gl.vm`` - - ``genlayer.vm`` - * - ``genlayer.gl.nondet`` - - ``genlayer.nondet`` - * - ``genlayer.gl.eq_principle`` - - ``genlayer.eq_principle`` - * - ``genlayer.gl.genvm_contracts`` - - ``genlayer.contract`` - * - ``genlayer.gl.annotations`` - - ``genlayer._internal.annotations`` - * - ``genlayer.gl.advanced`` - - removed (see below) - -.. note:: - Type aliases (``u8``..``u256``, ``i8``..``i256``) are no longer ``typing.NewType`` instances; they are now ``typing.Annotated[int, ...]``. They still work as type annotations, but they are no longer callable, so the ``u256(0)`` wrapping is gone β€” pass a plain ``int`` instead: - - .. code-block:: python - - # v0.2.x - gl.contract.deploy(code=source, value=u256(1000), salt_nonce=u256(42)) - - # v0.3.0 - gl.contract.deploy(code=source, value=1000, salt_nonce=42) - -Contract Declaration -~~~~~~~~~~~~~~~~~~~~~ - -``gl.Contract`` is now ``gl.contract.Contract``: - -.. code-block:: python - - # v0.2.x - class MyToken(gl.Contract): - ... - - # v0.3.0 - class MyToken(gl.contract.Contract): - ... - -Contract Interaction -~~~~~~~~~~~~~~~~~~~~~ - -Functions for interacting with other contracts have been renamed and moved into ``gl.contract``: - -.. code-block:: python - - # v0.2.x - contract = gl.get_contract_at(address) - gl.deploy_contract(code=source, args=[...]) - - @gl.contract_interface - class IToken: - class View: - def balance_of(self, owner: Address) -> u256: ... - - # v0.3.0 - contract = gl.contract.get_at(address) - gl.contract.deploy(code=source, args=[...]) - - @gl.contract.interface - class IToken: - class View: - def balance_of(self, owner: Address) -> u256: ... - -The ``ContractProxy`` type is renamed to ``gl.contract.Proxy``. - -Contract proxies, ``Contract`` itself, and the new ``gl.chain.Account`` all implement the ``gl.chain.IAccount`` protocol, exposing ``address``, ``balance``, and ``emit_transfer(value, *, on=...)``. - -Value Transfers and ``on=`` Parameter -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Write-call and deploy APIs now take an ``on`` keyword controlling when the message is applied, with values ``'accepted'`` or ``'finalized'`` (default ``'finalized'``): - -.. code-block:: python - - contract.emit(value=100, on='finalized').transfer(to, amount) - gl.contract.deploy(code=source, value=1000, on='finalized', salt_nonce=42) - -A new ``emit_transfer`` helper sends a plain value transfer without a method call. The receiver may catch it via ``Contract.__receive__`` (must be ``@gl.public.write.payable``): - -.. code-block:: python - - contract.emit_transfer(1000, on='finalized') - -Accounts -~~~~~~~~ - -``gl.chain.Account`` is a new lightweight wrapper around an ``Address`` that allows querying the balance of, or emitting a transfer to, any on-chain account (contract or EoA): - -.. code-block:: python - - acc = gl.chain.Account(some_address) - bal = acc.balance - acc.emit_transfer(100) - -Message Context -~~~~~~~~~~~~~~~ - -The ``gl.message`` object was a ``NamedTuple``. It is now a module (``genlayer.message``) with the same fields as module-level attributes: - -.. code-block:: python - - # v0.2.x - sender = gl.message.sender_address - value = gl.message.value - - # v0.3.0 (identical usage, but gl.message is a module now) - sender = gl.message.sender_address - value = gl.message.value - -``gl.message_raw`` is now ``gl.message.raw``. The chain ID is exposed as ``gl.message.chain_id`` (and as ``gl.chain.id``). - -Events -~~~~~~ - -The ``Event`` class has moved from ``genlayer.gl`` to ``genlayer.chain``: - -.. code-block:: python - - # v0.2.x - class Transfer(gl.Event): - def __init__(self, sender: Address, to: Address, /): ... - - # v0.3.0 - class Transfer(gl.chain.Event): - def __init__(self, sender: Address, to: Address, /): ... - -``gl.advanced.emit_raw_event(topics, blob)`` is now ``gl.chain.Event.emit_raw(topics, blob)``. - -Advanced / Error Handling -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The ``genlayer.gl.advanced`` module is removed. Its functionality has been relocated: - -.. code-block:: python - - # v0.2.x - gl.advanced.user_error_immediate("reason") - gl.advanced.emit_raw_event(topics, blob) - - # v0.3.0 - gl.vm.UserError.immediate("reason") - gl.chain.Event.emit_raw(topics, blob) - -``UserError`` now carries an arbitrary calldata-encodable payload instead of a string. The payload is accessed via ``.data`` (previously ``.message``), and ``UserError.immediate`` accepts any ``calldata.Encodable``: - -.. code-block:: python - - raise gl.vm.UserError({'kind': 'InsufficientBalance', 'have': have, 'need': need}) - - try: - ... - except gl.vm.UserError as e: - payload = e.data - -The error-message handler hook (``__on_errored_message__``) has been removed. - -VM Tracing -~~~~~~~~~~ - -Trace functions have moved from ``genlayer.gl`` to ``genlayer.vm``: - -.. code-block:: python - - # v0.2.x - gl.trace("debug message") - gl.trace_time_micro() - - # v0.3.0 - gl.vm.trace("debug message") - gl.vm.trace_time_micro() - -Non-deterministic Execution (``gl.vm``) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. warning:: - ``gl.vm.run_nondet`` **changed meaning**. The two ``run_nondet`` functions were renamed: - - * the old ``run_nondet_unsafe`` (no validator sandbox) is now ``run_nondet``; - * the old ``run_nondet`` (safe, validator runs in a sandbox) is now ``run_nondet_default``. - - Code that called ``gl.vm.run_nondet`` still compiles but now silently uses the - *unsafe* variant. To keep the previous behavior, switch to ``gl.vm.run_nondet_default``: - - .. code-block:: python - - # v0.2.x (safe) # v0.3.0 (same behavior) - gl.vm.run_nondet(...) gl.vm.run_nondet_default(...) - - # v0.2.x (unsafe) # v0.3.0 (same behavior) - gl.vm.run_nondet_unsafe(...) gl.vm.run_nondet(...) - -The high-level equivalence principles (``gl.eq_principle.*``) are unaffected β€” they were updated internally. - -``gl.vm.spawn_sandbox`` replaced the single ``allow_write_ops`` flag with three granular permissions, each effective only if the current VM holds it: - -.. code-block:: python - - # v0.2.x - gl.vm.spawn_sandbox(fn, allow_write_ops=True) - - # v0.3.0 - gl.vm.spawn_sandbox( - fn, - allow_write_storage=True, - allow_send_messages=True, - allow_register_runners=True, - ) - -Two runtime helpers were added: ``gl.vm.register_runner(code)`` registers a runner archive and returns its ``custom:`` id, and ``gl.vm.map_file(runner, path_in_runner, path_in_vfs)`` maps a file from a runner into the VM filesystem. - -Storage -~~~~~~~ - -Storage types (``DynArray``, ``Array``, ``TreeMap``) and the ``allow`` decorator are accessible via ``gl.storage``: - -.. code-block:: python - - # v0.2.x - x: gl.DynArray[str] - m: gl.TreeMap[str, u32] - @gl.allow - class MyRecord: ... - - # v0.3.0 - x: gl.storage.DynArray[str] - m: gl.storage.TreeMap[str, u32] - @gl.storage.allow - class MyRecord: ... - -``allow_storage`` was renamed to ``allow`` (accessible as ``gl.storage.allow``). A new ``gl.storage.Pickled[T]`` helper is available for storing arbitrary picklable objects. - -Decorators -~~~~~~~~~~ - -``gl.public`` and ``gl.private`` are still available directly on ``gl``: - -.. code-block:: python - - # Both v0.2.x and v0.3.0 - @gl.public.write - def transfer(self, to: Address, amount: u256): ... - - @gl.public.view - def balance_of(self, owner: Address) -> u256: ... - -Non-deterministic Operations -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -``gl.nondet`` now raises a dedicated ``gl.nondet.NondetException`` (with ``causes`` and ``ctx``) on errors instead of a bare exception. Web request helpers (``gl.nondet.web.get``/``post``/``put``/...) accept a ``sign: bool`` keyword to sign outbound requests with the contract's identity. - -Environment Detection -~~~~~~~~~~~~~~~~~~~~~~ - -A new top-level ``gl.IS_IN_VM`` boolean indicates whether code is running inside the GenVM, which is useful for code that is shared between contracts and off-chain tooling. The raw WASI module is available as ``gl.wasi``. A ``gl.gvm32`` module provides Crockford Base32 ``encode``/``decode`` helpers (mirroring the Rust ``genlayer_sdk::gvm32`` implementation). - -Summary of Renames -~~~~~~~~~~~~~~~~~~~ - -.. list-table:: - :header-rows: 1 - :widths: 50 50 - - * - v0.2.x - - v0.3.0 - * - ``gl.Contract`` - - ``gl.contract.Contract`` - * - ``gl.contract_interface`` - - ``gl.contract.interface`` - * - ``gl.deploy_contract(...)`` - - ``gl.contract.deploy(...)`` - * - ``gl.get_contract_at(addr)`` - - ``gl.contract.get_at(addr)`` - * - ``gl.ContractProxy`` - - ``gl.contract.Proxy`` - * - ``gl.Event`` - - ``gl.chain.Event`` - * - ``gl.advanced.user_error_immediate(...)`` - - ``gl.vm.UserError.immediate(...)`` - * - ``gl.advanced.emit_raw_event(...)`` - - ``gl.chain.Event.emit_raw(...)`` - * - ``gl.trace(...)`` - - ``gl.vm.trace(...)`` - * - ``gl.trace_time_micro()`` - - ``gl.vm.trace_time_micro()`` - * - ``gl.vm.run_nondet_unsafe(...)`` - - ``gl.vm.run_nondet(...)`` - * - ``gl.vm.run_nondet(...)`` - - ``gl.vm.run_nondet_default(...)`` - * - ``gl.message_raw`` - - ``gl.message.raw`` - * - ``gl.storage.allow_storage`` - - ``gl.storage.allow`` - * - ``gl.allow`` - - ``gl.storage.allow`` - * - ``gl.DynArray`` - - ``gl.storage.DynArray`` - * - ``gl.Array`` - - ``gl.storage.Array`` - * - ``gl.TreeMap`` - - ``gl.storage.TreeMap`` - * - ``UserError(msg: str).message`` - - ``UserError(data: Encodable).data`` diff --git a/docs/website/src/python-sdk/guides/boot_process.rst b/docs/website/src/python-sdk/guides/boot_process.rst deleted file mode 100644 index 02bb264e..00000000 --- a/docs/website/src/python-sdk/guides/boot_process.rst +++ /dev/null @@ -1,73 +0,0 @@ -.. _startup-process-reference: - -Startup Process -=============== - -Standard library provides a built-in bootloader that bridges :ref:`internal ABI ` -to a more Pythonic interface. - -Bootloader is located in module ``_genlayer_bootloader`` and is executed on import. - -Bootloader Module ------------------ - -``_genlayer_bootloader`` handles contract execution in three distinct modes: - -Entry Points -~~~~~~~~~~~~ - -The bootloader dispatches execution based on ``entry_kind`` from the VM message: - -MAIN -^^^^ -Handles standard contract method calls and initialization. - -* Loads user ``contract`` module and resolves the target method of a declared ``Contract`` class -* Validates method access permissions (public/private, payable) -* Handles special methods: ``__receive__``, ``__handle_undefined_method__`` -* Routes initialization calls to ``__init__`` method -* Enforces security restrictions on dunder methods - -SANDBOX -^^^^^^^ -Executes pickled function objects. - -CONSENSUS_STAGE -^^^^^^^^^^^^^^^ -Runs consensus stage functions with additional stage data parameter (leader non-deterministic blocks outputs or ``None``). - -Method Resolution -~~~~~~~~~~~~~~~~~ - -For MAIN entry kind, the bootloader implements comprehensive method resolution: - -* **Initialization**: Routes to private ``__init__`` method during contract deployment -* **Named Methods**: Resolves public method calls by name with validation -* **Special Handling**: Supports ``__receive__`` for direct calls and ``__handle_undefined_method__`` for undefined method calls -* **Security**: Blocks calls to methods starting with ``__`` and unknown special methods starting with ``#`` - -Error Handling -~~~~~~~~~~~~~~ - -The bootloader provides centralized error management: - -* Catches :py:class:`genlayer.gl.vm.UserError` exceptions and triggers rollback with error message -* Validates contract structure and method accessibility -* Returns appropriate error messages for invalid method calls - -Profiling Support -~~~~~~~~~~~~~~~~~ - -Optional performance profiling when ``GENLAYER_ENABLE_PROFILER`` environment variable is set to ``true``: - -* Uses ``cProfile`` with microsecond timing precision -* Outputs compressed, base64-encoded statistics to stderr on exit -* Integrates with :term:`GenVM` :ref:`debugging infrastructure ` - -Storage Integration -~~~~~~~~~~~~~~~~~~~ - -Manages contract storage lifecycle: - -* Locks default :term:`Storage Slot`\s during contract initialization -* Provides contract instance from the root slot diff --git a/docs/website/src/python-sdk/guides/floating_point.rst b/docs/website/src/python-sdk/guides/floating_point.rst deleted file mode 100644 index 173fbc24..00000000 --- a/docs/website/src/python-sdk/guides/floating_point.rst +++ /dev/null @@ -1,14 +0,0 @@ -Floating Point in Python API -============================ - -GenVM handles floating point arithmetic differently depending on the execution mode. - -Non-Deterministic Mode ----------------------- - -In non-deterministic mode, floating point operations are unrestricted. Standard hardware-native IEEE 754 floating point is used directly, providing full performance with no additional overhead. - -Deterministic Mode ------------------- - -In deterministic mode, GenVM uses a software-implemented IEEE 754 floating point via integer arithmetic. This ensures that floating point computations produce bit-exact identical results across all validator nodes, regardless of the underlying hardware or platform. Native floating point instructions can yield slightly different results depending on the CPU architecture, compiler optimizations, or rounding behavior, which would break consensus. The software implementation eliminates this source of non-determinism at the cost of **significant** performance overhead. diff --git a/docs/website/src/python-sdk/guides/index.rst b/docs/website/src/python-sdk/guides/index.rst deleted file mode 100644 index f1b97eaa..00000000 --- a/docs/website/src/python-sdk/guides/index.rst +++ /dev/null @@ -1,7 +0,0 @@ -Guides -====== - -.. toctree:: - - boot_process - floating_point diff --git a/docs/website/src/python-sdk/index.rst b/docs/website/src/python-sdk/index.rst deleted file mode 100644 index 8aeb5405..00000000 --- a/docs/website/src/python-sdk/index.rst +++ /dev/null @@ -1,14 +0,0 @@ -Python SDK -========== - -The ``genlayer`` Python package is the intelligent contract SDK for writing -contracts that run inside GenVM. For tutorials and getting started guides, -see the `main documentation `_. - -.. toctree:: - - introduction - reference/index - runners/index - guides/index - changelog diff --git a/docs/website/src/python-sdk/introduction.rst b/docs/website/src/python-sdk/introduction.rst deleted file mode 100644 index fba5380d..00000000 --- a/docs/website/src/python-sdk/introduction.rst +++ /dev/null @@ -1,15 +0,0 @@ -Introduction -============ - -The ``genlayer`` package provides the Python SDK for writing intelligent -contracts that run inside GenVM. It includes storage primitives, contract -interfaces, non-deterministic operations, equivalence principles, and EVM -interoperability -- everything needed to build contracts in Python. - -Under the hood, Python code executes via CPython compiled to WebAssembly as -a "runner" inside GenVM. The SDK abstracts the VM interface so contract -authors can write idiomatic Python while GenVM handles deterministic execution -and consensus. - -For tutorials and getting started, see the -`main documentation `_. diff --git a/docs/website/src/python-sdk/reference/genlayer.rst b/docs/website/src/python-sdk/reference/genlayer.rst deleted file mode 100644 index bdf4b2eb..00000000 --- a/docs/website/src/python-sdk/reference/genlayer.rst +++ /dev/null @@ -1,127 +0,0 @@ -Package genlayer -================ - -######### -Top level -######### - -.. automodule:: genlayer - :members: - :imported-members: - :undoc-members: - :show-inheritance: - -Integer aliases -^^^^^^^^^^^^^^^ - -It also has aliases for signed and unsigned integer types (such as :py:obj:`~genlayer.types.u256`) and a :py:obj:`~genlayer.types.bigint` alias that can be used in storage unlike regular :py:class:`int` - -######## -contract -######## - -.. automodule:: genlayer.contract - :members: - :imported-members: - :undoc-members: - :show-inheritance: - -####### -message -####### - -.. automodule:: genlayer.message - :members: - :imported-members: - :undoc-members: - :show-inheritance: - -##### -chain -##### - -.. automodule:: genlayer.chain - :members: - :imported-members: - :undoc-members: - :show-inheritance: - -## -vm -## - -.. automodule:: genlayer.vm - :members: - :imported-members: - :undoc-members: - :show-inheritance: - -### -evm -### - -.. automodule:: genlayer.evm - :members: - :imported-members: - :undoc-members: - :show-inheritance: - -###### -nondet -###### - -.. automodule:: genlayer.nondet - :members: - :imported-members: - :undoc-members: - :show-inheritance: - -########## -nondet.web -########## - -.. automodule:: genlayer.nondet.web - :members: - :imported-members: - :undoc-members: - :show-inheritance: - -############ -eq_principle -############ - -.. automodule:: genlayer.eq_principle - :members: - :imported-members: - :undoc-members: - :show-inheritance: - -######## -calldata -######## - -.. automodule:: genlayer.calldata - :members: - :imported-members: - :undoc-members: - :show-inheritance: - -####### -storage -####### - -.. automodule:: genlayer.storage - :members: - :imported-members: - :undoc-members: - :show-inheritance: - -##### -types -##### - -.. automodule:: genlayer.types - :members: - :imported-members: - :undoc-members: - :show-inheritance: diff --git a/docs/website/src/python-sdk/reference/genlayer_embeddings.rst b/docs/website/src/python-sdk/reference/genlayer_embeddings.rst deleted file mode 100644 index 06811a73..00000000 --- a/docs/website/src/python-sdk/reference/genlayer_embeddings.rst +++ /dev/null @@ -1,8 +0,0 @@ -Package genlayer_embeddings -=========================== - -.. automodule:: genlayer_embeddings - :members: - :imported-members: - :undoc-members: - :show-inheritance: diff --git a/docs/website/src/python-sdk/reference/index.rst b/docs/website/src/python-sdk/reference/index.rst deleted file mode 100644 index 6666f445..00000000 --- a/docs/website/src/python-sdk/reference/index.rst +++ /dev/null @@ -1,7 +0,0 @@ -API Reference -============= - -.. toctree:: - - genlayer - genlayer_embeddings diff --git a/docs/website/src/python-sdk/runners/index.rst b/docs/website/src/python-sdk/runners/index.rst deleted file mode 100644 index 07c9b492..00000000 --- a/docs/website/src/python-sdk/runners/index.rst +++ /dev/null @@ -1,38 +0,0 @@ -Runners -======= - -Runners are language runtimes compiled to WebAssembly that execute inside GenVM. -For the Python SDK, this includes CPython and supporting libraries (cloudpickle, -protobuf, embeddings, etc.) compiled to WASM. - -Each runner is identified by a content hash. When GenVM executes a contract, it -loads the specific runner versions declared in the contract's dependencies. - -Using Runners -------------- - -In your contract's first line, specify the runner dependency: - -.. code-block:: python - - # { "Depends": "py-genlayer:1jb45aa8ynh2a9c9xn3b7qqh8sm5q93hwfp7jqmwsfhh8jpz09h6" } - -For multi-file contracts, use ``py-genlayer-multi`` instead. - -For contracts using embeddings or semantic search, add ``py-lib-genlayer-embeddings`` -before the main runner using a Seq block: - -.. code-block:: python - - # { - # "Seq": [ - # { "Depends": "py-lib-genlayer-embeddings:" }, - # { "Depends": "py-genlayer:" } - # ] - # } - -For the full runner specification, see :doc:`/spec/02-execution-environment/04-runners`. - -For version history and what changed, see the :doc:`Changelog `. - -For list of available runners and their hashes, see the :doc:`Available Runners ` in the appendix. diff --git a/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/02-gl_call.rst b/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/02-gl_call.rst index 3ee2d854..8d7833f5 100644 --- a/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/02-gl_call.rst +++ b/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/02-gl_call.rst @@ -337,7 +337,7 @@ The outcomes, in check order, are: ------------------- Maps a file from a runner into the VM filesystem at runtime, behaving the same as -the ``MapFile`` runner action (see :doc:`../../../python-sdk/index` runners). If +the ``MapFile`` runner action (see the Python SDK runners documentation). If ``path_in_runner`` ends with ``/`` the whole directory subtree is mapped. Payload diff --git a/docs/website/yabuild.rb b/docs/website/yabuild.rb deleted file mode 100644 index 396407b6..00000000 --- a/docs/website/yabuild.rb +++ /dev/null @@ -1,11 +0,0 @@ -POETRY_RUN = ['poetry', 'run', '-C', cur_src] - -docs_out = root_build.join('out', 'docs') -docs_out.mkpath - -docs_out_txt = cur_build.join('txt') -docs_out_txt.mkpath - -LIB_SRC = root_src.join('runners', 'genlayer-py-std', 'src') - -codegen_src = cur_src.parent.parent.join('executor', 'codegen') diff --git a/executors/v0.2.x b/executors/v0.2.x index e749d2aa..b32cc4c9 160000 --- a/executors/v0.2.x +++ b/executors/v0.2.x @@ -1 +1 @@ -Subproject commit e749d2aa8201ab1a291d803d7de05674a2c70772 +Subproject commit b32cc4c96876fde1ec323f7cb40b9d31173bb2e5 diff --git a/executors/v0.3.x b/executors/v0.3.x index a7d97df0..be5cfe91 160000 --- a/executors/v0.3.x +++ b/executors/v0.3.x @@ -1 +1 @@ -Subproject commit a7d97df0e27927a4a2b12673718cf9d45872b522 +Subproject commit be5cfe91304a40e421e13197892c020ca1f2dd40 diff --git a/flake.lock b/flake.lock index e0532b21..0156faf8 100644 --- a/flake.lock +++ b/flake.lock @@ -1,5 +1,21 @@ { "nodes": { + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1767039857, + "narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=", + "owner": "NixOS", + "repo": "flake-compat", + "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "flake-compat", + "type": "github" + } + }, "flake-utils": { "inputs": { "systems": [ @@ -20,6 +36,27 @@ "type": "github" } }, + "git-hooks": { + "inputs": { + "flake-compat": "flake-compat", + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1783008725, + "narHash": "sha256-jGiy6+sxjNWXSjp25uoJuNfyH9zBK1PEDY0lVoL4ibQ=", + "owner": "cachix", + "repo": "git-hooks.nix", + "rev": "bca82caa46d5ec0f5d422c61fb1e30bc51313cbe", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "git-hooks.nix", + "type": "github" + } + }, "nixpkgs": { "locked": { "lastModified": 1778443072, @@ -39,6 +76,7 @@ "root": { "inputs": { "flake-utils": "flake-utils", + "git-hooks": "git-hooks", "nixpkgs": "nixpkgs", "systems": "systems" } diff --git a/flake.nix b/flake.nix index 7cef9949..97484525 100644 --- a/flake.nix +++ b/flake.nix @@ -8,6 +8,10 @@ url = "github:numtide/flake-utils"; inputs.systems.follows = "systems"; }; + git-hooks = { + url = "github:cachix/git-hooks.nix"; + inputs.nixpkgs.follows = "nixpkgs"; + }; }; outputs = @@ -16,6 +20,7 @@ nixpkgs, flake-utils, systems, + git-hooks, }: let for-systems = flake-utils.lib.eachDefaultSystem ( @@ -126,8 +131,22 @@ pkgs.python312Packages.aiohttp wabt ]; + # Docs toolchain from nix instead of a poetry venv: sphinx + the + # extensions conf.py loads. mermaid-cli is the `mmdc` the + # sphinxcontrib.mermaid svg output shells out to. The Python SDK + # autodoc (and its numpy dep) and the textβ†’txt merge (formerly ruby, + # now docs/website/merge_txts.py) moved to the executor sub-sites. + docs-python = pkgs.python312.withPackages ( + ps: with ps; [ + sphinx + myst-parser + pydata-sphinx-theme + sphinxcontrib-mermaid + sphinxcontrib-openapi + ] + ); packages-gen-docs = with pkgs; [ - lua-language-server + docs-python mermaid-cli ]; packages-py-test = with pkgs; [ @@ -149,6 +168,145 @@ export CARGO_TARGET_DIR="$(pwd)/build/ya-build/rust-target" ''; + # ---- Commit hooks (git-hooks.nix) ------------------------ + # Formatters/linters + local guards for the MANAGER tree only. + # Executor submodules ship their own top-level flake + hooks + # (each pinned to its own toolchain), so `^executors/` is dropped + # here and every executor's own `nix flake check` covers its + # files. Wired into CI by support/ci/pipelines/commit-hooks.sh and + # installed into .git/hooks by the `full` dev shell's shellHook. + hooks-python = pkgs.python312; + # cargo fmt per crate: each Cargo.toml picks up its own edition, so + # the manager's independent workspaces all format correctly. + cargo-fmt-all = pkgs.writeShellScript "cargo-fmt-all" '' + set -euo pipefail + export PATH="${pkgs.cargo}/bin:${pkgs.rustfmt}/bin:$PATH" + rc=0 + # git ls-files never descends into the executor submodule gitlinks, + # but exclude executors/ explicitly too: their Rust is formatted by + # each executor's own flake, against its own pinned toolchain. + while IFS= read -r toml; do + ( cd "$(dirname -- "$toml")" && cargo fmt ) || rc=1 + done < <(git ls-files '*Cargo.toml' ':(exclude)executors/') + exit $rc + ''; + pre-commit-check = git-hooks.lib.${system}.run { + src = ./.; + excludes = [ + "^executors/" + "^build/" + "^\\.direnv/" + ]; + hooks = { + # --- generic checks --------------------------------------- + trim-trailing-whitespace = { + enable = true; + excludes = [ + "^\\.git-third-party" + "/fuzz/" + ]; + }; + end-of-file-fixer = { + enable = true; + excludes = [ + "^\\.git-third-party" + "/fuzz/" + ]; + }; + check-added-large-files.enable = true; + check-json = { + enable = true; + excludes = [ + "^\\.git-third-party" + "/tsconfig\\.json$" + ]; + }; + check-yaml.enable = true; + check-toml.enable = true; + check-merge-conflicts.enable = true; + taplo.enable = true; + editorconfig-checker = { + enable = true; + # text-only (the built-in defaults to every file): keep binary + # blobs out of it, mirroring the old engine's `text` pseudo-type. + types = [ "text" ]; + excludes = [ + "git-third-party" + "/fuzz/" + ]; + }; + # --- manager-owned languages: python, lua, ts, nix -------- + # git-third-party is a vendored helper tool (late imports by + # design); the old engine skipped it (keyed on the `.py` ext), + # so keep ruff off it. + ruff = { + enable = true; + excludes = [ "^support/tools/git-third-party/" ]; + }; + ruff-format = { + enable = true; + excludes = [ "^support/tools/git-third-party/" ]; + }; + stylua = { + enable = true; + files = "\\.lua$"; + }; + prettier = { + enable = true; + files = "\\.(ts|tsx)$"; + excludes = [ "^\\.git-third-party" ]; + }; + nixfmt = { + enable = true; + files = "\\.nix$"; + }; + # --- rust (per-crate, edition-aware) ---------------------- + cargo-fmt = { + enable = true; + name = "cargo-fmt"; + entry = toString cargo-fmt-all; + files = "\\.rs$"; + pass_filenames = false; + }; + # --- github workflow / action schemas --------------------- + check-github-workflows = { + enable = true; + name = "check-github-workflows"; + entry = "${pkgs.check-jsonschema}/bin/check-jsonschema --builtin-schema vendor.github-workflows"; + files = "^\\.github/workflows/.*\\.ya?ml$"; + }; + check-github-actions = { + enable = true; + name = "check-github-actions"; + entry = "${pkgs.check-jsonschema}/bin/check-jsonschema --builtin-schema vendor.github-actions"; + files = "^\\.github/(actions/.+/)?action\\.ya?ml$"; + }; + # --- local guards ----------------------------------------- + # Keep the manager crate's [package] version in lockstep with + # .genvm-monorepo-root (executor lines version independently). + check-cargo-versions = { + enable = true; + name = "check-cargo-versions"; + entry = "${hooks-python}/bin/python3 -B ./support/ci tool check-versions sync"; + files = "^(implementation/Cargo\\.toml|\\.genvm-monorepo-root)$"; + pass_filenames = false; + }; + markdown-local-links = { + enable = true; + name = "markdown-local-links"; + entry = "${hooks-python}/bin/python3 support/scripts/md-local-links.py"; + types = [ "markdown" ]; + }; + # --- commit-msg ------------------------------------------- + check-commit-message = { + enable = true; + name = "check-commit-message"; + entry = "${hooks-python}/bin/python3 support/scripts/check-commit-message.py --message-file"; + stages = [ "commit-msg" ]; + }; + }; + }; + # ---- Active executor lines ------------------------------- # `.genvm-monorepo-root` carries two independent versions: # * `active-versions` β€” the executor lines built here; each @@ -218,7 +376,7 @@ exec-prefix = "executors/${key}.x"; exec-src = self + "/${exec-prefix}"; manifest = builtins.fromJSON (builtins.readFile (exec-src + "/manifest.json")); - clamped = clamp-version manifest.executor-version; + clamped = builtins.replaceStrings [ "." ] [ "_" ] (clamp-version manifest.executor-version); release-args = make-release-args exec-src exec-prefix; # executors/.x/default.nix reads its own # manifest.json for build-config (executor-version). @@ -237,15 +395,28 @@ # The manager is a single binary tagged with its own `version` # (independent of the executor lines it ships against). - manager-packages = import ./implementation ( - manager-release-args - // { - inherit compiled-libs genvm-tool; - build-config = { - executor-version = manager-version; - }; - } - ); + manager-packages = + pkgs.lib.mapAttrs + ( + _: package: + package.overrideAttrs (_: { + # These are portable release trees, not Nix-only executables. + # Keep install/bin/genvm-manager's `#!/bin/sh` instead of + # rewriting it to a host Nix store path during fixup. + dontPatchShebangs = true; + }) + ) + ( + import ./implementation ( + manager-release-args + // { + inherit compiled-libs genvm-tool; + build-config = { + executor-version = manager-version; + }; + } + ) + ); # ---- Runners --------------------------------------------- # The executor lines only export their own current runner @@ -272,24 +443,37 @@ universal-of = import ./runners/views/universal.nix runners-args; + # `cp -rsf src/. $out/` lays out a shadow tree: real directories and + # store-backed symlinks for regular files. Existing relative file + # symlinks are then restored verbatim so links such as + # `bin/post-install -> genvm-post-install` remain portable. It merges + # the top-level trees WITHOUT copying, and `-f` keeps the old + # last-one-wins overwrite semantics. Real files are materialised only + # later by the artifact packer. + symlink-merge-srcs = '' + for src in $srcs; do + cp -rsf --no-preserve=mode,ownership "$src"/. $out/ + while IFS= read -r -d "" link; do + if [ -d "$link" ]; then + echo "refusing to create directory symlink: $link" >&2 + exit 1 + fi + target="$(readlink "$link")" + if [[ "$target" != /* ]]; then + rel="''${link#"$src"/}" + ln -sfn "$target" "$out/$rel" + fi + done < <(find "$src" -type l -print0) + done + ''; + # Merge a { uid -> `//.tar` tree } set into one tree. merge-runner-trees = name: uni: - pkgs.stdenvNoCC.mkDerivation { - inherit name; - srcs = builtins.attrValues uni; - dontUnpack = true; - dontConfigure = true; - dontBuild = true; - dontFixup = true; - installPhase = '' - mkdir -p $out - for src in $srcs; do - cp --no-preserve=ownership -r $src/. $out/. - chmod -R u+w $out - done - ''; - }; + pkgs.runCommand name { srcs = builtins.attrValues uni; } '' + mkdir -p $out + ${symlink-merge-srcs} + ''; runners-all = merge-runner-trees "genvm-runners-all" (universal-of shared-runners-list); @@ -318,7 +502,7 @@ in '' mkdir -p "$out/$(dirname -- "${dst}")" - cp "${r.derivation}" "$out/${dst}" + ln -s "${r.derivation}" "$out/${dst}" '' ) legacy-runners-list; }; @@ -351,18 +535,13 @@ dontBuild = true; dontFixup = true; installPhase = '' - mkdir -p $out - for src in $srcs; do - cp --no-preserve=ownership -r $src/. $out/. - chmod -R u+w $out - done - mkdir -p $out/runners - cp --no-preserve=ownership -r $runnersAll/. $out/runners/. - chmod -R u+w $out/runners + mkdir -p $out $out/runners + ${symlink-merge-srcs} + # runners-all nested under runners/ (matching the CI layout). + cp -rsf "$runnersAll"/. $out/runners/ # Legacy lines carry their runners under their own executor root # (executor//legacy-runners); overlay that tree here. - cp --no-preserve=ownership -r $legacyRunnersAll/. $out/. - chmod -R u+w $out + cp -rsf "$legacyRunnersAll"/. $out/ ''; }; genvm-packages = builtins.listToAttrs ( @@ -373,55 +552,85 @@ # ---- Release distribution bundles ------------------------- # `executor[-]` (no version) merges every active line's - # executor tree (executor//...) β€” the release asset - # genvm---executor.tar.xz. Runners are NOT included; - # they ship platform-independently via runners-all-dist. + # executor tree (executor//...) into one bundle. The release + # publishes each line on its own (`executor-[-]`, + # one release per line in the executor repo), so this merged bundle has + # no CI consumer; it stays as the convenient "give me every line" + # target for local use. Runners are NOT included; they ship + # platform-independently via artifact-prepack-genvm-universal. combine-executors = suffix: - pkgs.stdenvNoCC.mkDerivation { - name = "genvm-executor${suffix}"; - srcs = builtins.map (line: executor-packages."executor-${line.clamped}${suffix}") executor-lines; - dontUnpack = true; - dontConfigure = true; - dontBuild = true; - dontFixup = true; - installPhase = '' + pkgs.runCommand "genvm-executor${suffix}" + { + srcs = builtins.map (line: executor-packages."executor-${line.clamped}${suffix}") executor-lines; + } + '' mkdir -p $out - for src in $srcs; do - cp --no-preserve=ownership -r $src/. $out/. - chmod -R u+w $out - done + ${symlink-merge-srcs} ''; - }; executor-dist-packages = builtins.listToAttrs ( builtins.map ( suffix: pkgs.lib.nameValuePair "executor${suffix}" (combine-executors suffix) ) platform-suffixes ); - # The release asset genvm-runners-all.tar.xz: consumers extract it at - # the install root, so the shared runners sit under runners/ and the - # legacy lines' runners at their executor//legacy-runners - # destination (the same overlay combine-genvm applies). - runners-all-dist = pkgs.stdenvNoCC.mkDerivation { - name = "genvm-runners-all-dist"; - runnersAll = runners-all; - legacyRunnersAll = legacy-runners-all; - dontUnpack = true; - dontConfigure = true; - dontBuild = true; - dontFixup = true; - installPhase = '' - mkdir -p $out/runners - cp --no-preserve=ownership -r $runnersAll/. $out/runners/. - chmod -R u+w $out/runners - cp --no-preserve=ownership -r $legacyRunnersAll/. $out/. - chmod -R u+w $out - ''; + # Trees consumed by the artifact packer. Platform artifacts contain + # the manager and every active executor line for that platform. The + # universal artifact contains the platform-independent runners at + # their final `runners/` destination, plus the legacy lines' runners at + # theirs. Keep every leaf as a symlink to its component derivation so + # prepacking does not duplicate outputs. + artifact-prepack-platforms = [ + "amd64-linux" + "arm64-linux" + "arm64-macos" + ]; + artifact-prepack-platform-packages = builtins.listToAttrs ( + builtins.map ( + platform: + let + suffix = "-${platform}"; + in + pkgs.lib.nameValuePair "artifact-prepack-genvm-${platform}" ( + pkgs.runCommand "artifact-prepack-genvm-${platform}" + { + srcs = + builtins.map (line: executor-packages."executor-${line.clamped}${suffix}") executor-lines + ++ [ manager-packages."manager${suffix}" ]; + } + '' + mkdir -p $out + ${symlink-merge-srcs} + '' + ) + ) artifact-prepack-platforms + ); + artifact-prepack-genvm-universal = + pkgs.runCommand "artifact-prepack-genvm-universal" + { + runnersAll = runners-all; + legacyRunnersAll = legacy-runners-all; + } + '' + mkdir -p $out/runners + cp -rsf "$runnersAll"/. $out/runners/ + # Legacy lines carry their runners under their own executor root + # (executor//legacy-runners), already laid out at that + # destination β€” overlay the tree as-is. + cp -rsf "$legacyRunnersAll"/. $out/ + ''; + artifact-prepack-packages = artifact-prepack-platform-packages // { + inherit artifact-prepack-genvm-universal; }; in { runners = runners-list; + checks.pre-commit-check = pre-commit-check; + # `nix fmt` runs the same generated hook config as `nix flake check`, + # in the working tree (manager files only β€” executors have their own). + formatter = pkgs.writeShellScriptBin "pre-commit-run" '' + exec ${pre-commit-check.config.package}/bin/pre-commit run --all-files --config ${pre-commit-check.config.configFile} + ''; packages = # Combined distribution: all executors + manager + runners. genvm-packages @@ -432,13 +641,19 @@ # Manager: manager[-]. // manager-packages // { - inherit runners-all legacy-runners-all runners-all-dist; + inherit runners-all legacy-runners-all; } + # Inputs for producing the three platform artifacts and the + # platform-independent universal artifact. + // artifact-prepack-packages # Utility packages (grouped separately). // { inherit genvm-tool; }; + devShells.minimal = pkgs.mkShell { + packages = packages-0 ++ packages-py-test ++ [ pkgs.ruby ]; + }; devShells.py-test = pkgs.mkShell { packages = packages-py-test ++ [ pkgs.ruby ]; shellHook = shell-hook-base + '' @@ -446,7 +661,9 @@ ''; }; devShells.gen-docs = pkgs.mkShell { - packages = packages-py-test ++ packages-gen-docs ++ [ pkgs.ruby ]; + # Self-contained docs toolchain (sphinx + extensions via nix, no + # poetry). generate.py also runs `nix eval`, so keep the base tools. + packages = packages-0 ++ packages-gen-docs; shellHook = shell-hook-base; }; devShells.initial-check = pkgs.mkShell { @@ -470,8 +687,14 @@ ++ packages-py-test ++ packages-rust ++ packages-gen-docs - ++ [ pkgs.nodejs ]; - shellHook = shell-hook-base; + ++ [ pkgs.nodejs ] + # pre-commit + the hook tools, so `pre-commit run` works in-shell. + # (No rustfmt/cargo here β€” cargo-fmt is a custom hook β€” so the + # custom-rust toolchain is not shadowed.) + ++ pre-commit-check.enabledPackages; + # The interactive dev shell (.envrc β†’ `.#full`) also installs the + # git-hooks pre-commit/commit-msg stubs into .git/hooks. + shellHook = shell-hook-base + pre-commit-check.shellHook; }; devShells.check-qemu = pkgs.mkShell { packages = packages-0 ++ [ pkgs.qemu ]; diff --git a/implementation/Cargo.lock b/implementation/Cargo.lock index e95b91ae..89914f17 100644 --- a/implementation/Cargo.lock +++ b/implementation/Cargo.lock @@ -1021,6 +1021,7 @@ dependencies = [ "dashmap", "genlayer_calldata", "genlayer_sdk", + "genvm_modules_interfaces", "hex", "libc", "num-bigint", @@ -1055,7 +1056,7 @@ dependencies = [ "futures-util", "genlayer_calldata", "genvm-common", - "genvm-modules-interfaces", + "genvm_modules_interfaces", "hex", "hickory-resolver", "hyper 0.14.32", @@ -1090,7 +1091,7 @@ dependencies = [ ] [[package]] -name = "genvm-modules-interfaces" +name = "genvm_modules_interfaces" version = "0.1.0" dependencies = [ "anyhow", diff --git a/implementation/Cargo.toml b/implementation/Cargo.toml index 4d88ece5..deeb9636 100644 --- a/implementation/Cargo.toml +++ b/implementation/Cargo.toml @@ -18,7 +18,9 @@ vendored-lua = ["mlua/vendored"] [dependencies] genlayer_calldata = { path = "../executors/v0.3.x/executor/crates/calldata" } genvm-common = { path = "../executors/v0.3.x/executor/crates/common" } -genvm-modules-interfaces = { path = "../crates/modules-interfaces" } +genvm_modules_interfaces = { path = "../executors/v0.3.x/executor/crates/modules-interfaces" } +# linked through common +# genvm_modules_interfaces = { path = "../crates/modules-interfaces" } anyhow = { version = "1.0.98", features = [] } async-trait = "0.1.88" diff --git a/implementation/tests/web_render_webdriver.rs b/implementation/tests/web_render_webdriver.rs new file mode 100644 index 00000000..0b267b85 --- /dev/null +++ b/implementation/tests/web_render_webdriver.rs @@ -0,0 +1,196 @@ +//! Regression tests for the web module's SSRF-filter exemption on the webdriver +//! connection. `Render` talks to the operator-configured `webdriver_host`, which +//! in Docker resolves to a private (RFC1918 / loopback) address the SSRF filter +//! would otherwise drop. That connection must therefore go through the unfiltered +//! client β€” while contract-driven `Request` traffic stays filtered. + +use std::collections::BTreeMap; +use std::sync::Arc; + +use anyhow::Context as _; +use genvm_common::*; + +use genvm_modules::common; +use genvm_modules::scripting::{self, Metrics}; +use tokio::io::AsyncWriteExt; + +struct TestCtx { + scripting: scripting::CtxPart, +} + +// Minimal stand-in for the webdriver: replies `200` with the `resulting-status` +// header `Render` reads, then closes. Bound to loopback but reached over public +// DNS (`localhost.direct`) so the SSRF filter's resolver sees a loopback answer. +async fn serve_webdriver(body: &'static [u8]) -> std::net::SocketAddr { + let server = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = server.local_addr().unwrap(); + + tokio::spawn(async move { + let (mut client, _) = server.accept().await.unwrap(); + let response = format!( + "HTTP/1.1 200 OK\r\nresulting-status: 200\r\nContent-Type: text/plain\r\nContent-Length: {}\r\n\r\n", + body.len() + ); + client.write_all(response.as_bytes()).await.unwrap(); + client.write_all(body).await.unwrap(); + client.shutdown().await.unwrap(); + }); + + addr +} + +// A web-module VM running the real `genvm-web-default.lua`. `__web` must exist +// before the script's `require("lib-web")` reads it, so it is set inside the +// data getter ahead of `load_script`. +async fn create_web_vm(webdriver_host: String) -> scripting::UserVM<(), (), TestCtx> { + let mut lua_dir = std::env::current_dir().unwrap(); + lua_dir.pop(); + lua_dir.push("install"); + lua_dir.push("lib"); + lua_dir.push("genvm-lua"); + let lua_dir = lua_dir + .canonicalize() + .with_context(|| format!("canonicalizing {:?}", lua_dir)) + .unwrap(); + let mut lua_path = lua_dir.to_str().unwrap().to_owned(); + lua_path.push_str("/?.lua"); + + let mut script_path = std::env::current_dir().unwrap(); + script_path.pop(); + script_path.push("install"); + script_path.push("config"); + script_path.push("genvm-web-default.lua"); + let script_path = script_path + .canonicalize() + .with_context(|| format!("canonicalizing {:?}", script_path)) + .unwrap(); + + let conf = sync::DArc::new(common::ModuleBaseConfig { + bind_address: None, + vm_count: 1, + lua_script_path: "".to_owned(), + lua_path, + signer_headers: Arc::new(BTreeMap::new()), + signer_url: Arc::from(""), + data_dir: String::new(), + }); + + scripting::UserVM::create( + &conf.clone(), + move |vm: mlua::Lua| async move { + // Stand in for `web::ctx::create_global`: only the fields the script + // touches (`config.webdriver_host`, `config.always_allow_hosts`, + // `allowed_tld`) are populated. + let web = vm.create_table()?; + let config = vm.create_table()?; + config.set("webdriver_host", webdriver_host)?; + config.set("always_allow_hosts", vm.create_table()?)?; + web.set("config", config)?; + + let tld = vm.create_table()?; + tld.set("com", true)?; + tld.set("direct", true)?; + web.set("allowed_tld", tld)?; + + vm.globals().set("__web", web)?; + + scripting::load_script(&vm, script_path.to_str().unwrap().to_owned()).await?; + + Ok(()) + }, + Box::new(move |vm, table, ctx: &sync::DArc| { + let scripting = ctx.gep(|x| &x.scripting); + scripting::setup_lua_default_ctx(scripting, vm, table)?; + Ok(()) + }), + ) + .await + .unwrap() +} + +// A context with `filter_dns = true`, i.e. the production Docker configuration: +// `client` is the SSRF-filtering client, `client_unfiltered` the plain one. +fn create_filtered_ctx() -> sync::DArc { + let hello = common::tests::get_hello(); + let metrics = sync::DArc::new(scripting::Metrics::default()); + let conf = sync::DArc::new(common::ModuleBaseConfig { + bind_address: None, + vm_count: 1, + lua_script_path: "".to_owned(), + lua_path: "".to_owned(), + signer_headers: Arc::new(BTreeMap::new()), + signer_url: Arc::from(""), + data_dir: String::new(), + }); + let scripting = scripting::create_ctx_part(&hello, &conf, metrics, true).unwrap(); + sync::DArc::new(TestCtx { scripting }) +} + +#[tokio::test] +async fn test_render_webdriver_bypasses_dns_filter() { + common::tests::setup(); + + let body = b"rendered by webdriver"; + let addr = serve_webdriver(body).await; + + // `localhost.direct` resolves publicly to loopback, where our stub is bound. + // With `filter_dns = true` the SSRF resolver would drop it β€” so this only + // succeeds because `Render` sends the webdriver request `unfiltered`. + let webdriver_host = format!("http://localhost.direct:{}", addr.port()); + let uvm = create_web_vm(webdriver_host).await; + + let render: mlua::Function = uvm.vm.globals().get("Render").unwrap(); + + let test_ctx = create_filtered_ctx(); + let (_, ctx_lua) = uvm.create_ctx(&test_ctx).unwrap(); + + let payload = uvm.vm.create_table().unwrap(); + payload.set("url", "http://example.com/").unwrap(); + payload.set("mode", "text").unwrap(); + payload.set("wait_after_loaded", 0).unwrap(); + + let res: mlua::Table = uvm + .call_fn(&render, (ctx_lua, payload)) + .await + .expect("Render must reach the loopback webdriver through the unfiltered client"); + + let text: mlua::String = res.get("text").unwrap(); + assert_eq!(text.as_bytes().as_ref(), body); +} + +#[tokio::test] +async fn test_web_request_still_filtered() { + common::tests::setup(); + + // Complementary check: general (contract-driven) `Request` traffic is NOT + // exempt. The same loopback-resolving host, not allowlisted, must be dropped + // by the SSRF resolver as a non-fatal `ADDRESS_FORBIDDEN`. This also proves + // the ctx above genuinely has filtering on, so the Render success is due to + // the exemption, not a disabled filter. + let uvm = create_web_vm("http://unused.example.com".to_owned()).await; + + let request: mlua::Function = uvm.vm.globals().get("Request").unwrap(); + + let test_ctx = create_filtered_ctx(); + let (_, ctx_lua) = uvm.create_ctx(&test_ctx).unwrap(); + + let payload = uvm.vm.create_table().unwrap(); + payload.set("url", "http://localhost.direct/").unwrap(); + payload.set("method", "GET").unwrap(); + payload + .set("headers", uvm.vm.create_table().unwrap()) + .unwrap(); + + let err = uvm + .call_fn::(&request, (ctx_lua, payload)) + .await + .expect_err("a non-allowlisted loopback host must stay filtered"); + + let module_err = scripting::try_unwrap_any_err(err).expect("expected a module error"); + assert!( + module_err.causes.contains(&"ADDRESS_FORBIDDEN".to_owned()), + "unexpected causes: {:?}", + module_err.causes + ); + assert!(!module_err.fatal, "ADDRESS_FORBIDDEN must be non-fatal"); +} diff --git a/install/bin/genvm-post-install b/install/bin/genvm-post-install new file mode 100755 index 00000000..3c4de8e7 --- /dev/null +++ b/install/bin/genvm-post-install @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 + +# NOTE: This script is a wrapper for the actual post-install script. + +import argparse +import logging +import subprocess +import sys +from pathlib import Path + +logging.basicConfig(level=logging.INFO) + +logger = logging.getLogger(__name__) + + +def str_to_bool(value): + if value.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif value.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise argparse.ArgumentTypeError('Boolean value expected.') + + +parser = argparse.ArgumentParser(add_help=False) +parser.add_argument( + '--create-venv', + type=str_to_bool, + default=True, + help='Whether to create a virtual environment (true/false)', +) +parser.add_argument( + '--use-patchelf', + type=str_to_bool, + default=False, + help='Set the ELF interpreter with patchelf instead of lief (true/false)', +) + +args, rest_args = parser.parse_known_args() + +has_help_arg = '-h' in rest_args or '--help' in rest_args + +if has_help_arg: + parser.print_help() + +genvm_root_dir = Path(__file__).parent.parent + + +def requirements_for(base: Path) -> Path: + """The requirements to install into the venv. + + lief is a heavy binary wheel whose only job is rewriting the ELF interpreter, + so `--use-patchelf` drops it β€” patchelf does that instead. The venv is keyed + by the hash of its requirements, so the two variants get separate venvs and + neither invalidates the other. + """ + if not args.use_patchelf: + return base + + kept = [line for line in base.read_text().splitlines() if 'lief' not in line] + filtered = genvm_root_dir.joinpath('data', 'venvs', 'requirements-patchelf.txt') + filtered.parent.mkdir(parents=True, exist_ok=True) + filtered.write_text('\n'.join(kept) + '\n') + return filtered + + +if not has_help_arg and args.create_venv: + venvs_path = genvm_root_dir.joinpath('data', 'venvs') + logger.info(f'venvs path: {venvs_path}') + + requirements_path = requirements_for( + genvm_root_dir.joinpath('lib', 'python', 'post-install', 'requirements.txt') + ) + + create_venv_path = Path(genvm_root_dir).joinpath( + 'lib', 'python', 'post-install', 'create_venv.py' + ) + + create_venv_src = create_venv_path.read_text() + create_venv_src_globals = {} + create_venv_code = compile(create_venv_src, str(create_venv_path), 'exec') + exec(create_venv_code, create_venv_src_globals) + create_venv = create_venv_src_globals['create_venv'] + venv_path: Path = create_venv(genvm_root_dir, requirements_path) + actual_python = venv_path.joinpath('bin', 'python') +else: + actual_python = Path(sys.executable) + + +subprocess.run( + [ + actual_python, + '-B', + genvm_root_dir.joinpath('lib', 'python', 'post-install', '__main__.py'), + # argparse consumed this above, so it is no longer in rest_args, but the + # real script needs it too. + f'--use-patchelf={args.use_patchelf}', + ] + + rest_args, + check=True, + text=True, +) diff --git a/install/bin/post-install.py b/install/bin/post-install.py deleted file mode 100755 index c652b3e1..00000000 --- a/install/bin/post-install.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env python3 - -# NOTE: This script is a wrapper for the actual post-install script. - -import argparse -import logging -import subprocess -import sys -from pathlib import Path - -logging.basicConfig(level=logging.INFO) - -logger = logging.getLogger(__name__) - - -def str_to_bool(value): - if value.lower() in ('yes', 'true', 't', 'y', '1'): - return True - elif value.lower() in ('no', 'false', 'f', 'n', '0'): - return False - else: - raise argparse.ArgumentTypeError('Boolean value expected.') - - -parser = argparse.ArgumentParser(add_help=False) -parser.add_argument( - '--create-venv', - type=str_to_bool, - default=True, - help='Whether to create a virtual environment (true/false)', -) - -args, rest_args = parser.parse_known_args() - -has_help_arg = '-h' in rest_args or '--help' in rest_args - -if has_help_arg: - parser.print_help() - -genvm_root_dir = Path(__file__).parent.parent - -if not has_help_arg and args.create_venv: - venvs_path = genvm_root_dir.joinpath('data', 'venvs') - logger.info(f'venvs path: {venvs_path}') - - requirements_path = genvm_root_dir.joinpath( - 'lib', 'python', 'post-install', 'requirements.txt' - ) - - create_venv_path = Path(genvm_root_dir).joinpath( - 'lib', 'python', 'post-install', 'create_venv.py' - ) - - create_venv_src = create_venv_path.read_text() - create_venv_src_globals = {} - create_venv_code = compile(create_venv_src, str(create_venv_path), 'exec') - exec(create_venv_code, create_venv_src_globals) - create_venv = create_venv_src_globals['create_venv'] - venv_path: Path = create_venv(genvm_root_dir, requirements_path) - actual_python = venv_path.joinpath('bin', 'python') -else: - actual_python = Path(sys.executable) - - -subprocess.run( - [ - actual_python, - '-B', - genvm_root_dir.joinpath('lib', 'python', 'post-install', '__main__.py'), - ] - + rest_args, - check=True, - text=True, -) diff --git a/install/bin/post-install.py b/install/bin/post-install.py new file mode 120000 index 00000000..1d8a8f29 --- /dev/null +++ b/install/bin/post-install.py @@ -0,0 +1 @@ +genvm-post-install \ No newline at end of file diff --git a/install/config/genvm-web-default.lua b/install/config/genvm-web-default.lua index 3f8d3f37..47055ff0 100644 --- a/install/config/genvm-web-default.lua +++ b/install/config/genvm-web-default.lua @@ -22,6 +22,7 @@ function Render(ctx, payload) headers = {}, error_on_status = true, response_body_max_size = payload.size_limit, + unfiltered = true, }) lib.log { diff --git a/install/lib/python/post-install/__main__.py b/install/lib/python/post-install/__main__.py index 996c13cc..3d492862 100644 --- a/install/lib/python/post-install/__main__.py +++ b/install/lib/python/post-install/__main__.py @@ -1,12 +1,16 @@ import argparse import hashlib +import io import json import logging import os import platform import shlex +import shutil import subprocess +import tarfile import traceback +import typing from pathlib import Path target_os = platform.system().lower() @@ -59,6 +63,12 @@ def str_to_bool_or_none(value): default=True, help='Whether to error on missing executor', ) +parser.add_argument( + '--use-patchelf', + type=str_to_bool, + default=False, + help='Set the ELF interpreter with patchelf instead of lief', +) parser.add_argument( '--log-level', type=str, @@ -74,6 +84,7 @@ def str_to_bool_or_none(value): ) step_names = [ + 'executor-download', 'runners-download', 'bin-patch', 'bin-check', @@ -92,6 +103,12 @@ def str_to_bool_or_none(value): default=None, help='Default value for download steps (default: use --default-steps)', ) +parser.add_argument( + '--executor-download', + type=str_to_bool_or_none, + default=None, + help='Enable/disable executor download step (default: use --default-download)', +) parser.add_argument( '--runners-download', type=str_to_bool_or_none, @@ -123,6 +140,8 @@ def str_to_bool_or_none(value): if args.default_download is None: args.default_download = args.default_steps +if args.executor_download is None: + args.executor_download = args.default_download if args.runners_download is None: args.runners_download = args.default_download if args.bin_patch is None: @@ -150,10 +169,13 @@ def str_to_bool_or_none(value): # Interpreter patching rewrites the ELF interpreter (dynamic loader) to the # bundled one, whose path is only known at install time. It applies to ELF # binaries (Linux) only; on other targets the loader is fixed and rpath/needed -# entries are already set correctly at build time, so lief is not needed. +# entries are already set correctly at build time, so nothing is needed. patch_interpreter = args.bin_patch and args.os == 'linux' -if patch_interpreter: +# `--use-patchelf` shells out to patchelf instead, so lief β€” a heavy binary +# wheel pulled in for this one job β€” is never imported (nor installed into the +# venv; see the wrapper). +if patch_interpreter and not args.use_patchelf: import lief lief.logging.set_level(lief.logging.LEVEL.ERROR) @@ -208,12 +230,7 @@ def get_interpreter_path(): return interpreter_path -def patch_executable(path: Path): - logger.info(f'Patching interpreter for {path}') - if not path.exists(): - logger.warning(f'Path {path} does not exist, skipping patching') - return - +def _patch_executable_lief(path: Path): binary = lief.parse(path) if not binary: logger.error(f'Failed to parse binary at {path}') @@ -240,6 +257,103 @@ def patch_executable(path: Path): logger.info(f'Successfully patched interpreter: {path}') +def _patch_executable_patchelf(path: Path): + patchelf = shutil.which('patchelf') + if patchelf is None: + raise RuntimeError('--use-patchelf was given but patchelf is not on PATH') + + if detect_executable_platform(path) != 'linux': + logger.info(f'{path} is not ELF, nothing to patch') + return + + current = subprocess.run( + [patchelf, '--print-interpreter', str(path)], + capture_output=True, + text=True, + ) + if current.returncode == 0: + current_interpreter = current.stdout.strip() + logger.info(f'Current interpreter: {current_interpreter}') + if current_interpreter and Path(current_interpreter).exists(): + logger.info( + f'Interpreter {current_interpreter} exists, skipping interpreter patching' + ) + return + else: + # A static binary has no interpreter to print; there is nothing to patch. + logger.info(f'{path} has no interpreter, nothing to patch') + return + + interpreter = get_interpreter_path() + subprocess.run( + [patchelf, '--set-interpreter', str(interpreter), str(path)], + check=True, + text=True, + ) + logger.info(f'Successfully patched interpreter: {path} -> {interpreter}') + + +def patch_executable(path: Path): + logger.info(f'Patching interpreter for {path}') + if not path.exists(): + logger.warning(f'Path {path} does not exist, skipping patching') + return + + if args.use_patchelf: + _patch_executable_patchelf(path) + else: + _patch_executable_lief(path) + + +# linux ships ELF, macos ships Mach-O; these are the only OSes we target. +BinaryOS = typing.Literal['linux', 'macos'] + +# Mach-O magics (thin 32/64-bit and universal/fat), in both stored byte orders. +_MACHO_MAGICS = frozenset( + { + b'\xfe\xed\xfa\xce', + b'\xce\xfa\xed\xfe', + b'\xfe\xed\xfa\xcf', + b'\xcf\xfa\xed\xfe', + b'\xca\xfe\xba\xbe', + b'\xbe\xba\xfe\xca', + b'\xca\xfe\xba\xbf', + b'\xbf\xba\xfe\xca', + } +) + + +def detect_executable_platform(path: Path) -> BinaryOS | None: + """Sniff a binary's header to identify its target OS: ELF -> linux, Mach-O + (any variant/byte order) -> macos. Returns None for anything unrecognized + (wrapper scripts, truncated/empty files, unknown formats).""" + try: + with open(path, 'rb') as f: + magic = f.read(4) + except OSError: + return None + if magic == b'\x7fELF': + return 'linux' + if magic in _MACHO_MAGICS: + return 'macos' + return None + + +def check_executable_platform(path: Path): + """Refuse a wrong-OS binary before we trust it. A bad release once shipped a + macOS (Mach-O) executor inside a linux tarball; the missing-only download + guard accepted it and it surfaced as an opaque `Exec format error` at + runtime. Here we error on a clear OS mismatch instead. Unrecognized formats + (None) are left alone rather than guessed at.""" + detected = detect_executable_platform(path) + if detected is not None and detected != args.os: + logger.error( + f'{path} is a {detected} binary but target OS is {args.os}; ' + 'wrong-platform executor shipped' + ) + raise RuntimeError(f'{path} is a {detected} binary but target OS is {args.os}') + + def run_check_command(command: list[str | Path]): env = os.environ.copy() env['LLVM_PROFILE_FILE'] = '/dev/null' @@ -251,6 +365,7 @@ def run_check_command(command: list[str | Path]): modules_executable = genvm_root_dir.joinpath('bin', 'genvm-modules') +check_executable_platform(modules_executable) if patch_interpreter: patch_executable(modules_executable) @@ -370,6 +485,35 @@ def download_runners_from_json( cur_dst.write_bytes(data) +def download_executor(executor_version: str): + """Fetch an executor line from its own release and unpack it at the install root. + + Since the repo split the manager release carries no executors: each line is + released separately, under its own executor-version. The tarball is laid out + as `executor//...`, i.e. relative to the install root, so it is + unpacked there. + """ + templates = manifest.get('executor_download_urls', []) + if not templates: + raise RuntimeError('manifest has no executor_download_urls') + + data = _download_template( + f'executor {executor_version}', + templates, + { + 'version': executor_version, + 'platform': f'{args.arch}-{args.os}', + 'arch': args.arch, + 'os': args.os, + }, + ) + + with tarfile.open(fileobj=io.BytesIO(data), mode='r:xz') as tar: + tar.extractall(genvm_root_dir, filter='data') + + logger.info(f'Unpacked executor {executor_version} into {genvm_root_dir}') + + all_executor_versions = list(manifest.get('executor_versions', {}).keys()) all_executor_versions.sort() @@ -399,6 +543,23 @@ def process_executor_version(executor_version: str): executor_root_dir = genvm_root_dir.joinpath('executor', executor_version) executor_executable = executor_root_dir.joinpath('bin', 'genvm') + # The manager bundle no longer ships the executors, so fetch what is missing + # before anything below expects it on disk. A failure here is only fatal if a + # missing executor is (--error-on-missing-executor). + if args.executor_download and not executor_executable.exists(): + logger.info(f'Executor {executor_version} is not installed, downloading it') + try: + download_executor(executor_version) + except Exception as e: + if args.error_on_missing_executor: + raise + logger.warning(f'Could not download executor {executor_version}: {e}') + + # Refuse a wrong-OS executor before anything trusts it, regardless of which + # steps run (a missing file is a no-op here; missing-file policy is handled + # per-step below). Mirrors the unconditional modules_executable check. + check_executable_platform(executor_executable) + if patch_interpreter or args.bin_check: if not executor_executable.exists(): if args.error_on_missing_executor: diff --git a/libs/deps/dependency-urls.json b/libs/deps/dependency-urls.json index 6ea59db5..52b56d10 100644 --- a/libs/deps/dependency-urls.json +++ b/libs/deps/dependency-urls.json @@ -77,7 +77,7 @@ "must_preload": false, "needed-for": [ "runners-all", - "runners-all-dist" + "artifact-prepack-genvm-universal" ], "fetcher": "fetchzip" }, @@ -91,7 +91,7 @@ "must_preload": false, "needed-for": [ "runners-all", - "runners-all-dist" + "artifact-prepack-genvm-universal" ], "fetcher": "fetchzip" }, @@ -106,7 +106,7 @@ "must_preload": true, "needed-for": [ "runners-all", - "runners-all-dist" + "artifact-prepack-genvm-universal" ], "fetcher": "fetchzip" }, @@ -121,7 +121,7 @@ "must_preload": false, "needed-for": [ "runners-all", - "runners-all-dist" + "artifact-prepack-genvm-universal" ], "fetcher": "fetchzip" }, @@ -135,7 +135,7 @@ "must_preload": false, "needed-for": [ "runners-all", - "runners-all-dist" + "artifact-prepack-genvm-universal" ], "fetcher": "fetchzip" }, @@ -150,7 +150,7 @@ "must_preload": false, "needed-for": [ "runners-all", - "runners-all-dist" + "artifact-prepack-genvm-universal" ], "fetcher": "fetchzip" }, @@ -245,7 +245,7 @@ "must_preload": false, "needed-for": [ "runners-all", - "runners-all-dist" + "artifact-prepack-genvm-universal" ], "fetcher": "fetchzip" }, diff --git a/runners/docs.nix b/runners/docs.nix index 48b72fcb..7201c0ab 100644 --- a/runners/docs.nix +++ b/runners/docs.nix @@ -10,10 +10,13 @@ let hash32 = r: - if r.hash == "test" then + # `hash` is the dev-mode marker, and only the forward-rolling lines carry it: + # the legacy v0.2 line exposes just `uid`, so this must not assume it exists. + if (r.hash or null) == "test" then "vTEST" - # gvm32 (Crockford Base32) β€” the encoding the executor uses for runner - # paths. Extracted from `uid` (`id:gvm32hash`); NOT Nix base32. + # The hash the executor actually uses for the runner's on-disk path, taken + # from `uid` (`id:hash32`) β€” gvm32 (Crockford Base32) on the forward lines, + # Nix base32 on the legacy one. Either way `uid` is the source of truth. else builtins.head (builtins.match "[^:]+:(.*)" r.uid); diff --git a/support/ci/README.md b/support/ci/README.md new file mode 100644 index 00000000..fba4fcbc --- /dev/null +++ b/support/ci/README.md @@ -0,0 +1,10 @@ +# Disclaimer + +This is an stdlib-only portion of `genvm-tool` to invoke on CI before nix + +## Requirements + +1. `/bin/sh` +2. `python3` interpreter in `$PATH` +3. working `gh` CLI +4. `nix` for most commands diff --git a/support/ci/__main__.py b/support/ci/__main__.py new file mode 100644 index 00000000..f0d743ab --- /dev/null +++ b/support/ci/__main__.py @@ -0,0 +1,60 @@ +import argparse +import subprocess +import sys +import typing + +import ci_lib +from pipelines import build, checks, docs, release, tests +from tools import branches, pr_branches, script_adapters, versions + +PIPELINES: list[ci_lib.Pipeline] = [ + *build.COMMANDS, + *docs.COMMANDS, + *tests.COMMANDS, + *checks.COMMANDS, + *release.COMMANDS, +] +TOOLS: list[ci_lib.Tool] = [ + *versions.COMMANDS, + *branches.COMMANDS, + *pr_branches.COMMANDS, + *script_adapters.COMMANDS, +] + + +def register_commands( + subparsers: argparse._SubParsersAction, + commands: typing.Sequence[ci_lib.Command], +) -> None: + for command in commands: + sub = subparsers.add_parser(command.name(), help=command.__doc__) + command.add_to(sub) + sub.set_defaults(func=command.handler) + + +def main() -> int: + parser = argparse.ArgumentParser(description='Run CI pipelines and tools') + top = parser.add_subparsers(dest='category', required=True) + + pipeline = top.add_parser('pipeline', help='Run a CI pipeline') + register_commands(pipeline.add_subparsers(dest='pipeline', required=True), PIPELINES) + + tool = top.add_parser('tool', help='Run a CI support tool') + register_commands(tool.add_subparsers(dest='tool', required=True), TOOLS) + + args = parser.parse_args() + return args.func(args) or 0 + + +if __name__ == '__main__': + try: + sys.exit(main()) + except subprocess.CalledProcessError as e: + import shlex + import traceback + + traceback.print_exc() + print(' '.join(shlex.quote(a) for a in e.cmd), file=sys.stderr) + print(e.stdout, file=sys.stdout) + print(e.stderr, file=sys.stderr) + sys.exit(1) diff --git a/support/ci/branch-versions.py b/support/ci/branch-versions.py deleted file mode 100755 index 85a9c5f2..00000000 --- a/support/ci/branch-versions.py +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python3 - -""" -Read the active version trains from .genvm-monorepo-root. - -The `active-versions` key holds every release train that is currently -live (e.g. ["v0.3", "v0.6"]). Entries may carry a leading `v`; it is -stripped on output so callers can build branch names directly: - - version branch: v.x (release-ready) - - dev branch: v-dev (integration) - -Usage: - branch-versions.py list # all active versions, ascending, one per line - branch-versions.py latest # the highest active version -""" - -import argparse -import json -import os -import sys -from pathlib import Path - -# MONOREPO_ROOT lets a caller point at a specific .genvm-monorepo-root -# (e.g. one extracted from origin/main) instead of the working tree's. -_default = Path(__file__).resolve().parents[2] / '.genvm-monorepo-root' -_conf = json.loads(Path(os.environ.get('MONOREPO_ROOT', _default)).read_bytes()) -_versions = _conf.get('active-versions', []) - - -def _bare(v: str) -> str: - """Strip an optional leading 'v', e.g. 'v0.3' -> '0.3'.""" - return v.lstrip('v') - - -def _key(v: str): - return tuple(int(x) for x in _bare(v).split('.')) - - -_versions = sorted((_bare(v) for v in _versions), key=_key) - - -def main() -> int: - parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter - ) - parser.add_argument('cmd', nargs='?', choices=['list', 'latest'], default='list') - args = parser.parse_args() - - if args.cmd == 'list': - print('\n'.join(_versions)) - return 0 - if not _versions: - print('no active versions configured in .genvm-monorepo-root', file=sys.stderr) - return 1 - print(_versions[-1]) - return 0 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/support/ci/cargo-clippy.sh b/support/ci/cargo-clippy.sh deleted file mode 100755 index 893e6a07..00000000 --- a/support/ci/cargo-clippy.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash - -set -e - -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) - -cd "$SCRIPT_DIR/../.." - -if ! cargo clippy --version -then - echo "ERROR: cargo clippy not installed" - exit 1 -fi - -for dir in $(git ls-files | grep -P 'Cargo\.toml') -do - if [ "$dir" == runners/nix/trg/py/modules/genvm-cpython-ext/Cargo.toml ] - then - continue - fi - pushd "$(dirname -- $dir)" 2> /dev/null > /dev/null - echo "clippy in $dir" - # cargo clippy --target-dir "$SCRIPT_DIR/../build/ya-build/rust-target/" -- -A clippy::upper_case_acronyms -Dwarnings - popd 2> /dev/null > /dev/null -done diff --git a/support/ci/check-versions.py b/support/ci/check-versions.py deleted file mode 100755 index 8ee52dcc..00000000 --- a/support/ci/check-versions.py +++ /dev/null @@ -1,205 +0,0 @@ -#!/usr/bin/env python3 - -""" -Keep the Rust crate versions in sync with .genvm-monorepo-root. - - check-versions.py [sync] - # pre-commit fixer: rewrite each crate's - # [package] major.minor to match - # .genvm-monorepo-root `version` (patch - # kept). Exits non-zero if it changed - # anything, so the commit is re-staged. - check-versions.py bump minor # cut next train: 0.3 -> 0.4 (default) - check-versions.py bump major # 0.3 -> 1.0 - check-versions.py bump --set 0.6 # explicit major.minor - -`bump` updates `version`, appends the new train to `active-versions`, -sets each crate's [package] version to .0 (preserving file -formatting) and prints the new major.minor. - -The `version` / `active-versions` values may carry a leading `v` (and -`version` may carry a `-suffix`); both are stripped when a bare -major.minor is needed (crate versions, the printed train name). -""" - -import argparse -import json -import pathlib -import re -import sys - -root = pathlib.Path(__file__).resolve().parents[2] -conf_path = root / '.genvm-monorepo-root' - -# Manager-owned crates only. The executor lives in a git submodule and is -# versioned independently (its own repo owns those crate versions); we never -# rewrite or mirror them from here. -CARGO_FILES = [ - 'implementation/Cargo.toml', -] - - -def load_conf(): - return json.loads(conf_path.read_bytes()) - - -def bare(v): - """Strip an optional leading 'v' and any '-suffix', e.g. - 'v0.3' -> '0.3', 'v0.3.2-dev' -> '0.3.2'.""" - return v.lstrip('v').split('-', 1)[0] - - -def to_mm(v): - """Normalize a version string to a bare major.minor.""" - return '.'.join(bare(v).split('.')[:2]) - - -def package_version(path): - section = None - for line in path.read_text().splitlines(): - s = line.strip() - m = re.match(r'\[(.+?)\]', s) - if m: - section = m.group(1) - continue - if section == 'package': - m = re.match(r'version\s*=\s*"([^"]+)"', s) - if m: - return m.group(1) - return None - - -def set_package_version(rel, full): - """Rewrite the [package] version of a crate to `full`. Returns the old - version, or None if no [package] version line was found.""" - p = root / rel - old = None - out = [] - section = None - for line in p.read_text().splitlines(keepends=True): - m = re.match(r'\s*\[(.+?)\]', line) - if m: - section = m.group(1) - elif section == 'package' and old is None: - m = re.match(r'(version\s*=\s*)"([^"]*)"', line) - if m: - old = m.group(2) - line = re.sub(r'"[^"]*"', f'"{full}"', line, count=1) - out.append(line) - if old is None: - return None - p.write_text(''.join(out)) - return old - - -def vkey(v): - return tuple(int(x) for x in to_mm(v).split('.')) - - -def sync(): - """Pre-commit fixer: force crate [package] major.minor to match - `version`, preserving the patch component. Non-zero exit on any - change (or error) so pre-commit re-stages the rewrite.""" - mm = to_mm(load_conf()['version']) - rc = 0 - for f in CARGO_FILES: - cur = package_version(root / f) - if cur is None: - print(f'{f}: could not find [package] version') - rc = 1 - continue - parts = cur.split('.', 2) - cur_mm = '.'.join(parts[:2]) - if cur_mm == mm: - continue - patch = parts[2] if len(parts) == 3 else '0' - target = f'{mm}.{patch}' - set_package_version(f, target) - print(f'{f}: bumped {cur} -> {target} to match .genvm-monorepo-root version ({mm})') - rc = 1 - return rc - - -def bump(level, set_version): - conf = load_conf() - active = conf.get('active-versions', []) - if not active: - print('no active versions configured in .genvm-monorepo-root', file=sys.stderr) - return 1 - latest = sorted(active, key=vkey)[-1] - - if set_version is not None: - new_mm = bare(set_version) - if not re.fullmatch(r'\d+\.\d+', new_mm): - print(f'--set expects major.minor, got {set_version!r}', file=sys.stderr) - return 2 - else: - major, minor = (int(x) for x in to_mm(latest).split('.')) - if level == 'minor': - minor += 1 - else: - major += 1 - minor = 0 - new_mm = f'{major}.{minor}' - - # Stored form keeps the leading `v` used by `active-versions` entries. - new = f'v{new_mm}' - - if new in active: - print(f'version {new} is already active', file=sys.stderr) - return 1 - if vkey(new) <= vkey(latest): - print(f'new version {new} is not greater than latest {latest}', file=sys.stderr) - return 1 - - # .genvm-monorepo-root β€” patch the two values in place, keep formatting. - text = conf_path.read_text() - new_active = '[' + ', '.join(f'"{v}"' for v in active + [new]) + ']' - text, n1 = re.subn(r'("version"\s*:\s*)"[^"]*"', rf'\g<1>"{new}"', text) - text, n2 = re.subn( - r'("active-versions"\s*:\s*)\[[^\]]*\]', lambda m: m.group(1) + new_active, text - ) - if n1 != 1 or n2 != 1: - print( - f'failed to patch .genvm-monorepo-root (version={n1}, active-versions={n2})', - file=sys.stderr, - ) - return 1 - conf_path.write_text(text) - - for f in CARGO_FILES: - if set_package_version(f, f'{new_mm}.0') is None: - print(f'{f}: could not find [package] version to bump', file=sys.stderr) - return 1 - - # Print the bare major.minor: callers build branch names as v${NEW}. - print(new_mm) - return 0 - - -def main(): - parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter - ) - sub = parser.add_subparsers(dest='cmd') - sub.add_parser( - 'sync', - help='rewrite crate versions to match version (default; pre-commit fixer)', - ) - bp = sub.add_parser('bump', help='cut the next version train') - bp.add_argument('level', nargs='?', choices=['minor', 'major'], default='minor') - bp.add_argument( - '--set', - dest='set_version', - metavar='MAJOR.MINOR', - help='explicit version (overrides level)', - ) - - args = parser.parse_args() - if args.cmd == 'bump': - return bump(args.level, args.set_version) - return sync() - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/support/ci/ci_lib.py b/support/ci/ci_lib.py new file mode 100644 index 00000000..394f76b0 --- /dev/null +++ b/support/ci/ci_lib.py @@ -0,0 +1,260 @@ +import abc +import argparse +import contextlib +import hashlib +import io +import lzma +import os +import shlex +import subprocess +import tarfile +import typing +from pathlib import Path + + +class Command(abc.ABC): + @abc.abstractmethod + def name(self) -> str: ... + + def add_to(self, parser: argparse.ArgumentParser) -> None: + pass + + @abc.abstractmethod + def handler(self, args: argparse.Namespace) -> int | None: ... + + +class Pipeline(Command): + pass + + +class Tool(Command): + pass + + +ROOT_DIR = Path(__file__).resolve().parent + +while ROOT_DIR != ROOT_DIR.parent: + if (ROOT_DIR / '.genvm-monorepo-root').exists(): + break + ROOT_DIR = ROOT_DIR.parent + +if not (ROOT_DIR / '.genvm-monorepo-root').exists(): + raise RuntimeError( + f'Could not find .genvm-monorepo-root in {ROOT_DIR} or any parent directory' + ) + +_group_stack: list[str] = [] + + +def github_group_start(name: str): + if len(_group_stack) > 0: + print('::endgroup::') + _group_stack.append(name) + print(f'::group::{name}', flush=True) + + +def github_group_end(): + if len(_group_stack) > 0: + _group_stack.pop() + print('::endgroup::', flush=True) + if len(_group_stack) > 0: + print(f'::group::{_group_stack[-1]} (continue)', flush=True) + + +@contextlib.contextmanager +def github_group(name: str): + github_group_start(name) + try: + yield + finally: + github_group_end() + + +@contextlib.contextmanager +def github_group_opt(name: str | None): + if name is not None: + github_group_start(name) + try: + yield + finally: + if name is not None: + github_group_end() + + +def github_error( + message: str, + *, + file: str | Path | None = None, + line: int | None = None, + col: int | None = None, +) -> None: + if line is not None: + assert file is not None, 'line number provided but no file' + if col is not None: + assert line is not None, 'column number provided but no line number' + + msg_prefix = '' + if file is not None: + msg_prefix = f' file={file}' + if line is not None: + msg_prefix += f',line={line}' + if col is not None: + msg_prefix += f',col={col}' + + print(f'::error{msg_prefix}::{message.replace(chr(10), " ")}') + + +def run( + command: list[str | Path], + *, + env: dict[str, str] | None = None, + cwd: Path | None = None, + check: bool = True, + stdout=None, + capture_output: bool = False, +) -> subprocess.CompletedProcess: + if cwd is None: + cwd = ROOT_DIR + else: + print(f'+ cd {shlex.quote(str(cwd.resolve()))}') + print(f'+ {" ".join(shlex.quote(str(arg)) for arg in command)}', flush=True) + + full_env = None + if env is not None: + full_env = os.environ.copy() + full_env.update(env) + + return subprocess.run( + [str(arg) for arg in command], + check=check, + env=full_env, + cwd=cwd, + stdout=stdout, + capture_output=capture_output, + text=True, + ) + + +def output(command: list[str | Path], *, cwd: Path | None = None) -> str: + return run(command, cwd=cwd, capture_output=True).stdout + + +_nix_develop_cache: set[str] = set() + + +def nix_develop_pre_cache(installable: str, add_group: bool = True) -> Path: + cache_path = ROOT_DIR / '.direnv' / 'profile-cache' + if not cache_path.exists(): + cache_path.mkdir(parents=True, exist_ok=True) + digest = hashlib.sha3_256(installable.encode()).hexdigest()[:8] + profile_path = cache_path / f'{digest}' + if installable in _nix_develop_cache and profile_path.exists(): + return profile_path + + with github_group_opt(f'nix-develop pre-cache: {installable}' if add_group else None): + run( + ['nix', 'develop', installable, '--profile', profile_path, '--command', 'true'], + ) + _nix_develop_cache.add(installable) + + return profile_path + + +def nix_develop( + installable: str, + cmd: typing.Sequence[str | Path], + *, + check: bool = True, + fresh_env: bool = False, + subcommand_group: str | None = None, + add_group: bool = True, +): + profile_path = nix_develop_pre_cache(installable, add_group=add_group) + if not profile_path.exists(): + raise RuntimeError(f'nix-develop profile cache {profile_path} does not exist') + + print(f'# using {installable}') + + fresh_env_args = [] + if fresh_env: + fresh_env_args = ['-i'] + + if subcommand_group is not None: + github_group_start(subcommand_group) + try: + return run( + ['nix', 'develop', profile_path, *fresh_env_args, '--command', *cmd], + check=check, + ) + finally: + if subcommand_group is not None: + github_group_end() + + +ARTIFACTS_DIR = ROOT_DIR / 'build' / 'artifacts' + + +def bundle_artifact( + *, + root_path: Path, + result_path: Path, + globs: list[str], + compression_level: int, +): + all_files: set[Path] = set() + for pat in globs: + all_files.update(root_path.glob(pat)) + + all_files_det = list(sorted(all_files, key=lambda p: str(p))) + + ext = result_path.name.split('.')[-1] + + def _process_with_tar(tar_file: tarfile.TarFile): + for path in all_files_det: + if path.is_dir() and not path.is_symlink(): + continue + stats = path.stat() + rel_path = path.relative_to(root_path) + info = tarfile.TarInfo() + info.name = str(rel_path) + info.type = tarfile.REGTYPE + if stats.st_mode & 0o111: + info.mode = 0o755 + else: + info.mode = 0o644 + # if we can preserve as local symlink we preserve + # otherwise we dereference + if path.is_symlink(): + target = path.readlink() + if not target.is_absolute(): + normalized = os.path.normpath(rel_path.parent / target) + if normalized != '..' and not normalized.startswith('..' + os.sep): + info.type = tarfile.SYMTYPE + info.linkname = target.as_posix() + info.size = 0 + tar_file.addfile(info) + continue + data = path.read_bytes() + info.size = path.stat().st_size + tar_file.addfile(info, io.BytesIO(data)) + + match ext: + case 'tar': + file = open(result_path, 'wb') + case 'xz': + file = lzma.open( + result_path, mode='wb', preset=compression_level, format=lzma.FORMAT_XZ + ) + case _: + raise ValueError(f'unsupported archive extension: {ext}') + + try: + with tarfile.open( + fileobj=file, + mode='w', + format=tarfile.USTAR_FORMAT, + encoding='utf-8', + ) as tar_file: + _process_with_tar(tar_file) + finally: + file.close() diff --git a/support/ci/genvm-merge-into-dev.py b/support/ci/genvm-merge-into-dev.py deleted file mode 100755 index 92d0ca76..00000000 --- a/support/ci/genvm-merge-into-dev.py +++ /dev/null @@ -1,367 +0,0 @@ -#!/usr/bin/env python3 -"""Gate and perform a Merge of a PR into a v-dev branch. - -Invoked by the reusable .github/workflows/branch_merge_into_dev.yaml when a -maintainer ticks the "Merge" box on the PR action panel. It re-checks every -gate against the EXACT PR head commit and then advances the dev branch by a -plain (fast-forward-only) push, so what lands is byte-identical to what CI -and E2E validated. - -Gates (all required, all on the head commit): -1. base branch is a v-dev branch -2. PR carries the `rtm` (ready-to-merge) label -3. full GenVM CI (queue.yaml) concluded success -4. the cross-repo E2E check concluded success -5. the PR is 0 commits behind base - -Strategy: 1 commit -> fast-forward the original commit (SHA preserved); -more -> squash into one commit on top of base, then fast-forward. The PR -is closed afterwards. - -Executor mirror: the manager release line (e.g. v0.6) is independent of the -executor lines it ships (v0.2, v0.3), so a single manager commit carries a -gitlink into EACH active line's submodule. We mirror every active line by -fast-forwarding its own `-dev` branch (`v0.2-dev`, `v0.3-dev`, ...) to -that line's gitlink commit. True cross-repo atomicity is impossible, so we -(1) fast-forward-check all sides up front, then (2) push the executors first β€” -they are the dependencies the manager commit points at β€” and (3) push the -manager. If the manager push then loses a race the executors are left one -fast-forward ahead, which is harmless (monotonic) and reconciled when Merge is -re-ticked. Lines whose gitlink is unchanged are skipped. - -After a successful merge each line's executor mirror branch -(`pr//`, opened by branch_executor_prs.yaml) is deleted β€” its -tip now lives in the executor's `-dev`, so the branch and its PR are -redundant. - -(The dev -> version release-gate merge is a separate, not-yet-automated -flow; on that one the executor's version branch is fast-forwarded and its -dev/version branches are kept at the same point.) - -Talks to GitHub through the `gh` CLI and moves refs through `git`; it -never executes PR code. The dev branches are protected, so the workflow -checks out with the GENVM_CI_PRIVATE_KEY deploy key and pushes non-force -(a base that advanced is safely rejected). The active executor submodules -must be checked out with a remote `origin` that can push the executor repo -(the workflow wires its own deploy key); the active lines come from -.genvm-monorepo-root via branch-versions.py. - -Env: GITHUB_REPOSITORY, PR_NUMBER, GH_TOKEN, E2E_CHECK_PATTERN. -""" - -import json -import os -import re -import subprocess -import sys -from pathlib import Path - -REPO = os.environ['GITHUB_REPOSITORY'] -PR = os.environ['PR_NUMBER'] -E2E_PATTERN = os.environ.get('E2E_CHECK_PATTERN', 'e2e') -HERE = Path(__file__).resolve().parent - - -def run(*args, check=True): - return subprocess.run(args, check=check, text=True, capture_output=True) - - -def gh(*args): - return run('gh', *args).stdout - - -def git(*args, check=True): - return run('git', *args, check=check) - - -def egit(submodule, *args, check=True): - """git inside an executor submodule checkout.""" - return run('git', '-C', submodule, *args, check=check) - - -def active_lines(): - """Active executor lines as `v` tags (e.g. ['v0.2', 'v0.3']). The manager - release line is independent of these; a manager PR ships every active line.""" - out = run(sys.executable, str(HERE / 'branch-versions.py'), 'list').stdout - return [f'v{v}' for v in out.split()] - - -def block(msg): - """Comment the reason on the PR and fail the job.""" - run( - 'gh', - 'pr', - 'comment', - PR, - '--repo', - REPO, - '--body', - f'❌ Merge blocked: {msg}', - check=False, - ) - sys.exit(1) - - -def pr_view(*fields): - out = gh('pr', 'view', PR, '--repo', REPO, '--json', ','.join(fields)) - return json.loads(out) - - -def check_gates(pr): - if pr['state'] != 'OPEN': - block(f'PR is not open (state: {pr["state"]}).') - if pr['isDraft']: - block('PR is a draft.') - - base = pr['baseRefName'] - if not re.fullmatch(r'v.*-dev', base): - block(f'base branch `{base}` is not a `v-dev` branch.') - - # 2. rtm label - if not any(label['name'] == 'rtm' for label in pr['labels']): - block('missing the `rtm` (ready-to-merge) label.') - - head_sha = pr['headRefOid'] - - # 3. full GenVM CI (queue.yaml) green on the head commit. The same head may - # carry skipped runs (label events that didn't run full tests) alongside the - # real one, and the run may have been started by either a push - # (event=pull_request) or the action panel (event=workflow_dispatch) β€” so we - # query all events and require ANY completed run on this exact commit to have - # succeeded, not just the most recent. - runs = json.loads( - gh( - 'api', - f'repos/{REPO}/actions/workflows/queue.yaml/runs?head_sha={head_sha}', - ) - )['workflow_runs'] - if not any(r['status'] == 'completed' and r['conclusion'] == 'success' for r in runs): - got = ', '.join(f'{r["status"]}/{r["conclusion"]}' for r in runs) or 'no run' - block(f'GenVM CI (queue.yaml) is not green on `{head_sha}` (runs: {got}).') - - # 4. cross-repo E2E check green on the head commit - checks = json.loads( - gh( - 'api', - f'repos/{REPO}/commits/{head_sha}/check-runs?per_page=100', - ) - )['check_runs'] - e2e = [c for c in checks if re.search(E2E_PATTERN, c['name'], re.I)] - if not e2e: - block(f'no E2E check found on `{head_sha}` (run it on this PR first).') - bad = [c for c in e2e if c['conclusion'] != 'success'] - if bad: - conclusions = ' '.join(c['conclusion'] or 'pending' for c in e2e) - block(f'E2E is not green on `{head_sha}` (conclusions: {conclusions}).') - - return base, head_sha - - -def delete_executor_mirror_branch(submodule, line, head_ref): - """Delete the executor mirror branch `pr//` after a merge. - - The manager PR's work for this line lived on that namespaced branch; its tip is - now contained in the executor's `-dev` (we just fast-forwarded it there), - so the branch β€” and the auto-opened executor PR it backed (branch_executor_prs.yaml) - β€” are redundant. Deleting the branch also closes that PR. Best-effort: a failure - here never unwinds an otherwise-complete merge. The branch is unprotected, so the - executor deploy key can delete it. - """ - branch = f'pr/{line}/{head_ref}' - r = egit(submodule, 'push', 'origin', '--delete', branch, check=False) - if r.returncode == 0: - print(f'deleted executor mirror branch {branch}') - else: - print(f'note: could not delete executor mirror branch {branch}: {r.stderr.strip()}') - - -def executor_gitlink(commit, submodule): - """Executor submodule commit (gitlink) recorded at `commit`'s tree.""" - out = git('ls-tree', commit, submodule).stdout - # "160000 commit \t" - m = re.match(r'\S+\s+commit\s+([0-9a-f]+)\t', out) - if not m: - block( - f'could not resolve the executor submodule pointer ({submodule}) at `{commit}`.' - ) - return m.group(1) - - -def executor_mirror_action(submodule, exec_base, exec_sha): - """Decide how to mirror `exec_sha` onto the executor's `` branch. - - Returns 'push' (fast-forward, including a fresh create) or 'skip' (the - branch already contains exec_sha because the executor advanced past what - this PR pins). Blocks if exec_sha is unknown to the executor repo or the - branch has diverged from it.""" - # Bring every executor head (and thus exec_sha, if pushed) local. - egit(submodule, 'fetch', '--no-tags', 'origin', '+refs/heads/*:refs/remotes/origin/*') - - if ( - egit(submodule, 'cat-file', '-e', f'{exec_sha}^{{commit}}', check=False).returncode - != 0 - ): - block( - f'executor commit `{exec_sha}` ({submodule}) is not in the executor repo; ' - f'push the executor side of this change first, then re-tick Merge.' - ) - - ref = f'refs/remotes/origin/{exec_base}' - if ( - egit(submodule, 'rev-parse', '--verify', '--quiet', ref, check=False).returncode - != 0 - ): - return 'push' # branch absent -> push creates it - - tip = egit(submodule, 'rev-parse', ref).stdout.strip() - if tip == exec_sha: - return 'skip' # already there - if ( - egit( - submodule, 'merge-base', '--is-ancestor', exec_sha, tip, check=False - ).returncode - == 0 - ): - # Executor moved ahead of what this PR pins; the pinned commit is - # already contained, so there is nothing to push. - return 'skip' - if ( - egit( - submodule, 'merge-base', '--is-ancestor', tip, exec_sha, check=False - ).returncode - == 0 - ): - return 'push' # clean fast-forward - block( - f'executor `{exec_base}` (`{tip[:12]}`) has diverged from the PR executor ' - f'commit `{exec_sha[:12]}`; mirror is not a fast-forward.' - ) - - -def mirror_executor_lines(push_sha): - """Fast-forward each active executor line's `-dev` to the gitlink this - merge pins, BEFORE the manager push (the manager commit references them). Lines - whose gitlink is unchanged are skipped. Returns the lines that were pinned so - their mirror branches can be cleaned up after the manager push lands.""" - pinned = [] - for line in active_lines(): - submodule = f'executors/{line}.x' - exec_base = f'{line}-dev' - exec_sha = executor_gitlink(push_sha, submodule) - if executor_mirror_action(submodule, exec_base, exec_sha) == 'push': - print(f'Mirroring executor {exec_base} -> {exec_sha}') - if ( - egit( - submodule, 'push', 'origin', f'{exec_sha}:refs/heads/{exec_base}', check=False - ).returncode - != 0 - ): - block( - f'fast-forward push of executor `{exec_base}` was rejected (it advanced); ' - f're-tick Merge.' - ) - else: - print(f'Executor {exec_base} already contains {exec_sha}; nothing to mirror') - pinned.append((submodule, line)) - return pinned - - -def merge(pr, base, head_sha): - # Fetch the exact head commit (works for fork PRs too) and live base tip. - git( - 'fetch', - '--no-tags', - 'origin', - f'refs/pull/{PR}/head:refs/prhead', - f'+refs/heads/{base}:refs/remotes/origin/{base}', - ) - - fetched = git('rev-parse', 'refs/prhead').stdout.strip() - if fetched != head_sha: - block( - f'head moved during merge (expected `{head_sha}`, got `{fetched}`); re-tick Merge.' - ) - - # 5. authoritative 0-commits-behind check at merge time. - if ( - git( - 'merge-base', '--is-ancestor', f'origin/{base}', 'refs/prhead', check=False - ).returncode - != 0 - ): - behind = git('rev-list', '--count', f'refs/prhead..origin/{base}').stdout.strip() - block( - f'PR is {behind} commit(s) behind `{base}`; update the branch and re-tick Merge.' - ) - - git('config', 'user.name', 'genvm-ci') - git('config', 'user.email', 'genvm-ci@genlayer.com') - - if len(pr['commits']) == 1: - print(f'Single commit: fast-forwarding {base} to {head_sha}') - push_sha = head_sha - else: - print(f'Squashing {len(pr["commits"])} commits onto {base}') - author = git('log', '-1', '--format=%an <%ae>', head_sha).stdout.strip() - git('checkout', '-B', '_merge', f'origin/{base}') - git('merge', '--squash', head_sha) - message = f'{pr["title"]} (#{PR})\n\n{pr["body"] or ""}\n' - git('commit', '--author', author, '-m', message) - push_sha = git('rev-parse', 'HEAD').stdout.strip() - - # Mirror every active executor line onto its own `-dev` branch. The - # manager line (e.g. v0.6) is independent of the executor lines (v0.2, v0.3) - # it ships, so one merge fast-forwards each changed line. Fast-forward-check - # all of them BEFORE the manager push (the manager side was checked above), - # pushing the executors first: the manager commit gitlinks into them, so the - # dependencies must land before the referrer. - pinned = mirror_executor_lines(push_sha) - - # Non-force FF push; rejected if base advanced since the checks. The - # executors are already advanced at this point; a rejection here leaves them - # one fast-forward ahead (harmless, monotonic) and re-ticking reconciles. - if ( - git('push', 'origin', f'{push_sha}:refs/heads/{base}', check=False).returncode != 0 - ): - block( - f'fast-forward push to manager `{base}` was rejected (base advanced) β€” the ' - f'executor lines were already fast-forwarded; re-tick Merge to land the ' - f'manager side.' - ) - - run( - 'gh', - 'pr', - 'comment', - PR, - '--repo', - REPO, - '--body', - f'βœ… Merged into `{base}` (`{push_sha}`) via fast-forward.', - check=False, - ) - run('gh', 'pr', 'close', PR, '--repo', REPO, check=False) - - # Clean up the now-redundant executor mirror branches (and the PRs they backed). - for submodule, line in pinned: - delete_executor_mirror_branch(submodule, line, pr['headRefName']) - - -def main(): - pr = pr_view( - 'baseRefName', - 'headRefName', - 'headRefOid', - 'commits', - 'labels', - 'state', - 'isDraft', - 'title', - 'body', - ) - base, head_sha = check_gates(pr) - merge(pr, base, head_sha) - - -if __name__ == '__main__': - main() diff --git a/support/ci/pipelines/README.md b/support/ci/pipelines/README.md deleted file mode 100644 index af7b24f8..00000000 --- a/support/ci/pipelines/README.md +++ /dev/null @@ -1,9 +0,0 @@ -```mermaid -graph TD -PreCommit -.-> BuildUniversal --> RustTest -PreCommit -.-> PythonTest -PreCommit -.-> BuildAmd64 -BuildAmd64 --> NodePipeline -BuildUniversal --> NodePipeline -RustTest -.-> NodePipeline -``` diff --git a/support/ci/pipelines/_common.sh b/support/ci/pipelines/_common.sh deleted file mode 100755 index 07d0d7d1..00000000 --- a/support/ci/pipelines/_common.sh +++ /dev/null @@ -1,7 +0,0 @@ -set -ex - -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) - -cd "$SCRIPT_DIR/../../.." - -NIX=$(dirname $(which nix)) diff --git a/support/ci/pipelines/build.py b/support/ci/pipelines/build.py new file mode 100644 index 00000000..20516847 --- /dev/null +++ b/support/ci/pipelines/build.py @@ -0,0 +1,100 @@ +import argparse +import sys +from pathlib import Path + +import ci_lib + + +def build_and_pack( + *, + installable: str, + artifact_name: str, + globs: list[str] | None = None, + compression_level: int = 9, + show_trace: bool = False, +) -> Path: + """Realize `installable` and pack its output tree as ARTIFACTS_DIR/`artifact_name`. + + The out-link is keyed by artifact, not by installable: one installable may be + packed into several artifacts that differ only in `globs`, and each needs its + own `result-*` link. + + `show_trace` only helps on flake *evaluation* errors β€” build failures are + already covered by `-L`. + """ + build = artifact_name.removesuffix('.xz').removesuffix('.tar') + build_trg = ci_lib.ROOT_DIR / 'build' / f'result-{build}' + # nix does not create the out-link's parent + build_trg.parent.mkdir(exist_ok=True, parents=True) + ci_lib.run( + [ + 'nix', + 'build', + '-v', + '-L', + installable, + '--out-link', + str(build_trg), + *(['--show-trace'] if show_trace else []), + ], + ) + print(f'# built {build_trg} from {installable}') + + ci_lib.ARTIFACTS_DIR.mkdir(exist_ok=True, parents=True) + tar_path = ci_lib.ARTIFACTS_DIR / artifact_name + ci_lib.bundle_artifact( + root_path=build_trg, + result_path=tar_path, + globs=globs if globs is not None else ['**/*'], + compression_level=compression_level, + ) + print(f'# packed {tar_path} with {compression_level} for {installable}') + + return tar_path + + +class Build(ci_lib.Pipeline): + """Build and package a GenVM target.""" + + def name(self) -> str: + return 'build' + + def add_to(self, parser: argparse.ArgumentParser) -> None: + parser.add_argument('--target', required=True) + parser.add_argument('--compression', default=9, type=int) + parser.add_argument('--no-refetch', action='store_true') + parser.add_argument('--show-trace', action='store_true') + + def handler(self, args: argparse.Namespace) -> int: + head_revision = ci_lib.output(['git', 'rev-parse', 'HEAD']).strip() + print(f'$TARGET = {args.target}') + print(f'$HEAD_REVISION = {head_revision}') + + if not args.no_refetch: + with ci_lib.github_group('prefetch dependencies'): + ci_lib.run( + [ + sys.executable, + 'support/runner-script.py', + 'dependencies', + 'prefetch-needed', + '--all', + '--target', + args.target, + ] + ) + + with ci_lib.github_group(f'nix build ({args.target})'): + # `?submodules=1`: the executor lines are git submodules; without it the + # flake source tree has empty executors/.x dirs and evaluation fails. + build_and_pack( + installable=f'.?submodules=1#{args.target}', + artifact_name=f'genvm-{args.target}.tar.xz', + compression_level=args.compression, + show_trace=args.show_trace, + ) + + return 0 + + +COMMANDS = [Build()] diff --git a/support/ci/pipelines/build.sh b/support/ci/pipelines/build.sh deleted file mode 100755 index 41921215..00000000 --- a/support/ci/pipelines/build.sh +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env bash - -TARGET="" -EXECUTOR_VERSION="" -EXTRA_BUNDLE="" - -while [[ $# -gt 0 ]]; do - case $1 in - --target) - TARGET="$2" - shift 2 - ;; - --target=*) - TARGET="${1#*=}" - shift - ;; - --executor-version) - EXECUTOR_VERSION="$2" - shift 2 - ;; - --executor-version=*) - EXECUTOR_VERSION="${1#*=}" - shift - ;; - --extra-bundle) - EXTRA_BUNDLE="$2" - EXTRA_BUNDLE="$(readlink -f "$EXTRA_BUNDLE")" - shift 2 - ;; - --extra-bundle=*) - EXTRA_BUNDLE="${1#*=}" - EXTRA_BUNDLE="$(readlink -f "$EXTRA_BUNDLE")" - shift - ;; - --no-refetch) - NO_REFETCH=1 - shift - ;; - --compression) - COMPRESSION="$2" - shift 2 - ;; - --compression=*) - COMPRESSION="${1#*=}" - shift - ;; - -h|--help) - echo "Usage: $0 [--target TARGET] [--executor-version VERSION]" - echo " --target TARGET Specify the target platform" - echo " --executor-version VERSION Specify the executor version" - exit 0 - ;; - *) - echo "Unknown option: $1" >&2 - exit 1 - ;; - esac -done - -# Validate arguments -if [[ -n "$TARGET" && -z "$TARGET" ]]; then - echo "Error: --target cannot be empty" >&2 - exit 1 -fi - -if [[ -n "$EXECUTOR_VERSION" && -z "$EXECUTOR_VERSION" ]]; then - echo "Error: --executor-version cannot be empty" >&2 - exit 1 -fi - -if [[ -n "${COMPRESSION:-}" && "$COMPRESSION" != "skip" ]]; then - if ! [[ "$COMPRESSION" =~ ^[1-9]$ ]]; then - echo "Error: --compression must be 1-9 or 'skip'" >&2 - exit 1 - fi -fi - -HEAD_REVISION=$(git rev-parse HEAD) - -echo "\$TARGET = $TARGET" -echo "\$HEAD_REVISION = $HEAD_REVISION" -echo "\$EXECUTOR_VERSION = $EXECUTOR_VERSION" -echo "\$EXTRA_BUNDLE = $EXTRA_BUNDLE" - -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -source "$SCRIPT_DIR/_common.sh" - -cat < flake-config.json -{ - "executor-version": "$EXECUTOR_VERSION", - "repo-url": "https://github.com/genlayerlabs/genvm-manager.git", - "head-revision": "$HEAD_REVISION" -} -EOF - -if [[ -z "${NO_REFETCH:-}" ]]; then - python3 "$SCRIPT_DIR/../../runner-script.py" dependencies prefetch-needed --all --target "$TARGET" -fi - -mkdir -p build -# `?submodules=1`: the executor lines are git submodules; without it the flake -# source tree has empty executors/.x dirs and evaluation fails. -nix build -o "build/out-$TARGET" -v -L ".?submodules=1#$TARGET" --show-trace - -PREV=$(readlink -f .) -pushd "build/out-$TARGET" -find . -type f -print0 | sort -z | \ - xargs -0 tar --transform 's,^\./,,' --mode=ug+w -cf "$PREV/build/genvm-$TARGET.tar" - -if [[ "${COMPRESSION:-9}" != "skip" ]]; then - xz -z -"${COMPRESSION:-9}" --force "$PREV/build/genvm-$TARGET.tar" -fi - -popd diff --git a/support/ci/pipelines/checks.py b/support/ci/pipelines/checks.py new file mode 100644 index 00000000..d06b4e7d --- /dev/null +++ b/support/ci/pipelines/checks.py @@ -0,0 +1,168 @@ +import argparse +import json +import os +import sys +import tempfile +from pathlib import Path + +import ci_lib + + +class CommitHooks(ci_lib.Pipeline): + """Run pre-commit hooks for manager and executor repos.""" + + def name(self) -> str: + return 'commit-hooks' + + def handler(self, args: argparse.Namespace) -> int: + system = ci_lib.output( + ['nix', 'eval', '--impure', '--raw', '--expr', 'builtins.currentSystem'] + ).strip() + repos = [ + ('.', 'manager', '.?submodules=1'), + ('executors/v0.3.x', 'executor v0.3.x', '.'), + ('executors/v0.2.x', 'executor v0.2.x', '.'), + ] + for directory, label, flakeref in repos: + with ci_lib.github_group(f'pre-commit: {label}'): + cwd = ci_lib.ROOT_DIR / directory + cfg = ci_lib.output( + [ + 'nix', + 'build', + '--no-link', + '--print-out-paths', + f'{flakeref}#checks.{system}.pre-commit-check.config.configFile', + ], + cwd=cwd, + ).strip() + pc = ( + Path( + ci_lib.output( + [ + 'nix', + 'build', + '--no-link', + '--print-out-paths', + f'{flakeref}#checks.{system}.pre-commit-check.config.package', + ], + cwd=cwd, + ).strip() + ) + / 'bin' + / 'pre-commit' + ) + with tempfile.TemporaryDirectory() as home: + ci_lib.run( + [ + pc, + 'run', + '--all-files', + '--config', + cfg, + '--hook-stage', + 'pre-commit', + '--show-diff-on-failure', + '--color', + 'always', + ], + cwd=cwd, + env={'PRE_COMMIT_HOME': home}, + ) + return 0 + + +class CommitMessages(ci_lib.Pipeline): + """Check commit messages for changed repos.""" + + def name(self) -> str: + return 'commit-messages' + + def handler(self, args: argparse.Namespace) -> int: + if not os.environ.get('CHANGES'): + ci_lib.github_error( + "CHANGES is empty; it must hold the get-src action's `changes` output" + ) + return 1 + + rc = 0 + for directory, info in json.loads(os.environ['CHANGES']).items(): + if not info.get('has_changes'): + continue + base = info.get('base_commit') + head = info.get('branch_commit') + if not base or not head: + continue + + label = 'manager' if directory == '.' else directory + checker = ( + ci_lib.ROOT_DIR / directory / 'support' / 'scripts' / 'check-commit-message.py' + ) + if not checker.is_file(): + ci_lib.github_error(f'{label} has no support/scripts/check-commit-message.py') + rc = 1 + continue + + if ( + ci_lib.run( + ['git', '-C', directory, 'cat-file', '-e', f'{base}^{{commit}}'], check=False + ).returncode + != 0 + ): + ci_lib.run( + [ + 'git', + '-C', + directory, + 'fetch', + '--no-tags', + '--force', + 'origin', + '+refs/heads/*:refs/remotes/origin/*', + ] + ) + + with ci_lib.github_group(f'commit messages: {label} ({base}..{head})'): + commits = ci_lib.output( + ['git', '-C', directory, 'rev-list', f'{base}..{head}'] + ).split() + if not commits: + print('no new commits') + for sha in commits: + subject = ci_lib.output( + ['git', '-C', directory, 'log', '-1', '--format=%s', sha] + ).strip() + message = ci_lib.output( + ['git', '-C', directory, 'log', '-1', '--format=%B', sha] + ) + result = ci_lib.run( + [sys.executable, checker, '--message-text', message], check=False + ) + if result.returncode == 0: + print(f'ok {sha} {subject}') + else: + ci_lib.github_error(f'{label} {sha}: bad commit message: {subject}') + rc = 1 + return rc + + +class CargoClippy(ci_lib.Pipeline): + """Check cargo clippy availability and enumerate crates.""" + + def name(self) -> str: + return 'cargo-clippy' + + def handler(self, args: argparse.Namespace) -> int: + if ci_lib.run(['cargo', 'clippy', '--version'], check=False).returncode != 0: + print('ERROR: cargo clippy not installed') + return 1 + for path in ci_lib.output(['git', 'ls-files']).splitlines(): + if not path.endswith('Cargo.toml'): + continue + if path == 'runners/nix/trg/py/modules/genvm-cpython-ext/Cargo.toml': + continue + print(f'clippy in {path}') + return 0 + + +COMMANDS = [CommitHooks(), CommitMessages(), CargoClippy()] diff --git a/support/ci/pipelines/docs.py b/support/ci/pipelines/docs.py new file mode 100644 index 00000000..67a1cdce --- /dev/null +++ b/support/ci/pipelines/docs.py @@ -0,0 +1,120 @@ +import argparse +import json +import os +import sys + +import ci_lib + + +class Docs(ci_lib.Pipeline): + """Generate and build documentation.""" + + def name(self) -> str: + return 'docs' + + def handler(self, args: argparse.Namespace) -> int: + # Every sphinx conf.py (manager + executor sub-sites) reads COPYRIGHT_YEAR + # from the environment; set it once so all the builds below inherit it. + os.environ['COPYRIGHT_YEAR'] = ci_lib.output( + ['git', 'log', '-1', '--date=format:%Y', '--format=%ad'] + ).strip() + + monorepo = json.loads((ci_lib.ROOT_DIR / '.genvm-monorepo-root').read_text()) + + # Runner versions are per-executor-line data (the runner manifest), so the + # generated manifest + available-runners page land in the primary line's + # SDK sub-site, not the manager. generate.py writes available-runners.rst + # next to this json (json_path.with_name). + primary_line = monorepo['active-versions'][0] + runners_versions_json = ( + f'executors/{primary_line}.x/docs/website/src/python-sdk/runners-versions.json' + ) + with ci_lib.github_group('generate runner-versions'): + ci_lib.run( + [sys.executable, 'docs/website/generate.py', runners_versions_json], + ) + + # Manager site first: it owns the intersphinx inventory the executor + # sub-sites cross-link against (SDK β†’ spec/impl-spec). Cross-linking is + # one-way, so a single manager pass is enough β€” nothing here references + # the sub-sites back. + # + # `?submodules=1`: the manager flake imports the executor lines, so the + # executors/.x paths must be in the flake source tree. + ci_lib.nix_develop( + '.?submodules=1#gen-docs', + ['./support/ci/run.sh', 'pipeline', 'docs-src'], + ) + + # Each active executor line builds its own standalone docs sub-site in its + # own `gen-docs` shell (self-contained sphinx toolchain), accumulated + # under build/doc/html/executors// and linked from the manager's + # generated executors.rst page. Built after the manager site so its + # objects.inv exists (intersphinx) and its html root is not clobbered. + doc_root = ci_lib.ROOT_DIR / 'build' / 'doc' + for line in monorepo['active-versions']: + exec_prefix = f'executors/{line}.x' + src = (ci_lib.ROOT_DIR / exec_prefix / 'docs' / 'website' / 'src').resolve() + installable = f'./{exec_prefix}#gen-docs' + ci_lib.nix_develop( + installable, + ['sphinx-build', '-b', 'html', src, doc_root / 'html' / 'executors' / line], + subcommand_group=f'sphinx build executor sub-site: {line}', + ) + + # Lines that ship the Python SDK also feed the manager's "docs for + # LLMs" bundle: build the text output and fold python-sdk into + # _static/ai/api.txt (where the manager's llms.txt points). + if (src / 'python-sdk').is_dir(): + text_out = doc_root / 'text' / 'executors' / line + ci_lib.nix_develop( + installable, + ['sphinx-build', '-b', 'text', src, text_out], + subcommand_group=f'sphinx text (ai bundle): {line}', + ) + ci_lib.run( + [ + sys.executable, + 'docs/website/merge_txts.py', + text_out / 'python-sdk', + doc_root / 'html' / '_static' / 'ai' / 'api.txt', + ] + ) + return 0 + + +class DocsSrc(ci_lib.Pipeline): + """Build docs inside the docs Nix environment.""" + + def name(self) -> str: + return 'docs-src' + + def handler(self, args: argparse.Namespace) -> int: + out_base = (ci_lib.ROOT_DIR / 'build' / 'doc').resolve() + out_base.mkdir(parents=True, exist_ok=True) + + # sphinx and its extensions come from the gen-docs Nix shell (the + # `sphinx-build` on PATH is wrapped against that env), so there is no + # venv/poetry step: conf.py lives at docs/website/src/conf.py, which + # sphinx-build discovers from the source dir. + src = (ci_lib.ROOT_DIR / 'docs' / 'website' / 'src').resolve() + with ci_lib.github_group('sphinx build (html + text)'): + ci_lib.run(['sphinx-build', '-b', 'html', src, out_base / 'html']) + ci_lib.run(['sphinx-build', '-b', 'text', src, out_base / 'text']) + + # The Python SDK moved to the executor sub-sites, so its api.txt bundle is + # produced there (see Docs above); the manager owns spec + impl-spec. + with ci_lib.github_group('merge AI txt bundles'): + for section in ('spec', 'impl-spec'): + ci_lib.run( + [ + sys.executable, + 'docs/website/merge_txts.py', + out_base / 'text' / section, + out_base / 'html' / '_static' / 'ai' / f'{section}.txt', + ] + ) + return 0 + + +COMMANDS = [Docs(), DocsSrc()] diff --git a/support/ci/pipelines/docs.sh b/support/ci/pipelines/docs.sh deleted file mode 100755 index 4180e94f..00000000 --- a/support/ci/pipelines/docs.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -source "$SCRIPT_DIR/_common.sh" - -COPYRIGHT_YEAR="$(git log -1 --date=format:%Y --format=%ad)" -export COPYRIGHT_YEAR - -python3 ./docs/website/generate.py docs/website/src/impl-spec/appendix/runners-versions.json - -nix develop -i .#gen-docs --command bash ./support/ci/pipelines/src/docs.sh diff --git a/support/ci/pipelines/release.py b/support/ci/pipelines/release.py new file mode 100644 index 00000000..d7a703d5 --- /dev/null +++ b/support/ci/pipelines/release.py @@ -0,0 +1,299 @@ +import argparse +import json +import os +import re + +import ci_lib +from pipelines.tests import GithubStep, SubStep, step_nix_develop, step_run + +# Platforms are named the way nix names them everywhere: cells, flake attrs and +# published assets. +PLATFORMS = ('amd64-linux', 'arm64-linux', 'arm64-macos') + +# One cell per release asset. A platform's prepack tree already merges the +# manager with every active executor line, and `universal` holds the +# platform-independent runners β€” so nothing here has to know about executor +# lines, and a cell is a single build-and-pack. +CELLS = (*PLATFORMS, 'universal') + +RELEASE_PRESET = 'tests/presets/release.txt' + +# The generated release notes travel as their own artifact so a PR-label run +# produces them too β€” they are the one release output worth eyeballing before +# the release happens. +NOTES_ARTIFACT = 'release-notes' +NOTES_FILE = 'release-notes.md' + + +def artifact_of(cell: str) -> str: + """Artifact name a build cell uploads and a test cell downloads. + + Keyed by cell rather than minted at upload time: every leg of a matrix job + writes the *same* `outputs` key, so a per-leg name cannot be handed back as + an output. A name both sides derive from the plan needs no output at all. + """ + return f'release-build-{cell}' + + +def asset_of(cell: str) -> str: + """The tarball a cell packs β€” already named as the published release asset. + + Nothing downstream renames it: the publisher attaches what it downloads. + """ + return f'genvm-{cell}.tar.xz' + + +def _build_cell(name: str) -> dict: + step: SubStep = { + # `?submodules=1`: the executor lines are git submodules; without it the + # flake source tree has empty executors/.x dirs. + 'type': 'build-and-pack', + 'installable': f'.?submodules=1#artifact-prepack-genvm-{name}', + 'artifact_name': asset_of(name), + } + return { + 'name': name, + # Every platform cross-compiles from linux (zig does the darwin cross), + # so even arm64-macos builds here; only its *test* needs a mac. + 'runs_on': 'ubuntu-latest', + 'artifact_name': artifact_of(name), + 'job_json': json.dumps({'build': [step]}), + } + + +def build_cells() -> list[dict]: + return [_build_cell(cell) for cell in CELLS] + + +# --- test cells ---------------------------------------------------------------- +# Each cell exercises the *downloaded* artifacts: the tarballs are extracted into +# build/out, and `genvm-tool configure` writes build/info.json pointing there, so +# the runner never builds. +# +# No step goes through a shell: the replay engine execs argv directly, so +# anything a shell would have done is resolved here instead β€” the preset is read +# at plan time rather than via `$(cat ...)`, `|| true` is check=False, and +# `|| test $? -eq 124` is expect_exit_codes. + +POST_INSTALL = './build/out/bin/genvm-post-install' + +# Starting the llm module with no backends proves the binary runs and its config +# parses. It is a daemon, so it never exits on its own: timeout kills it and 124 +# is the success signal (0 would mean it died early). +SMOKE_TIMEOUT_SECONDS = 5 +SMOKE_EXIT_CODES = [0, 124] +SMOKE_MODULES = [ + './build/out/bin/genvm-modules', + 'llm', + '--allow-empty-backends', + '--config', + './build/out/config/genvm-module-llm.yaml', +] + + +def release_filter_tag() -> str: + """The release preset's tag expression, read at plan time. + + The old jobs inlined `$(cat ...)` into the shell; resolving it here keeps the + steps shell-free and puts the expression in the plan, where it is visible. + """ + return (ci_lib.ROOT_DIR / RELEASE_PRESET).read_text().strip() + + +def stable_test_steps(installable: str) -> GithubStep: + """configure + run the release preset inside `installable`'s shell. + + `configure` must come first: it writes build/info.json, which is what points + the runner at the extracted build/out tree. + """ + commands = [ + (['genvm-tool', 'configure'], 'configure'), + ( + ['genvm-tool', 'test', 'run', '--ci', '--filter-tag', release_filter_tag()], + 'stable tests', + ), + ] + return [ + step_nix_develop(installable, command, subcommand_group=group) + for command, group in commands + ] + + +def _test_cell( + name: str, + steps: GithubStep, + *, + runs_on: str, + with_nix: bool = True, + setup_python: bool = False, + runners: bool = True, +) -> dict: + return { + 'name': name, + 'runs_on': runs_on, + 'platform_artifact': artifact_of(name), + # '' means "this cell does not need the runners artifact". + 'runners_artifact': artifact_of('universal') if runners else '', + 'with_nix': with_nix, + 'setup_python': setup_python, + 'job_json': json.dumps({'test': steps}), + } + + +def _smoke_step(timeout_bin: str) -> SubStep: + return step_run( + [timeout_bin, str(SMOKE_TIMEOUT_SECONDS), *SMOKE_MODULES], + expect_exit_codes=SMOKE_EXIT_CODES, + ) + + +def _cell_amd64_linux() -> dict: + return _test_cell( + 'amd64-linux', + [ + step_run([POST_INSTALL, '--default-download=false', '--precompile=true']), + {'type': 'group-start', 'name': 'smoke test modules startup'}, + _smoke_step('timeout'), + {'type': 'group-end'}, + *stable_test_steps('.?submodules=1#mock-tests'), + ], + runs_on='ubuntu-latest', + ) + + +def _cell_arm64_linux() -> dict: + qemu_shell = '.?submodules=1#check-qemu' + qemu = 'qemu-aarch64' + return _test_cell( + 'arm64-linux', + [ + # aarch64 binaries on an amd64 host: post-install's bin-check and + # precompile steps exec them and die with Exec format error, and + # without the runners artifact the registry is absent too. Neither is + # a real failure here β€” the qemu smoke below is what proves the build. + step_run( + [POST_INSTALL, '--error-on-missing-executor=false', '--default-download=false'], + check=False, + ), + step_nix_develop( + qemu_shell, + [qemu, './build/out/bin/genvm-modules', '--version'], + subcommand_group='version under qemu', + ), + step_nix_develop( + qemu_shell, + ['timeout', str(SMOKE_TIMEOUT_SECONDS), qemu, *SMOKE_MODULES], + subcommand_group='smoke test modules startup', + expect_exit_codes=SMOKE_EXIT_CODES, + ), + ], + runs_on='ubuntu-latest', + # No stable suite: the whole tree is foreign-arch, so there is nothing to + # run it with beyond the qemu smoke above. + runners=False, + ) + + +# No nix on the mac runner, so genvm-tool comes from a venv β€” and `activate` also +# puts the venv's python3 on PATH, which calling .venv/bin/genvm-tool by path +# would not. That needs a shell, so this leg keeps one. +MACOS_STABLE_TESTS = """\ +set -euo pipefail +python3 -m venv .venv +source .venv/bin/activate +pip install ./support/tools/genvm-tool +genvm-tool configure +genvm-tool test run --ci --filter-tag "$1" +""" + + +def _cell_arm64_macos() -> dict: + return _test_cell( + 'arm64-macos', + [ + step_run(['brew', 'install', 'wabt', 'coreutils']), + step_run(['rustup', 'target', 'add', 'wasm32-wasip1']), + step_run([POST_INSTALL, '--default-download=false']), + {'type': 'group-start', 'name': 'smoke test modules startup'}, + # coreutils' gtimeout: macOS ships no timeout(1). + _smoke_step('gtimeout'), + {'type': 'group-end'}, + {'type': 'group-start', 'name': 'stable tests'}, + # The filter goes in as $1 rather than interpolated: it is an + # expression full of & | ! and must not meet the shell. + step_run(['bash', '-c', MACOS_STABLE_TESTS, 'bash', release_filter_tag()]), + {'type': 'group-end'}, + ], + runs_on='macos-latest', + with_nix=False, + setup_python=True, + ) + + +def test_cells() -> list[dict]: + return [_cell_amd64_linux(), _cell_arm64_linux(), _cell_arm64_macos()] + + +class PlanReleaseMatrix(ci_lib.Pipeline): + """Emit the release build/test matrices and the tag they are built at. + + One build cell per release artifact (runners + one per platform) and one test + cell per platform, all keyed by names both sides derive from this plan. + """ + + def name(self) -> str: + return 'plan-release-matrix' + + def add_to(self, parser: argparse.ArgumentParser) -> None: + parser.add_argument( + '--version-override', + default='', + help='release tag; defaults to `version` from .genvm-monorepo-root', + ) + + def handler(self, args: argparse.Namespace) -> int: + monorepo = json.loads((ci_lib.ROOT_DIR / '.genvm-monorepo-root').read_text()) + tag = args.version_override or monorepo['version'] + + # Shape only. Whether the tag is already taken depends on the remote and + # is the publisher's business β€” a PR-label run plans the same tag every + # time and must not care. + if not re.match(r'^v[0-9]+\.[0-9]+\.[0-9]+', tag): + ci_lib.github_error(f'invalid version {tag!r}') + return 1 + + builds = build_cells() + tests = test_cells() + + outputs = { + 'tag': tag, + 'build_matrix': json.dumps({'include': builds}), + 'test_matrix': json.dumps({'include': tests}), + 'artifact_notes': NOTES_ARTIFACT, + 'notes_file': NOTES_FILE, + } + # The artifact names are the caller's whole view of the run: it downloads + # by them and never learns the matrices produced them. Emitted here (from + # a plain job) rather than from the build matrix, whose legs would all + # write the same `outputs` key. + for cell in builds: + outputs[f'artifact_{cell["name"].replace("-", "_")}'] = cell['artifact_name'] + + out = os.environ.get('GITHUB_OUTPUT') + if not out: + ci_lib.github_error('GITHUB_OUTPUT is not set') + print(json.dumps(outputs, indent=2)) + return 1 + with open(out, 'a') as f: + for key, value in outputs.items(): + f.write(f'{key}={value}\n') + + print(f'planned release {tag}') + for cell in builds: + print(f' build {cell["name"]} -> {cell["artifact_name"]}') + for cell in tests: + print(f' test {cell["name"]} on {cell["runs_on"]}') + return 0 + + +COMMANDS = [PlanReleaseMatrix()] diff --git a/support/ci/pipelines/src/docs.sh b/support/ci/pipelines/src/docs.sh deleted file mode 100644 index 831c52a3..00000000 --- a/support/ci/pipelines/src/docs.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash - -set -ex - -pushd docs/website -poetry install --no-root -popd - -mkdir -p build/doc - -OUT_BASE="$(readlink -f build/doc)" - -poetry run -C docs/website -- sphinx-build -b html "$(readlink -f docs/website/src)" "$OUT_BASE/html" -poetry run -C docs/website -- sphinx-build -b text "$(readlink -f docs/website/src)" "$OUT_BASE/text" - -ruby ./docs/website/merge-txts.rb "$OUT_BASE/text/python-sdk" "$OUT_BASE/html/_static/ai/api.txt" -ruby ./docs/website/merge-txts.rb "$OUT_BASE/text/spec" "$OUT_BASE/html/_static/ai/spec.txt" -ruby ./docs/website/merge-txts.rb "$OUT_BASE/text/impl-spec" "$OUT_BASE/html/_static/ai/impl-spec.txt" diff --git a/support/ci/pipelines/src/test-python.sh b/support/ci/pipelines/src/test-python.sh deleted file mode 100755 index c423294a..00000000 --- a/support/ci/pipelines/src/test-python.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -set -ex - -cd ./executors/v0.3.x/runners/genlayer-py-std - -# Dependency env comes from the standalone nix flake (was `poetry install`); the -# packages under test are imported from src/ + src-emb/ via PYTHONPATH (was the -# poetry editable install). Mirrors tests/runner/genvm_tool_plugins/pytest.py. -env_dir="$(nix build --no-link --print-out-paths path:../../support/nix/py-test)" - -PYTHONPATH="$PWD/src:$PWD/src-emb${PYTHONPATH:+:$PYTHONPATH}" \ - "$env_dir/bin/pytest" diff --git a/support/ci/pipelines/src/test-rust-fuzz.sh b/support/ci/pipelines/src/test-rust-fuzz.sh deleted file mode 100755 index ffe21d14..00000000 --- a/support/ci/pipelines/src/test-rust-fuzz.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash - -export PATH="$NIX:$PATH" - -set -ex - -genvm-tool configure - -nix develop '.?submodules=1#mock-tests' --command genvm-tool test run --ci --filter-tag "$(cat tests/presets/rust-fuzz.txt)" diff --git a/support/ci/pipelines/src/test-rust.sh b/support/ci/pipelines/src/test-rust.sh deleted file mode 100755 index cfe561c5..00000000 --- a/support/ci/pipelines/src/test-rust.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash - -export PATH="$NIX:$PATH" - -set -ex - -genvm-tool configure - -ninja -v -C build all/bin - -python3 ./build/out/bin/post-install.py \ - --error-on-missing-executor=false \ - --default-download=false - -nix develop '.?submodules=1#mock-tests' --command genvm-tool test run --ci --filter-tag "$(cat tests/presets/rust.txt)" diff --git a/support/ci/pipelines/test-python.sh b/support/ci/pipelines/test-python.sh deleted file mode 100755 index 310adaf8..00000000 --- a/support/ci/pipelines/test-python.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -source "$SCRIPT_DIR/_common.sh" - -nix develop '.?submodules=1#py-test' --command bash ./support/ci/pipelines/src/test-python.sh diff --git a/support/ci/pipelines/test-rust-fuzz.sh b/support/ci/pipelines/test-rust-fuzz.sh deleted file mode 100755 index b43faa67..00000000 --- a/support/ci/pipelines/test-rust-fuzz.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash - -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -source "$SCRIPT_DIR/_common.sh" - -export ORIGINAL_PATH="$PATH" -export ORIGINAL_LD_LIBRARY_PATH="$LD_LIBRARY_PATH" - -nix develop '.?submodules=1#rust-test' --command bash ./support/ci/pipelines/src/test-rust-fuzz.sh diff --git a/support/ci/pipelines/test-rust.sh b/support/ci/pipelines/test-rust.sh deleted file mode 100755 index 9b646059..00000000 --- a/support/ci/pipelines/test-rust.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env bash - -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -source "$SCRIPT_DIR/_common.sh" - -export ORIGINAL_PATH="$PATH" -export ORIGINAL_LD_LIBRARY_PATH="$LD_LIBRARY_PATH" - -mkdir -p build/out/executor/vTEST/data - -# gvm32 (Crockford Base32) is the encoding the executor uses for runner paths; -# it is carried in `x.uid` as `id:gvm32hash`. Do NOT use `toHashFormat = "nix32"` -# here: Nix base32 has a different alphabet, so the registry names would never -# match the on-disk tars / GCS objects (e.g. nix32 `04l343…` vs gvm32 `5tnhg…`). -nix eval --verbose --impure --read-only --show-trace --json --expr \ - 'let drv = import ./executors/v0.3.x/runners { host-system = builtins.currentSystem; } ; in builtins.listToAttrs (builtins.map (x: { name = x.id; value = builtins.head (builtins.match "[^:]+:(.*)" x.uid); }) drv)' \ - > build/out/executor/vTEST/data/latest.json - -nix eval --verbose --impure --read-only --show-trace --json --expr \ - 'let drv = import ./executors/v0.3.x/runners { host-system = builtins.currentSystem; } ; in builtins.listToAttrs (builtins.map (x: { name = x.id; value = [ (builtins.head (builtins.match "[^:]+:(.*)" x.uid)) ]; }) drv)' \ - > build/out/executor/vTEST/data/all.json - -# we can't run it within nix because it uses `nix add` which sigsegvs -python3 ./support/runner-script.py \ - download \ - --nix-preload --allow-partial --dest build/out/runners --registry build/out/executor/vTEST/data/all.json - -nix build -v -L -o build/out-runners '.?submodules=1#runners-all' -mkdir -p ./build/out/runners -cp -r ./build/out-runners/. ./build/out/runners/. -chmod -R +w ./build/out/runners/. - -nix develop '.?submodules=1#rust-test' --command python3 \ - ./support/runner-script.py \ - upload \ - --root build/out/runners --registry build/out/executor/vTEST/data/all.json || true - -nix develop '.?submodules=1#rust-test' --command bash ./support/ci/pipelines/src/test-rust.sh diff --git a/support/ci/pipelines/tests.py b/support/ci/pipelines/tests.py new file mode 100644 index 00000000..588ecfaa --- /dev/null +++ b/support/ci/pipelines/tests.py @@ -0,0 +1,444 @@ +import argparse +import json +import os +import subprocess +import typing +from dataclasses import dataclass, field + +import ci_lib +from pipelines.build import build_and_pack + +type SubStep = dict[str, typing.Any] + +type GithubStep = list[SubStep] + +type MultiGithubStep = dict[str, GithubStep] + + +def merge_with_none[T]( + l: T | None, r: T | None, fn: typing.Callable[[T, T], T] +) -> T | None: + if l is None: + return r + if r is None: + return l + return fn(l, r) + + +def merge_str_bar(l: str, r: str) -> str: + return f'({l})|({r})' + + +# --- step builders ------------------------------------------------------------- +# A cell's work is a JSON list of steps that TestMatrixCell replays. The shapes +# below match the ci_lib call each step maps to, so keep them in sync with +# TestMatrixCell.handler. + + +def step_run( + command: list[str], *, check: bool = True, expect_exit_codes: list[int] | None = None +) -> SubStep: + ret = {'type': 'run', 'command': command, 'check': check} + if expect_exit_codes is not None: + ret['expect_exit_codes'] = expect_exit_codes + return ret + + +def step_nix_develop( + installable: str, + command: list[str], + *, + check: bool = True, + subcommand_group: str | None = None, + expect_exit_codes: list[int] | None = None, +) -> SubStep: + ret = { + 'type': 'nix-develop', + 'installable': installable, + 'command': command, + 'check': check, + } + if subcommand_group is not None: + ret['subcommand_group'] = subcommand_group + if expect_exit_codes is not None: + ret['expect_exit_codes'] = expect_exit_codes + return ret + + +def _check_of(step: SubStep) -> bool: + # A step asserting exact exit codes must survive a non-zero one to assert on + # it, so `expect_exit_codes` implies check=False and _finish does the judging. + if step.get('expect_exit_codes') is not None: + return False + return step.get('check', True) + + +def _finish(step: SubStep, result: subprocess.CompletedProcess) -> None: + """Enforce a step's `expect_exit_codes`, if it declared any. + + Lets a step assert an exact non-zero exit β€” e.g. a daemon smoke test killed by + `timeout` must exit 124 β€” without a shell to write `|| test $? -eq 124` in. + `check` is what tolerates *any* failure; this tolerates a named set. + """ + expected = step.get('expect_exit_codes') + if expected is None: + return + if result.returncode not in expected: + raise ValueError( + f'{step["command"]} exited {result.returncode}, expected one of {expected}' + ) + + +@dataclass +class ToolTest: + tests_name_filter_for_or: str | None + tests_tags_filter_for_or: str | None + + def merge(self, r: 'ToolTest') -> 'ToolTest': + return ToolTest( + tests_name_filter_for_or=merge_with_none( + self.tests_name_filter_for_or, r.tests_name_filter_for_or, merge_str_bar + ), + tests_tags_filter_for_or=merge_with_none( + self.tests_tags_filter_for_or, r.tests_tags_filter_for_or, merge_str_bar + ), + ) + + +@dataclass +class Stage[T]: + data: T + commands_pre: GithubStep = field(default_factory=list) + commands_post: GithubStep = field(default_factory=list) + + def merge(self, r: 'Stage[T]', fn: typing.Callable[[T, T], T]) -> 'Stage[T]': + return Stage( + commands_pre=self.commands_pre + r.commands_pre, + commands_post=self.commands_post + r.commands_post, + data=fn(self.data, r.data), + ) + + +@dataclass +class DefaultStepInfo: + configure_step: Stage[None] | None = None + build_step: Stage[frozenset[str]] | None = None + tool_step: Stage[ToolTest] | None = None + + def merge(self, r: 'DefaultStepInfo') -> 'DefaultStepInfo': + return DefaultStepInfo( + configure_step=merge_with_none( + self.configure_step, + r.configure_step, + lambda a, b: a.merge(b, lambda _l, _r: None), + ), + build_step=merge_with_none( + self.build_step, r.build_step, lambda a, b: a.merge(b, lambda l, r: l | r) + ), + tool_step=merge_with_none( + self.tool_step, r.tool_step, lambda a, b: a.merge(b, ToolTest.merge) + ), + ) + + def into_commands(self) -> MultiGithubStep: + multi_step: MultiGithubStep = { + 'configure': [], + 'build': [], + 'tool': [], + } + + def extend_step(gh_step: str, stage: Stage, body: GithubStep) -> None: + multi_step[gh_step].extend(stage.commands_pre) + multi_step[gh_step].extend(body) + multi_step[gh_step].extend(stage.commands_post) + + def group(gh_step: str, name: str, stage: Stage, body: GithubStep) -> None: + multi_step[gh_step].append({'type': 'group-start', 'name': name}) + extend_step(gh_step, stage, body) + multi_step[gh_step].append({'type': 'group-end'}) + + if (stage := self.configure_step) is not None: + extend_step( + 'configure', + stage, + [ + step_nix_develop( + '.?submodules=1#minimal', ['genvm-tool', 'configure', '--ci'] + ) + ], + ) + + if (stage := self.build_step) is not None: + multi_step['build'].extend(stage.commands_pre) + for subtarget in list(sorted(stage.data)): + multi_step['build'].append( + {'type': 'group-start', 'name': f'build {subtarget}'} + ) + multi_step['build'].append( + step_nix_develop( + '.?submodules=1#rust-test', + ['ninja', '--verbose', '-C', 'build', subtarget], + ) + ) + multi_step['build'].append({'type': 'group-end'}) + multi_step['build'].extend(stage.commands_post) + + if (stage := self.tool_step) is not None: + filters: list[str] = [] + if stage.data.tests_name_filter_for_or is not None: + filters += ['--filter-name', stage.data.tests_name_filter_for_or] + if stage.data.tests_tags_filter_for_or is not None: + filters += ['--filter-tag', stage.data.tests_tags_filter_for_or] + group( + 'tool', + f'tool {stage.data.tests_name_filter_for_or or ""}', + stage, + [ + step_nix_develop( + '.?submodules=1#mock-tests', ['genvm-tool', 'test', 'run', '--ci', *filters] + ) + ], + ) + + return multi_step + + +# in future we can shard it +TEST_RUST_EXECUTORS = DefaultStepInfo( + tool_step=Stage( + data=ToolTest( + tests_name_filter_for_or='^executors/', + tests_tags_filter_for_or='rust', + ), + ), +) + +TEST_RUST_MANAGER = DefaultStepInfo( + configure_step=Stage(None), + tool_step=Stage( + data=ToolTest( + tests_name_filter_for_or='^(?!executors/)', + tests_tags_filter_for_or='rust', + ), + ), +) + +TEST_PYTHON = DefaultStepInfo( + tool_step=Stage( + data=ToolTest( + tests_name_filter_for_or=None, + tests_tags_filter_for_or='python', + ), + ) +) + +TEST_MOCK_BUILD_BINS = DefaultStepInfo( + configure_step=Stage(None), + build_step=Stage(data=frozenset(['all/bin'])), +) + +TEST_MOCK_BUILD_RUNNERS = DefaultStepInfo( + configure_step=Stage(None), + build_step=Stage(data=frozenset(['all/runners', 'all/data'])), +) + +TEST_MOCK_BUILD_ALL = TEST_MOCK_BUILD_BINS.merge(TEST_MOCK_BUILD_RUNNERS) + +TEST_MOCK_PR = DefaultStepInfo( + tool_step=Stage( + commands_pre=[ + {'type': 'group-start', 'name': 'precompile'}, + step_run(['./build/out/bin/genvm-manager', 'check-install', '--precompile']), + {'type': 'group-end'}, + ], + data=ToolTest( + tests_name_filter_for_or=None, + tests_tags_filter_for_or='integration', + ), + ) +) + +TEST_MOCK_RELEASE = DefaultStepInfo( + tool_step=Stage( + commands_pre=[ + step_run(['./build/out/bin/genvm-manager', 'check-install', '--precompile']) + ], + data=ToolTest( + tests_name_filter_for_or=None, + tests_tags_filter_for_or='integration & stable & !bench', + ), + ) +) + + +def _cell( + name: str, + info: DefaultStepInfo, + *, + runs_on: str = 'ubuntu-latest', + disk_reclaim: bool = False, + fuzz_host: bool = False, + buildx: bool = False, + gcp: bool = False, +) -> dict: + # `name` labels the cell in the UI; `job_json` is the step list TestMatrixCell + # replays. The booleans drive test_cell.yaml's setup steps β€” every key is always + # present because matrix-tests passes each to a typed reusable-workflow input. + return { + 'name': name, + 'job_json': json.dumps(info.into_commands()), + 'runs_on': runs_on, + 'disk_reclaim': disk_reclaim, + 'fuzz_host': fuzz_host, + 'buildx': buildx, + 'gcp': gcp, + } + + +QUEUE_CELLS = [ + _cell('python', TEST_PYTHON), + _cell('rust-executors', TEST_RUST_EXECUTORS, fuzz_host=True, disk_reclaim=True), + _cell('rust-manager', TEST_RUST_MANAGER, fuzz_host=True, disk_reclaim=True), + _cell( + 'mock-pr', + TEST_MOCK_BUILD_ALL.merge(TEST_MOCK_PR), + buildx=True, + disk_reclaim=True, + fuzz_host=True, + gcp=True, + ), +] + + +class PlanQueueMatrix(ci_lib.Pipeline): + """Emit queue.yaml's test matrix as a GitHub `matrix` output. + + The heavy cells run only when a marker is set (RUN_FULL_TESTS, from the rtm / + run-full-tests labels or a manual dispatch); otherwise the matrix is empty and + matrix-tests is skipped. + """ + + def name(self) -> str: + return 'plan-queue-matrix' + + def handler(self, args: argparse.Namespace) -> int: + run_full = os.environ.get('RUN_FULL_TESTS', '').strip().lower() == 'true' + matrix = json.dumps({'include': QUEUE_CELLS if run_full else []}) + out = os.environ.get('GITHUB_OUTPUT') + if not out: + print(f'GITHUB_OUTPUT not set; matrix would be:\n{matrix}') + return 1 + with open(out, 'a') as f: + f.write(f'matrix={matrix}\n') + print(f'planned matrix (RUN_FULL_TESTS={run_full}): {matrix}') + return 0 + + +class TestMatrixCell(ci_lib.Pipeline): + """Replay one step list of a matrix cell (from plan-queue-matrix / into_commands). + + into_commands emits a list per key ('configure', 'build', 'tool'); test_cell.yaml + slices it with fromJSON and hands us one of those lists, so --job-json here is a + flat step list, not the whole mapping. The runner-side setup (disk reclaim, fuzz + host, buildx, gcp) is done by test_cell.yaml before this runs. + """ + + def name(self) -> str: + return 'test-cell' + + def add_to(self, parser: argparse.ArgumentParser) -> None: + parser.add_argument('--job-json', required=True, type=str) + + def handler(self, args: argparse.Namespace) -> int: + all_steps: GithubStep = json.loads(args.job_json) + + nix_develop_installables = set() + for step in all_steps: + if step['type'] == 'nix-develop': + nix_develop_installables.add(step['installable']) + + if len(nix_develop_installables) > 1: + all_installables = list(sorted(nix_develop_installables)) + with ci_lib.github_group('nix-develop installables pre-cache'): + for inst in all_installables: + ci_lib.nix_develop_pre_cache(inst, add_group=False) + for step in all_steps: + match step['type']: + case 'group-start': + ci_lib.github_group_start(step['name']) + case 'group-end': + ci_lib.github_group_end() + case 'run': + _finish( + step, + ci_lib.run(step['command'], check=_check_of(step)), + ) + case 'nix-develop': + _finish( + step, + ci_lib.nix_develop( + step['installable'], + step['command'], + check=_check_of(step), + subcommand_group=step.get('subcommand_group'), + ), + ) + case 'build-and-pack': + build_and_pack( + installable=step['installable'], + artifact_name=step['artifact_name'], + globs=step.get('globs'), + compression_level=step.get('compression_level', 9), + show_trace=step.get('show_trace', False), + ) + case other: + raise ValueError(f'unknown step type: {other!r}') + return 0 + + +class CheckCellKeys(ci_lib.Pipeline): + """Fail unless a cell's step mapping holds exactly the keys the caller replays. + + into_commands' keys and test_cell.yaml's "Job:" steps must stay in sync in both + directions: a key nobody replays would silently drop its steps, and a step whose + key is gone would silently do nothing. Neither fails on its own, so the workflow + passes the keys it handles and we require an exact match. + """ + + def name(self) -> str: + return 'check-cell-keys' + + def add_to(self, parser: argparse.ArgumentParser) -> None: + parser.add_argument('--job-json', required=True, type=str) + parser.add_argument('keys', nargs='+', help='keys the caller replays') + + def handler(self, args: argparse.Namespace) -> int: + multi_step = json.loads(args.job_json) + if not isinstance(multi_step, dict): + ci_lib.github_error( + f'job_json must be a mapping of key to step list, got {type(multi_step).__name__}' + ) + return 1 + + expected = set(args.keys) + actual = set(multi_step.keys()) + for key in sorted(actual - expected): + ci_lib.github_error( + f'job_json key {key!r} is replayed by no step: add one to test_cell.yaml or drop the key from into_commands' + ) + for key in sorted(expected - actual): + ci_lib.github_error( + f'no job_json key {key!r}: drop its step from test_cell.yaml or emit the key from into_commands' + ) + if actual != expected: + return 1 + + print(f'cell keys are in sync: {sorted(actual)}') + return 0 + + +COMMANDS = [ + PlanQueueMatrix(), + TestMatrixCell(), + CheckCellKeys(), +] diff --git a/support/ci/pr_branches_info.py b/support/ci/pr_branches_info.py new file mode 100644 index 00000000..d7040519 --- /dev/null +++ b/support/ci/pr_branches_info.py @@ -0,0 +1,357 @@ +#!/usr/bin/env python3 +"""Collect per-repo branch movement for a manager PR. + +A manager PR (e.g. on the v0.6 line) bundles work across several repos: the +manager itself and every active executor line it ships (v0.2, v0.3, ...), each +an `executors/v.x` submodule. This module reports, for each of those repos, +how the PR head moved relative to the base branch: + +- base_sha: where the base side sits +- head_sha: where the PR head sits +- ahead_by: commits the head is ahead of base (new work) +- behind_by: commits the head is behind base (base advanced underneath it) +- pr_url: the open PR for this repo's head_ref -> base_ref, if any (the manager PR +itself for "."; the executor mirror PR for a line) + +The manager is keyed as "." and compared branch-to-branch (base branch tip vs +PR head). Each executor line is keyed by its submodule path and compared +gitlink-to-gitlink: base_sha/head_sha are the submodule commits recorded at the +manager base tip and at the PR head, and ahead_by/behind_by are measured inside +the executor repo. A line whose gitlink is unchanged reports 0/0; use `.moved` +to test that. + +Everything is read through the `gh` API, so no manager or submodule checkout is +needed β€” this is safe to call from a `pull_request_target` workflow that never +runs PR code. The manager repo is read with a manager-scoped token; the executor +repo needs a token with access to it (the default GITHUB_TOKEN is manager-scoped +and cannot reach another repo), exactly as tools/open_executor_prs.py does. + +Two shas that differ but cannot be compared (the executor head commit was not +pushed to the executor repo, or the line is brand new / removed at one side) +leave ahead_by/behind_by as None β€” `.moved` is still meaningful. + +Env (for `from_env`): PR_NUMBER, MANAGER_REPO (or GITHUB_REPOSITORY), +EXECUTOR_REPO, MANAGER_TOKEN, EXECUTOR_TOKEN. +""" + +import dataclasses +import json +import os +import subprocess +import sys +import time + +from tools.versions import active_versions as configured_active_versions + +MANAGER_REPO_DEFAULT = 'genlayerlabs/genvm-manager' +EXECUTOR_REPO_DEFAULT = 'genlayerlabs/genvm-executor' + +MANAGER_PATH = '.' + + +@dataclasses.dataclass(frozen=True) +class RepoInfo: + """How one repo's PR head moved relative to its base branch. + + `base_sha`/`head_sha` are None when the path is absent at that side (a line + added or removed by the PR). `ahead_by`/`behind_by` are None when the two + shas differ but are not comparable (the head commit is not on the executor + remote, or one side is absent). + """ + + path: str # "." for the manager, "executors/v.x" for an executor line + repo: str # GitHub slug the shas live in + line: str | None # None for the manager, "v" for an executor line + base_ref: str # base branch the head is compared against + head_ref: str # head branch (manager PR head; `pr//` for a line) + base_sha: str | None + head_sha: str | None + ahead_by: int | None + behind_by: int | None + pr_url: str | None # open PR head_ref -> base_ref, or None if none is open + + @property + def moved(self) -> bool: + """The head points somewhere other than the base.""" + return self.head_sha != self.base_sha + + @property + def has_pr(self) -> bool: + """An open PR for this repo's head_ref -> base_ref exists.""" + return self.pr_url is not None + + @property + def rebased(self) -> bool: + """Head contains the base tip (not behind). A positive behind_by means it + needs a rebase; unknown (None) is left to the synced check.""" + return not self.behind_by + + @property + def synced(self) -> bool: + """The head gitlink is present in the executor repo (comparable to base), so + it can land together with the manager commit. Being ahead of base is normal β€” + manager and executors land almost-atomically β€” so only a head commit missing + from the executor repo (ahead_by is None) is unsynced.""" + return self.ahead_by is not None + + def as_dict(self) -> dict: + """Plain-dict form (all fields plus the derived flags) for JSON output.""" + return { + **dataclasses.asdict(self), + 'moved': self.moved, + 'has_pr': self.has_pr, + 'rebased': self.rebased, + 'synced': self.synced, + } + + +# Every `gh` call here is an idempotent read (`gh api` GET, `gh pr list`), so a +# transient failure is safe to retry. The one we actually hit is GitHub handing +# back a truncated/empty body, which gh reports as `unexpected end of JSON input` +# and which killed the whole executor-precondition gate on a single hiccup; 5xx, +# rate-limit and network blips are retried for the same reason. A terminal 4xx +# (notably the 404 that `allow_missing` callers expect for an absent line) is not +# retried, so that fast path stays fast. +_RETRY_ATTEMPTS = 4 +_RETRY_BACKOFF_S = 2.0 + +_TRANSIENT_MARKERS = ( + 'unexpected end of json input', + 'http 500', + 'http 502', + 'http 503', + 'http 504', + 'rate limit', + 'timeout', + 'timed out', + 'connection reset', + 'connection refused', + 'eof', +) + + +def _is_transient(r: subprocess.CompletedProcess) -> bool: + blob = f'{r.stdout}\n{r.stderr}'.lower() + return any(marker in blob for marker in _TRANSIENT_MARKERS) + + +def gh(*args: str, token: str, check: bool = True) -> subprocess.CompletedProcess: + """`gh` with GH_TOKEN bound to the given token (manager- or executor-scoped). + + Retries transient failures (see `_TRANSIENT_MARKERS`) a few times with linear + backoff; a terminal failure is returned as-is (or raised, when `check`), + exactly as a single `subprocess.run(check=...)` would. + """ + for attempt in range(_RETRY_ATTEMPTS): + result = subprocess.run( + ['gh', *args], + env={**os.environ, 'GH_TOKEN': token}, + check=False, + text=True, + capture_output=True, + ) + if result.returncode == 0 or not _is_transient(result): + break + if attempt + 1 < _RETRY_ATTEMPTS: + delay = _RETRY_BACKOFF_S * (attempt + 1) + print( + f'gh {" ".join(args)}: transient failure, retrying in {delay:.0f}s ' + f'(attempt {attempt + 1}/{_RETRY_ATTEMPTS})', + file=sys.stderr, + ) + time.sleep(delay) + if check and result.returncode != 0: + raise subprocess.CalledProcessError( + result.returncode, result.args, output=result.stdout, stderr=result.stderr + ) + return result + + +def _api(*args: str, token: str, allow_missing: bool = False): + """`gh api` returning parsed JSON, or None on a 404 when allow_missing.""" + r = gh('api', *args, token=token, check=not allow_missing) + if r.returncode != 0: + # allow_missing: treat any failure (typically HTTP 404 β€” a ref/path that + # does not exist on that remote) as "absent" rather than an error. + return None + return json.loads(r.stdout) + + +def compare( + repo: str, base: str, head: str, token: str, *, allow_missing: bool = False +): + """GitHub `compare` of `base...head` in `repo`. + + Returns `(base_tip_sha, ahead_by, behind_by)`, or None when the comparison is + not possible (allow_missing). `base_tip_sha` is the base ref's own commit; the + ahead/behind counts are relative to the merge base, matching git's + `rev-list --left-right base...head`. Only the counts are used, so the 250-commit + cap on the returned `commits` list is irrelevant. + """ + data = _api( + f'repos/{repo}/compare/{base}...{head}', token=token, allow_missing=allow_missing + ) + if data is None: + return None + return data['base_commit']['sha'], data['ahead_by'], data['behind_by'] + + +def submodule_sha(manager_repo: str, path: str, ref: str, token: str) -> str | None: + """The executor gitlink (submodule commit) recorded at `manager_repo`'s `ref`. + + Reads the submodule pointer through the contents API (`type: submodule`), so no + checkout is needed. None when the path is absent at that ref (a line the PR adds + or removes). + """ + data = _api( + f'repos/{manager_repo}/contents/{path}?ref={ref}', token=token, allow_missing=True + ) + if data is None: + return None + return data['sha'] + + +def existing_pr(repo: str, head: str, base: str, token: str) -> str | None: + """URL of the open PR `head -> base` in `repo`, or None if none is open.""" + url = gh( + 'pr', + 'list', + '--repo', + repo, + '--head', + head, + '--base', + base, + '--state', + 'open', + '--json', + 'url', + '--jq', + '.[0].url // ""', + token=token, + ).stdout.strip() + return url or None + + +def manager_info(pr_number: str, manager_repo: str, manager_token: str) -> RepoInfo: + """RepoInfo for the manager repo itself (base branch tip vs PR head). + + The manager PR always exists (it is the one being inspected), so its `pr_url` + is taken straight from the PR object rather than queried. + """ + pr = _api(f'repos/{manager_repo}/pulls/{pr_number}', token=manager_token) + base_ref = pr['base']['ref'] + head_ref = pr['head']['ref'] + head_sha = pr['head']['sha'] + base_sha, ahead_by, behind_by = compare( + manager_repo, base_ref, head_sha, manager_token + ) + return RepoInfo( + path=MANAGER_PATH, + repo=manager_repo, + line=None, + base_ref=base_ref, + head_ref=head_ref, + base_sha=base_sha, + head_sha=head_sha, + ahead_by=ahead_by, + behind_by=behind_by, + pr_url=pr['html_url'], + ) + + +def executor_info( + line: str, + manager_repo: str, + manager_base_sha: str, + manager_head_sha: str, + manager_head_ref: str, + executor_repo: str, + manager_token: str, + executor_token: str, +) -> RepoInfo: + """RepoInfo for one executor line: its gitlink at the manager base vs head. + + The executor mirror branch is deterministically named `pr//` off + the manager PR's head ref (genvm_tool.common.Repo.feature_branch), so its + `pr_url` is the open PR of that branch into `-dev`, if any. + """ + path = f'executors/{line}.x' + base_ref = f'{line}-dev' + head_ref = f'pr/{line}/{manager_head_ref}' + base_sha = submodule_sha(manager_repo, path, manager_base_sha, manager_token) + head_sha = submodule_sha(manager_repo, path, manager_head_sha, manager_token) + + if base_sha is not None and head_sha is not None: + if base_sha == head_sha: + ahead_by, behind_by = 0, 0 # gitlink unchanged; no need to ask the executor + else: + # The gitlink moved. Ask the executor repo how far. allow_missing: the + # head gitlink may not be pushed to the executor yet (a PR that bumped the + # submodule pointer without pushing the executor side). + cmp = compare( + executor_repo, base_sha, head_sha, executor_token, allow_missing=True + ) + ahead_by, behind_by = (None, None) if cmp is None else (cmp[1], cmp[2]) + else: + # Line added (no base) or removed (no head): moved, but nothing to compare. + ahead_by, behind_by = None, None + + return RepoInfo( + path=path, + repo=executor_repo, + line=line, + base_ref=base_ref, + head_ref=head_ref, + base_sha=base_sha, + head_sha=head_sha, + ahead_by=ahead_by, + behind_by=behind_by, + pr_url=existing_pr(executor_repo, head_ref, base_ref, executor_token), + ) + + +def collect( + pr_number: str, + *, + manager_repo: str = MANAGER_REPO_DEFAULT, + executor_repo: str = EXECUTOR_REPO_DEFAULT, + manager_token: str, + executor_token: str, +) -> dict[str, RepoInfo]: + """Branch movement for every repo a manager PR touches, keyed by path. + + "." is the manager; "executors/v.x" is each active executor line (from + .genvm-monorepo-root via tools.versions). + """ + manager = manager_info(pr_number, manager_repo, manager_token) + assert manager.base_sha is not None # manager base branch always exists + infos: dict[str, RepoInfo] = {manager.path: manager} + for version in configured_active_versions(): + info = executor_info( + f'v{version}', + manager_repo, + manager.base_sha, + manager.head_sha, + manager.head_ref, + executor_repo, + manager_token, + executor_token, + ) + infos[info.path] = info + return infos + + +def from_env(pr_number: str | None = None) -> dict[str, RepoInfo]: + """`collect` with arguments read from the workflow environment. + + `pr_number` overrides $PR_NUMBER when given (e.g. from a CLI argument). + """ + return collect( + pr_number or os.environ['PR_NUMBER'], + manager_repo=os.environ.get('MANAGER_REPO') + or os.environ.get('GITHUB_REPOSITORY', MANAGER_REPO_DEFAULT), + executor_repo=os.environ.get('EXECUTOR_REPO', EXECUTOR_REPO_DEFAULT), + manager_token=os.environ['MANAGER_TOKEN'], + executor_token=os.environ['EXECUTOR_TOKEN'], + ) diff --git a/support/ci/provision-branches.sh b/support/ci/provision-branches.sh deleted file mode 100755 index 1f14bd8e..00000000 --- a/support/ci/provision-branches.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env bash - -# Provision the branch model for every active version train. -# -# For each version X in .genvm-monorepo-root's `active-versions`: -# 1. ensure the version branch v.x exists (created from main if absent) -# 2. ensure the dev branch v-dev exists (created from v.x if absent) -# 3. ensure a standing release-gate PR v-dev -> v.x is open -# -# Branch creation pushes over SSH using the GENVM_CI_PRIVATE_KEY deploy -# key (the checkout step wires it up), so it works even though the -# version/dev branches are protected. PR creation uses GH_TOKEN. -# -# Idempotent: re-running only creates what is missing. - -set -x -euo pipefail - -git fetch origin --prune - -remote_has() { - git show-ref --verify --quiet "refs/remotes/origin/$1" -} - -versions="$(python3 support/ci/branch-versions.py list)" - -# Collect every missing branch as a refspec and create them all in one -# atomic push, so we never leave a train half-provisioned. -pushrefs=() - -for V in $versions; do - VER="v${V}.x" - DEV="v${V}-dev" - - if ! remote_has "$VER"; then - echo "Will create version branch ${VER} from main" - ver_commit="$(git rev-parse origin/main)" - pushrefs+=("origin/main:refs/heads/${VER}") - else - echo "Version branch ${VER} already exists" - ver_commit="$(git rev-parse "origin/${VER}")" - fi - - if ! remote_has "$DEV"; then - # Seed the dev branch one (empty) commit ahead of the version branch: - # a fresh ${DEV} identical to ${VER} has no diff, so the standing - # release-gate PR below would fail with "No commits between". - echo "Will create dev branch ${DEV} from ${VER} (with seed commit)" - seed="$(GIT_AUTHOR_NAME='github-actions[bot]' \ - GIT_AUTHOR_EMAIL='41898282+github-actions[bot]@users.noreply.github.com' \ - GIT_COMMITTER_NAME='github-actions[bot]' \ - GIT_COMMITTER_EMAIL='41898282+github-actions[bot]@users.noreply.github.com' \ - git commit-tree "${ver_commit}^{tree}" -p "${ver_commit}" \ - -m "chore(ci): seed ${DEV} release-gate branch 🌱")" - pushrefs+=("${seed}:refs/heads/${DEV}") - else - echo "Dev branch ${DEV} already exists" - fi -done - -if [ "${#pushrefs[@]}" -gt 0 ]; then - echo "Creating ${#pushrefs[@]} branch(es) in one atomic push" - git push --atomic origin "${pushrefs[@]}" - git fetch origin --prune -else - echo "All branches already exist; nothing to push" -fi - -for V in $versions; do - VER="v${V}.x" - DEV="v${V}-dev" - - existing="$(gh pr list --repo "$GITHUB_REPOSITORY" --base "$VER" --head "$DEV" --state open --json number --jq '.[0].number // ""')" - if [ -z "$existing" ]; then - echo "Opening standing release-gate PR ${DEV} -> ${VER}" - gh pr create --repo "$GITHUB_REPOSITORY" --base "$VER" --head "$DEV" \ - --title "Release gate: ${DEV} β†’ ${VER}" \ - --body "Standing release-gate PR. \`${DEV}\` accumulates incremental v${V} work; this PR merges into the release-ready \`${VER}\` branch only once the cross-repo E2E pipeline is green. Comment \`/run-e2e v${V}\` to fire it (genlayerlabs/genlayer-e2e). It may stay red while the v${V} train is in progress." - else - echo "Standing PR ${DEV} -> ${VER} already open (#${existing})" - fi -done diff --git a/support/ci/run.sh b/support/ci/run.sh new file mode 100755 index 00000000..5868c4c2 --- /dev/null +++ b/support/ci/run.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +export PYTHONPATH="$SCRIPT_DIR/:$PYTHONPATH" + +exec python3 -B "$SCRIPT_DIR/__main__.py" "$@" diff --git a/support/ci/tools/__init__.py b/support/ci/tools/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/support/ci/tools/branches.py b/support/ci/tools/branches.py new file mode 100644 index 00000000..ebbceeab --- /dev/null +++ b/support/ci/tools/branches.py @@ -0,0 +1,118 @@ +import argparse +import os +import subprocess + +import ci_lib + +from tools.versions import active_versions + + +def gh(*args: str, check: bool = True) -> subprocess.CompletedProcess: + return ci_lib.run(['gh', *args], check=check, capture_output=True) + + +class ProvisionBranches(ci_lib.Tool): + """Provision version/dev branches and standing release-gate PRs.""" + + def name(self) -> str: + return 'provision-branches' + + def remote_has(self, branch: str) -> bool: + return ( + ci_lib.run( + ['git', 'show-ref', '--verify', '--quiet', f'refs/remotes/origin/{branch}'], + check=False, + ).returncode + == 0 + ) + + def handler(self, args: argparse.Namespace) -> int: + ci_lib.run(['git', 'fetch', 'origin', '--prune']) + + pushrefs: list[str] = [] + for version in active_versions(): + version_branch = f'v{version}.x' + dev_branch = f'v{version}-dev' + + if not self.remote_has(version_branch): + print(f'Will create version branch {version_branch} from main') + version_commit = ci_lib.output(['git', 'rev-parse', 'origin/main']).strip() + pushrefs.append(f'origin/main:refs/heads/{version_branch}') + else: + print(f'Version branch {version_branch} already exists') + version_commit = ci_lib.output( + ['git', 'rev-parse', f'origin/{version_branch}'] + ).strip() + + if not self.remote_has(dev_branch): + print( + f'Will create dev branch {dev_branch} from {version_branch} (with seed commit)' + ) + seed = ci_lib.output( + [ + 'git', + 'commit-tree', + f'{version_commit}^{{tree}}', + '-p', + version_commit, + '-m', + f'chore(ci): seed {dev_branch} release-gate branch 🌱', + ], + cwd=ci_lib.ROOT_DIR, + ).strip() + pushrefs.append(f'{seed}:refs/heads/{dev_branch}') + else: + print(f'Dev branch {dev_branch} already exists') + + if pushrefs: + print(f'Creating {len(pushrefs)} branch(es) in one atomic push') + ci_lib.run(['git', 'push', '--atomic', 'origin', *pushrefs]) + ci_lib.run(['git', 'fetch', 'origin', '--prune']) + else: + print('All branches already exist; nothing to push') + + repo = os.environ['GITHUB_REPOSITORY'] + for version in active_versions(): + version_branch = f'v{version}.x' + dev_branch = f'v{version}-dev' + existing = gh( + 'pr', + 'list', + '--repo', + repo, + '--base', + version_branch, + '--head', + dev_branch, + '--state', + 'open', + '--json', + 'number', + '--jq', + '.[0].number // ""', + ).stdout.strip() + if existing: + print( + f'Standing PR {dev_branch} -> {version_branch} already open (#{existing})' + ) + continue + + print(f'Opening standing release-gate PR {dev_branch} -> {version_branch}') + gh( + 'pr', + 'create', + '--repo', + repo, + '--base', + version_branch, + '--head', + dev_branch, + '--title', + f'Release gate: {dev_branch} -> {version_branch}', + '--body', + f'Standing release-gate PR. `{dev_branch}` accumulates incremental v{version} work; this PR merges into the release-ready `{version_branch}` branch only once the cross-repo E2E pipeline is green. Comment `/run-e2e v{version}` to fire it (genlayerlabs/genlayer-e2e). It may stay red while the v{version} train is in progress.', + ) + return 0 + + +COMMANDS = [ProvisionBranches()] diff --git a/support/ci/tools/genvm_merge_into_dev.py b/support/ci/tools/genvm_merge_into_dev.py new file mode 100755 index 00000000..9a8a092f --- /dev/null +++ b/support/ci/tools/genvm_merge_into_dev.py @@ -0,0 +1,585 @@ +#!/usr/bin/env python3 +"""Gate and perform a Merge of a PR into a v-dev branch. + +Invoked by the reusable .github/workflows/branch_merge_into_dev.yaml when a +maintainer ticks the "Merge" box on the PR action panel. It re-checks every +gate against the EXACT PR head commit and then advances the dev branch by a +plain (fast-forward-only) push, so what lands is byte-identical to what CI +and E2E validated. + +Gates (all required, all on the head commit): +1. base branch is a v-dev branch +2. PR carries the `rtm` (ready-to-merge) label +3. full GenVM CI (queue.yaml) concluded success +4. the cross-repo E2E check concluded success +5. the PR is 0 commits behind base + +Strategy: every repo ends up with ONE commit per merged PR. The manager's +commits are squashed onto base (a single commit that needs no gitlink rewrite is +fast-forwarded as-is, preserving its SHA); each executor line's commits are +squashed onto its own dev branch. The PR is closed afterwards. + +Executor mirror: the manager release line (e.g. v0.6) is independent of the +executor lines it ships (v0.2, v0.3), so a single manager commit carries a +gitlink into EACH active line's submodule. Every active line is squashed onto +its own `-dev` branch (`v0.2-dev`, `v0.3-dev`, ...). Squashing gives the +line a NEW commit, so the sha the PR pinned never lands β€” the manager commit is +rewritten to gitlink the squashed commit instead (`update-index --cacheinfo`). +Lines whose gitlink is unchanged, or that already sit at a single commit on +their dev tip, are left alone. + +True cross-repo atomicity is impossible, so we (1) plan and fast-forward-check +all sides up front, then (2) push the executors first β€” they are the +dependencies the manager commit points at β€” and (3) push the manager. If the +manager push then loses a race the executors are left one fast-forward ahead, +which is harmless (monotonic) and reconciled when Merge is re-ticked. + +Squashing collapses several authors into one, so `Co-authored-by:` trailers are +added for every other human author in the range (never tool/AI attribution β€” the +commit-message hook rejects that). + +After a successful merge each line's executor mirror branch +(`pr//`, opened by branch_executor_prs.yaml) is deleted β€” its +tip now lives in the executor's `-dev`, so the branch and its PR are +redundant. + +(The dev -> version release-gate merge is a separate, not-yet-automated +flow; on that one the executor's version branch is fast-forwarded and its +dev/version branches are kept at the same point.) + +Talks to GitHub through the `gh` CLI and moves refs through `git`; it does +not exec arbitrary PR code, though the workflow checks out this script itself +from the PR head (gated on the `ci-safe` label). The dev branches are +protected, so the workflow checks out with the GENVM_CI_PRIVATE_KEY deploy key +and pushes non-force +(a base that advanced is safely rejected). The active executor submodules +must be checked out with a remote `origin` that can push the executor repo +(the workflow wires its own deploy key); the active lines come from +.genvm-monorepo-root via tools.versions. + +Env: GITHUB_REPOSITORY, PR_NUMBER, GH_TOKEN, E2E_CHECK_PATTERN. +""" + +import json +import os +import re +import subprocess +import sys + +from tools.versions import active_versions as configured_active_versions + +REPO = os.environ['GITHUB_REPOSITORY'] +PR = os.environ['PR_NUMBER'] +E2E_PATTERN = os.environ.get('E2E_CHECK_PATTERN', 'e2e') + + +def run(*args, check=True, env=None): + full_env = None + if env is not None: + full_env = os.environ.copy() + full_env.update(env) + return subprocess.run(args, check=check, text=True, capture_output=True, env=full_env) + + +def gh(*args): + return run('gh', *args).stdout + + +def git(*args, check=True): + return run('git', *args, check=check) + + +def egit(submodule, *args, check=True, env=None): + """git inside an executor submodule checkout.""" + return run('git', '-C', submodule, *args, check=check, env=env) + + +def is_ancestor(repo, maybe_ancestor, descendant): + args = ('merge-base', '--is-ancestor', maybe_ancestor, descendant) + if repo is None: + return git(*args, check=False).returncode == 0 + return egit(repo, *args, check=False).returncode == 0 + + +# Identities the commit-message hook rejects as AI/tool attribution +# (support/scripts/check-commit-message.py AI_PATTERNS). A squash must not +# resurrect one as a Co-authored-by trailer β€” the hook would then reject the +# message it produced, and nothing re-validates a generated squash message. +AI_IDENTITY_RE = re.compile( + r'claude|gpt|copilot|anthropic|openai|\bai\b|noreply@anthropic', re.IGNORECASE +) + + +def commit_author(repo, sha): + """(name, email, author-date) of `sha`. NUL-separated so a name containing + `<` or spaces round-trips.""" + out = ( + egit(repo, 'log', '-1', '--format=%an%x00%ae%x00%aI', sha) + if repo is not None + else git('log', '-1', '--format=%an%x00%ae%x00%aI', sha) + ).stdout.rstrip('\n') + name, email, date = out.split('\x00') + return name, email, date + + +def range_authors(repo, rev_range): + """Distinct (name, email) commit authors in `rev_range`, first-seen order.""" + args = ('log', '--format=%an%x00%ae', rev_range) + out = (git(*args) if repo is None else egit(repo, *args)).stdout + + authors = [] + for line in out.split('\n'): + if '\x00' not in line: + continue + name, _, email = line.partition('\x00') + if (name, email) not in authors: + authors.append((name, email)) + return authors + + +def coauthor_trailers(repo, rev_range, exclude): + """`Co-authored-by:` for every human author in the range except `exclude`. + + A squash keeps only one author, so everyone else who wrote a commit in the + range would otherwise lose attribution. Authors whose identity the + commit-message hook rejects as AI/tool attribution are dropped β€” otherwise the + generated message would fail that hook, and nothing re-checks it. + """ + trailers = [] + for name, email in range_authors(repo, rev_range): + if (name, email) == exclude: + continue + ident = f'{name} <{email}>' + if AI_IDENTITY_RE.search(ident): + print(f'dropping AI/tool co-author {ident}') + continue + trailers.append(f'Co-authored-by: {ident}') + return trailers + + +def active_lines(): + """Active executor lines as `v` tags (e.g. ['v0.2', 'v0.3']). The manager + release line is independent of these; a manager PR ships every active line.""" + return [f'v{v}' for v in configured_active_versions()] + + +def block(msg): + """Comment the reason on the PR and fail the job.""" + run( + 'gh', + 'pr', + 'comment', + PR, + '--repo', + REPO, + '--body', + f'❌ Merge blocked: {msg}', + check=False, + ) + sys.exit(1) + + +def pr_view(*fields): + out = gh('pr', 'view', PR, '--repo', REPO, '--json', ','.join(fields)) + return json.loads(out) + + +def check_gates(pr): + if pr['state'] != 'OPEN': + block(f'PR is not open (state: {pr["state"]}).') + if pr['isDraft']: + block('PR is a draft.') + + base = pr['baseRefName'] + if not re.fullmatch(r'v.*-dev', base): + block(f'base branch `{base}` is not a `v-dev` branch.') + + # 2. rtm label + if not any(label['name'] == 'rtm' for label in pr['labels']): + block('missing the `rtm` (ready-to-merge) label.') + + head_sha = pr['headRefOid'] + + # 3. full GenVM CI (queue.yaml) green on the head commit. The same head may + # carry skipped runs (label events that didn't run full tests) alongside the + # real one, and the run may have been started by either a push + # (event=pull_request) or the action panel (event=workflow_dispatch) β€” so we + # query all events and require ANY completed run on this exact commit to have + # succeeded, not just the most recent. + runs = json.loads( + gh( + 'api', + f'repos/{REPO}/actions/workflows/queue.yaml/runs?head_sha={head_sha}', + ) + )['workflow_runs'] + if not any(r['status'] == 'completed' and r['conclusion'] == 'success' for r in runs): + got = ', '.join(f'{r["status"]}/{r["conclusion"]}' for r in runs) or 'no run' + block(f'GenVM CI (queue.yaml) is not green on `{head_sha}` (runs: {got}).') + + # 4. cross-repo E2E check green on the head commit + checks = json.loads( + gh( + 'api', + f'repos/{REPO}/commits/{head_sha}/check-runs?per_page=100', + ) + )['check_runs'] + e2e = [c for c in checks if re.search(E2E_PATTERN, c['name'], re.I)] + if not e2e: + block(f'no E2E check found on `{head_sha}` (run it on this PR first).') + bad = [c for c in e2e if c['conclusion'] != 'success'] + if bad: + conclusions = ' '.join(c['conclusion'] or 'pending' for c in e2e) + block(f'E2E is not green on `{head_sha}` (conclusions: {conclusions}).') + + return base, head_sha + + +def delete_executor_mirror_branch(submodule, line, head_ref): + """Delete the executor mirror branch `pr//` after a merge. + + The manager PR's work for this line lived on that namespaced branch; its tip is + now contained in the executor's `-dev` (we just fast-forwarded it there), + so the branch β€” and the auto-opened executor PR it backed (branch_executor_prs.yaml) + β€” are redundant. Deleting the branch also closes that PR. Best-effort: a failure + here never unwinds an otherwise-complete merge. The branch is unprotected, so the + executor deploy key can delete it. + """ + branch = f'pr/{line}/{head_ref}' + r = egit(submodule, 'push', 'origin', '--delete', branch, check=False) + if r.returncode == 0: + print(f'deleted executor mirror branch {branch}') + else: + print(f'note: could not delete executor mirror branch {branch}: {r.stderr.strip()}') + + +def executor_gitlink(commit, submodule): + """Executor submodule commit (gitlink) recorded at `commit`'s tree.""" + out = git('ls-tree', commit, submodule).stdout + # "160000 commit \t" + m = re.match(r'\S+\s+commit\s+([0-9a-f]+)\t', out) + if m is None: + block( + f'could not resolve the executor submodule pointer ({submodule}) at `{commit}`.' + ) + else: + return m.group(1) + + +# Provenance trailer on a squashed executor commit: the PR-side commit whose +# content it carries. A squash keeps only the TREE of that commit, so the sha the +# manager PR pins never becomes an ancestor of the dev branch β€” ancestry alone can +# no longer answer "did this line already land?". Re-ticking Merge after a failed +# manager push (the executors are pushed first) would otherwise see a tip that is +# neither ancestor nor descendant of the pinned sha and block the PR as diverged, +# forever. This trailer is how the second run recognises its own work. +SQUASHED_FROM = 'GenVM-Squashed-From' + + +def build_messages(pr): + """Return `(manager_message, executor_message)` for this PR. + + The body is the PR body with CRLF normalized (GitHub returns CRLF, and + `commit-tree` β€” unlike `git commit` β€” does no cleanup, so without this the + executor and manager commits for one PR would differ byte-for-byte) and with + any pre-existing provenance-trailer line stripped, so a body that quotes + `GenVM-Squashed-From:` (a revert PR quoting what it reverts) cannot poison the + re-tick lookup. The executor commit lands in another repo, so its PR reference + is fully qualified; the manager's bare `#N` resolves in-repo. + """ + body = (pr['body'] or '').replace('\r\n', '\n').replace('\r', '\n') + body = '\n'.join( + ln for ln in body.split('\n') if not ln.strip().startswith(f'{SQUASHED_FROM}:') + ).strip() + title = pr['title'] + tail = f'\n\n{body}\n' if body else '\n' + return f'{title} (#{PR}){tail}', f'{title} ({REPO}#{PR}){tail}' + + +def already_squashed(submodule, exec_base, exec_sha): + """The commit on `` that already carries `exec_sha`'s content, if any.""" + out = egit( + submodule, + 'log', + f'refs/remotes/origin/{exec_base}', + f'--grep=^{SQUASHED_FROM}: {exec_sha}$', + '--format=%H', + '-1', + ).stdout.strip() + return out or None + + +def plan_executor_line(submodule, exec_base, exec_sha, message): + """Work out what this line should land on its `-dev`. + + Returns `(sha, needs_push)`. The PR's commits on this line are squashed into + one commit on top of the line's dev tip, so the executor history mirrors the + manager's: one commit per merged PR. `sha` is that squashed commit β€” or + `exec_sha` itself when there is nothing to squash (a single commit already + sitting on the tip, or a branch that has to be created). + + Blocks if exec_sha is unknown to the executor repo or the branch has diverged. + """ + # Bring every executor head (and thus exec_sha, if pushed) local. + egit(submodule, 'fetch', '--no-tags', 'origin', '+refs/heads/*:refs/remotes/origin/*') + + if ( + egit(submodule, 'cat-file', '-e', f'{exec_sha}^{{commit}}', check=False).returncode + != 0 + ): + block( + f'executor commit `{exec_sha}` ({submodule}) is not in the executor repo; ' + f'push the executor side of this change first, then re-tick Merge.' + ) + + ref = f'refs/remotes/origin/{exec_base}' + if ( + egit(submodule, 'rev-parse', '--verify', '--quiet', ref, check=False).returncode + != 0 + ): + return exec_sha, True # branch absent -> push creates it + + tip = egit(submodule, 'rev-parse', ref).stdout.strip() + if tip == exec_sha: + return exec_sha, False # already there + + landed = already_squashed(submodule, exec_base, exec_sha) + if landed is not None: + # A previous run squashed this line and pushed it; only the manager push + # is left. Reuse that commit rather than squashing a second one. + print(f'Executor {exec_base} already carries {exec_sha[:12]} as {landed[:12]}') + return landed, False + + if is_ancestor(submodule, exec_sha, tip): + # Executor moved ahead of what this PR pins; the pinned commit is + # already contained, so there is nothing to push. + return exec_sha, False + + tree = egit(submodule, 'rev-parse', f'{exec_sha}^{{tree}}').stdout.strip() + if not is_ancestor(submodule, tip, exec_sha): + # A maintainer may have merged the auto-opened executor PR by hand: the + # tree is already on the tip under a different sha and without our trailer. + # Same content already landed, so mirror the tip rather than block. + if egit(submodule, 'rev-parse', f'{tip}^{{tree}}').stdout.strip() == tree: + print( + f'Executor {exec_base} already carries {exec_sha[:12]} as its tip {tip[:12]}' + ) + return tip, False + block( + f'executor `{exec_base}` (`{tip[:12]}`) has diverged from the PR executor ' + f'commit `{exec_sha[:12]}`; mirror is not a fast-forward.' + ) + + commits = egit(submodule, 'rev-list', f'{tip}..{exec_sha}').stdout.split() + if len(commits) == 1: + # Already a single commit on top of the tip: squashing it would only + # rewrite it into an identical tree under a new sha. + return exec_sha, True + + name, email, date = commit_author(submodule, exec_sha) + + trailers = coauthor_trailers(submodule, f'{tip}..{exec_sha}', (name, email)) + # The provenance trailer must be there for a re-tick to recognise this commit. + trailers.append(f'{SQUASHED_FROM}: {exec_sha}') + full_message = message + '\n' + '\n'.join(trailers) + '\n' + + # A squash is just the PR's tree on top of the tip; commit-tree builds it + # without touching the working tree or index. + squashed = egit( + submodule, + 'commit-tree', + tree, + '-p', + tip, + '-m', + full_message, + env={ + 'GIT_AUTHOR_NAME': name, + 'GIT_AUTHOR_EMAIL': email, + 'GIT_AUTHOR_DATE': date, + 'GIT_COMMITTER_NAME': 'genvm-ci', + 'GIT_COMMITTER_EMAIL': 'genvm-ci@genlayer.com', + }, + ).stdout.strip() + print(f'Squashed {len(commits)} executor commits ({exec_base}) into {squashed}') + return squashed, True + + +def plan_executor_lines(head_sha, message): + """Plan every active line off the PR head's gitlinks. + + Returns a list of `(submodule, line, exec_base, pinned_sha, landed_sha, + needs_push)`. `landed_sha` differs from `pinned_sha` when the line's commits + were squashed β€” the manager commit must then be rewritten to gitlink the + squashed commit, since the one the PR pinned never lands on the dev branch. + """ + plans = [] + for line in active_lines(): + submodule = f'executors/{line}.x' + exec_base = f'{line}-dev' + pinned = executor_gitlink(head_sha, submodule) + landed, needs_push = plan_executor_line(submodule, exec_base, pinned, message) + plans.append((submodule, line, exec_base, pinned, landed, needs_push)) + return plans + + +def push_executor_lines(plans): + """Push each line's landed commit onto its `-dev`, BEFORE the manager + push (the manager commit gitlinks into them, so the dependencies land first).""" + for submodule, _line, exec_base, _pinned, landed, needs_push in plans: + if not needs_push: + print(f'Executor {exec_base} already contains {landed}; nothing to mirror') + continue + print(f'Mirroring executor {exec_base} -> {landed}') + if ( + egit( + submodule, 'push', 'origin', f'{landed}:refs/heads/{exec_base}', check=False + ).returncode + != 0 + ): + block( + f'fast-forward push of executor `{exec_base}` was rejected (it advanced); ' + f're-tick Merge.' + ) + + +def merge(pr, base, head_sha): + # Fetch the exact head commit (works for fork PRs too) and live base tip. + git( + 'fetch', + '--no-tags', + 'origin', + f'refs/pull/{PR}/head:refs/prhead', + f'+refs/heads/{base}:refs/remotes/origin/{base}', + ) + + fetched = git('rev-parse', 'refs/prhead').stdout.strip() + if fetched != head_sha: + block( + f'head moved during merge (expected `{head_sha}`, got `{fetched}`); re-tick Merge.' + ) + + # 5. authoritative 0-commits-behind check at merge time. + if not is_ancestor(None, f'origin/{base}', 'refs/prhead'): + behind = git('rev-list', '--count', f'refs/prhead..origin/{base}').stdout.strip() + block( + f'PR is {behind} commit(s) behind `{base}`; update the branch and re-tick Merge.' + ) + + git('config', 'user.name', 'genvm-ci') + git('config', 'user.email', 'genvm-ci@genlayer.com') + + manager_message, executor_message = build_messages(pr) + + # Plan every active executor line first: each line's commits are squashed into + # one commit on its `-dev`, so a line that was squashed lands under a + # NEW sha and the manager commit has to gitlink that one instead of the sha the + # PR pinned (which never lands on the dev branch). + plans = plan_executor_lines(head_sha, executor_message) + + # Refuse to move any line's gitlink backward. The 0-behind gate is on manager + # commits, not the gitlink, so a PR can be current on commits yet stale on the + # pointer (another PR advanced the line after this one branched); landing it + # would silently revert that PR's executor work. + for submodule, _line, exec_base, _pinned, landed, _push in plans: + base_link = executor_gitlink(f'origin/{base}', submodule) + if landed != base_link and is_ancestor(submodule, landed, base_link): + block( + f'merging would move the `{submodule}` gitlink backward on `{base}` ' + f'(base pins `{base_link[:12]}`, this PR would set `{landed[:12]}`); ' + f'another change advanced it β€” update the PR and re-tick Merge.' + ) + + rewritten = { + submodule: landed + for submodule, _line, _base, pinned, landed, _push in plans + if landed != pinned + } + + if len(pr['commits']) == 1 and not rewritten: + print(f'Single commit: fast-forwarding {base} to {head_sha}') + push_sha = head_sha + else: + if rewritten: + print( + f'Executor lines were squashed; rewriting the manager gitlinks: {rewritten}' + ) + print(f'Squashing {len(pr["commits"])} commits onto {base}') + name, email, date = commit_author(None, head_sha) + git('checkout', '-B', '_merge', f'origin/{base}') + git('merge', '--squash', head_sha) + + # Repoint each squashed line at the commit that actually lands. The index + # holds the PR's gitlinks after `merge --squash`; overwrite them in place + # (no submodule checkout involved). + for submodule, landed in rewritten.items(): + git('update-index', '--cacheinfo', f'160000,{landed},{submodule}') + + trailers = coauthor_trailers(None, f'origin/{base}..{head_sha}', (name, email)) + full_message = manager_message + if trailers: + full_message += '\n' + '\n'.join(trailers) + '\n' + + git( + 'commit', + '--author', + f'{name} <{email}>', + '--date', + date, + '-m', + full_message, + ) + push_sha = git('rev-parse', 'HEAD').stdout.strip() + + # Push the executors first: the manager commit gitlinks into them, so the + # dependencies must land before the referrer. + push_executor_lines(plans) + pinned = [(submodule, line) for submodule, line, _b, _p, _l, _push in plans] + + # Non-force FF push; rejected if base advanced since the checks. The + # executors are already advanced at this point; a rejection here leaves them + # one fast-forward ahead (harmless, monotonic) and re-ticking reconciles. + if ( + git('push', 'origin', f'{push_sha}:refs/heads/{base}', check=False).returncode != 0 + ): + block( + f'fast-forward push to manager `{base}` was rejected (base advanced) β€” the ' + f'executor lines were already fast-forwarded; re-tick Merge to land the ' + f'manager side.' + ) + + run( + 'gh', + 'pr', + 'comment', + PR, + '--repo', + REPO, + '--body', + f'βœ… Merged into `{base}` (`{push_sha}`) via fast-forward.', + check=False, + ) + run('gh', 'pr', 'close', PR, '--repo', REPO, check=False) + + # Clean up the now-redundant executor mirror branches (and the PRs they backed). + for submodule, line in pinned: + delete_executor_mirror_branch(submodule, line, pr['headRefName']) + + +def main(): + pr = pr_view( + 'baseRefName', + 'headRefName', + 'headRefOid', + 'commits', + 'labels', + 'state', + 'isDraft', + 'title', + 'body', + ) + base, head_sha = check_gates(pr) + merge(pr, base, head_sha) + + +if __name__ == '__main__': + main() diff --git a/support/ci/make-relase-notes.py b/support/ci/tools/make_release_notes.py similarity index 93% rename from support/ci/make-relase-notes.py rename to support/ci/tools/make_release_notes.py index 2f6a56a1..31281bbd 100755 --- a/support/ci/make-relase-notes.py +++ b/support/ci/tools/make_release_notes.py @@ -8,6 +8,7 @@ import unicodedata from collections import defaultdict from dataclasses import dataclass +from pathlib import Path def strip_trailing_emoji(text: str) -> str: @@ -58,7 +59,7 @@ class CommitEntry: email: str -def parse_git_log(rev_range: str) -> list[CommitEntry]: +def parse_git_log(rev_range: str, *, dir: Path | None = None) -> list[CommitEntry]: """Run git log and return parsed commit entries. Body lines that look like ``* type: description`` are promoted to top-level @@ -67,11 +68,14 @@ def parse_git_log(rev_range: str) -> list[CommitEntry]: squash/PR umbrella title and dropped β€” the bullets already carry the real changes, so keeping the subject would double-count them. """ + if dir is None: + dir = Path('.') result = subprocess.run( ['git', 'log', '--pretty=format:%ae%x01%B%x00', rev_range], capture_output=True, text=True, check=True, + cwd=dir, ) entries: list[CommitEntry] = [] for raw_commit in result.stdout.split('\0'): @@ -159,7 +163,14 @@ def main() -> None: sys.exit(1) categories = categorize(entries) - print(format_notes(categories, args.rev_range)) + + all_notes = [] + + all_notes.append('# GenVM Manager Release Notes\n') + + all_notes.append(format_notes(categories, args.rev_range)) + + print(''.join(all_notes)) if __name__ == '__main__': diff --git a/support/ci/open-executor-prs.py b/support/ci/tools/open_executor_prs.py similarity index 93% rename from support/ci/open-executor-prs.py rename to support/ci/tools/open_executor_prs.py index d92680da..96788c4f 100644 --- a/support/ci/open-executor-prs.py +++ b/support/ci/tools/open_executor_prs.py @@ -8,7 +8,7 @@ genvm-executor repo (`pr//`, see genvm_tool.common.Repo.feature_branch). So we fan out over the active executor lines (.genvm-monorepo-root via -branch-versions.py) rather than deriving a single line from the manager base. For +tools.versions) rather than deriving a single line from the manager base. For each active line whose mirror branch `pr//` exists on genvm-executor we open (or reuse) a PR `pr//` -> `-dev` there, then upsert a marked comment on the manager PR listing each as an `executor: ` line. @@ -24,8 +24,8 @@ import json import os import subprocess -import sys -from pathlib import Path + +from tools.versions import active_versions as configured_active_versions MANAGER_REPO = os.environ['MANAGER_REPO'] EXECUTOR_REPO = os.environ.get('EXECUTOR_REPO', 'genlayerlabs/genvm-executor') @@ -34,7 +34,6 @@ MANAGER_TOKEN = os.environ['MANAGER_TOKEN'] EXECUTOR_TOKEN = os.environ['EXECUTOR_TOKEN'] -HERE = Path(__file__).resolve().parent # Marker so re-runs update the same comment instead of stacking new ones. COMMENT_MARKER = '' @@ -52,13 +51,7 @@ def gh(*args, token, check=True): def active_lines() -> list[str]: """Active executor lines as `v` tags (e.g. ['v0.2', 'v0.3']).""" - out = subprocess.run( - [sys.executable, str(HERE / 'branch-versions.py'), 'list'], - check=True, - text=True, - capture_output=True, - ).stdout - return [f'v{v}' for v in out.split()] + return [f'v{v}' for v in configured_active_versions()] def executor_branch_exists(branch: str) -> bool: diff --git a/support/ci/pr-action-panel.py b/support/ci/tools/pr_action_panel.py similarity index 86% rename from support/ci/pr-action-panel.py rename to support/ci/tools/pr_action_panel.py index bd5d43cf..a6003faf 100644 --- a/support/ci/pr-action-panel.py +++ b/support/ci/tools/pr_action_panel.py @@ -15,6 +15,9 @@ label, and an already-set label means no re-dispatch on unrelated edits. - "Rerun full tests" is a momentary button: dispatch a fresh queue.yaml run on the current head, then untick. +- "Provision executor PRs" is a momentary button: dispatch +branch_provision_executor_prs.yaml for this PR (force-push each moved executor +line's mirror branch and open its PR), then untick. - "Merge" is a momentary button: expose `merge=true` so the caller runs the reusable merge workflow, then untick. 4. untick the momentary boxes (not the sticky Force one). @@ -93,6 +96,22 @@ def dispatch_full_tests(): print(f'dispatched queue.yaml on `{branch}`') +def dispatch_provision(): + # Momentary: fire the executor-provisioning workflow for this PR. It runs from + # the default branch (trusted YAML) and checks out the PR head for the scripts + # behind its own ci-safe guard. A workflow_dispatch from GITHUB_TOKEN runs. + gh( + 'workflow', + 'run', + 'branch_provision_executor_prs.yaml', + '--repo', + REPO, + '-f', + f'pr={PR}', + ) + print(f'dispatched executor provisioning for PR #{PR}') + + def ticked_boxes(body): return [ m.group(1).strip().lower() @@ -152,6 +171,10 @@ def main(): if any('rerun' in b for b in boxes): dispatch_full_tests() + # Provision: momentary -> force-push executor mirror branches and open their PRs. + if any('provision' in b for b in boxes): + dispatch_provision() + if any('merge' in b for b in boxes): set_output('merge', 'true') diff --git a/support/ci/tools/pr_branches.py b/support/ci/tools/pr_branches.py new file mode 100644 index 00000000..55d01cd6 --- /dev/null +++ b/support/ci/tools/pr_branches.py @@ -0,0 +1,243 @@ +import argparse +import json +import os + +import ci_lib +import pr_branches_info +from pr_branches_info import RepoInfo + + +def force_push_head(repo: str, branch: str, sha: str, token: str) -> str: + """Point `refs/heads/` in `repo` at `sha`, creating or force-moving it. + + `sha` must already be an object in `repo` (the executor commit the manager + gitlink pins). Returns 'created' or 'updated'. + """ + # Force-update an existing branch first (the common re-provision case); if the + # branch does not exist yet, create it. + updated = pr_branches_info.gh( + 'api', + '--method', + 'PATCH', + f'repos/{repo}/git/refs/heads/{branch}', + '-f', + f'sha={sha}', + '-F', + 'force=true', + token=token, + check=False, + ) + if updated.returncode == 0: + return 'updated' + created = pr_branches_info.gh( + 'api', + '--method', + 'POST', + f'repos/{repo}/git/refs', + '-f', + f'ref=refs/heads/{branch}', + '-f', + f'sha={sha}', + token=token, + check=False, + ) + if created.returncode == 0: + return 'created' + raise SystemExit( + f'could not push {branch} -> {sha} in {repo} ' + f'(is the executor commit pushed there?): ' + f'{updated.stderr.strip()} / {created.stderr.strip()}' + ) + + +def open_pr( + repo: str, + head: str, + base: str, + title: str, + manager_repo: str, + manager_pr: str, + token: str, +) -> str: + """Open the executor mirror PR `head -> base`, or reuse one that races in.""" + body = ( + f'Auto-opened executor mirror of {manager_repo}#{manager_pr}.\n\n' + f'Carries the executor-side work for that manager PR.' + ) + r = pr_branches_info.gh( + 'pr', + 'create', + '--repo', + repo, + '--base', + base, + '--head', + head, + '--title', + title, + '--body', + body, + token=token, + check=False, + ) + if r.returncode != 0: + url = pr_branches_info.existing_pr(repo, head, base, token) + if url: + return url + raise SystemExit( + f'could not create PR {head} -> {base} in {repo}: {r.stderr.strip()}' + ) + return r.stdout.strip().splitlines()[-1].strip() + + +class PrBranches(ci_lib.Tool): + """Inspect and provision per-repo branch movement for a manager PR.""" + + def name(self) -> str: + return 'pr-branches' + + def add_to(self, parser: argparse.ArgumentParser) -> None: + sub = parser.add_subparsers(dest='cmd', required=True) + show = sub.add_parser( + 'show-info', + help='print each repo base/head/ahead_by/behind_by as JSON', + ) + show.add_argument('pr', nargs='?', help='manager PR number (default: $PR_NUMBER)') + prov = sub.add_parser( + 'provision', + help='force-push each moved executor line mirror branch and open its PR', + ) + prov.add_argument('pr', nargs='?', help='manager PR number (default: $PR_NUMBER)') + check = sub.add_parser( + 'check', + help='fail unless every repo is rebased and every executor pinned commit is landable', + ) + check.add_argument('pr', nargs='?', help='manager PR number (default: $PR_NUMBER)') + + def show_info(self, args: argparse.Namespace) -> int: + infos = pr_branches_info.from_env(args.pr) + print(json.dumps({path: info.as_dict() for path, info in infos.items()}, indent=2)) + return 0 + + def provision(self, args: argparse.Namespace) -> int: + infos = pr_branches_info.from_env(args.pr) + manager = infos[pr_branches_info.MANAGER_PATH] + pr_number = args.pr or os.environ['PR_NUMBER'] + manager_token = os.environ['MANAGER_TOKEN'] + executor_token = os.environ['EXECUTOR_TOKEN'] + + title = pr_branches_info.gh( + 'api', + f'repos/{manager.repo}/pulls/{pr_number}', + '--jq', + '.title', + token=manager_token, + ).stdout.strip() + + for info in infos.values(): + if info.line is None: + continue # the manager itself; only executor lines are mirrored + if not self._provisionable(info): + continue + + state = force_push_head(info.repo, info.head_ref, info.head_sha, executor_token) + print(f'{info.line}: {state} {info.head_ref} -> {info.head_sha[:12]}') + + # If a PR already exists, only the force-push above is needed; reuse it. + url = info.pr_url or open_pr( + info.repo, + info.head_ref, + info.base_ref, + title, + manager.repo, + pr_number, + executor_token, + ) + print(f'{info.line}: PR {url}') + return 0 + + def check(self, args: argparse.Namespace) -> int: + """Gate for the manager PR's CI. Records every problem, then fails if any. + + The manager PR and its executor PRs land almost-atomically via the panel + (Provision then Merge), so this checks that the land is POSSIBLE, not that the + executor side already landed: + + 1. Every repo (manager + each executor line) must be rebased on its base β€” a + repo behind base needs a rebase. + 2. Every executor line that existed at base (base_sha set) must be landable β€” + its pinned head commit is present in the executor repo, so the merge can + squash/push it. Being ahead of base is normal (that is the unlanded work that + lands with the PR); only a commit missing from the executor repo fails. A line + the PR adds (base_sha null) is not gated. + + No PR number (a bare manual dispatch not tied to a PR) is a no-op pass β€” + there is nothing to gate. + """ + pr = args.pr or os.environ.get('PR_NUMBER') or '' + if not pr.strip(): + print('no PR number provided; nothing to check') + return 0 + infos = pr_branches_info.from_env(pr) + + errors: list[str] = [] + + def record(label: str, check: str, ok: bool, detail: str) -> None: + # One line per (repo, check): " OK" or an ::error:: with why. + if ok: + print(f'{label} {check} OK') + else: + print(f'::error::{label} {check}: {detail}') + errors.append(f'{label} {check}: {detail}') + + # 1. every repo must be rebased (not behind base) + for info in infos.values(): + record( + self._label(info), + 'rebased', + info.rebased, + f'{info.behind_by} commit(s) behind base; rebase it', + ) + + # 2. every executor line present at base must be landable: its pinned commit + # is in the executor repo. Ahead-of-base is fine β€” it lands with the PR. + for info in infos.values(): + if info.line is None or info.base_sha is None: + continue + detail = ( + 'present at base but missing at the PR head' + if info.head_sha is None + else f'pinned commit {info.head_sha[:12]} is not on the executor repo; ' + 'push/provision it so it can land with the manager PR' + ) + record(self._label(info), 'landable', info.synced, detail) + + return 1 if errors else 0 + + def _label(self, info: RepoInfo) -> str: + return 'manager' if info.line is None else f'executor {info.line}' + + def _provisionable(self, info: RepoInfo) -> bool: + """Whether an executor line has a mirror branch + PR to provision.""" + if info.base_sha is None: + print(f'{info.line}: base sha is null (line absent at base); skipping') + return False + if info.head_sha is None: + print(f'{info.line}: no head gitlink (line removed); skipping') + return False + if not info.moved: + print(f'{info.line}: gitlink unchanged; skipping') + return False + return True + + def handler(self, args: argparse.Namespace) -> int: + if args.cmd == 'show-info': + return self.show_info(args) + if args.cmd == 'provision': + return self.provision(args) + if args.cmd == 'check': + return self.check(args) + return 1 + + +COMMANDS = [PrBranches()] diff --git a/support/ci/tools/script_adapters.py b/support/ci/tools/script_adapters.py new file mode 100644 index 00000000..4185ef68 --- /dev/null +++ b/support/ci/tools/script_adapters.py @@ -0,0 +1,44 @@ +import argparse +import importlib +import sys + +import ci_lib + + +class ModuleTool(ci_lib.Tool): + def __init__(self, name: str, module: str): + self._name = name + self._module = module + + def name(self) -> str: + return self._name + + def add_to(self, parser: argparse.ArgumentParser) -> None: + parser.add_argument('args', nargs=argparse.REMAINDER) + + def handler(self, args: argparse.Namespace) -> int: + module = importlib.import_module(self._module) + old_argv = sys.argv + try: + sys.argv = [self._name, *args.args] + try: + result = module.main() + except SystemExit as exc: + if exc.code is None: + return 0 + if isinstance(exc.code, int): + return exc.code + print(exc.code, file=sys.stderr) + return 1 + finally: + sys.argv = old_argv + return result or 0 + + +COMMANDS = [ + ModuleTool('genvm-merge-into-dev', 'tools.genvm_merge_into_dev'), + ModuleTool('make-release-notes', 'tools.make_release_notes'), + ModuleTool('make-relase-notes', 'tools.make_release_notes'), + ModuleTool('open-executor-prs', 'tools.open_executor_prs'), + ModuleTool('pr-action-panel', 'tools.pr_action_panel'), +] diff --git a/support/ci/tools/versions.py b/support/ci/tools/versions.py new file mode 100644 index 00000000..91dfd68a --- /dev/null +++ b/support/ci/tools/versions.py @@ -0,0 +1,188 @@ +import argparse +import json +import os +import re +import sys + +import ci_lib + + +def bare(version: str) -> str: + return version.lstrip('v').split('-', 1)[0] + + +def major_minor(version: str) -> str: + return '.'.join(bare(version).split('.')[:2]) + + +def version_key(version: str) -> tuple[int, ...]: + return tuple(int(part) for part in major_minor(version).split('.')) + + +def active_versions() -> list[str]: + conf_path = os.environ.get('MONOREPO_ROOT', ci_lib.ROOT_DIR / '.genvm-monorepo-root') + conf = json.loads(open(conf_path, 'rb').read()) + return sorted( + (bare(version) for version in conf.get('active-versions', [])), key=version_key + ) + + +class BranchVersions(ci_lib.Tool): + """Read active version trains from .genvm-monorepo-root.""" + + def name(self) -> str: + return 'branch-versions' + + def add_to(self, parser: argparse.ArgumentParser) -> None: + parser.add_argument('cmd', nargs='?', choices=['list', 'latest'], default='list') + + def handler(self, args: argparse.Namespace) -> int: + versions = active_versions() + if args.cmd == 'list': + print('\n'.join(versions)) + return 0 + if not versions: + print('no active versions configured in .genvm-monorepo-root', file=sys.stderr) + return 1 + print(versions[-1]) + return 0 + + +class CheckVersions(ci_lib.Tool): + """Keep manager-owned Rust crate versions in sync with .genvm-monorepo-root.""" + + CARGO_FILES = ['implementation/Cargo.toml'] + + def name(self) -> str: + return 'check-versions' + + def add_to(self, parser: argparse.ArgumentParser) -> None: + sub = parser.add_subparsers(dest='cmd') + sub.add_parser('sync', help='rewrite crate versions to match version') + bump = sub.add_parser('bump', help='cut the next version train') + bump.add_argument('level', nargs='?', choices=['minor', 'major'], default='minor') + bump.add_argument('--set', dest='set_version', metavar='MAJOR.MINOR') + + def conf_path(self): + return ci_lib.ROOT_DIR / '.genvm-monorepo-root' + + def load_conf(self): + return json.loads(self.conf_path().read_bytes()) + + def package_version(self, path): + section = None + for line in path.read_text().splitlines(): + stripped = line.strip() + match = re.match(r'\[(.+?)\]', stripped) + if match: + section = match.group(1) + continue + if section == 'package': + match = re.match(r'version\s*=\s*"([^"]+)"', stripped) + if match: + return match.group(1) + return None + + def set_package_version(self, rel: str, full: str): + path = ci_lib.ROOT_DIR / rel + old = None + out = [] + section = None + for line in path.read_text().splitlines(keepends=True): + match = re.match(r'\s*\[(.+?)\]', line) + if match: + section = match.group(1) + elif section == 'package' and old is None: + match = re.match(r'(version\s*=\s*)"([^"]*)"', line) + if match: + old = match.group(2) + line = re.sub(r'"[^"]*"', f'"{full}"', line, count=1) + out.append(line) + if old is None: + return None + path.write_text(''.join(out)) + return old + + def sync(self) -> int: + mm = major_minor(self.load_conf()['version']) + rc = 0 + for cargo_file in self.CARGO_FILES: + current = self.package_version(ci_lib.ROOT_DIR / cargo_file) + if current is None: + print(f'{cargo_file}: could not find [package] version') + rc = 1 + continue + parts = current.split('.', 2) + if '.'.join(parts[:2]) == mm: + continue + patch = parts[2] if len(parts) == 3 else '0' + target = f'{mm}.{patch}' + self.set_package_version(cargo_file, target) + print( + f'{cargo_file}: bumped {current} -> {target} to match .genvm-monorepo-root version ({mm})' + ) + rc = 1 + return rc + + def bump(self, level: str, set_version: str | None) -> int: + conf = self.load_conf() + active = conf.get('active-versions', []) + if not active: + print('no active versions configured in .genvm-monorepo-root', file=sys.stderr) + return 1 + latest = sorted(active, key=version_key)[-1] + + if set_version is not None: + new_mm = bare(set_version) + if not re.fullmatch(r'\d+\.\d+', new_mm): + print(f'--set expects major.minor, got {set_version!r}', file=sys.stderr) + return 2 + else: + major, minor = (int(part) for part in major_minor(latest).split('.')) + if level == 'minor': + minor += 1 + else: + major += 1 + minor = 0 + new_mm = f'{major}.{minor}' + + new = f'v{new_mm}' + if new in active: + print(f'version {new} is already active', file=sys.stderr) + return 1 + if version_key(new) <= version_key(latest): + print(f'new version {new} is not greater than latest {latest}', file=sys.stderr) + return 1 + + text = self.conf_path().read_text() + new_active = '[' + ', '.join(f'"{version}"' for version in active + [new]) + ']' + text, n1 = re.subn(r'("version"\s*:\s*)"[^"]*"', rf'\g<1>"{new}"', text) + text, n2 = re.subn( + r'("active-versions"\s*:\s*)\[[^\]]*\]', + lambda match: match.group(1) + new_active, + text, + ) + if n1 != 1 or n2 != 1: + print( + f'failed to patch .genvm-monorepo-root (version={n1}, active-versions={n2})', + file=sys.stderr, + ) + return 1 + self.conf_path().write_text(text) + + for cargo_file in self.CARGO_FILES: + if self.set_package_version(cargo_file, f'{new_mm}.0') is None: + print( + f'{cargo_file}: could not find [package] version to bump', file=sys.stderr + ) + return 1 + print(new_mm) + return 0 + + def handler(self, args: argparse.Namespace) -> int: + if args.cmd == 'bump': + return self.bump(args.level, args.set_version) + return self.sync() + + +COMMANDS = [BranchVersions(), CheckVersions()] diff --git a/support/manifest-base.yaml b/support/manifest-base.yaml index 3c6f2b16..25b07e31 100644 --- a/support/manifest-base.yaml +++ b/support/manifest-base.yaml @@ -4,3 +4,8 @@ # build-independent and lives under version control. runners_download_urls: - "https://storage.googleapis.com/gh-af/genvm_runners/{name}/{hash}.tar" +# Each executor line is released on its own, in its own repo, tagged with its +# executor-version β€” the manager bundle does not carry them. `{version}` is that +# tag and `{platform}` is the nix-style token (amd64-linux, arm64-macos, ...). +executor_download_urls: + - "https://github.com/genlayerlabs/genvm-executor/releases/download/{version}/genvm-{platform}-executor.tar.xz" diff --git a/support/nix/precommit/flake.lock b/support/nix/precommit/flake.lock deleted file mode 100644 index 2d3322f9..00000000 --- a/support/nix/precommit/flake.lock +++ /dev/null @@ -1,61 +0,0 @@ -{ - "nodes": { - "flake-utils": { - "inputs": { - "systems": "systems" - }, - "locked": { - "lastModified": 1731533236, - "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, - "nixpkgs": { - "locked": { - "lastModified": 1782467914, - "narHash": "sha256-pGvFkM8N0xEkIIXDe5YYfbEAvHrk4IxBrjB/x8OomhE=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "e73de5be04e0eff4190a1432b946d469c794e7b4", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixos-unstable", - "repo": "nixpkgs", - "type": "github" - } - }, - "root": { - "inputs": { - "flake-utils": "flake-utils", - "nixpkgs": "nixpkgs" - } - }, - "systems": { - "locked": { - "lastModified": 1681028828, - "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", - "owner": "nix-systems", - "repo": "default", - "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", - "type": "github" - }, - "original": { - "owner": "nix-systems", - "repo": "default", - "type": "github" - } - } - }, - "root": "root", - "version": 7 -} diff --git a/support/nix/precommit/flake.nix b/support/nix/precommit/flake.nix deleted file mode 100644 index 98d2d775..00000000 --- a/support/nix/precommit/flake.nix +++ /dev/null @@ -1,47 +0,0 @@ -{ - description = "genvm-manager pre-commit lint tools (pinned, independent of the umbrella flake)"; - - inputs = { - nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; - flake-utils.url = "github:numtide/flake-utils"; - }; - - outputs = - { - self, - nixpkgs, - flake-utils, - }: - flake-utils.lib.eachDefaultSystem ( - system: - let - pkgs = import nixpkgs { inherit system; }; - # One named output per tool: hooks.toml `nix = ""` references - # these, and they are what `default` aggregates. - tools = { - inherit (pkgs) - ruff - stylua - prettier - editorconfig-checker - check-jsonschema - rustfmt - nixfmt - cargo - taplo - ; - pre-commit-hooks = pkgs.python3Packages.pre-commit-hooks; - }; - in - { - packages = tools // { - default = pkgs.buildEnv { - name = "genvm-precommit-tools-manager"; - # pre-commit-hooks ships a `python`/scripts that can collide. - ignoreCollisions = true; - paths = builtins.attrValues tools; - }; - }; - } - ); -} diff --git a/support/runner-script.py b/support/runner-script.py index a2df2447..39bfc1b9 100755 --- a/support/runner-script.py +++ b/support/runner-script.py @@ -3,6 +3,7 @@ import hashlib import json import os +import re import sys import typing import urllib.parse @@ -268,6 +269,25 @@ def run_merge(args): print(json.dumps(all_regs_list, sort_keys=True)) +EXECUTOR_LINE_TARGET_RE = re.compile(r'^executor-v\d+\.\d+-(?P.+)$') + + +def _expand_targets(targets: set[str]) -> set[str]: + """Add the per-platform alias of every per-line executor target. + + Every executor line builds against the same dependency set, so the registry + keys them by platform (`executor-amd64-linux`), while the build targets name + a line too (`executor-v0.3-amd64-linux`). Without this, a per-line target + matches no dependency and the prefetch is a silent no-op. + """ + expanded = set(targets) + for target in targets: + match = EXECUTOR_LINE_TARGET_RE.match(target) + if match: + expanded.add(f'executor-{match.group("platform")}') + return expanded + + def _iter_needed_deps(targets: set[str] | None, all_deps: bool): deps_file = ( Path(__file__).resolve().parent.parent / 'libs' / 'deps' / 'dependency-urls.json' @@ -276,6 +296,9 @@ def _iter_needed_deps(targets: set[str] | None, all_deps: bool): with open(deps_file, 'r') as f: deps = json.load(f) + if targets is not None: + targets = _expand_targets(targets) + for dep in deps: if not all_deps and not dep.get('must_preload', False): continue diff --git a/support/scripts/ci-changes.py b/support/scripts/ci-changes.py new file mode 100755 index 00000000..92d4581b --- /dev/null +++ b/support/scripts/ci-changes.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +"""Report what each repo changed in this CI run, as JSON. + +Emits one entry per repo: the manager keyed as ``.``, every executor submodule by +its path, each holding ``base_commit``, ``branch_commit`` and ``has_changes``. + +A submodule's commits are not visible from the manager's history, so its +endpoints are the gitlinks recorded on either side of the pull request. + +Outside a pull request there is nothing to compare against, so ``base_commit == +branch_commit`` and ``has_changes`` is false everywhere. +""" + +import argparse +import json +import os +import subprocess +import sys +from pathlib import Path + +MANAGER = '.' + +BASE_REF = 'refs/genvm-ci/base' +HEAD_REF = 'refs/genvm-ci/prhead' + + +def git(*args: str, check: bool = True) -> str: + proc = subprocess.run( + ['git', *args], + check=check, + capture_output=True, + text=True, + ) + return proc.stdout.strip() + + +def submodule_paths() -> list[str]: + if not Path('.gitmodules').exists(): + return [] + out = git('config', '--file', '.gitmodules', '--get-regexp', r'^submodule\..*\.path$') + return sorted(line.split(' ', 1)[1] for line in out.splitlines() if line) + + +def gitlink_at(commit: str, path: str) -> str | None: + """The submodule commit `path` points at, as recorded in `commit`'s tree.""" + entry = git('ls-tree', commit, path) + if not entry: + return None + meta, _, _name = entry.partition('\t') + mode, obj_type, sha = meta.split() + if obj_type != 'commit': + return None + return sha + + +def endpoints(base: str, head: str, path: str) -> tuple[str | None, str | None]: + if path == MANAGER: + return base, head + return gitlink_at(base, path), gitlink_at(head, path) + + +def fetch_pr_refs(base_ref: str, pr_number: str) -> tuple[str, str]: + """Materialize the base tip and the PR head, and return (merge_base, head). + + A `pull_request` checkout leaves HEAD on the merge commit, and the base + branch is usually not fetched at all, so both sides are fetched by explicit + refspec (which also works for forks, where the head sha is not fetchable on + its own). + + `--no-recurse-submodules` is required: we only read gitlink SHAs out of the + superproject trees (`git ls-tree`), never any submodule object. Once the + submodules are populated (get-all-git.py runs before us), git's default + `fetch.recurseSubmodules=on-demand` would otherwise try to fetch every + gitlink the newly-fetched PR range touches β€” and a historical commit in the + range may pin an executor commit that was never pushed (or since deleted) + on its remote, which aborts the whole fetch with `upload-pack: not our ref`. + """ + git( + 'fetch', + '--no-tags', + '--no-recurse-submodules', + 'origin', + f'+refs/heads/{base_ref}:{BASE_REF}', + f'+refs/pull/{pr_number}/head:{HEAD_REF}', + ) + head = git('rev-parse', HEAD_REF) + # The repo requires PR branches to be 0 commits behind, so this is normally + # the base tip; taking the merge base anyway keeps the range honest if that + # gate is ever relaxed. + merge_base = git('merge-base', BASE_REF, HEAD_REF) + return merge_base, head + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + '--github-output', + action='store_true', + help='also append `changes=` to $GITHUB_OUTPUT', + ) + args = parser.parse_args() + + event = os.environ.get('GITHUB_EVENT_NAME', '') + base_ref = os.environ.get('GITHUB_BASE_REF', '') + pr_number = os.environ.get('GITHUB_PR_NUMBER', '') + + paths = [MANAGER, *submodule_paths()] + + if event == 'pull_request' and base_ref and pr_number: + base, head = fetch_pr_refs(base_ref, pr_number) + else: + # Not a pull request: nothing to diff against. + base = head = git('rev-parse', 'HEAD') + + result = {} + for path in paths: + base_commit, branch_commit = endpoints(base, head, path) + result[path] = { + 'base_commit': base_commit, + 'branch_commit': branch_commit, + # A submodule that only exists on one side (added or removed) counts + # as changed; one that exists on neither does not. + 'has_changes': base_commit != branch_commit + and not (base_commit is None and branch_commit is None), + } + + encoded = json.dumps(result) + print(encoded) + + if args.github_output: + github_output = os.environ.get('GITHUB_OUTPUT') + if not github_output: + print( + '::error::--github-output given but GITHUB_OUTPUT is unset', file=sys.stderr + ) + return 1 + with open(github_output, 'a', encoding='utf-8') as out: + out.write(f'changes={encoded}\n') + + return 0 + + +if __name__ == '__main__': + try: + sys.exit(main()) + except subprocess.CalledProcessError as e: + import shlex + import traceback + + traceback.print_exc() + print(' '.join(shlex.quote(a) for a in e.cmd), file=sys.stderr) + print(e.stdout, file=sys.stdout) + print(e.stderr, file=sys.stderr) + sys.exit(1) diff --git a/support/scripts/get-all-git.py b/support/scripts/get-all-git.py new file mode 100755 index 00000000..89104875 --- /dev/null +++ b/support/scripts/get-all-git.py @@ -0,0 +1,273 @@ +#!/usr/bin/env python3 + +import argparse +import hashlib +import os +import posixpath +import re +import shlex +import subprocess +from dataclasses import dataclass +from pathlib import Path +from urllib.parse import urlparse, urlunparse + + +@dataclass(frozen=True) +class RepoGitInfo: + path: str + remote_url: str + gitlink_hash: str + + +@dataclass(frozen=True) +class SubmoduleGitInfo: + name: str + path: str + url: str + gitlink_hash: str + + +def run(cmd: list[str], cwd: Path, env: dict[str, str]) -> None: + print(f'+ {shlex.join(cmd)}', flush=True) + subprocess.run(cmd, check=True, cwd=cwd, env=env) + + +def git_output(cmd: list[str], cwd: Path) -> str: + return subprocess.run( + cmd, check=True, cwd=cwd, text=True, capture_output=True + ).stdout.strip() + + +def git_output_bytes(cmd: list[str], cwd: Path) -> bytes: + return subprocess.run(cmd, check=True, cwd=cwd, stdout=subprocess.PIPE).stdout + + +def repo_root(value: str | None) -> Path: + if value is not None: + return Path(value).resolve() + + workspace = os.environ.get('GITHUB_WORKSPACE') + if workspace: + return Path(workspace).resolve() + + return Path(__file__).resolve().parents[2] + + +def resolve_submodule_url(url: str, parent_url: str) -> str: + if not (url.startswith('./') or url.startswith('../')): + return url + + parsed = urlparse(parent_url) + if parsed.scheme: + path = posixpath.normpath(posixpath.join(parsed.path, url)) + return urlunparse(parsed._replace(path=path)) + + scp_like = re.fullmatch(r'([^/][^:]*):(.*)', parent_url) + if scp_like: + prefix, path = scp_like.groups() + return f'{prefix}:{posixpath.normpath(posixpath.join(path, url))}' + + return str((Path(parent_url) / url).resolve()) + + +def _read_gitmodules_entries(root: Path) -> dict[str, dict[str, str]]: + out = git_output_bytes( + [ + 'git', + 'config', + '--file', + '.gitmodules', + '--null', + '--get-regexp', + r'^submodule\..*\.(path|url)$', + ], + root, + ) + result: dict[str, dict[str, str]] = {} + for item in out.rstrip(b'\0').split(b'\0'): + if not item: + continue + key_raw, value_raw = item.split(b'\n', 1) + key = key_raw.decode() + value = value_raw.decode() + prefix, field = key.rsplit('.', 1) + name = prefix.removeprefix('submodule.') + result.setdefault(name, {})[field] = value + return result + + +def _gitlink_hash(root: Path, path: str) -> str: + ls_tree = git_output(['git', 'ls-tree', 'HEAD', path], root) + if not ls_tree: + raise RuntimeError(f'submodule path is not present in HEAD: {path}') + fields = ls_tree.split() + if len(fields) < 3 or fields[0] != '160000': + raise RuntimeError(f'path is not a submodule gitlink in HEAD: {path}') + return fields[2] + + +def get_submodule_git_info(root: Path) -> list[SubmoduleGitInfo]: + root = root.resolve() + parent_url = git_output(['git', 'config', '--get', 'remote.origin.url'], root) + result = [] + for name, data in _read_gitmodules_entries(root).items(): + path = data['path'] + url = resolve_submodule_url(data['url'], parent_url) + result.append( + SubmoduleGitInfo( + name=name, + path=path, + url=url, + gitlink_hash=_gitlink_hash(root, path), + ) + ) + return result + + +def get_all_repo_git_info(root: Path) -> list[RepoGitInfo]: + root = root.resolve() + result = [ + RepoGitInfo( + path='.', + remote_url=git_output(['git', 'config', '--get', 'remote.origin.url'], root), + gitlink_hash=git_output(['git', 'rev-parse', 'HEAD'], root), + ) + ] + + for submodule in get_submodule_git_info(root): + result.append( + RepoGitInfo( + path=submodule.path, + remote_url=submodule.url, + gitlink_hash=submodule.gitlink_hash, + ) + ) + + return result + + +def cache_path_for_url(root: Path, url: str) -> Path: + parsed = urlparse(url) + url_path = parsed.path if parsed.scheme else url.rsplit(':', 1)[-1] + name = Path(url_path).name or 'repo' + name = re.sub(r'\.git$', '', name) + name = re.sub(r'[^A-Za-z0-9._-]+', '-', name).strip('-') or 'repo' + digest = hashlib.sha256(url.encode()).hexdigest()[:16] + return root / '.git' / 'genvm-submodule-cache' / f'{name}-{digest}.git' + + +def ensure_cache(root: Path, url: str, hashes: list[str], env: dict[str, str]) -> Path: + cache = cache_path_for_url(root, url) + cache.parent.mkdir(parents=True, exist_ok=True) + if not (cache / 'HEAD').exists(): + run(['git', 'init', '--bare', str(cache)], root, env) + + remote_exists = ( + subprocess.run( + ['git', '-C', str(cache), 'remote', 'get-url', 'origin'], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ).returncode + == 0 + ) + if remote_exists: + run(['git', '-C', str(cache), 'remote', 'set-url', 'origin', url], root, env) + else: + run(['git', '-C', str(cache), 'remote', 'add', 'origin', url], root, env) + + run( + ['git', '-C', str(cache), 'fetch', '--no-tags', 'origin', *hashes], + root, + env, + ) + return cache + + +def set_remote(repo: Path, name: str, url: str, env: dict[str, str]) -> None: + remote_exists = ( + subprocess.run( + ['git', '-C', str(repo), 'remote', 'get-url', name], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ).returncode + == 0 + ) + if remote_exists: + run(['git', '-C', str(repo), 'remote', 'set-url', name, url], repo, env) + else: + run(['git', '-C', str(repo), 'remote', 'add', name, url], repo, env) + + +def update_submodules_with_cache(root: Path, env: dict[str, str]) -> None: + submodules = get_submodule_git_info(root) + by_url: dict[str, list[SubmoduleGitInfo]] = {} + for submodule in submodules: + by_url.setdefault(submodule.url, []).append(submodule) + + cache_by_url = { + url: ensure_cache( + root, + url, + sorted({submodule.gitlink_hash for submodule in url_submodules}), + env, + ) + for url, url_submodules in by_url.items() + } + + for submodule in submodules: + cache = cache_by_url[submodule.url] + run( + [ + 'git', + 'submodule', + 'update', + '--init', + '--recursive', + '--reference', + str(cache), + '--depth', + '1', + '--no-fetch', + '--', + submodule.path, + ], + root, + env, + ) + set_remote(root / submodule.path, 'origin', submodule.url, env) + set_remote(root / submodule.path, 'cache', str(cache), env) + + +def main() -> None: + parser = argparse.ArgumentParser( + description='Checkout git submodules and vendored third-party trees.' + ) + parser.add_argument( + '--repo-root', + help='repository root; defaults to GITHUB_WORKSPACE or this script location', + ) + parser.add_argument( + '--third-party', + default='--all', + help='arguments for `git third-party update`; use "none" to skip', + ) + args = parser.parse_args() + + root = repo_root(args.repo_root) + env = os.environ.copy() + git_third_party = root / 'support' / 'tools' / 'git-third-party' + env['PATH'] = f'{git_third_party}{os.pathsep}{env.get("PATH", "")}' + + update_submodules_with_cache(root, env) + + if args.third_party == 'none': + return + + third_party_args = shlex.split(args.third_party) + for cfg in sorted(root.glob('executors/*/.git-third-party/config.json')): + executor = cfg.parent.parent + run(['git', 'third-party', 'update', *third_party_args], executor, env) + + +if __name__ == '__main__': + main() diff --git a/support/tools/genvm-tool/genvm_tool/hook/mdlinks.py b/support/scripts/md-local-links.py similarity index 74% rename from support/tools/genvm-tool/genvm_tool/hook/mdlinks.py rename to support/scripts/md-local-links.py index 97fa2cb3..f1c75760 100644 --- a/support/tools/genvm-tool/genvm_tool/hook/mdlinks.py +++ b/support/scripts/md-local-links.py @@ -1,13 +1,19 @@ -"""Local Markdown link checking (pure, no I/O orchestration). +#!/usr/bin/env python3 +"""Pre-commit guard: verify local file-path links in Markdown resolve. -Validates that **local** link targets (and reference definitions) resolve to +Validates that **local** link targets (and reference definitions) point at existing files; ignores external links (http(s)/mailto/other schemes), -protocol-relative `//host` links, and in-page `#anchor` links. Used by the -genvm-tool `md-local-links` builtin in place of the network `markdown-link-check`. +protocol-relative `//host` links, and in-page `#anchor` links. Replaces the +network `markdown-link-check`. + +Invoked by the git-hooks `markdown-local-links` hook with the changed markdown +files as arguments; the repo root is the current working directory (pre-commit +runs hooks from the repo top level). """ import os import re +import sys import urllib.parse from pathlib import Path @@ -67,3 +73,17 @@ def broken_links(md: Path, root: Path) -> list[str]: f'{_rel(md, root)}: broken local link {raw!r} -> {_rel(dest, root)} (not found)' ) return out + + +def main(argv: list[str]) -> int: + root = Path.cwd() + problems = [] + for arg in argv: + problems += broken_links(Path(arg), root) + for p in problems: + print(p, file=sys.stderr) + return 1 if problems else 0 + + +if __name__ == '__main__': + raise SystemExit(main(sys.argv[1:])) diff --git a/support/tools/genvm-tool/default.nix b/support/tools/genvm-tool/default.nix index 9ff5fd3e..c152cddf 100644 --- a/support/tools/genvm-tool/default.nix +++ b/support/tools/genvm-tool/default.nix @@ -1,4 +1,4 @@ -# genvm-tool: the multi-repo git-hook runner, `configure`, `git ls`, and the +# genvm-tool: the multi-repo `configure` / `git ls` helpers and the # `genvm-tool test` runner (under `genvm_tool/tests`, shipped with the package). { pkgs ? import { }, diff --git a/support/tools/genvm-tool/genvm_tool/__main__.py b/support/tools/genvm-tool/genvm_tool/__main__.py index 86e64c56..494cfbfa 100644 --- a/support/tools/genvm-tool/genvm_tool/__main__.py +++ b/support/tools/genvm-tool/genvm_tool/__main__.py @@ -3,9 +3,10 @@ """Command-line interface for genvm-tool. Top-level commands (`configure`, `test`) sit alongside command *groups* (`git` -with `git ls` / `git list-repo`, `hook` with `hook run` / `hook install`). Each group is a -sub-package exposing `COMMANDS`; every leaf module exposes `NAME`, `HELP`, -`configure(parser)` and `main(ctx, args)` (sync or async). +with `git ls` / `git list-repo`). Each group is a sub-package exposing +`COMMANDS`; every leaf module exposes `NAME`, `HELP`, `configure(parser)` and +`main(ctx, args)` (sync or async). Commit hooks are handled by git-hooks.nix +(each repo's flake), not this tool β€” see `support/ci/run.sh pipeline commit-hooks`. """ import argparse @@ -24,11 +25,10 @@ common, formatter, git, - hook, ) TOPLEVEL = [cmd_configure, cmd_test, cmd_build_manifest, cmd_codegen] -GROUPS = [git, hook] +GROUPS = [git] def _add_leaf(subparsers, mod) -> None: @@ -40,7 +40,7 @@ def _add_leaf(subparsers, mod) -> None: def _create_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog='genvm-tool', - description='GenVM monorepo git helper (build, tests, hooks, ls)', + description='GenVM monorepo git helper (build, tests, ls)', ) # `--print-completion {bash,zsh,tcsh}` emits a shell completion script for the # whole command tree and exits (shtab walks the subparsers below at call time). @@ -126,4 +126,16 @@ def main() -> None: if __name__ == '__main__': - main() + import subprocess + + try: + main() + except subprocess.CalledProcessError as e: + import shlex + import traceback + + traceback.print_exc() + print(' '.join(shlex.quote(a) for a in e.cmd), file=sys.stderr) + print(e.stdout, file=sys.stdout) + print(e.stderr, file=sys.stderr) + sys.exit(1) diff --git a/support/tools/genvm-tool/genvm_tool/cmd_configure.py b/support/tools/genvm-tool/genvm_tool/cmd_configure.py index 07985369..6a6d4854 100644 --- a/support/tools/genvm-tool/genvm_tool/cmd_configure.py +++ b/support/tools/genvm-tool/genvm_tool/cmd_configure.py @@ -72,6 +72,24 @@ def _load_versions(source_dir: Path) -> tuple[str, list[ExecVersion]]: return cfg['version'], versions +def build_independent_info(source_dir: Path) -> dict: + """The parts of ``info.json`` derivable from source alone β€” no build needed. + + ``executor_versions`` maps each active line (e.g. ``v0.3``) to its built + ``out/executor/`` directory name; ``primary_executor_version`` is the + primary line's. Both come from ``.genvm-monorepo-root`` and each line's + committed ``manifest.json``, so the test harness can synthesize a usable + ``info.json`` (see the manager-root ``.genvm-tool.py``) without first running + ``configure``. ``configure`` merges this into the full, build-dependent info. + """ + primary_key, versions = _load_versions(source_dir) + primary = next((v for v in versions if v.key == primary_key), versions[0]) + return { + 'executor_versions': {v.key: v.real for v in versions}, + 'primary_executor_version': primary.real, + } + + def configure(parser): parser.add_argument( '--test-always-fail-module', @@ -203,10 +221,11 @@ def main(ctx: common.Context, args) -> int: tool_src = source_dir / 'support' / 'tools' / 'genvm-tool' genvm_tool_sh = build_dir / 'genvm_tool.sh' build_dir.mkdir(parents=True, exist_ok=True) + pp = ':'.join(sys.path + [str(tool_src)]) genvm_tool_sh.write_text( '#!/bin/sh\n' '# Generated by `genvm-tool configure`, DO NOT EDIT MANUALLY.\n' - f'export PYTHONPATH="{tool_src}:${{PYTHONPATH:-}}"\n' + f'export PYTHONPATH="{pp}:${{PYTHONPATH:-}}"\n' f'exec "{sys.executable}" -m genvm_tool "$@"\n' ) genvm_tool_sh.chmod(0o755) @@ -362,14 +381,18 @@ def main(ctx: common.Context, args) -> int: data_phony = n.build('phony', 'all/data') # all/bin: build the binaries and `cp` every install-tree file into out/. + all_manager = n.build('phony', 'all/manager') + all_manager.add_dependency('out/bin/genvm-modules') all_bin = n.build('phony', 'all/bin') - all_bin.add_dependency('out/bin/genvm-modules') + all_bin.add_dependency(all_manager) # Per-line executor build, runner data, codegen, and install tree, each # landing under its own `out/executor/` directory. Each line's # `.genvm-tool.py:configure(line)` hook owns its registrations (the lines # already diverge β€” e.g. frozen runner registry vs nix-derived manifests). for v in versions: + all_exec_line = n.build('phony', f'all/executor/{v.key}') + all_bin.add_dependency(all_exec_line) line = ninja.LineContext( n=n, source_dir=source_dir, @@ -378,11 +401,13 @@ def main(ctx: common.Context, args) -> int: real=v.real, codegen_phony=codegen_phony, data_phony=data_phony, - all_bin=all_bin, + all_bin=all_exec_line, ) configurator = _line_configurator(ctx, source_dir, v.exec_rel) configurator(line) + all_exec_line.finish() + codegen_phony.finish() data_phony.finish() @@ -400,6 +425,8 @@ def _runner_file(f: Path) -> bool: if _runner_file(f) ] + n.build('phony', 'all/runners').add_dependency('target/runners').finish() + runners_build = n.build('CUSTOM_COMMAND', 'target/runners') runners_build.var( 'command', @@ -459,7 +486,8 @@ def _runner_file(f: Path) -> bool: n.build('phony', 'cargo/clippy').add_dependency(n.all_clippy).finish() n.build('phony', 'cargo/clippy/fix').add_dependency(n.all_clippy_fix).finish() - n.install('install', 'out', all_bin) + n.install('install', 'out', all_manager) + all_manager.finish() all_bin.finish() all_phony = n.build('phony', 'all') @@ -476,11 +504,10 @@ def _runner_file(f: Path) -> bool: 'coverage_dir': str(build_dir / 'cov'), 'build_dir': str(build_dir), 'rust_target_dir': str(rust_target_dir), - # Maps each active line (e.g. `v0.3`) to its built `out/executor/` - # directory name. Consumers (test harness) read this to reroute runs to a - # locally-built executor. - 'executor_versions': {v.key: v.real for v in versions}, - 'primary_executor_version': primary.real, + # executor_versions / primary_executor_version β€” derivable without a build; + # the test harness synthesizes the same keys from source when info.json is + # absent (see build_independent_info). + **build_independent_info(source_dir), } (build_dir / 'info.json').write_text(json.dumps(info, indent=2)) diff --git a/support/tools/genvm-tool/genvm_tool/cmd_test.py b/support/tools/genvm-tool/genvm_tool/cmd_test.py index 4592146b..5888c475 100644 --- a/support/tools/genvm-tool/genvm_tool/cmd_test.py +++ b/support/tools/genvm-tool/genvm_tool/cmd_test.py @@ -4,8 +4,8 @@ its `run` / `show plan|test|services|tags` subcommands plus the suite-provided filter flags. The runner reuses genvm-tool's `Context` (logger/printer) and the top-level `-C` / `--log-format` / `--log-level`; the runner package is imported -lazily so the stdlib-only paths (`configure`, `git ls`, `hook run`, -`hook install`) never pull in its `aiohttp` / `jsonnet` dependency closure. +lazily so the stdlib-only paths (`configure`, `git ls`) never pull in its +`aiohttp` / `jsonnet` dependency closure. """ import argparse diff --git a/support/tools/genvm-tool/genvm_tool/codegen/rust.py b/support/tools/genvm-tool/genvm_tool/codegen/rust.py index 0d056d90..44ed3499 100644 --- a/support/tools/genvm-tool/genvm_tool/codegen/rust.py +++ b/support/tools/genvm-tool/genvm_tool/codegen/rust.py @@ -211,6 +211,8 @@ def render(defs: list[Definition], **_opts) -> str: buf.append('}\n\n') elif isinstance(d, StrTrie): _trie(d, buf) + + buf.append('\n// EOF\n') return ''.join(buf) diff --git a/support/tools/genvm-tool/genvm_tool/common.py b/support/tools/genvm-tool/genvm_tool/common.py index f9f8bfc8..900c3bfe 100644 --- a/support/tools/genvm-tool/genvm_tool/common.py +++ b/support/tools/genvm-tool/genvm_tool/common.py @@ -1,9 +1,7 @@ """Shared primitives: repo discovery and git queries. A "repo" is the manager root plus every executor submodule -(`executors/.x/`, discovered from `.genvm-monorepo-root`). The hook-execution -engine (tool profiles, filtering, runner, builtins) lives in -`genvm_tool.hook.engine`. +(`executors/.x/`, discovered from `.genvm-monorepo-root`). """ import functools @@ -19,13 +17,8 @@ MONOREPO_MARKER = '.genvm-monorepo-root' # Per-repo project file (manager + every executor). Exec'd once before any -# subcommand runs; subcommands ask it for what they need β€” `hooks(ctx)` (commit -# hooks for that repo) and, on the manager, `tests(ctx)` (the test suite). +# subcommand runs; on the manager it also provides `tests(ctx)` (the test suite). PROJECT_FILE = '.genvm-tool.py' -PRECOMMIT_SUBDIR = 'support/nix/precommit' -# Kept verbatim per the agreed cache location; sits in the gitignored `.direnv`. -CACHE_SUBDIR = '.direnv/genvm-git-helper' -DEFAULT_STAGE = 'pre-commit' @functools.lru_cache(maxsize=None) @@ -118,14 +111,6 @@ def __init__(self, name: str, path: Path, rel: str): self.path = path # absolute working-tree root self.rel = rel # '' for manager, else the manager-relative prefix - @property - def precommit_dir(self) -> Path: - return self.path / PRECOMMIT_SUBDIR - - @property - def cache_dir(self) -> Path: - return self.path / CACHE_SUBDIR - def is_checked_out(self) -> bool: # A submodule's `.git` is a gitdir file; the manager's is a directory. return (self.path / '.git').exists() @@ -180,32 +165,6 @@ def executor_rels(root: Path) -> set[str]: # --- git ------------------------------------------------------------------- -# git scopes itself to one repo through these variables. A pre-commit / commit-msg -# hook runs with them set to the *parent* repo (GIT_INDEX_FILE points at the -# in-progress commit's index, GIT_DIR at the parent .git, ...). Because we operate -# on each repo β€” and its submodules β€” by explicit path (`git -C `), those -# inherited values must be dropped: otherwise a submodule `git` reads the parent's -# index/objects and dies (e.g. `git diff --cached` -> exit 128, "unable to read -# "). See `strip_inherited_git_env`. -_INHERITED_GIT_ENV_VARS = ( - 'GIT_DIR', - 'GIT_WORK_TREE', - 'GIT_INDEX_FILE', - 'GIT_PREFIX', - 'GIT_COMMON_DIR', - 'GIT_OBJECT_DIRECTORY', - 'GIT_ALTERNATE_OBJECT_DIRECTORIES', - 'GIT_NAMESPACE', -) - - -def strip_inherited_git_env() -> None: - """Drop the repo-scoping GIT_* variables inherited from a parent git hook, so - our per-repo `git -C ` calls discover each repo from its own path - instead of being pinned to the invoking repo.""" - for var in _INHERITED_GIT_ENV_VARS: - os.environ.pop(var, None) - def _git_z(repo: Repo, *args: str) -> list[str]: out = subprocess.run( @@ -223,10 +182,3 @@ def ls_files(repo: Repo, drop_rels: set[str] | None = None) -> list[str]: if drop_rels: files = [f for f in files if f not in drop_rels] return files - - -def staged_files(repo: Repo) -> list[str]: - """Staged added/copied/modified files (repo-relative), regular files only.""" - files = _git_z(repo, 'diff', '--cached', '--name-only', '--diff-filter=ACM') - # Drop the executor gitlink (a staged submodule bump is not a file). - return [f for f in files if (repo.path / f).is_file()] diff --git a/support/tools/genvm-tool/genvm_tool/git/check_for_push.py b/support/tools/genvm-tool/genvm_tool/git/check_for_push.py index 4ebcbc21..2d31fd51 100644 --- a/support/tools/genvm-tool/genvm_tool/git/check_for_push.py +++ b/support/tools/genvm-tool/genvm_tool/git/check_for_push.py @@ -11,6 +11,10 @@ - in_sync_with_origin β€” the branch tip vs its branch on `origin` - action β€” the next step in the add β†’ commit β†’ push pipeline +After all repo rows, it emits one aggregated `suggested_push_command` block: a +newline-joined shell snippet that pushes the ready repos in order, or `none` +when anything is not ready. + By default the origin comparison is a LIVE `git ls-remote` query. The queries are launched up front β€” one per distinct origin URL (the executor submodules share a remote, so they batch into a single multi-ref call) and all in parallel β€” before @@ -25,13 +29,15 @@ `same_branch_as_manager` is shown for submodules as context (the line-namespaced mirror convention: `feat/x` on the manager maps to `pr/v0.3/feat/x` in the executor β€” see `common.Repo.feature_branch`) but does not gate `ready`. Exit -status is non-zero when any repo is not ready, so the command can gate a push. +status is non-zero when any repo is not ready. An `absent` origin branch is +still push-ready because `git push` will create it. This NEVER fetches, resets, switches, or writes anything β€” `ls-remote` reads the remote without updating any local ref. """ import os +import shlex import subprocess from .. import common @@ -242,6 +248,13 @@ def _suggest(detached: bool, staged: bool, unstaged: bool, origin_kind: str) -> }.get(origin_kind, 'fetch (cannot compare with origin)') +def _suggested_push_command(repo: common.Repo, branch: str, ready: bool) -> str | None: + """A copy-pasteable shell command for pushing the current branch.""" + if not ready or not branch: + return None + return f'cd {shlex.quote(str(repo.path))} && git push origin {shlex.quote(branch)}' + + def main(ctx: common.Context, args) -> int: manager = common.discover_repos(ctx.root, 'manager')[0] manager_branch = _current_branch(manager) @@ -280,13 +293,22 @@ def main(ctx: common.Context, args) -> int: ) local[repo.name] = row + # Present executor repos first so the manager row naturally comes last, matching + # the push order in the submodules guide. + entries.sort(key=lambda item: item[0].rel == '') + lookup = _collect_origin_queries(launched) if launched is not None else {} all_ready = True + push_commands: list[str] = [] for repo, checked_out, branch in entries: if not checked_out: # Not checked out: nothing local to push, and nothing we can verify. - ctx.printer.put(repo.name, checked_out='no', action='checkout submodule') + ctx.printer.put( + repo.name, + checked_out='no', + action='checkout submodule', + ) continue is_sub = repo.rel != '' @@ -297,7 +319,7 @@ def main(ctx: common.Context, args) -> int: else: sync_msg, kind = _origin_state_remote(repo, branch, lookup) - ready = row['changed'] == 0 and not detached and kind == 'synced' + ready = row['changed'] == 0 and not detached and kind in {'synced', 'absent'} fields = { 'branch': branch or 'DETACHED', 'dirty': f'yes ({row["changed"]})' if row['changed'] else 'no', @@ -321,7 +343,17 @@ def main(ctx: common.Context, args) -> int: fields['in_sync_with_origin'] = sync_msg fields['action'] = _suggest(detached, row['staged'], row['unstaged'], kind) fields['ready'] = 'yes' if ready else 'NO' + if ready: + push_commands.append(_suggested_push_command(repo, branch, ready)) all_ready = all_ready and ready ctx.printer.put(repo.name, **fields) + if all_ready and push_commands: + ctx.printer.put( + 'suggested_push_command', + command='\n'.join(push_commands), + ) + else: + ctx.printer.put('suggested_push_command', command='none') + return 0 if all_ready else 1 diff --git a/support/tools/genvm-tool/genvm_tool/hook/__init__.py b/support/tools/genvm-tool/genvm_tool/hook/__init__.py deleted file mode 100644 index ec88c34b..00000000 --- a/support/tools/genvm-tool/genvm_tool/hook/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -"""`hook` command group: run / install the per-repo lint hooks.""" - -from . import install, run - -NAME = 'hook' -HELP = 'run and install the per-repo lint hooks' -COMMANDS = [run, install] diff --git a/support/tools/genvm-tool/genvm_tool/hook/engine.py b/support/tools/genvm-tool/genvm_tool/hook/engine.py deleted file mode 100644 index fa446085..00000000 --- a/support/tools/genvm-tool/genvm_tool/hook/engine.py +++ /dev/null @@ -1,300 +0,0 @@ -"""Hook-execution engine: nix tool profiles, file filtering, runner, builtins. - -Repos are processed concurrently (disjoint working trees) and each repo's tool -profile is built concurrently; hooks *within* a repo run sequentially so the -in-place fixers (trailing-whitespace, clang-format, cargo fmt) never race on the -same file. -""" - -import asyncio -import hashlib -import os -import re -from dataclasses import dataclass -from pathlib import Path - -from .. import common -from . import mdlinks - -# Extension -> identify-style type classes used by hooks' `types_or` filter. -# `.h` is ambiguous, so it matches both c and c++. -EXT_TYPES = { - '.py': {'python'}, - '.pyi': {'python'}, - '.c': {'c'}, - '.h': {'c', 'c++'}, - '.cc': {'c++'}, - '.cpp': {'c++'}, - '.cxx': {'c++'}, - '.hpp': {'c++'}, - '.hh': {'c++'}, - '.rs': {'rust'}, - '.lua': {'lua'}, - '.ts': {'ts'}, - '.tsx': {'tsx'}, - '.json': {'json'}, - '.yaml': {'yaml'}, - '.yml': {'yaml'}, - '.toml': {'toml'}, - '.md': {'markdown'}, - '.markdown': {'markdown'}, -} - - -# --- subprocess ------------------------------------------------------------ - - -async def _run(ctx: common.Context, argv: list[str], **kw) -> tuple[int, str]: - """Spawn `argv`, capture combined output, return (rc, output).""" - ctx.logger.trace('exec', argv=argv, cwd=kw.get('cwd')) - proc = await asyncio.create_subprocess_exec( - *argv, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.STDOUT, - **kw, - ) - out, _ = await proc.communicate() - return proc.returncode, out.decode('utf-8', 'replace') - - -# --- nix tool profile ------------------------------------------------------ - -_PROFILE_KEY_RE = re.compile(r'[0-9a-f]{16}$') - - -def _profile_key(precommit_dir: Path) -> str: - h = hashlib.sha256() - for name in ('flake.nix', 'flake.lock'): - f = precommit_dir / name - if f.is_file(): - h.update(f.read_bytes()) - return h.hexdigest()[:16] - - -async def build_profile(ctx: common.Context, repo: common.Repo) -> Path | None: - """Realise the repo's lint-tool buildEnv; return its `bin/` (or None). - - Cached as a gc-root out-link named after the hash of flake.nix+flake.lock; - a present link with a live store target is reused, stale profile siblings are - pruned (the `genvm-tool` install link, a non-hash name, is left alone). - """ - pcd = repo.precommit_dir - if not (pcd / 'flake.nix').is_file(): - return None - key = _profile_key(pcd) - cache = repo.cache_dir - cache.mkdir(parents=True, exist_ok=True) - link = cache / key - if link.exists(): # present and store target alive - ctx.logger.debug('tool profile cached', repo=repo.name, key=key) - return link / 'bin' - for sib in cache.iterdir(): - if sib.name != key and _PROFILE_KEY_RE.fullmatch(sib.name) and sib.is_symlink(): - sib.unlink() - ctx.logger.info('building tool profile', repo=repo.name) - rc, out = await _run(ctx, ['nix', 'build', f'path:{pcd}', '--out-link', str(link)]) - if rc != 0: - raise common.ToolError(f'nix build failed for {repo.name}:\n{out}') - return link / 'bin' - - -# --- hook manifests + file filtering --------------------------------------- - - -def load_hooks(ctx: common.Context, repo: common.Repo, stage: str) -> list[dict]: - """Hooks for `repo`/`stage`, sourced from that repo's `.genvm-tool.py:hooks`.""" - project = common.load_project(repo.path) - hooks_fn = getattr(project, 'hooks', None) if project else None - raw = list(hooks_fn(ctx)) if hooks_fn else [] - return [h for h in raw if stage in h.get('stages', [common.DEFAULT_STAGE])] - - -def _file_types(path: str) -> set[str]: - return EXT_TYPES.get(Path(path).suffix.lower(), set()) - - -def _is_binary(path: Path) -> bool: - try: - with open(path, 'rb') as fh: - return b'\0' in fh.read(8192) - except OSError: - return True - - -def filter_files(hook: dict, files: list[str], repo: common.Repo) -> list[str]: - inc = re.compile(hook['files']) if hook.get('files') else None - exc = re.compile(hook['exclude']) if hook.get('exclude') else None - types_or = set(hook['types_or']) if hook.get('types_or') else None - # `text` is a pseudo-type (any non-binary file) β€” mirrors pre-commit's - # default `types: [text]`, which keeps the in-place fixers off binaries. - want_text = bool(types_or) and 'text' in types_or - ext_types = (types_or - {'text'}) if types_or else None - out = [] - for f in files: - if inc and not inc.search(f): - continue - if exc and exc.search(f): - continue - if types_or is not None: - ok = bool(ext_types and (_file_types(f) & ext_types)) - if not ok and want_text and not _is_binary(repo.path / f): - ok = True - if not ok: - continue - out.append(f) - return out - - -# --- runner ---------------------------------------------------------------- - - -def _hook_args(hook: dict, fix: bool) -> list[str]: - """Args for the hook in the requested mode. In fix mode a hook's - `fix_args` (the rewrite-in-place invocation) replace its `args`; a hook - without `fix_args` has no fix mode and runs its plain `args` either way.""" - if fix and 'fix_args' in hook: - return list(hook['fix_args']) - return list(hook.get('args', [])) - - -def _argv(repo: common.Repo, hook: dict, hook_args: list[str]) -> list[str]: - entry = hook['entry'] - # A local hook whose entry is a path is resolved against the repo root; - # a bare command (local or not) is resolved from PATH (profile first). - if hook.get('local') and '/' in entry: - argv = [str(repo.path / entry)] - else: - argv = [entry] - return argv + hook_args - - -def _profile_env(profile_bin: Path | None) -> dict: - env = dict(os.environ) - if profile_bin is not None: - env['PATH'] = f'{profile_bin}{os.pathsep}' + env.get('PATH', '') - return env - - -@dataclass -class HookResult: - repo: str - hook_id: str - status: str # 'pass' | 'skip' | 'fail' - output: str = '' - - -async def _builtin_cargo_fmt(ctx, repo, env, files, fix) -> tuple[int, str]: - """Format (or in check mode verify) every cargo crate tracked in `repo`. - - Per-crate `cargo fmt` picks up each crate's edition from its Cargo.toml, so - this is correct across the repo's independent workspaces. In fix mode it - rewrites in place; in check mode it adds `--check` and fails on drift. - """ - mode = [] if fix else ['--check'] - dirs = sorted( - {str(Path(m).parent) for m in common.ls_files(repo) if m.endswith('Cargo.toml')} - ) - rc_all, chunks = 0, [] - for d in dirs: - rc, out = await _run(ctx, ['cargo', 'fmt', *mode], cwd=str(repo.path / d), env=env) - if rc != 0: - rc_all = 1 - chunks.append(f'# {d}\n{out}'.rstrip()) - return rc_all, '\n'.join(chunks) - - -async def _builtin_md_local_links(ctx, repo, env, files, fix) -> tuple[int, str]: - """Verify local file-path links in the given markdown files resolve. - - Check-only; broken links cannot be auto-fixed, so `fix` is ignored.""" - problems = [] - for f in files: - problems += mdlinks.broken_links(repo.path / f, repo.path) - return (1 if problems else 0), '\n'.join(problems) - - -# Hooks may set `builtin = ""` instead of `entry` to run logic that does -# not map to a single file-list invocation (per-crate cargo fmt; in-process -# markdown link checking). Each receives (ctx, repo, env, selected_files, fix). -BUILTINS = { - 'cargo-fmt': _builtin_cargo_fmt, - 'md-local-links': _builtin_md_local_links, -} - - -async def execute_hook( - ctx: common.Context, - repo: common.Repo, - hook: dict, - files: list[str], - profile_bin: Path | None, - *, - stage: str, - msg_file: str | None = None, - fix: bool = True, -) -> HookResult: - """Run one hook, capturing its output.""" - has_selector = bool(hook.get('files') or hook.get('types_or')) - label = HookResult(repo.name, hook['id'], 'pass') - env = _profile_env(profile_bin) - - if stage == 'commit-msg': - selected = [] - else: - selected = filter_files(hook, files, repo) - # Nothing to do: a filename-passing hook with no files would otherwise - # get zero args and scan the whole tree; a selector hook with no match - # is simply skipped. (Matches pre-commit.) - if not selected and (hook.get('pass_filenames', True) or has_selector): - label.status = 'skip' - return label - - if builtin := hook.get('builtin'): - rc, out = await BUILTINS[builtin](ctx, repo, env, selected, fix) - else: - argv = _argv(repo, hook, _hook_args(hook, fix)) - if stage == 'commit-msg': - if msg_file is not None: - argv.append(msg_file) - elif hook.get('pass_filenames', True): - argv += selected - rc, out = await _run(ctx, argv, cwd=str(repo.path), env=env) - - label.status = 'pass' if rc == 0 else 'fail' - label.output = out - return label - - -async def process_repo( - ctx: common.Context, - repo: common.Repo, - stage: str, - files: list[str], - *, - msg_file: str | None = None, - fix: bool = True, -) -> list[HookResult]: - """Build the profile, then run every `stage` hook of `repo` sequentially.""" - hooks = load_hooks(ctx, repo, stage) - if not hooks: - return [] - profile_bin = await build_profile(ctx, repo) - results = [] - for hook in hooks: - res = await execute_hook( - ctx, repo, hook, files, profile_bin, stage=stage, msg_file=msg_file, fix=fix - ) - _report(ctx, res) - results.append(res) - return results - - -def _report(ctx: common.Context, res: HookResult) -> None: - tag = {'pass': 'βœ“', 'skip': '-', 'fail': 'βœ—'}[res.status] - name = f'{res.repo}:{res.hook_id}' - if res.status == 'fail': - ctx.printer.put(f'{tag} {name} FAILED', output=res.output.rstrip()) - elif res.status == 'skip': - ctx.logger.debug(f'{tag} {name} (skipped)') - else: - ctx.logger.info(f'{tag} {name}') diff --git a/support/tools/genvm-tool/genvm_tool/hook/install.py b/support/tools/genvm-tool/genvm_tool/hook/install.py deleted file mode 100644 index 2f3b9ecf..00000000 --- a/support/tools/genvm-tool/genvm_tool/hook/install.py +++ /dev/null @@ -1,109 +0,0 @@ -"""`genvm-tool hook install` β€” build the tool and install git-hook stubs. - -Stubs exec the nix-built `genvm-tool` (so commits work without the dev shell), -falling back to the in-tree wrapper, and `exit 0` in a standalone checkout where -the manager root cannot be located. -""" - -import stat -import subprocess -from pathlib import Path - -from .. import common - -NAME = 'install' -HELP = 'build genvm-tool and install the git-hook stubs into every repo' - -STAGES = ('pre-commit', 'commit-msg') -TOOL_LINK = 'genvm-tool' # out-link name under the manager cache dir - -_STUB = """\ -#!/usr/bin/env bash -# Installed by `genvm-tool hook install`. Do not edit. -set -euo pipefail -dir="$(git rev-parse --show-toplevel)" -while [ "$dir" != / ] && [ ! -f "$dir/.genvm-monorepo-root" ]; do - dir="$(dirname "$dir")" -done -[ -f "$dir/.genvm-monorepo-root" ] || exit 0 # standalone checkout: skip -tool="$dir/{cache}/{link}/bin/genvm-tool" -[ -x "$tool" ] || {{ echo "genvm-tool not built (run 'genvm-tool hook install'); skipping {stage}" >&2; exit 0; }} -# The dev shell exports PYTHONPATH pointing at *its* genvm-tool build; left set, -# the hook's python would import genvm_tool from there instead of the tool we -# just resolved (running stale code). The nix wrapper bakes in every dep, so drop -# the inherited python path and let it use its own. -unset PYTHONPATH NIX_PYTHONPATH -exec "$tool" hook run --repo {repo} --hook-stage {stage} -- "$@" -""" - - -def configure(parser): - parser.add_argument( - '--repo', - default='all', - help='repo selector for hook install (default: all)', - ) - - -def _hooks_dir(repo: common.Repo) -> Path: - out = subprocess.run( - ['git', '-C', str(repo.path), 'rev-parse', '--git-path', 'hooks'], - capture_output=True, - text=True, - check=True, - ).stdout.strip() - p = Path(out) - return p if p.is_absolute() else repo.path / p - - -def _repo_selector(repo: common.Repo, stage: str) -> str: - """Repo selector baked into the git-hook stub for `repo`/`stage`. - - The manager's `pre-commit` fans out to `all` so a manager commit also lints - any staged executor changes. `commit-msg` stays manager-local: the message - belongs to the manager commit, not the children. - """ - if repo.rel == '' and stage == 'pre-commit': - return 'all' - return repo.name - - -def _write_stub(path: Path, repo_selector: str, stage: str): - path.write_text( - _STUB.format( - cache=common.CACHE_SUBDIR, link=TOOL_LINK, repo=repo_selector, stage=stage - ) - ) - path.chmod(path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) - - -def main(ctx: common.Context, args) -> int: - root = ctx.root - - # Build genvm-tool from its standalone default.nix (not the umbrella flake) - # into a gc-root out-link the stubs point at. - (root / common.CACHE_SUBDIR).mkdir(parents=True, exist_ok=True) - ctx.logger.info('building genvm-tool') - subprocess.run( - [ - 'nix-build', - 'support/tools/genvm-tool/default.nix', - '-o', - str(root / common.CACHE_SUBDIR / TOOL_LINK), - ], - cwd=str(root), - check=True, - ) - - for repo in common.discover_repos(root, args.repo): - if not repo.is_checked_out(): - ctx.logger.warning('skipping repo (not checked out)', repo=repo.name) - continue - hooks_dir = _hooks_dir(repo) - hooks_dir.mkdir(parents=True, exist_ok=True) - for stage in STAGES: - _write_stub(hooks_dir / stage, _repo_selector(repo, stage), stage) - ctx.printer.put( - 'installed hooks', repo=repo.name, stages=list(STAGES), dir=str(hooks_dir) - ) - return 0 diff --git a/support/tools/genvm-tool/genvm_tool/hook/run.py b/support/tools/genvm-tool/genvm_tool/hook/run.py deleted file mode 100644 index b4358980..00000000 --- a/support/tools/genvm-tool/genvm_tool/hook/run.py +++ /dev/null @@ -1,98 +0,0 @@ -"""`genvm-tool hook run` β€” run per-repo hooks (CI, manual, and git-hook entry). - -This is also the git-hook entry point: the installed stubs invoke -`hook run --repo --hook-stage -- "$@"`, where the trailing args -carry e.g. the commit-msg file path. Without `--all-files` only staged files -are linted; a manager commit stages only manager files (the executor gitlink is -dropped), an executor commit only executor files. The manager's `pre-commit` -stub selects `--repo all`, so a manager commit also fans out to the children's -`pre-commit` over their respective staged files. - -Hooks fix in place by default (a tool with a `fix_args` invocation rewrites; -others fall back to their check invocation). Pass `--check` to only verify and -never modify β€” CI uses it so unformatted code fails instead of being silently -rewritten. - -Repos run concurrently (disjoint working trees); each repo builds its tool -profile then runs its hooks sequentially. -""" - -import asyncio - -from .. import common -from . import engine - -NAME = 'run' -HELP = 'run hooks across the manager and executor submodules' - - -def configure(parser): - parser.add_argument('--repo', default='all', help='repo selector (default: all)') - parser.add_argument( - '--all-files', - action='store_true', - help='run over every tracked file instead of just the staged ones', - ) - # Fix in place by default; tools without a fix mode just run as a check. - # `--check` forces check-only (CI uses it so unformatted code fails instead - # of being silently rewritten). - fix = parser.add_mutually_exclusive_group() - fix.add_argument( - '--fix', - dest='fix', - action='store_true', - default=True, - help='apply fixes in place where the tool supports it (default)', - ) - fix.add_argument( - '--check', - dest='fix', - action='store_false', - help='only check; never modify files', - ) - parser.add_argument('--hook-stage', default=common.DEFAULT_STAGE) - # Trailing args git passes to the hook (e.g. the commit-msg file path). - parser.add_argument('hook_args', nargs='*') - - -async def _one_repo(ctx, repo, stage, all_files, msg_file, fix): - if not engine.load_hooks(ctx, repo, stage): - return [] - if not repo.is_checked_out(): - raise common.ToolError(f'{repo.name} is not checked out (submodule missing)') - if stage == 'commit-msg': - files = [] - elif all_files: - # Drop submodule gitlinks from the manager listing (they are not files). - drop = common.executor_rels(ctx.root) if repo.rel == '' else None - files = common.ls_files(repo, drop) - else: - files = common.staged_files(repo) - return await engine.process_repo(ctx, repo, stage, files, msg_file=msg_file, fix=fix) - - -async def main(ctx: common.Context, args) -> int: - # When invoked as a git hook, git exports GIT_DIR / GIT_INDEX_FILE / ... for - # the *parent* repo. We fan out to every repo (and its submodules) by explicit - # path, so leaving those set would pin our `git -C ` calls to the - # parent's index/objects and crash them (exit 128). Strip them up front. - common.strip_inherited_git_env() - repos = common.discover_repos(ctx.root, args.repo) - msg_file = ( - args.hook_args[0] if (args.hook_stage == 'commit-msg' and args.hook_args) else None - ) - batches = await asyncio.gather( - *( - _one_repo(ctx, r, args.hook_stage, args.all_files, msg_file, args.fix) - for r in repos - ) - ) - failed = [res for batch in batches for res in batch if res.status == 'fail'] - if failed: - ctx.printer.put( - 'hooks failed', - count=len(failed), - hooks=[f'{r.repo}:{r.hook_id}' for r in failed], - ) - return 1 - return 0 diff --git a/support/tools/genvm-tool/genvm_tool/tests/__init__.py b/support/tools/genvm-tool/genvm_tool/tests/__init__.py index c8b7e191..5709aaf4 100644 --- a/support/tools/genvm-tool/genvm_tool/tests/__init__.py +++ b/support/tools/genvm-tool/genvm_tool/tests/__init__.py @@ -60,6 +60,8 @@ class SharedContext: # collectors / CLI args. suite: Any = None + ci: bool = False + _git_files: list[Path] | None = None _interrupted: threading.Event = field(default_factory=threading.Event) _config: dict[str, Any] | None = None @@ -107,8 +109,14 @@ def git_files(self) -> list[Path]: if self._git_files is not None: return self._git_files + # --recurse-submodules so executor crates (each `executors/.x` is a + # submodule since the monorepo split) are listed too: the umbrella's + # collect_rust is the only path that discovers Cargo.toml/fuzz targets, and + # executors carry no tests() hook of their own. Without it only the two + # manager crates are seen and every executor rust crate goes untested. + # Uninitialized submodules are simply skipped (not an error). r = subprocess.run( - ['git', 'ls-files'], + ['git', 'ls-files', '--recurse-submodules'], check=True, capture_output=True, text=True, diff --git a/support/tools/genvm-tool/genvm_tool/tests/cli.py b/support/tools/genvm-tool/genvm_tool/tests/cli.py index 03189e9a..da95999e 100755 --- a/support/tools/genvm-tool/genvm_tool/tests/cli.py +++ b/support/tools/genvm-tool/genvm_tool/tests/cli.py @@ -425,6 +425,9 @@ async def foo(): conf_env = asyncio.run(foo()) + # --ci is a `run`-subcommand flag; absent for other subcommands (default False). + shared_context.ci = getattr(conf_env.args, 'ci', False) + if 'func' not in conf_env.args: logger.error('subcommand not given') parser_result.parser.print_help() diff --git a/support/tools/genvm-tool/genvm_tool/tests/stage/execution.py b/support/tools/genvm-tool/genvm_tool/tests/stage/execution.py index c99ad256..e007d35a 100644 --- a/support/tools/genvm-tool/genvm_tool/tests/stage/execution.py +++ b/support/tools/genvm-tool/genvm_tool/tests/stage/execution.py @@ -380,10 +380,17 @@ async def _run_case_locked(ctx: _ExecutionContext, case: genvm_tool.tests.test.C # Mirror the full result (with context) into the per-test log file if file_fmt is not None: sign = 'PASS' if success else 'FAIL' + retract_without_ci_flag = [] if ctx.shared.ci else ['raw_result'] + retracted = {} + for k in retract_without_ci_flag: + if k in context: + retracted[k] = context.pop(k) file_fmt.put( f'{sign} {case.description.name} in {elapsed:.3f}s', **context, ) + for k, v in retracted.items(): + context[k] = v if log_file is not None: log_file.close() diff --git a/support/tools/git-third-party/git-third-party b/support/tools/git-third-party/git-third-party index 9843b7fe..e0b436a5 100755 --- a/support/tools/git-third-party/git-third-party +++ b/support/tools/git-third-party/git-third-party @@ -17,58 +17,73 @@ # along with this program. If not, see . # -EXE_NAME = 'git third-party' - -import subprocess -import os -import sys import abc -import typing -import json -import textwrap import io +import json import logging +import os +import re +import subprocess +import sys +import textwrap +import typing from pathlib import Path +EXE_NAME = 'git third-party' + logging.basicConfig(stream=sys.stderr, level=logging.INFO) -GIT = os.getenv('GIT', 'git') +# Runtime context, populated by `_init_context()` at the start of each `main()` +# call rather than at import time. Keeping import side-effect free lets this file +# be imported directly (e.g. by the test-suite) and driven in-process. +GIT = 'git' +top_dir: Path +cur_dir: Path +SAVED_FILES_PATH: Path +PATCHES_PATH: Path +CONFIG_PATH: Path +config: dict -try: - top_dir_str = subprocess.check_output( - [GIT, 'rev-parse', '--show-toplevel'], text=True - ) -except Exception as e: - print( - f'could not execute `{GIT} rev-parse --show-toplevel`. You may wish to set `GIT` env variable', - file=sys.stderr, - ) - exit(1) -top_dir = Path(top_dir_str.strip()) -cur_dir = Path(os.getcwd()) +def _init_context() -> None: + global GIT, top_dir, cur_dir, SAVED_FILES_PATH, PATCHES_PATH, CONFIG_PATH, config -SAVED_FILES_PATH = top_dir.joinpath('.git-third-party') + GIT = os.getenv('GIT', 'git') -PATCHES_PATH = SAVED_FILES_PATH.joinpath('patches') -CONFIG_PATH = SAVED_FILES_PATH.joinpath('config.json') + try: + top_dir_str = subprocess.check_output( + [GIT, 'rev-parse', '--show-toplevel'], text=True + ) + except Exception: + print( + f'could not execute `{GIT} rev-parse --show-toplevel`. You may wish to set `GIT` env variable', + file=sys.stderr, + ) + sys.exit(1) -if not PATCHES_PATH.exists(): - logging.info(f'patches directory does not exist, creating path=`{PATCHES_PATH}`') - PATCHES_PATH.mkdir(parents=True, exist_ok=True) + top_dir = Path(top_dir_str.strip()) + cur_dir = Path(os.getcwd()) -if not CONFIG_PATH.exists(): - config = {'repos': {}} -else: - config = json.loads(CONFIG_PATH.read_text()) + SAVED_FILES_PATH = top_dir.joinpath('.git-third-party') + PATCHES_PATH = SAVED_FILES_PATH.joinpath('patches') + CONFIG_PATH = SAVED_FILES_PATH.joinpath('config.json') + if not PATCHES_PATH.exists(): + logging.info(f'patches directory does not exist, creating path=`{PATCHES_PATH}`') + PATCHES_PATH.mkdir(parents=True, exist_ok=True) + + if not CONFIG_PATH.exists(): + config = {'repos': {}} + else: + config = json.loads(CONFIG_PATH.read_text()) -class Command: + +class Command(metaclass=abc.ABCMeta): name: typing.ClassVar[str] help: str @abc.abstractmethod - def handle(args: list[str]) -> None: ... + def handle(self, args: list[str]) -> None: ... def show_help(self, file: io.StringIO): file.write(f'subcommand `{self.name}`:\n') @@ -82,12 +97,127 @@ def _get_patches_dir(name: str) -> Path: def _dirty_check(name: str) -> None: target_dir = top_dir.joinpath(*name.split('/')) - for opts in ([], ['--cached']): - rs = subprocess.run( - [GIT, 'diff', '--exit-code', '--quiet', *opts], check=False, cwd=target_dir + # `--untracked-files=all` so a brand-new local file counts as dirt, and + # `--ignore-submodules=none` so a hostile `diff.ignoreSubmodules` config can + # not hide a modified submodule. A non-empty porcelain listing is dirt; it + # also names the offending paths, which we surface to the user. + rs = subprocess.run( + [GIT, 'status', '--porcelain', '--untracked-files=all', '--ignore-submodules=none'], + check=True, + cwd=target_dir, + text=True, + capture_output=True, + ) + if rs.stdout.strip(): + raise GitThirdPartyException( + f'repo {name} is dirty ( {target_dir} ):\n{rs.stdout.rstrip()}' + ) + + +def _patch_id(diff: str, cwd: Path) -> str | None: + """Stable patch-id of a diff/patch, or None when there is nothing to hash. + + `--stable` hashes only the diff hunks, ignoring the commit hash, author, + date and message β€” so a patch re-applied by `git am` compares equal to the + saved patch file it came from, which is exactly the metadata-independence the + rest of the tool relies on. + """ + rs = subprocess.run( + [GIT, 'patch-id', '--stable'], + input=diff, + text=True, + capture_output=True, + check=True, + cwd=cwd, + ) + out = rs.stdout.strip() + # Empty for a patch with no textual diff (e.g. an empty or merge commit). + return out.split()[0] if out else None + + +def _unsaved_commits_check(name: str) -> None: + """Refuse to overwrite local commits that were never `save`d. + + `update` detaches to the base commit and replays the *saved* patch series, so + any commit on top that no patch file records would be silently orphaned. We + compare the current history above the base against the saved patches by + patch-id, keep the longest matching prefix (K), and error when the checkout + is *ahead* β€” carrying `commits(base..HEAD) - K` unrecorded commits. + """ + conf = config['repos'][name] + target_dir = top_dir.joinpath(*name.split('/')) + base = conf['commit'] + + # Nothing checked out yet (fresh `add` / freshly `init`-ed dir): nothing to + # protect. + if ( + subprocess.run( + [GIT, 'rev-parse', '--verify', '--quiet', 'HEAD'], + check=False, + cwd=target_dir, + capture_output=True, + ).returncode + != 0 + ): + return + + # Base not in this checkout: we cannot relate HEAD to the saved series (most + # likely the pin was bumped). Let update fetch + reapply; the worktree dirty + # check still protects uncommitted work. + if ( + subprocess.run( + [GIT, 'cat-file', '-e', f'{base}^{{commit}}'], + check=False, + cwd=target_dir, + capture_output=True, + ).returncode + != 0 + ): + logging.warning( + f'base commit {base} not present in {name}, skipping unsaved-commits check' + ) + return + + revs = subprocess.run( + [GIT, 'rev-list', '--reverse', f'{base}..HEAD'], + check=True, + cwd=target_dir, + text=True, + capture_output=True, + ).stdout.split() + current_ids = [ + _patch_id( + subprocess.run( + [GIT, 'diff-tree', '--patch', '--no-commit-id', rev], + check=True, + cwd=target_dir, + text=True, + capture_output=True, + ).stdout, + target_dir, + ) + for rev in revs + ] + + patches_dir = _get_patches_dir(name) + saved_ids = [ + _patch_id(patches_dir.joinpath(str(i)).read_text(), target_dir) + for i in range(1, conf['patches'] + 1) + ] + + # Longest common prefix of patch-ids; a None (unhashable commit) stops it. + matched = 0 + for cur, saved in zip(current_ids, saved_ids): + if cur is None or cur != saved: + break + matched += 1 + + ahead = len(current_ids) - matched + if ahead > 0: + raise GitThirdPartyException( + f'repo {name} is {ahead} commit(s) ahead of its saved patches ' + f'( {target_dir} ); run `save` first or reset those commits' ) - if rs.returncode != 0: - raise GitThirdPartyException(f'repo {name} is dirty ( {target_dir} )') def _update_at(name: str) -> None: @@ -102,6 +232,7 @@ def _update_at(name: str) -> None: ) _dirty_check(name) + _unsaved_commits_check(name) rs = subprocess.run( [GIT, '-c', 'advice.detachedHead=false', 'checkout', conf['commit']], check=False, @@ -145,7 +276,7 @@ def _update_at(name: str) -> None: if len(patch_files) != 0: subprocess.run([GIT, 'am', *patch_files], cwd=target_dir, check=True) else: - logging.warning(f'no patches detected') + logging.warning('no patches detected') class GitThirdPartyException(Exception): @@ -174,17 +305,16 @@ class AddCommand(Command): if len(args) != 3: print(f'Unexpected arguments {args}', file=sys.stderr) self.show_help(sys.stderr) - exit(1) + sys.exit(1) path, repo, commit = args - import re if re.search(r'[\\:;]', path): print(f'bad path `{path}`', file=sys.stderr) - exit(1) + sys.exit(1) name = _get_path_rel_to_dir(path) if name in config['repos']: print(f'repo ({name}) is already in config', file=sys.stderr) - exit(1) + sys.exit(1) target_dir = top_dir.joinpath(*name.split('/')) res = subprocess.run([GIT, 'check-ignore', target_dir], check=False) if res.returncode != 0: @@ -278,31 +408,40 @@ def _save_at(name: str) -> None: conf['patches'] = patches_cnt -mode = sys.argv[1] if len(sys.argv) > 1 else 'help' +def main(argv: list[str] | None = None) -> int: + argv = sys.argv[1:] if argv is None else list(argv) -handlers: dict[str, Command] = {} + _init_context() -for c in [UpdateCommand, SaveCommand, AddCommand]: - handlers[c.name] = c() + mode = argv[0] if len(argv) > 0 else 'help' -mode_handler = handlers.get(mode, None) -if mode_handler is None: - print( - f'{EXE_NAME} expected one of following subcommands: ' + '|'.join(handlers.keys()), - file=sys.stderr, - ) - for k in handlers.values(): - k.show_help(sys.stderr) - exit(1) + handlers: dict[str, Command] = {} -try: - mode_handler.handle(sys.argv[2:]) -except GitThirdPartyException as e: - print(*e.args, file=sys.stderr) - exit(1) + for c in [UpdateCommand, SaveCommand, AddCommand]: + handlers[c.name] = c() + + mode_handler = handlers.get(mode, None) + if mode_handler is None: + print( + f'{EXE_NAME} expected one of following subcommands: ' + '|'.join(handlers.keys()), + file=sys.stderr, + ) + for k in handlers.values(): + k.show_help(sys.stderr) + return 1 + + try: + mode_handler.handle(argv[1:]) + except GitThirdPartyException as e: + print(*e.args, file=sys.stderr) + return 1 + + with open(CONFIG_PATH, 'wt') as f: + json.dump(config, f, indent='\t') + f.write('\n') + + return 0 -import json -with open(CONFIG_PATH, 'wt') as f: - json.dump(config, f, indent='\t') - f.write('\n') +if __name__ == '__main__': # pragma: no cover + sys.exit(main()) diff --git a/tests/runner/genvm_tool_plugins/cargo.py b/tests/runner/genvm_tool_plugins/cargo.py index d6917eb6..ea23afac 100644 --- a/tests/runner/genvm_tool_plugins/cargo.py +++ b/tests/runner/genvm_tool_plugins/cargo.py @@ -278,6 +278,7 @@ def cargo_test( base_cmd = [ 'cargo', 'test', + '--message-format=short', '--color=always', '--target-dir', str(TARGET_DIR), diff --git a/tests/runner/genvm_tool_plugins/integration.py b/tests/runner/genvm_tool_plugins/integration.py index 7b1db4e9..0be01f89 100644 --- a/tests/runner/genvm_tool_plugins/integration.py +++ b/tests/runner/genvm_tool_plugins/integration.py @@ -163,6 +163,15 @@ def _short_failure_context( return short +def _emit_ci_error_annotation(jsonnet_path: Path, reason: str) -> None: + """Write a GitHub Actions error annotation directly to stdout.""" + # GitHub Actions command parsing uses percent-encoded newlines and carriage + # returns inside the annotation body. + reason = reason.replace('%', '%25').replace('\r', '%0D').replace('\n', '%0A') + sys.stdout.write(f'::error file={jsonnet_path}::Test failed {reason}\n') + sys.stdout.flush() + + def _make_log_adapter(formatter: genvm_tool.tests.Formatter) -> 'origin_logger.Logger': class _FormatterLoggerAdapter(origin_logger.Logger): """Adapts genvm_tool.formatter.Formatter to base_host.Logger interface.""" @@ -365,6 +374,11 @@ async def run( tmp_dir, ci=self._case.ci, ) + if self._case.ci: + _emit_ci_error_annotation( + self._case.jsonnet_path, + str(context.get('reason', 'prepare script failed')), + ) raise genvm_tool.tests.test.FinishedEarlyException( result=genvm_tool.tests.test.Result( passed=False, @@ -471,6 +485,7 @@ async def run( self, previous_results: list[typing.Any] ) -> genvm_tool.tests.test.Result: empty_storage = self._tmp_dir.joinpath('empty-storage.pickle') + jsonnet_path = self._test_case.jsonnet_path for attempt in range(self._max_attempts): sub_logger = _make_log_adapter(local_ctx.shared.logger) @@ -491,6 +506,11 @@ async def run( self._tmp_dir.joinpath(self._tree_path), ci=self._ci, ) + if self._ci: + _emit_ci_error_annotation( + jsonnet_path, + str(context.get('reason', 'unknown error')), + ) raise genvm_tool.tests.test.FinishedEarlyException( result=genvm_tool.tests.test.Result( passed=False, @@ -765,6 +785,7 @@ async def _run_single_step( 'passed': False, 'context': { 'exception': e, + 'reason': f'{e.__class__.__name__}: {e}', 'tree_path': tree_path, }, } diff --git a/tests/runner/genvm_tool_plugins/ninja.py b/tests/runner/genvm_tool_plugins/ninja.py index 3306b9f9..9a274875 100644 --- a/tests/runner/genvm_tool_plugins/ninja.py +++ b/tests/runner/genvm_tool_plugins/ninja.py @@ -214,7 +214,14 @@ def _resolve_path(self, value: Path) -> str: return str(value) def _scalar(self, value) -> None: - if isinstance(value, RawStr): + if isinstance(value, Build): + # A Build used as an output/dependency reference means "this edge's + # output" β€” depend on its primary output (which for a multi-output edge + # is enough to force the whole edge; aggregator phony edges have one). + # Without this a Build passed to add_dependency would leak its repr. + assert value.outputs, 'cannot reference a Build that has no output' + self._scalar(value.outputs[0]) + elif isinstance(value, RawStr): self.buf.append(str(value)) elif isinstance(value, Path): self.buf.append(_escape(self._resolve_path(value))) From 30594129f2ab72037cd38b9cd5ba999a08e54409 Mon Sep 17 00:00:00 2001 From: kp2pml30 Date: Fri, 24 Jul 2026 22:17:26 +0900 Subject: [PATCH 04/45] =?UTF-8?q?chore:=20fix=20tests=20*=20docs:=20deslop?= =?UTF-8?q?ify=20*=20fix:=20disable=20codegen=20for=20legacy=20versions=20?= =?UTF-8?q?*=20fix(llm):=20charge=20provider=20usage=20when=20the=20respon?= =?UTF-8?q?se=20shape=20check=20fails=20=F0=9F=94=92=20*=20feat(scripting)?= =?UTF-8?q?:=20grow=20the=20vm=20pool=20on=20demand=20instead=20of=20waiti?= =?UTF-8?q?ng=20=E2=9C=A8=20*=20feat(llm):=20delegate=20backend=20selectio?= =?UTF-8?q?n=20and=20retries=20to=20unhardcoded=20=E2=9C=A8=20*=20chore(ci?= =?UTF-8?q?):=20consolidate=20tools=20under=20=5F=5Fmain=5F=5F=20and=20rew?= =?UTF-8?q?ork=20merge=20landing=20=E2=99=BB=20*=20chore:=20add=20check-so?= =?UTF-8?q?urce-text=20guard;=20keep=20source=20ASCII=20and=20citation-fre?= =?UTF-8?q?e=20=F0=9F=A7=B9=20*=20feat(nondet):=20validate=20leader=20resu?= =?UTF-8?q?lts=20and=20entry=20calldata=20=E2=9C=A8=F0=9F=94=92?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .genvm-monorepo-root | 12 +- .genvm-tool.py | 14 +- .github/actions/postmortem/action.yaml | 8 +- .github/workflows/branch_executor_prs.yaml | 30 +- .github/workflows/branch_forward.yaml | 14 +- .github/workflows/branch_retarget.yaml | 6 +- .github/workflows/queue.yaml | 11 + .gitmodules | 4 + CLAUDE.md | 43 ++- .../modules-interfaces/src/domain/fees/abi.rs | 4 +- crates/modules-interfaces/src/lib.rs | 8 +- ...pre-validating untrusted decoded inputs.md | 264 ++++++++++++++ docs/contributing/README.md | 5 +- docs/contributing/howto/README.md | 2 + docs/contributing/howto/building/build.md | 8 + .../howto/committing/submodules.md | 4 +- docs/contributing/howto/genvm-tool.md | 85 ++--- .../howto/releasing/release-build.md | 1 - .../howto/releasing/versioning.md | 6 +- docs/contributing/howto/testing/rust.md | 9 + docs/contributing/howto/write-a-script.md | 43 +++ .../tutorial/first-contribution.md | 73 ++++ .../03-wasi_genlayer_sdk/02-gl_call.rst | 9 + .../03-wasi_genlayer_sdk/03-schemas.rst | 60 ++++ docs/website/src/spec/03-vm/01-startup.rst | 17 + .../03-vm/04-determinism-mode-switching.rst | 48 ++- docs/website/src/spec/03-vm/05-result.rst | 15 + .../src/spec/04-contract-interface/02-abi.rst | 17 + .../spec/04-contract-interface/06-gvm32.rst | 2 + .../src/spec/appendix/constants-pending.rst | 31 ++ executors/v0.3.x | 2 +- flake.nix | 6 + implementation/default.nix | 16 + implementation/src/common/mod.rs | 15 +- implementation/src/llm/handler.rs | 2 +- implementation/src/llm/providers.rs | 60 +++- implementation/src/manager/run.rs | 2 +- implementation/src/scripting/ctx/dflt.rs | 126 +++++++ implementation/src/scripting/pool.rs | 135 ++++++-- implementation/src/web/handler.rs | 2 +- implementation/tests/overloaded.rs | 2 + implementation/tests/policy_dispatch.rs | 276 +++++++++++++++ implementation/tests/providers.rs | 2 +- implementation/tests/timeout.rs | 2 + implementation/tests/web_render_webdriver.rs | 4 +- install/config/genvm-llm-default.lua | 324 +++++++++++++----- install/lib/genvm-lua/lib-genvm.lua | 4 + install/lib/genvm-lua/lib-llm.lua | 30 +- libs/unhardcoded-engine | 1 + support/ci/__main__.py | 15 +- support/ci/gh_common.py | 207 +++++++++++ support/ci/pr_branches_info.py | 104 +----- support/ci/tools/branches.py | 9 +- support/ci/tools/genvm_merge_into_dev.py | 192 ++++++++--- support/ci/tools/make_release_notes.py | 47 +-- support/ci/tools/open_executor_prs.py | 117 +++---- support/ci/tools/pr_action_panel.py | 99 +++++- support/ci/tools/pr_branches.py | 47 +-- support/ci/tools/script_adapters.py | 44 --- support/ci/tools/versions.py | 47 ++- support/scripts/check-source-text.py | 73 ++++ support/tools/genvm-tool/default.nix | 9 +- .../tools/genvm-tool/genvm_tool/__main__.py | 117 ++++++- .../genvm_tool/cmd_build_manifest.py | 7 +- .../genvm-tool/genvm_tool/cmd_codegen.py | 16 +- .../genvm-tool/genvm_tool/cmd_configure.py | 104 ++++-- .../tools/genvm-tool/genvm_tool/cmd_howto.py | 94 +++++ .../tools/genvm-tool/genvm_tool/cmd_test.py | 13 +- .../genvm-tool/genvm_tool/codegen/model.py | 21 +- .../genvm-tool/genvm_tool/codegen/rst.py | 26 +- .../genvm-tool/genvm_tool/codegen/rust.py | 93 ++++- .../genvm_tool/git/check_for_push.py | 42 +-- .../genvm_tool/git/create_branches.py | 16 +- .../genvm_tool/tests/stage/filter.py | 2 +- support/tools/genvm-tool/pyproject.toml | 1 + tests/runner/genvm_tool_plugins/cargo.py | 83 ++++- .../runner/genvm_tool_plugins/integration.py | 27 +- tests/runner/genvm_tool_plugins/ninja.py | 45 ++- tests/runner/genvm_tool_plugins/pytest.py | 4 - tests/system/permits/test.py | 1 + tests/templates/util.jsonnet | 4 +- 81 files changed, 2871 insertions(+), 719 deletions(-) create mode 100644 docs/adr/013. pre-validating untrusted decoded inputs.md create mode 100644 docs/contributing/howto/write-a-script.md create mode 100644 docs/contributing/tutorial/first-contribution.md create mode 100644 implementation/tests/policy_dispatch.rs create mode 160000 libs/unhardcoded-engine create mode 100644 support/ci/gh_common.py delete mode 100644 support/ci/tools/script_adapters.py create mode 100644 support/scripts/check-source-text.py create mode 100644 support/tools/genvm-tool/genvm_tool/cmd_howto.py diff --git a/.genvm-monorepo-root b/.genvm-monorepo-root index f691af1d..08b44276 100644 --- a/.genvm-monorepo-root +++ b/.genvm-monorepo-root @@ -1,6 +1,14 @@ { "version": "v0.6.0-rc1", - "active-versions": ["v0.3", "v0.2"], + "active-versions": [ + "v0.3", + "v0.2" + ], + "support-only-versions": [ + "v0.2" + ], "artifacts_dir": "build/test-artifacts", - "extra_python_paths": ["tests/runner"] + "extra_python_paths": [ + "tests/runner" + ] } diff --git a/.genvm-tool.py b/.genvm-tool.py index 9d82383c..681d1429 100644 --- a/.genvm-tool.py +++ b/.genvm-tool.py @@ -37,17 +37,27 @@ def tests(ctx): ctx.shared.logger.warning('build/info.json not found, generating default') _build_dir = ctx.shared.root_dir / 'build' _build_dir.mkdir(parents=True, exist_ok=True) + _monorepo_cfg = json.loads( + (ctx.shared.root_dir / genvm_tool.cmd_configure.MONOREPO_ROOT_FILE).read_text() + ) + _rust_target_dir = _build_dir / 'ya-build' / 'rust-target' _info_path.write_text( json.dumps( { 'coverage_dir': str(_build_dir / 'cov'), 'build_dir': str(_build_dir), - 'rust_target_dir': str(_build_dir / 'ya-build' / 'rust-target'), + 'rust_target_dir': str(_rust_target_dir), + 'rust_target': genvm_tool.cmd_configure.detect_rust_target(), # executor_versions / primary_executor_version: resolved from # source (.genvm-monorepo-root + committed manifests), so a # build-less test run still knows which out/executor/ dir # to reroute to. configure emits the same keys. - **genvm_tool.cmd_configure.build_independent_info(ctx.shared.root_dir), + **genvm_tool.cmd_configure.build_independent_info( + _monorepo_cfg, ctx.shared.root_dir + ), + **genvm_tool.cmd_configure.rust_target_dirs_info( + _monorepo_cfg, ctx.shared.root_dir, _rust_target_dir + ), }, indent=2, ) diff --git a/.github/actions/postmortem/action.yaml b/.github/actions/postmortem/action.yaml index 661bc1b7..574cf316 100644 --- a/.github/actions/postmortem/action.yaml +++ b/.github/actions/postmortem/action.yaml @@ -10,9 +10,11 @@ description: >- workflow is a separate job on a fresh runner: it would report a pristine machine instead of the one that just failed. - Callers should add `if: ${{ !cancelled() }}` (the grace window on a cancel is - too short to spend here) and `continue-on-error: true`. Linux-only, matching - every cell that calls it. + Callers gate on `failure()` β€” a cancel's grace window is too short to spend + here β€” and add `continue-on-error: true`. Linux-only, matching every cell + that calls it. Keep double-brace expression syntax out of this file entirely: + the runner templates the whole manifest, description included, and rejects + workflow-only functions such as `failure()` there. runs: using: composite steps: diff --git a/.github/workflows/branch_executor_prs.yaml b/.github/workflows/branch_executor_prs.yaml index b613d2d8..2c9f109d 100644 --- a/.github/workflows/branch_executor_prs.yaml +++ b/.github/workflows/branch_executor_prs.yaml @@ -1,13 +1,22 @@ name: branch / open linked executor PRs -# When a manager PR is opened against a dev branch, open the matching executor -# PR(s) in the genvm-executor repo. The manager release line is independent of the -# executor lines it ships, so one manager PR fans out over EVERY active executor -# line (.genvm-monorepo-root): for each, `pr//` -> -dev. Each -# is listed on the manager PR as an `executor: ` line. The executor work lives -# on line-namespaced mirror branches the developer pushed via +# When a manager PR is opened (or updated) against a dev branch, open the matching +# executor PR(s) in the genvm-executor repo. The manager release line is independent +# of the executor lines it ships, so one manager PR fans out over EVERY active +# executor line (.genvm-monorepo-root): for each, `pr//` -> -dev. +# Each is listed on the manager PR as an `executor: ` line. The executor work +# lives on line-namespaced mirror branches the developer pushed via # `genvm-tool git create-branches`. See `support/ci/run.sh tool open-executor-prs`. # +# `synchronize` (a push to the PR head) re-runs it: executor mirror branches are +# often pushed after the manager PR is opened, so the initial `opened` run finds +# nothing to link. `edited` fires when the base branch changes, so a PR opened +# against `main` and retargeted onto a dev branch (branch_retarget.yaml) also +# runs β€” the `opened` event was filtered out because its base was still `main`. +# The script is idempotent β€” it reuses any existing executor PR and upserts the +# same marked comment β€” so re-running per event just fills in the links as the +# branches appear. +# # pull_request_target gives a manager-scoped write token for the comment; the # cross-repo executor PR needs a PAT with genvm-executor access # (GENVM_EXECUTOR_PR_TOKEN) β€” the default GITHUB_TOKEN cannot reach another repo. @@ -16,7 +25,14 @@ name: branch / open linked executor PRs on: pull_request_target: branches: ['v*-dev'] - types: [opened] + types: [opened, synchronize, reopened, edited] + +# Serialize per PR so overlapping pushes don't race on creating the executor PR or +# upserting the comment; don't cancel in-flight runs β€” each one is a cheap, safe +# reconcile and the last state should reflect the latest head. +concurrency: + group: executor-prs-${{ github.event.pull_request.number }} + cancel-in-progress: false permissions: pull-requests: write diff --git a/.github/workflows/branch_forward.yaml b/.github/workflows/branch_forward.yaml index fddaf7b7..bc9c6816 100644 --- a/.github/workflows/branch_forward.yaml +++ b/.github/workflows/branch_forward.yaml @@ -46,14 +46,14 @@ jobs: - name: Fast-forward main to latest dev branch run: | BRANCH="${GITHUB_REF_NAME}" - # Decide "latest" from main's active-versions (the authoritative - # alias), NOT the pushed branch's own copy β€” an older dev - # branch carries a stale active-versions and would otherwise try - # to drag main backwards once a newer train exists. + # Decide the current manager train from main's `version` field (the + # authoritative alias), NOT the pushed branch's own copy β€” an older + # dev branch carries a stale version and would otherwise try to drag + # main backwards once a newer train exists. git show origin/main:.genvm-monorepo-root > /tmp/main-monorepo-root.json - LATEST="$(MONOREPO_ROOT=/tmp/main-monorepo-root.json ./support/ci/run.sh tool branch-versions latest)" - if [ "$BRANCH" != "v${LATEST}-dev" ]; then - echo "Pushed branch ${BRANCH} is not the latest dev branch (v${LATEST}-dev); nothing to do" + MANAGER="$(MONOREPO_ROOT=/tmp/main-monorepo-root.json ./support/ci/run.sh tool branch-versions manager)" + if [ "$BRANCH" != "v${MANAGER}-dev" ]; then + echo "Pushed branch ${BRANCH} is not the latest dev branch (v${MANAGER}-dev); nothing to do" exit 0 fi echo "Fast-forwarding main to ${GITHUB_SHA} (from ${BRANCH})" diff --git a/.github/workflows/branch_retarget.yaml b/.github/workflows/branch_retarget.yaml index 7e2bea60..045929e6 100644 --- a/.github/workflows/branch_retarget.yaml +++ b/.github/workflows/branch_retarget.yaml @@ -35,8 +35,8 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - LATEST="$(./support/ci/run.sh tool branch-versions latest)" - DEV="v${LATEST}-dev" + MANAGER="$(./support/ci/run.sh tool branch-versions manager)" + DEV="v${MANAGER}-dev" PR="${{ github.event.pull_request.number }}" gh pr edit "$PR" --repo "$GITHUB_REPOSITORY" --base "$DEV" @@ -44,6 +44,6 @@ jobs: gh pr comment "$PR" --repo "$GITHUB_REPOSITORY" --body "$(cat <- GenVM full${{ github.event_name == 'workflow_dispatch' && format(' (manual, PR #{0})', inputs.pr) || '' }} +# One in-flight full run per PR head branch; a newer push cancels the stale run. +# `github.head_ref` is the PR head branch on pull_request events. Cancelling a +# superseded head's run is safe: the Merge gate only consumes CI that is green on +# the exact head it fast-forwards, which is always the newest one. The panel's +# workflow_dispatch has no head_ref (its `github.ref_name` is `/merge` on PRs, +# not a branch), so fall back to the unique run id β€” a manual rerun then gets its +# own group and never cancels (or is cancelled by) another PR's run. +concurrency: + group: genvm-full-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + defaults: run: shell: bash -x {0} diff --git a/.gitmodules b/.gitmodules index 57000707..8e1c6ef4 100644 --- a/.gitmodules +++ b/.gitmodules @@ -6,3 +6,7 @@ path = executors/v0.2.x url = ../genvm-executor branch = v0.2.x +[submodule "libs/unhardcoded-engine"] + path = libs/unhardcoded-engine + url = ../unhardcoded-engine + branch = main diff --git a/CLAUDE.md b/CLAUDE.md index bc887e19..f3eaaa09 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -3,15 +3,34 @@ Before doing build, test, submodule, release/versioning, or codegen work, read the matching guide in `docs/contributing/howto/`: -- [setup.md](docs/contributing/howto/setup.md) β€” first-time clone, nix/direnv dev shell -- [genvm-tool.md](docs/contributing/howto/genvm-tool.md) β€” genvm-tool commands, codegen, plugins -- [building/build.md](docs/contributing/howto/building/build.md) β€” debug build: configure + ninja -- [building/runners.md](docs/contributing/howto/building/runners.md) β€” getting runners -- [building/docs.md](docs/contributing/howto/building/docs.md) β€” website/spec docs, ADRs -- [testing/README.md](docs/contributing/howto/testing/README.md) β€” `genvm-tool test`, continue files (+ [rust.md](docs/contributing/howto/testing/rust.md), [python.md](docs/contributing/howto/testing/python.md), [integration.md](docs/contributing/howto/testing/integration.md), [fuzzing.md](docs/contributing/howto/testing/fuzzing.md)) -- [committing/submodules.md](docs/contributing/howto/committing/submodules.md) β€” multi-repo commits, gitlinks, hooks, push order -- [committing/runners.md](docs/contributing/howto/committing/runners.md) β€” runner hash hygiene before committing (dev-mode, hash-updater.py) -- [committing/git-third-party.md](docs/contributing/howto/committing/git-third-party.md) β€” vendored wasmtime/wasm-tools patch workflow -- [releasing/versioning.md](docs/contributing/howto/releasing/versioning.md) β€” release trains, `.genvm-monorepo-root`, version scripts -- [releasing/release-build.md](docs/contributing/howto/releasing/release-build.md) β€” nix packages, release assets -- [extending/](docs/contributing/howto/README.md#extending) β€” add LLM provider / WASI function / host function, modify a runner or wasmtime +- [genvm-tool.md](docs/contributing/howto/genvm-tool.md) β€” command overview, codegen data β†’ generated files, plugin/test-definition layout +- [setup.md](docs/contributing/howto/setup.md) β€” first-time clone: submodule init, nix/direnv dev shell, third-party materialization +- [write-a-script.md](docs/contributing/howto/write-a-script.md) β€” conventions for new helper scripts: python + argparse, integrate into genvm-tool / support/ci + +## Building +- [build.md](docs/contributing/howto/building/build.md) β€” debug build: configure + ninja, outputs, cargo quirks +- [docs.md](docs/contributing/howto/building/docs.md) β€” building the website, spec vs impl-spec, ADRs +- [runners.md](docs/contributing/howto/building/runners.md) β€” where runners come from (build vs download) + +## Committing +- [git-third-party.md](docs/contributing/howto/committing/git-third-party.md) β€” how vendored trees (wasmtime, …) are tracked, edited, and committed as patches +- [runners.md](docs/contributing/howto/committing/runners.md) β€” runner dev-mode and hash hygiene before committing an executor line +- [submodules.md](docs/contributing/howto/committing/submodules.md) β€” manager + executor lines, gitlink bumps, pre-commit hook, push order + +## Extending +- [add-host-function.md](docs/contributing/howto/extending/add-host-function.md) β€” new executor↔host protocol method +- [add-llm-provider.md](docs/contributing/howto/extending/add-llm-provider.md) β€” new LLM backend in the manager +- [add-wasi-function.md](docs/contributing/howto/extending/add-wasi-function.md) β€” new gl_call method or raw WASI function in an executor +- [modify-runner.md](docs/contributing/howto/extending/modify-runner.md) β€” runner dev-mode and hash refresh +- [modify-wasmtime.md](docs/contributing/howto/extending/modify-wasmtime.md) β€” patching vendored wasmtime, trap plumbing + +## Releasing +- [release-build.md](docs/contributing/howto/releasing/release-build.md) β€” nix packages, platforms, release assets +- [versioning.md](docs/contributing/howto/releasing/versioning.md) β€” release trains, `.genvm-monorepo-root`, check-versions/branch-versions scripts + +## Testing +- [README.md](docs/contributing/howto/testing/README.md) β€” `genvm-tool test` itself: filters, presets, continue files +- [fuzzing.md](docs/contributing/howto/testing/fuzzing.md) β€” AFL fuzz targets, host sysctl prep +- [integration.md](docs/contributing/howto/testing/integration.md) β€” running one executor line, `stable` (offline) subset, golden `.stdout`/`.hash` files, `--ignore-hash`, services +- [python.md](docs/contributing/howto/testing/python.md) β€” Python tests, direct pytest for genlayer-py-std +- [rust.md](docs/contributing/howto/testing/rust.md) β€” Rust tests, `#rust-test` shell, coverage diff --git a/crates/modules-interfaces/src/domain/fees/abi.rs b/crates/modules-interfaces/src/domain/fees/abi.rs index 26a99aff..93f293e7 100644 --- a/crates/modules-interfaces/src/domain/fees/abi.rs +++ b/crates/modules-interfaces/src/domain/fees/abi.rs @@ -136,7 +136,7 @@ fn encode_node(node: &MessageAllocationNode, parent_index: U256) -> Vec { buf } -/// `abi.encode(InternalMessageParams)` β€” a dynamic tuple (contains +/// `abi.encode(InternalMessageParams)` -- a dynamic tuple (contains /// `uint256[] rotations`), so it is prefixed with the offset word. `appealRounds` /// is reconstructed as `len(rotations) - 1` (the chain-derived value). /// @@ -171,7 +171,7 @@ fn encode_internal_params(params: &InternalMessageParams) -> Vec { buf } -/// `abi.encode(ExternalMessageParams)` β€” a fully static tuple, so it is just +/// `abi.encode(ExternalMessageParams)` -- a fully static tuple, so it is just /// the two inline words with no leading offset. fn encode_external_params(params: &ExternalMessageParams) -> Vec { let mut buf = Vec::new(); diff --git a/crates/modules-interfaces/src/lib.rs b/crates/modules-interfaces/src/lib.rs index 3d711cb2..5dc956ef 100644 --- a/crates/modules-interfaces/src/lib.rs +++ b/crates/modules-interfaces/src/lib.rs @@ -107,7 +107,7 @@ impl GenericValue { } } -// ── Manual Encode for GenericValue ────────────────────────────────── +// -- Manual Encode for GenericValue ---------------------------------- impl Encode for GenericValue { type Error = W::Error; @@ -145,7 +145,7 @@ impl Encode for GenericValue { } } -// ── Manual Decode for GenericValue ────────────────────────────────── +// -- Manual Decode for GenericValue ---------------------------------- impl Decode for GenericValue { fn decode(deserializer: D) -> std::result::Result { @@ -223,7 +223,7 @@ impl Decode for GenericValue { } } -// ── Encode / Decode for Result ───────────────────────────────── +// -- Encode / Decode for Result --------------------------------- #[derive(Clone, Deserialize, Serialize)] pub enum Result { @@ -936,7 +936,7 @@ fn decode_host_data(val: genlayer_calldata::Value) -> std::result::Result( diff --git a/docs/adr/013. pre-validating untrusted decoded inputs.md b/docs/adr/013. pre-validating untrusted decoded inputs.md new file mode 100644 index 00000000..229d36ef --- /dev/null +++ b/docs/adr/013. pre-validating untrusted decoded inputs.md @@ -0,0 +1,264 @@ +# ADR-013: Pre-Validating Untrusted Inputs β€” Leader Results and Message Payloads + +## Status + +- **Status**: Accepted +- **Date**: 2026-07-20 + +## Context + +Two inputs cross into the executor from outside the VM, neither produced nor +validated by it: + +1. **The leader-proposed non-deterministic result** β€” a raw `bytes::Bytes` the + host hands `run_nondet`, framed `[result_code: u8][data]` (spec: + `subvm-result-encoding`, [ADR-003](<003. host communication.md>)). +2. **The `Main` entry payload** β€” must follow the calling convention of + [ADR-005](<005. abi.md>): `{"": String?, "args": Array?, "kwargs": Map?}`. + +`calldata::decode` already rejects trailing bytes, unsorted/duplicate keys and +non-minimal uleb, and caps nesting at depth 128. The bug was that on these two +paths it was skipped or its failure mishandled: + +- A leader `Return` payload was **never decoded** β€” the tail was wrapped + `Maybe::Checked(Raw(..))` (a "validated" state nobody established) and flowed + verbatim into the execution hash. `UserError`/`VmError` were decoded but a + failure **trapped**, turning leader garbage into a validator internal error + and a likely timeout vote. Unknown result codes and absent results also + trapped. A `VmError` code was accepted as an arbitrary string. +- The entry shape was enforced per-runner and inconsistently: Python only + checked `isinstance(dict)` then splatted `args`/`kwargs` (wrong types failed + deep in the interpreter); Rust's `MainCalldata::parse` was *more* lenient β€” a + non-Map became a valid no-arg call and wrong-typed `args`/`kwargs` were + dropped. The same malformed call had two undefined outcomes. + +## Decision + +Both inputs are decoded and shape-checked at the boundary, before any runner +loads. Malformed input yields a deterministic, contract-visible outcome β€” +never a trap. No new primitive: call `calldata::decode` where it was skipped +and handle its failure correctly. + +### Rule 1 β€” accepting a proposed leader result (`Validator` and `Sync`) + +One total function, `parse_leader_result`, a pure function of the proposed +bytes so every honest node derives the same result: + +1. **Absent or empty** β†’ `absent_leader_nondet_output`. (Empty must equal + absent: `Supervisor::push_nondet_result` pads gaps with empty `Bytes`.) +2. Result code not in `Return`(0)/`UserError`(1)/`VmError`(2) β†’ + `leader_output malformed`. This includes `InternalError`(3): the enum admits + it but the wire format does not. +3. **`Return`/`UserError`**: the payload must satisfy `calldata::decode` + (canonical, depth ≀ 128, no trailing bytes) else `leader_output malformed`. + Accepted bytes are preserved **exactly** β€” validation never re-encodes, or + validators would hash a value other than the one proposed. +4. **`VmError`**: the payload must be UTF-8, carry **no ` # ` detail** (Rule 1b: + an honest leader never publishes one), then, *in order*: + 1. If the code is in the **derived-outcome namespace** (equals or + space-extends `absent_leader_nondet_output` or `leader_output`), it is + replaced by `leader_output uses_this_error `. Checked *before* validity + so proposing the literal `leader_output malformed` cannot echo verbatim. + 2. Otherwise the code must be a valid `vm_error` trie path, including the + canonical spelling of any parameter (`exit_code +7`/`007` are rejected β€” + same `i32`, different bytes, and codes reach the hash). The check is + **generated** from the trie data, so it cannot drift. Fail β†’ + `leader_output malformed`. Pending-ABI codes (below) are not trie paths, + hence not proposable. + +The parsed-or-substituted result then flows through the **standard consensus +comparison** β€” handed to the contract's validator function like any proposal. +No branch traps or bypasses the comparison: the substitution is deterministic +(honest validators vote identically) and the execution-hash comparison already +catches a lying leader. The old special case that marked a disagreement +directly is removed. Fees (`consume_nondet_output`) are charged on the +accepted/substituted result's **own** length β€” rejected bytes never enter the +result or the hash. + +**Derived-outcome namespace.** Validators derive `absent_leader_nondet_output`, +`leader_output malformed`, `leader_output uses_this_error ` and +`leader_output extra ` themselves. A proposal must never be byte-equal to +what a validator derives from rejecting it, or "leader proposed X" and +"proposal rejected as X" become indistinguishable (namespace hygiene, not +anti-spoofing β€” a dishonest leader controls its own hash regardless). So an +in-namespace proposal maps to `leader_output uses_this_error `, `` = first +6 chars of `gvm32(sha3_256(proposed code))`. A self-replacing fixed point +(~2³⁰ brute force) maps to the sentinel `…uses_this_error fix_point`, which +can't collide with an `` (9 chars vs always 6). `` collisions are +harmless β€” the value carries no meaning. + +### Rule 1.5 β€” surplus leader results (`Validator` and `Sync`) + +A run consumes one leader result per non-deterministic block it reaches. The +run reaching a block with no result is Rule 1.1; the complement β€” a leader +publishing **more** results than the run consumed β€” was silently tolerated, +hiding a leader whose execution diverged into extra blocks. Now: if the leader +supplied more results than blocks run, the run's result is replaced by +`leader_output extra `, `` = first 6 chars of `gvm32(sha3_256(b))` where +`b` is the **full `[result_code][data]` buffer** the run would otherwise have +returned (`RunOk::as_bytes()`) β€” fingerprinting the discarded result rather +than losing it. This is **not** a non-deterministic disagreement (it's a +property of the counts, not a re-executed block). No fix-point escape is needed: +a `leader_output` code is never accepted verbatim (Rule 1.4), so this can't be +byte-equal to a proposal. The outcome depends only on the two counts, so *when* +the check runs (after the block count is final) is unspecified. + +### Rule 1b β€” publishing the leader's own result (`Leader`) + +Before the computed result is pushed to the host (and hashed): + +- A `VMError`'s fused ` # ` suffix is **stripped** β€” the nondet channel + is detail-free (diagnostic, no compat promise, and a free-form byte channel + into the hash otherwise). The local `cause` is kept for logs. +- **Nothing else.** The leader must *not* run Rule 1 on its own output: + self-filtering can only rewrite an honest result into a derived-namespace code + that validators replace again β€” a guaranteed honest-vs-honest hash split. + Rule 1 is for hostile input, Rule 1b for trusted output. + +Stripping suffices because the rest is valid by construction: codes come from +the generated constructors and canonical `exit_code ` (a generated test +asserts every constructor passes the validity check). The one arbitrary-string +site, the `vmError "…"` fee builtin, is unreachable from a nondet child (every +bucket that raises it needs a permission the child lacks); the child also can't +reach the derived namespace (it has `spawn_nondet: false`). Both invariants must +be re-checked if the nondet child's permissions change β€” a non-trie code trips +the `debug_assert!` in debug and, in release, is an honest-node hash split. +Consequence: `leaders_result` is always detail-free while a validator's own +error keeps its detail; the default `compare_vm_errors` compares `public_code` +only, so it is unaffected β€” custom comparators must not read a detail from +`leaders_result`. + +### Rule 2 β€” message payload shape + +The shape is the **decode contract** of the `MainCallData` / `MainDeployData` +types: a Map whose every key is one of `""`, `"args"`, `"kwargs"` (**closed** β€” +unknown keys rejected, so two byte-different payloads can't describe one call), +with `""` a String (**absent** on a deployment β€” the deploy shape has no method +name), `"args"` an Array, `"kwargs"` a Map. Element *types* inside +`args`/`kwargs` stay the runner's job; the convention's extension point is the +*value* of `""` (`#get-schema`, `#error`), not the key set. + +Because it is a decode-time property of those types, it is enforced wherever +untrusted entry calldata is decoded: + +- **Top-level host entry** β€” decoded in `run_with_impl` before permissions are + read or any runner loads; failure (or a `""` present on `is_init`) is a + `malformed_entry` VM error **result**. `PostMessage`/`DeployContract` emit + messages that later execute as top-level entries, so they pass here too. +- **Every `gl_call` message** β€” `CallContract`, `PostMessage` and + `DeployContract` carry the calldata as a typed field, so a malformed payload + fails the `gl_call::Message` decode and answers **`Errno::Inval`**, like the + other argument checks; the caller can recover. + +### Run modes + +The `is_sync`/`is_leader()` boolean pair is replaced by `rt::RunMode { Sync, +Leader, Validator }`, fixed at startup from the execution data (`sync` flag; +else presence of `leader_nondet_results`). Rule 1 governs `Validator`/`Sync` +(in sync there is no vote β€” the derived result is simply the call's result); +Rule 1b governs `Leader`. + +### Where the constants live + +`malformed_entry`, `leader_output malformed`, `leader_output uses_this_error +{str}` and `leader_output extra {str}` go in the **pending** ABI data +(`executor/codegen/data/public-abi-pending.json`), the +[ADR-012](<012. runner memory accounting.md>) mechanism for public-ABI +constants that must not regenerate the runner-hashed `public_abi.py`. Generated +into `common` only β†’ hash-neutral. Cost: the `vm_error` taxonomy temporarily +lives in two shapes; fold the pending codes into the trie at the next +hash-moving release train. Until then the generated validity check rejects them, +which is what makes them unproposable under Rule 1.4. + +## Consequences + +**Positive.** Leader bytes are held to the same decoder as everything else; the +unbounded-nesting path into the hash is closed. A malicious leader can no longer +turn validators into timeout votes, invent VM error codes, propose internal +errors, or produce output byte-equal to its own rejection. The free-form detail +channel into the hash is closed. Malformed calls fail identically in every +runner language, at a defined point, with a defined error. Absent/empty/malformed +leader results collapse into one total parse feeding one comparison path. + +**Negative.** Consensus-visible behaviour change (results/calls accepted before +are now rejected; nondet `VMError` loses its ` # detail`); ships with a version +bump ([ADR-010](<010. versions.md>)), runner hashes unmoved thanks to the +pending mechanism. Rust-SDK contracts relying on `MainCalldata::parse`'s lenient +defaulting (non-Map payload, wrong-typed `args`) now get `malformed_entry`. A +`""` key on a deployment is now an error (was silently ignored on init) β€” eight +integration fixtures sent one and their goldens move to `malformed_entry`. The +`vm_error` taxonomy is split until the next hash-moving train. Validating +`Return` costs one decode pass over host-bounded leader bytes on the validator +path. + +**Neutral.** A leader still has an arbitrary-*content* channel via a +*well-formed* `Return`/`UserError` (inherent β€” the rules bound shape, not +content). A rejected proposal still runs the comparison stage (deterministic, so +a validator function that agrees does so on every honest node). Executor-produced +payloads (sandbox result, `Sandbox`/`ConsensusStage` pickles, `entry_stage_data`) +are trust boundaries only if the executor is already compromised. Other +untrusted decode sites keep their handling (gl_call buffers β†’ `Errno::Inval`; +host `ExecutionData`/RPC β†’ host errors). Nothing in-tree emits non-canonical +calldata, extra keys or observable nondet details. + +## Alternatives Considered + +- **Keep trapping on malformed proposals** β€” converts leader garbage into a + validator internal error, a leader-controlled availability attack. +- **Force a disagreement on rejection, bypassing comparison** (the old + empty-buffer path) β€” duplicates the comparison machinery, takes the vote from + the contract; the hash comparison already catches a liar. +- **Accept and strip a bounded ` # detail`** β€” honest leaders never publish one, + so it only adds a dead range and an edge where a valid code flips to malformed. +- **Run Rule 1 on the leader's own output** (symmetry) β€” can only rewrite an + honest result into a derived code validators re-replace: a consensus-split + hazard. +- **Re-encode the `Return` payload instead of rejecting** β€” silently normalizes + malformed consensus input; the validator's bytes would differ from the + proposal while claiming agreement. +- **Validate lazily when the contract reads `leaders_result`** β€” too late, the + payload is already in the hash. +- **Enforce the message shape in each SDK** β€” once per language, differently, + and a runner can't reject a call before it starts. +- **Tolerate unknown entry keys** β€” two byte-different payloads would describe + one call and hash differently. + +## Implementation Notes + +- Rule 1 is `parse_leader_result` + `validate_leader_vm_error`, Rule 1b the + leader branch of `run_nondet`, all in + `executor/src/wasi/genlayer_sdk/run.rs` (the pure functions are fuzz targets). + `RunMode` lives in `executor/src/rt/mod.rs`, set in `executor/src/exe/run.rs`. +- Rule 2 is the derived `Decode` of `MainCallData`/`MainDeployData` + (`executor/crates/sdk-rs/src/abi/entry.rs`), enforced at `run_with_impl` + (`executor/src/lib.rs`) and at the `gl_call::Message` decode + (`executor/src/wasi/genlayer_sdk/mod.rs`). The new `#[calldata(option_as_absence)]` + attribute models an optional key as `Option` (absent β‡’ `None`, present β‡’ + the inner `T`, never a `null`). +- The generated `VmError::is_valid_` and the derived-namespace prefixes are + emitted by the rust codegen backend (`codegen/rust.py`) from the same trie + data as the constructors; a parameter must round-trip (`v.to_string() == + rest`). Rust-only; `public_abi.py` does not move. +- `mark_nondet_disagreement` and `Supervisor::is_leader` lose their last callers + and are removed. +- Tests pin every branch of Rule 1 (canonical `exit_code`, generated-constants + round-trip) and Rule 2; integration tests inject malformed proposals via the + harness `leader_nondet: [{kind: raw, ...}]`. AFL fuzz targets + (`executor/fuzz/{leader-result,entry-data}.rs`) assert byte-faithful + acceptance / the closed derived grammar and diff the validate-only path against + a materializing reference. +- Spec: proposed-result rules and derived namespace in `subvm-result-encoding`; + `03-vm/04-determinism-mode-switching.rst` covers all three modes; Rule 2 on + the calling-convention page with its failure surfaces in `03-vm/01-startup.rst` + and under `CallContract`. The rst backend gained `--rst-anchor-ns` so the + pending appendix does not redefine the real `gvm-def-str-trie-vm-error` label. + +### Deferred + +- **Python SDK.** The `_decode_sub_vm_result_retn` empty-buffer guard and the + `_genlayer_bootloader.py` `CalldataSchema` `method`/`''` key touch the runner + tree and would move hashes. Rule 1 makes the guard defensive, not load-bearing. +- **`message.rs` `CallKey::UNNAMED` fallback** on a decode failure β€” should be an + error. +- **Folding the pending codes into the `vm_error` trie** at the next hash-moving + train. diff --git a/docs/contributing/README.md b/docs/contributing/README.md index 4c044f50..d48e8269 100644 --- a/docs/contributing/README.md +++ b/docs/contributing/README.md @@ -1,10 +1,13 @@ # Contributing to GenVM development ## Sub pages +- [first-contribution tutorial](./tutorial/first-contribution.md) β€” end-to-end walkthrough: patch an executor, branch, commit, push, open the PR. - [how-to guides](./howto/README.md) β€” setup, build, test, submodules, versioning, extension recipes, … ## PR requirements -Main requirement is that all tests must pass. It includes [pre-commit](https://pre-commit.com) and test suites. PRs are merged via a queue, that executes all tests and merges *iff* they all pass. It is done to ensure that `HEAD` of `main` branch is always stable +All checks must pass: [pre-commit](https://pre-commit.com) hooks and the test +suites. PRs land through a merge queue that runs the full test set and merges +*iff* everything passes β€” this keeps `HEAD` of `main` always stable. ## Security GenVM is consensus-critical. Do not open public issues for vulnerabilities β€” diff --git a/docs/contributing/howto/README.md b/docs/contributing/howto/README.md index 050379a8..af91dc13 100644 --- a/docs/contributing/howto/README.md +++ b/docs/contributing/howto/README.md @@ -2,6 +2,7 @@ - [setup.md](setup.md) β€” first-time clone: submodule init, nix/direnv dev shell, third-party materialization. - [genvm-tool.md](genvm-tool.md) β€” command overview, codegen data β†’ generated files, plugin/test-definition layout. +- [write-a-script.md](write-a-script.md) β€” conventions for new helper scripts: python + argparse, integrate into genvm-tool / support/ci. ## building/ - [build.md](building/build.md) β€” debug build: configure + ninja, outputs, cargo quirks. @@ -17,6 +18,7 @@ ## committing/ - [submodules.md](committing/submodules.md) β€” manager + executor lines, gitlink bumps, pre-commit hook, push order. +- [runners.md](committing/runners.md) β€” runner dev-mode and hash hygiene before committing an executor line. - [git-third-party.md](committing/git-third-party.md) β€” how vendored trees (wasmtime, …) are tracked, edited, and committed as patches. ## releasing/ diff --git a/docs/contributing/howto/building/build.md b/docs/contributing/howto/building/build.md index 32ea2266..2e24df26 100644 --- a/docs/contributing/howto/building/build.md +++ b/docs/contributing/howto/building/build.md @@ -38,3 +38,11 @@ Getting or changing runners: [runners.md](runners.md), [modify-runner.md](../ext - In this nix environment, `cargo check` inside an executor's `executor/` dir may need `LD_LIBRARY_PATH="$(nix eval --raw nixpkgs#zlib)/lib"` so rustc finds libz. +- Each executor line gets its own cargo target dir + (`build/ya-build/rust-target/`, listed in `build/info.json` under + `rust_target_dirs`); crates outside a line use the parent dir. Lines ship + crates with identical names and versions, and cargo's artifact hash ignores + where a package came from, so a shared dir lets them overwrite each other. +- Build and tests both pass `--target ` (`rust_target` in + `build/info.json`). Omitting it builds a second, unshared unit graph in the + same dir, so running cargo by hand without it recompiles everything. diff --git a/docs/contributing/howto/committing/submodules.md b/docs/contributing/howto/committing/submodules.md index 405eacb0..a8ab1fb4 100644 --- a/docs/contributing/howto/committing/submodules.md +++ b/docs/contributing/howto/committing/submodules.md @@ -48,14 +48,14 @@ into that repo's `.git/hooks`, so `git commit` runs the hooks for the repo you are committing in. - Run everything by hand: `nix develop -c pre-commit run --all-files` in the - repo (or `bash support/ci/pipelines/commit-hooks.sh` from the manager root to + repo (or `./support/ci/run.sh pipeline commit-hooks` from the manager root to check the manager + every executor at once, as CI does). - Do not `git commit --no-verify` β€” let the hooks run, even for a pure gitlink bump (they are cheap when nothing needs formatting). - The executor crates reach the manager's shared `crates/` via relative paths, so executor hooks must run against the **nested working tree** (submodule checked out under the manager), not an isolated flake sandbox β€” this is why - CI uses `commit-hooks.sh` (in-tree) instead of `nix flake check`. + CI uses the `commit-hooks` pipeline (in-tree) instead of `nix flake check`. ## Pushing diff --git a/docs/contributing/howto/genvm-tool.md b/docs/contributing/howto/genvm-tool.md index 5520cfe7..2209bd9b 100644 --- a/docs/contributing/howto/genvm-tool.md +++ b/docs/contributing/howto/genvm-tool.md @@ -1,59 +1,46 @@ # genvm-tool -Lives at `support/tools/genvm-tool` (on PATH as `genvm-tool` in the dev shell; +Lives at `support/tools/genvm-tool` (on `PATH` as `genvm-tool` in the dev shell; outside it, call `support/tools/genvm-tool/genvm-tool`). -## Commands +## Command reference -| Command | Purpose | -|---|---| -| `configure` | generate `build/build.ninja` + `build/info.json` β€” [build.md](building/build.md) | -| `test` | run the test suite β€” [testing/README.md](testing/README.md) | -| `codegen` | generate language bindings from data JSON (below) | -| `build-manifest` | generate the manager's `data/manifest.yaml` | -| `git ls` / `list-repo` / `create-branches` / `check-for-push` | git helpers across manager + submodules β€” [submodules.md](committing/submodules.md) | +The full man page is generated from the parser itself: + +```bash +genvm-tool --print-manpage | man -l - +``` -Command registration: `genvm_tool/__main__.py` (`TOPLEVEL`, `GROUPS`); each -command module has `NAME`/`HELP` β€” look there for details. +## genvm-tool test -## codegen +```sh +genvm-tool test run [--filter-name REGEX] [--filter-tag EXPR] [--filter-continue FILE] [--fail-fast] +``` -```bash -genvm-tool codegen --lang {rust,python,rst,go} -i -o +All given filters must match (they are AND-ed). `--filter-name` is an +unanchored regex over the test name; `--filter-tag` is a boolean tag +expression like `(a|b)&!c`. + +To list every test a filter selects (several hundred entries β€” pipe it, or use +the json log format for machine consumption): + +```sh +genvm-tool --log-format=json test show test ``` -(`--go-package` selects the go package name, default `genvm`.) - -Codegen data lives per executor line at -`executors/v.x/executor/codegen/data/{public-abi.json,host-fns.json}`. -After editing the JSON, regenerate via the build (`genvm-tool configure` then -`ninja -C build codegen`) or run `genvm-tool codegen` by hand per the tables: - -Per-line generated files (registered by `register_standard_codegen` in -`tests/runner/genvm_tool_plugins/ninja.py`, paths relative to the line root): - -| Output | Input | Lang | -|---|---|---| -| `executor/crates/sdk-rs/src/abi/consts.rs` | `public-abi.json` | rust | -| `runners/genlayer-py-std/src/genlayer/vm/public_abi.py` | `public-abi.json` | python | -| `executor/crates/common/src/host_fns.rs` | `host-fns.json` | rust | - -Manager-global generated files (from the **primary** line's data, wired in -`genvm_tool/cmd_configure.py`) β€” these go stale when a line's codegen JSON -changes and are refreshed by the same configure/ninja codegen step: - -| Output | Input | Lang | -|---|---|---| -| `tests/runner/origin/host_fns.py` | `host-fns.json` | python | -| `tests/runner/origin/public_abi.py` | `public-abi.json` | python | -| `docs/website/src/spec/appendix/constants.rst` | `public-abi.json` | rst | - -## Plugins and test definitions - -Plugins (`tests/runner/genvm_tool_plugins/`) are **importable libraries** β€” -reusable build/test helpers, no side effects on import. Test *definitions* are -kept out of the plugin package: each lives in `tests/system//test.py` -exposing `collect(ctx, **kwargs)`, and is registered from `.genvm-tool.py` via -`ctx.collect_dir('tests/system/', ...)` (discovery: -`genvm_tool/tests/stage/collection.py`). The project entry point overall is -`.genvm-tool.py` at the repo root (each executor line has its own). +Full testing workflow: [testing/README.md](testing/README.md). + +## genvm-tool git + +Multi-repo helpers over the manager and every executor submodule: + +| Command | Effect | +|---|---| +| `genvm-tool git ls` | list tracked files across all repos | +| `genvm-tool git list-repo` | list the manager and every executor submodule repo | +| `genvm-tool git create-branches` | create a feature branch across the sub-repos that have new content | +| `genvm-tool git check-for-push` | read-only pre-push status; prints the suggested push command | + +Where they fit in the workflow: +[first-contribution tutorial](../tutorial/first-contribution.md), +[submodules.md](committing/submodules.md). diff --git a/docs/contributing/howto/releasing/release-build.md b/docs/contributing/howto/releasing/release-build.md index 49588219..5c0edd82 100644 --- a/docs/contributing/howto/releasing/release-build.md +++ b/docs/contributing/howto/releasing/release-build.md @@ -14,7 +14,6 @@ Platform suffixes: none (native), `-amd64-linux`, `-arm64-linux`, `-arm64-macos` | `executor-[-]` | one executor line | | `artifact-prepack-genvm-` | a platform's release asset: manager + every active executor line | | `artifact-prepack-genvm-universal` | the universal release asset: runners (`runners/` prefix + legacy-line overlay) | -| `genvm-tool` | the tool itself | ```bash nix build -v -L '.?submodules=1#genvm-amd64-linux' diff --git a/docs/contributing/howto/releasing/versioning.md b/docs/contributing/howto/releasing/versioning.md index 5b87d353..90b4c109 100644 --- a/docs/contributing/howto/releasing/versioning.md +++ b/docs/contributing/howto/releasing/versioning.md @@ -26,8 +26,10 @@ them. manager crates to `.0`, and prints the bare major.minor (callers build branch names from it). -`support/ci/branch-versions.py list|latest` β€” prints active trains (leading `v` -stripped). Branch naming per train ``: +`support/ci/branch-versions.py list|manager` β€” `list` prints the active +executor lines from `active-versions`; `manager` prints the manager's own train +from `version` (both as bare major.minor). The manager dev/release branches are +named after `manager`, NOT the executor lines. Branch naming per train ``: - `v.x` β€” release branch (release-ready) - `v-dev` β€” integration branch diff --git a/docs/contributing/howto/testing/rust.md b/docs/contributing/howto/testing/rust.md index abbd655a..b8aff9f6 100644 --- a/docs/contributing/howto/testing/rust.md +++ b/docs/contributing/howto/testing/rust.md @@ -1,5 +1,14 @@ # Rust tests +## Where to put tests + +1. If it is trivial, put it as an integration test into `/tests/*.rs` +2. If the entire `mod tests` is below 30 lines, you can put it inline +3. Tests for `mod.rs` go in a `tests.rs` alongside it, declared with `mod tests;` +4. Tests for `abc.rs` go in `abc_test.rs`, referenced with `#[path = "abc_test.rs"] mod tests;` + +## How to run + ```bash genvm-tool test run --filter-tag rust # add --coverage for coverage ``` diff --git a/docs/contributing/howto/write-a-script.md b/docs/contributing/howto/write-a-script.md new file mode 100644 index 00000000..ce343e5f --- /dev/null +++ b/docs/contributing/howto/write-a-script.md @@ -0,0 +1,43 @@ +# Writing a script + +1. Prefer python over bash. +2. Use `argparse` for the CLI. +3. Integrate into `genvm-tool` or `support/ci` when it fits, rather than adding a loose entry point. +4. If the script checks something, print each rejection reason to `stdout`, so a failure needs no further digging. + +## Where it lives + +Standalone scripts go in `support/scripts/`. Keep them executable +(`chmod +x`) and pure-stdlib unless there is a strong reason otherwise, so they +run without the dev shell. + +## Wiring a check as a pre-commit hook + +Hooks are declared in `flake.nix` (the committed `.pre-commit-config.yaml` is +generated from it by `git-hooks.nix` on `nix develop` / `nix flake check` -- do +not edit it by hand). Add an entry under the local-guards block: + +```nix +check-source-text = { + enable = true; + name = "check-source-text"; + entry = "${hooks-python}/bin/python3 support/scripts/check-source-text.py"; + files = "\\.rs$"; # narrow to the file types you check +}; +``` + +pre-commit passes the changed files as arguments (`argparse` positional +`files`). Exit non-zero to reject the commit; per point 4, print each rejection +to stdout in a greppable form, ideally `path:lineno: `. See +`support/scripts/check-source-text.py` for a worked example (it also supports a +per-file `check-source-text: off` opt-out comment). + +## Scripts shared with a submodule + +Each executor submodule (`executors/vD.D.x`) is a separate git repo with its own +pre-commit config, and a commit made inside it runs those hooks from its own +checkout -- it cannot reach into this umbrella. (The submodule is not built +standalone; the umbrella drives its build. This is only about its independent +git hooks.) So a guard that must run in both places keeps an in-tree copy under +the submodule's own `support/scripts/`, wired from the submodule's `flake.nix`. +Keep the two copies identical. diff --git a/docs/contributing/tutorial/first-contribution.md b/docs/contributing/tutorial/first-contribution.md new file mode 100644 index 00000000..481b7ba8 --- /dev/null +++ b/docs/contributing/tutorial/first-contribution.md @@ -0,0 +1,73 @@ +# Your first contribution: patching an executor + +End-to-end walkthrough for the most common change β€” fixing something inside an +executor line (`executors/.x`). Each step links to the how-to guide with +the full details; read those when a step surprises you. + +## 1. Set up the checkout + +Follow [setup.md](../howto/setup.md): materialize submodules and vendored trees +(`python3 support/scripts/get-all-git.py`), then enter the dev shell +(`nix develop '.?submodules=1#full'`, usually auto-loaded via direnv). + +## 2. Make and verify the change + +Edit inside `executors/.x/`, then build and test: + +```bash +genvm-tool configure && ninja -C build all/bin # building/build.md +genvm-tool test run --filter-name 'executors/.x/' # testing/README.md +``` + +## 3. Create the branches + +```bash +genvm-tool git create-branches feat/ +``` + +It surveys which repos carry new content and creates the branch in the ones +you tick: the manager gets `feat/` verbatim, each executor gets +`pr//feat/` (all lines push to one shared executor remote, hence +the namespace). Branch model: +[submodules.md](../howto/committing/submodules.md). + +## 4. Commit, then update the gitlink + +The manager pins each submodule by a gitlink (a pinned commit), so the order +matters ([submodules.md](../howto/committing/submodules.md)): + +1. Commit **inside** the executor: `cd executors/.x && git add … && git commit`. +2. In the manager, stage the moved gitlink: `git add executors/.x`. +3. Commit the manager (gitlink bump + any manager-side changes together). + +If you touched runners, do the hash hygiene first: +[committing/runners.md](../howto/committing/runners.md). + +## 5. Pass pre-commit + +The dev shell installs each repo's pre-commit hooks, so `git commit` already +runs them where you commit β€” do **not** bypass with `--no-verify`. To run them +ahead of time: + +- manager only: `nix fmt` (runs the whole generated pre-commit config over the + working tree); +- everything at once, as CI does: `./support/ci/run.sh pipeline commit-hooks`. + +## 6. Push β€” executor first, then manager + +```bash +genvm-tool git check-for-push # readiness per repo + suggested push command +git -C executors/.x push origin pr//feat/ +git push origin feat/ +``` + +Submodules must be pushed before the manager so its pinned gitlinks resolve. + +## 7. Open one PR β€” in the manager only + +Open the manager PR against its dev base (e.g. `v0.6-dev`). Do **not** open +executor PRs yourself: CI automatically opens the matching executor PR for +every active line and links each on the manager PR as an `executor: ` +line. + +Merge requirements (tests, merge queue): [../README.md](../README.md). diff --git a/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/02-gl_call.rst b/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/02-gl_call.rst index 8d7833f5..ba7b5592 100644 --- a/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/02-gl_call.rst +++ b/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/02-gl_call.rst @@ -74,9 +74,16 @@ Requirements #. :ref:`gvm-perm-deterministic` #. :ref:`gvm-perm-call-others` +#. ``calldata`` satisfies :ref:`gvm-def-contract-call-conv` Creates a :term:`sub-VM`. See :ref:`gvm-meta-property-derivation`. +The calling convention is checked in the calling :term:`sub-VM`, before the +callee is spawned. A violation is the caller's own malformed argument, so it is +answered with ``Errno::Inval`` like the other argument checks and the caller can +recover β€” unlike a top-level entry, where the same violation is the execution's +result (:ref:`gvm-vm-startup-entry-validation`). + .. _gvm-def-post-message: ``PostMessage`` Message @@ -112,6 +119,7 @@ Requirements #. :ref:`gvm-perm-send-messages` #. Sufficient contract balance for value transfer #. When ``use_balance`` is set: :ref:`gvm-perm-use-balance-for-message-fees` and ``fee_params`` +#. ``calldata`` satisfies :ref:`gvm-def-contract-call-conv` .. _gvm-gl-call-balance-fees: @@ -206,6 +214,7 @@ Requirements #. :ref:`gvm-perm-send-messages` #. Sufficient contract balance for value transfer #. When ``use_balance`` is set: :ref:`gvm-perm-use-balance-for-message-fees` and ``fee_params`` +#. ``calldata`` satisfies :ref:`gvm-def-contract-call-conv` Supports CREATE2-style deployment with salt nonce for deterministic addressing. ``use_balance`` / ``fee_params`` behave as for :ref:`gvm-gl-call-balance-fees`. diff --git a/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/03-schemas.rst b/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/03-schemas.rst index c27c33f2..96074cae 100644 --- a/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/03-schemas.rst +++ b/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/03-schemas.rst @@ -30,6 +30,66 @@ The ``data`` portion depends on the result code: - **user_error**: :ref:`Calldata Encoded ` error value - **vm_error**: UTF-8 encoded :ref:`gvm-def-str-trie-vm-error` string +.. _gvm-def-proposed-result-validity: + +Validity Of A Proposed Result +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A result the executor did not compute itself β€” the leader-proposed +non-deterministic result consumed in :ref:`gvm-def-sync-mode` and +:ref:`gvm-def-validator-mode` β€” is accepted only if it satisfies all of the +following. A rejected proposal never traps and never bypasses the comparison +stage; it is replaced by a derived :ref:`gvm-def-vm-error` and handed to the +comparison as if it had been proposed. + +#. The buffer is non-empty. An absent or empty proposal yields + :ref:`gvm-def-str-trie-value-vm-error-absent-leader-nondet-output`. +#. ``result_code`` is one of + :ref:`gvm-def-enum-value-result-code-return`, + :ref:`gvm-def-enum-value-result-code-user-error` or + :ref:`gvm-def-enum-value-result-code-vm-error`. Any other byte β€” including + :ref:`gvm-def-enum-value-result-code-internal-error`, which the enumeration + admits but the wire format does not β€” yields + :ref:`gvm-def-pending-str-trie-value-vm-error-leader-output-malformed`. +#. For **return** and **user_error**, ``data`` is valid + :ref:`gvm-def-calldata-encoding` with no trailing bytes; otherwise + :ref:`gvm-def-pending-str-trie-value-vm-error-leader-output-malformed`. +#. For **vm_error**, ``data`` is UTF-8, carries no ``" # "`` detail, and names a + :ref:`gvm-def-str-trie-vm-error` path, including the canonical spelling of + any parameter. Codes outside the trie β€” among them every + :doc:`pending constant <../../appendix/constants-pending>` β€” yield + :ref:`gvm-def-pending-str-trie-value-vm-error-leader-output-malformed`. + +An accepted result is preserved **byte for byte**: validation never re-encodes, +so every node hashes the value that was proposed. + +.. _gvm-def-derived-outcome-namespace: + +Derived-Outcome Namespace +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Following :ref:`gvm-def-vm-error` codes are derived by the consuming executor +rather than proposed: + +#. :ref:`gvm-def-str-trie-value-vm-error-absent-leader-nondet-output` +#. :ref:`gvm-def-pending-str-trie-value-vm-error-leader-output-malformed` +#. :ref:`gvm-def-pending-str-trie-value-vm-error-leader-output-uses-this-error` +#. :ref:`gvm-def-pending-str-trie-value-vm-error-leader-output-extra` + +A proposed code that equals, or extends at a space boundary, either +``absent_leader_nondet_output`` or ``leader_output`` is replaced by +:ref:`gvm-def-pending-str-trie-value-vm-error-leader-output-uses-this-error` +whose parameter is the first 6 characters of the +:doc:`../../04-contract-interface/06-gvm32` encoding of ``sha3_256`` of the +proposed code. A proposal that is its own replacement maps to the parameter +``fix_point``, which cannot collide with a derived parameter because it is 9 +characters long and a derived one is always 6. + +This check runs **before** the trie-validity check, so proposing a derived code +verbatim can never produce output byte-equal to the proposal. Distinct proposals +may share a parameter; the mapping is deterministic and the value carries no +meaning beyond identifying the proposal as rejected. + .. _gvm-def-module-result-encoding: Module Result Encoding diff --git a/docs/website/src/spec/03-vm/01-startup.rst b/docs/website/src/spec/03-vm/01-startup.rst index 704eb3fd..1f4cbc4a 100644 --- a/docs/website/src/spec/03-vm/01-startup.rst +++ b/docs/website/src/spec/03-vm/01-startup.rst @@ -14,6 +14,8 @@ Top-Level Startup Before executing the root :term:`sub-VM`, :term:`GenVM`: +#. Validates the entry payload against :ref:`gvm-def-contract-call-conv` (see + below). #. Reads the contract's locked slots and root-slot data. #. Resolves the contract code from deployment input or from the contract's configured code slot. @@ -27,6 +29,21 @@ If code is supplied with the execution request, that code is the runner source for the deployment execution. Otherwise, the runner source is the contract code resolved from storage. +.. _gvm-vm-startup-entry-validation: + +Entry Payload Validation +~~~~~~~~~~~~~~~~~~~~~~~~ + +The entry payload is host-supplied and is checked against +:ref:`gvm-def-contract-call-conv` before permissions are read and before any +runner loads. A payload that violates the convention produces the +:ref:`gvm-def-vm-error` +:ref:`gvm-def-pending-str-trie-value-vm-error-malformed-entry` as the execution +result, like any other failure at this point. + +Messages emitted by :ref:`gvm-def-post-message` and ``DeployContract`` are +executed later as top-level entries, so they are validated here too. + .. _gvm-vm-startup-message: Startup Message diff --git a/docs/website/src/spec/03-vm/04-determinism-mode-switching.rst b/docs/website/src/spec/03-vm/04-determinism-mode-switching.rst index 858ad3a6..2d295c77 100644 --- a/docs/website/src/spec/03-vm/04-determinism-mode-switching.rst +++ b/docs/website/src/spec/03-vm/04-determinism-mode-switching.rst @@ -1,24 +1,58 @@ Switching To :ref:`gvm-def-non-det-mode` ======================================== -When requesting a non-deterministic execution, new :term:`sub-VM` is created. +When requesting a non-deterministic execution, a new :term:`sub-VM` is created. +Which of the three modes below applies is fixed for the whole execution: a node +runs as leader when it computes the non-deterministic result itself, and as +validator or in sync mode when it is handed one. + +.. _gvm-def-leader-mode: Leader Mode ----------- -Returns back :ref:`gvm-def-vm-result` produced by :term:`sub-VM` as-is. +Returns the :ref:`gvm-def-vm-result` produced by the :term:`sub-VM`, with one +change before it is published and before it enters the execution hash: a +:ref:`gvm-def-vm-error`'s ``" # "`` suffix is **stripped**. The +non-deterministic result channel carries bare codes only, so what the leader +publishes is exactly what an honest validator accepts under +:ref:`gvm-def-proposed-result-validity`. + +The leader does **not** apply that acceptance check to its own result. Doing so +could only rewrite an honest result into the +:ref:`gvm-def-derived-outcome-namespace`, which every validator would replace +again β€” an execution-hash mismatch between two honest nodes. + +.. _gvm-def-sync-mode: Sync Mode --------- -Returns back leaders result as is. +The proposed result is accepted or replaced per +:ref:`gvm-def-proposed-result-validity` and returned. There is no vote to cast, +so the resulting :ref:`gvm-def-vm-result` is simply the call's result. + +.. _gvm-def-validator-mode: Validator Mode -------------- -This :term:`sub-VM` must :ref:`gvm-def-return` a ``bool`` value, if validator -accepts the result of the leader :term:`sub-VM` execution. +The proposed result is accepted or replaced per +:ref:`gvm-def-proposed-result-validity`, then handed to the :term:`sub-VM` for +comparison. That :term:`sub-VM` must :ref:`gvm-def-return` a ``bool`` value: +whether the validator accepts the leader's result. Any other result has the same +effect as producing ``bool(false)``. + +A replaced proposal takes this same path β€” the derived code is compared like any +other, so the vote stays contract-controlled and no branch bypasses the +comparison. The replacement is a pure function of the proposed bytes, so every +honest validator derives the same value and votes identically. + +Returns the accepted or replaced result. -Any other results have the same effect as producing ``bool(false)``. +Fees +---- -Returns back leaders result. +The accepted or replaced result is charged on **its own** length, not on the +length of the proposed bytes: rejected bytes never enter the result or the +execution hash. diff --git a/docs/website/src/spec/03-vm/05-result.rst b/docs/website/src/spec/03-vm/05-result.rst index 88570044..e3147b0a 100644 --- a/docs/website/src/spec/03-vm/05-result.rst +++ b/docs/website/src/spec/03-vm/05-result.rst @@ -157,3 +157,18 @@ small hash is a SHA3-256 digest over a :ref:`gvm-def-calldata-encoding` encoded Non-deterministic sub-calls do not contribute. A run with no deterministic sub-calls finalizes its accumulator to a fixed digest, so that error and edge results hash uniformly. + +Post-Execution Result Validation +-------------------------------- + +A consumer in :ref:`gvm-def-sync-mode` or :ref:`gvm-def-validator-mode` +consumes one leader-proposed result per non-deterministic block it runs. If the +leader supplied **more** results than the run consumed, the surplus blocks were +never reached: the run's result is replaced by +:ref:`gvm-def-pending-str-trie-value-vm-error-leader-output-extra`, whose +parameter is the first 6 characters of the :ref:`gvm-def-gvm32` encoding of the +``sha3_256`` digest of the complete ``[result_code][data]`` +:ref:`sub-VM result buffer ` the run would +otherwise have returned β€” the whole wire buffer as emitted, not a decoded +payload or alternate representation. No non-deterministic disagreement is caused +by this. diff --git a/docs/website/src/spec/04-contract-interface/02-abi.rst b/docs/website/src/spec/04-contract-interface/02-abi.rst index 7488a8fb..6fa33522 100644 --- a/docs/website/src/spec/04-contract-interface/02-abi.rst +++ b/docs/website/src/spec/04-contract-interface/02-abi.rst @@ -32,6 +32,23 @@ Method calls use :ref:`gvm-def-calldata-encoding` format with following conventi The method name is carried under the empty key ``""``. Because calldata maps are encoded with sorted keys, the empty key always sorts first. +This shape is enforced by :term:`GenVM` before any runner is loaded, so a +malformed call fails identically in every runner language. The payload must +decode as :ref:`gvm-def-calldata-encoding` and yield a Map whose every key is +one of ``""``, ``"args"`` or ``"kwargs"``, with ``""`` a String, ``"args"`` an +Array and ``"kwargs"`` a Map. On a deployment the ``""`` key must be **absent**. + +The key set is **closed**: unknown keys are rejected rather than ignored, so two +byte-different payloads can never describe the same call. The extension point of +the convention is the *value* of ``""`` (see `Special Methods`_), not the key +set. Element types inside ``args``/``kwargs`` are matched against the method +signature by the runner, not here. + +Violations are reported as +:ref:`gvm-def-pending-str-trie-value-vm-error-malformed-entry`; where that +surfaces depends on the entry point, see :ref:`gvm-vm-startup-entry-validation` +and :ref:`gvm-def-gl-call-call-contract`. + .. _gvm-def-call-key: Call Key diff --git a/docs/website/src/spec/04-contract-interface/06-gvm32.rst b/docs/website/src/spec/04-contract-interface/06-gvm32.rst index dd658944..8b208434 100644 --- a/docs/website/src/spec/04-contract-interface/06-gvm32.rst +++ b/docs/website/src/spec/04-contract-interface/06-gvm32.rst @@ -1,3 +1,5 @@ +.. _gvm-def-gvm32: + GVM32 (Base32) ============== diff --git a/docs/website/src/spec/appendix/constants-pending.rst b/docs/website/src/spec/appendix/constants-pending.rst index 0b076926..d70b1b90 100644 --- a/docs/website/src/spec/appendix/constants-pending.rst +++ b/docs/website/src/spec/appendix/constants-pending.rst @@ -1,2 +1,33 @@ Constants ========= + +.. _gvm-def-pending-str-trie-vm-error: + +vm_error +-------- + +Type: str_trie + +.. _gvm-def-pending-str-trie-value-vm-error-malformed-entry: + +``malformed_entry`` +~~~~~~~~~~~~~~~~~~~ + +.. _gvm-def-pending-str-trie-value-vm-error-leader-output-malformed: + +``leader_output malformed`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. _gvm-def-pending-str-trie-value-vm-error-leader-output-uses-this-error: + +``leader_output uses_this_error`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Param: str + +.. _gvm-def-pending-str-trie-value-vm-error-leader-output-extra: + +``leader_output extra`` +~~~~~~~~~~~~~~~~~~~~~~~ + +Param: str diff --git a/executors/v0.3.x b/executors/v0.3.x index be5cfe91..7c50d3d9 160000 --- a/executors/v0.3.x +++ b/executors/v0.3.x @@ -1 +1 @@ -Subproject commit be5cfe91304a40e421e13197892c020ca1f2dd40 +Subproject commit 7c50d3d9d80e7ea065f3560fbe790b6664acb233 diff --git a/flake.nix b/flake.nix index 97484525..38f158f0 100644 --- a/flake.nix +++ b/flake.nix @@ -297,6 +297,12 @@ entry = "${hooks-python}/bin/python3 support/scripts/md-local-links.py"; types = [ "markdown" ]; }; + check-source-text = { + enable = true; + name = "check-source-text"; + entry = "${hooks-python}/bin/python3 support/scripts/check-source-text.py"; + files = "\\.rs$"; + }; # --- commit-msg ------------------------------------------- check-commit-message = { enable = true; diff --git a/implementation/default.nix b/implementation/default.nix index 4c3ed9bd..0694077a 100644 --- a/implementation/default.nix +++ b/implementation/default.nix @@ -17,6 +17,10 @@ let lib = pkgs.lib; + # The shipped LLM dispatch script requires the `llm_policy` package, which + # lives in the unhardcoded-engine submodule instead of under install/. + llm-policy-src = root-src + "/libs/unhardcoded-engine"; + make-for-target = target: let @@ -81,6 +85,18 @@ let chmod -R u+w "$out" fi done + # compiled-libs doubles as the link input and as a copy source, so its + # static archives land here too; they are already inside the binary. + rm -f "$out"/lib/*.a + + if [ ! -f "${llm-policy-src}/llm_policy.lua" ]; then + echo "${llm-policy-src} is missing or incomplete; run \`git submodule update --init libs/unhardcoded-engine\`" >&2 + exit 1 + fi + cp --no-preserve=ownership "${llm-policy-src}/llm_policy.lua" "$out/lib/genvm-lua/llm_policy.lua" + cp --no-preserve=ownership -r "${llm-policy-src}/llm_policy" "$out/lib/genvm-lua/llm_policy" + chmod -R u+w "$out/lib/genvm-lua" + patch-yaml-schema --tag ${build-config.executor-version} "$out" patch-llm-config --tag ${build-config.executor-version} "$out/config/genvm-module-llm.yaml" patch-web-config --tag ${build-config.executor-version} "$out/config/genvm-module-web.yaml" diff --git a/implementation/src/common/mod.rs b/implementation/src/common/mod.rs index 293ba3f6..c9709634 100644 --- a/implementation/src/common/mod.rs +++ b/implementation/src/common/mod.rs @@ -740,7 +740,7 @@ pub fn is_no_routable_address(err: &reqwest::Error) -> bool { /// Resolver that resolves via hickory and drops every address that is not /// globally routable. This is the SSRF guard for the web module: it prevents a /// contract from steering the node into the operator's internal network (e.g. -/// link-local, loopback, RFC1918) β€” including via a DNS-rebinding attack, since +/// link-local, loopback, RFC1918) -- including via a DNS-rebinding attack, since /// the filtering happens in the very resolver reqwest connects through, leaving /// no second, unfiltered lookup. The hostname stays in the URL, so TLS SNI, /// certificate verification and the `Host` header keep working over HTTPS. @@ -1064,6 +1064,19 @@ pub mod tests { super::base_client_builder().build().map_err(Into::into) } + /// A `package.path` fragment for the `llm_policy` package, which the shipped + /// llm dispatch script requires. It ships from the unhardcoded-engine + /// submodule; a clone without submodules gets told what to run. + pub fn llm_policy_lua_path() -> String { + let root = std::path::PathBuf::from("../libs/unhardcoded-engine") + .canonicalize() + .expect( + "libs/unhardcoded-engine is missing; \ + run `git submodule update --init libs/unhardcoded-engine`", + ); + format!("{}/?.lua", root.to_str().unwrap()) + } + pub fn get_hello() -> Arc { Arc::new(genvm_modules_interfaces::GenVMHello { genvm_id: genvm_modules_interfaces::GenVMId(999), diff --git a/implementation/src/llm/handler.rs b/implementation/src/llm/handler.rs index 2cf78d30..cb346e39 100644 --- a/implementation/src/llm/handler.rs +++ b/implementation/src/llm/handler.rs @@ -59,7 +59,7 @@ impl MessageHandlerProvider, > { let genvm_id = ctx.scripting.hello.genvm_id; - let user_vm = self.vm_pool.get().await; + let user_vm = self.vm_pool.get().await?; let (handler_ctx, ctx_val) = user_vm.create_ctx(&ctx)?; diff --git a/implementation/src/llm/providers.rs b/implementation/src/llm/providers.rs index a6d2eb9b..03f66a31 100644 --- a/implementation/src/llm/providers.rs +++ b/implementation/src/llm/providers.rs @@ -1,5 +1,4 @@ use crate::{common::ModuleResult, scripting}; -use anyhow::Context as _; use base64::Engine; use genvm_common::*; @@ -145,8 +144,12 @@ pub trait Provider { ) -> ModuleResult>> { let res = self.exec_prompt_json_as_text(ctx, prompt, model).await?; let json_str = sanitize_json_str(&res.result); - let parsed = - serde_json::from_str(&json_str).with_context(|| format!("parsing {json_str:?}"))?; + // Unparseable output still cost the operator; charge the usage with an + // empty object rather than dropping the tokens by erroring. + let parsed = serde_json::from_str(&json_str).unwrap_or_else(|e| { + log_warn!(json = json_str, error = e.to_string(); "unparseable json response, charging usage with empty object"); + serde_json::Map::new() + }); Ok(ProviderResponse::new(parsed, res.tokens)) } @@ -369,8 +372,10 @@ impl Provider for OpenAICompatible { return Ok(ProviderResponse::new("".into(), tokens)); } - let response = - response.ok_or_else(|| anyhow::anyhow!("can't get response field {}", &res.body))?; + let response = response.unwrap_or_else(|| { + log_warn!(body:serde = res.body; "openai: missing content, charging usage with empty response"); + "" + }); Ok(ProviderResponse::new(response.to_owned(), tokens)) } @@ -439,7 +444,10 @@ impl Provider for OpenAICompatible { .body .pointer("/choices/0/message/content") .and_then(|v| if v.is_null() { Some("") } else { v.as_str() }) - .ok_or_else(|| anyhow::anyhow!("can't get response field {}", &res.body))?; + .unwrap_or_else(|| { + log_warn!(body:serde = res.body; "openai: missing json content, charging usage with empty response"); + "" + }); Ok(ProviderResponse::new(sanitize_json_str(response), tokens)) } @@ -547,7 +555,10 @@ impl Provider for OLlama { .as_object() .and_then(|v| v.get("response")) .and_then(|v| v.as_str()) - .ok_or_else(|| anyhow::anyhow!("can't get response field {}", &res.body))?; + .unwrap_or_else(|| { + log_warn!(body:serde = res.body; "ollama: missing response, charging usage with empty response"); + "" + }); Ok(ProviderResponse::new(response.to_owned(), tokens)) } @@ -606,7 +617,10 @@ impl Provider for OLlama { .as_object() .and_then(|v| v.get("response")) .and_then(|v| v.as_str()) - .ok_or_else(|| anyhow::anyhow!("can't get response field {}", &res.body))?; + .unwrap_or_else(|| { + log_warn!(body:serde = res.body; "ollama: missing json response, charging usage with empty response"); + "" + }); Ok(ProviderResponse::new(sanitize_json_str(response), tokens)) } } @@ -699,8 +713,10 @@ impl Provider for Gemini { return Ok(ProviderResponse::new("".into(), tokens)); } - let res = - res.ok_or_else(|| anyhow::anyhow!("can't get response field {}", &res_json.body))?; + let res = res.unwrap_or_else(|| { + log_warn!(body:serde = res_json.body; "gemini: missing text, charging usage with empty response"); + "" + }); Ok(ProviderResponse::new(res.into(), tokens)) } @@ -761,8 +777,10 @@ impl Provider for Gemini { return Ok(ProviderResponse::new("{}".to_owned(), tokens)); } - let res = - res.ok_or_else(|| anyhow::anyhow!("can't get response field {}", &res_json.body))?; + let res = res.unwrap_or_else(|| { + log_warn!(body:serde = res_json.body; "gemini: missing json text, charging usage with empty response"); + "" + }); Ok(ProviderResponse::new(sanitize_json_str(res), tokens)) } @@ -872,7 +890,10 @@ impl Provider for Anthropic { .body .pointer("/content/0/text") .and_then(|x| x.as_str()) - .ok_or_else(|| anyhow::anyhow!("can't get response field {}", &res.body))?; + .unwrap_or_else(|| { + log_warn!(body:serde = res.body; "anthropic: missing text, charging usage with empty response"); + "" + }); Ok(ProviderResponse::new(String::from(text), tokens)) } @@ -945,9 +966,13 @@ impl Provider for Anthropic { .body .pointer("/content/0/input") .and_then(|x| x.as_object()) - .ok_or_else(|| anyhow::anyhow!("can't get response field {}", &res.body))?; + .cloned() + .unwrap_or_else(|| { + log_warn!(body:serde = res.body; "anthropic: missing json input, charging usage with empty object"); + serde_json::Map::new() + }); - Ok(ProviderResponse::new(val.clone(), tokens)) + Ok(ProviderResponse::new(val, tokens)) } async fn exec_prompt_bool_reason( @@ -1015,7 +1040,10 @@ impl Provider for Anthropic { .body .pointer("/content/0/input/result") .and_then(|x| x.as_bool()) - .ok_or_else(|| anyhow::anyhow!("can't get response field {}", &res.body))?; + .unwrap_or_else(|| { + log_error!(body:serde = res.body; "no result in reason, returning false"); + false + }); Ok(ProviderResponse::new(val, tokens)) } diff --git a/implementation/src/manager/run.rs b/implementation/src/manager/run.rs index 2bd76a09..6b3124b9 100644 --- a/implementation/src/manager/run.rs +++ b/implementation/src/manager/run.rs @@ -1014,7 +1014,7 @@ pub async fn start_genvm( }); // Resolve command path. A request may reroute to an explicit executor dir, - // but only when running with a debug mode of `safe` or above β€” production + // but only when running with a debug mode of `safe` or above -- production // (`disabled`) always uses the manifest-resolved version so consensus can't // be steered to a different binary. let mut command_path = ctx.executors_path.clone(); diff --git a/implementation/src/scripting/ctx/dflt.rs b/implementation/src/scripting/ctx/dflt.rs index 5a15957d..d3942f4e 100644 --- a/implementation/src/scripting/ctx/dflt.rs +++ b/implementation/src/scripting/ctx/dflt.rs @@ -14,6 +14,16 @@ use super::req::Request; use super::CtxPart; +static START: std::sync::LazyLock = + std::sync::LazyLock::new(std::time::Instant::now); + +/// Process-relative monotonic milliseconds: comparable only within one process, +/// not a wall clock. Exists because the Lua sandbox loads no os library. +fn monotonic_ms() -> mlua::Integer { + let elapsed = START.elapsed().as_millis(); + elapsed.min(mlua::Integer::MAX as u128) as mlua::Integer +} + impl CtxPart { async fn request(&self, vm: &mlua::Lua, req: Request) -> anyhow::Result { log_trace!(request:serde = req; "received request"); @@ -262,6 +272,38 @@ pub fn create_global( use rand::TryRngCore; + // All hashes return raw digest bytes. + dflt.set( + "sha2_256", + vm.create_function(|vm: &mlua::Lua, data: mlua::String| { + let digest = ring::digest::digest(&ring::digest::SHA256, &data.as_bytes()); + vm.create_string(digest.as_ref()) + })?, + )?; + + dflt.set( + "sha3_256", + vm.create_function(|vm: &mlua::Lua, data: mlua::String| { + use sha3::Digest as _; + let digest = sha3::Sha3_256::digest(data.as_bytes()); + vm.create_string(digest.as_slice()) + })?, + )?; + + dflt.set( + "keccak256", + vm.create_function(|vm: &mlua::Lua, data: mlua::String| { + use sha3::Digest as _; + let digest = sha3::Keccak256::digest(data.as_bytes()); + vm.create_string(digest.as_slice()) + })?, + )?; + + dflt.set( + "monotonic_ms", + vm.create_function(|_: &mlua::Lua, _: ()| Ok(monotonic_ms()))?, + )?; + dflt.set( "random_bytes", vm.create_function(|vm: &mlua::Lua, length: usize| { @@ -292,3 +334,87 @@ pub fn create_global( Ok(mlua::Value::Table(dflt)) } + +#[cfg(test)] +mod tests { + use std::{collections::BTreeMap, sync::Arc}; + + use super::*; + + fn config() -> crate::common::ModuleBaseConfig { + crate::common::ModuleBaseConfig { + bind_address: None, + lua_script_path: String::new(), + vm_count: 1, + lua_path: String::new(), + signer_url: Arc::from(""), + signer_headers: Arc::new(BTreeMap::new()), + data_dir: String::new(), + } + } + + fn dflt_table(vm: &mlua::Lua) -> mlua::Table { + match create_global(vm, &config()).unwrap() { + mlua::Value::Table(table) => table, + _ => panic!("expected table"), + } + } + + fn hash_hex(name: &str, input: &[u8]) -> String { + let vm = mlua::Lua::new(); + let dflt = dflt_table(&vm); + let hash: mlua::Function = dflt.get(name).unwrap(); + let input = vm.create_string(input).unwrap(); + let digest: mlua::String = hash.call(input).unwrap(); + hex::encode(digest.as_bytes()) + } + + #[test] + fn sha2_256_known_vectors() { + assert_eq!( + hash_hex("sha2_256", b""), + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ); + assert_eq!( + hash_hex("sha2_256", b"abc"), + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); + } + + #[test] + fn sha3_256_known_vectors() { + assert_eq!( + hash_hex("sha3_256", b""), + "a7ffc6f8bf1ed76651c14756a061d662f580ff4de43b49fa82d80a4b80f8434a" + ); + assert_eq!( + hash_hex("sha3_256", b"abc"), + "3a985da74fe225b2045c172d6bd390bd855f086e3e9d525b46bfe24511431532" + ); + } + + #[test] + fn keccak256_known_vectors() { + // The Ethereum keccak256 of the empty input. + assert_eq!( + hash_hex("keccak256", b""), + "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" + ); + assert_eq!( + hash_hex("keccak256", b"abc"), + "4e03657aea45a94fc7d47ba826c8d667c0d1e6e33a64a036ec44f58fa12d6c45" + ); + } + + #[test] + fn monotonic_ms_is_non_decreasing() { + let vm = mlua::Lua::new(); + let dflt = dflt_table(&vm); + let monotonic_ms: mlua::Function = dflt.get("monotonic_ms").unwrap(); + let first: mlua::Integer = monotonic_ms.call(()).unwrap(); + let second: mlua::Integer = monotonic_ms.call(()).unwrap(); + + assert!(first >= 0); + assert!(second >= first); + } +} diff --git a/implementation/src/scripting/pool.rs b/implementation/src/scripting/pool.rs index 8d54b40f..19aaa2da 100644 --- a/implementation/src/scripting/pool.rs +++ b/implementation/src/scripting/pool.rs @@ -1,24 +1,32 @@ +use genvm_common::*; use std::sync::Arc; +type VmFuture = std::pin::Pin< + Box>> + Send>, +>; + +/// Builds a fresh VM. Kept so the pool can grow on demand rather than block when +/// every pre-allocated VM is checked out. +type VmFactory = dyn Fn() -> VmFuture + Send + Sync; + struct Inner { free: crossbeam::queue::ArrayQueue>>, + factory: Box>, } -/// A fixed-size pool of Lua VMs. A VM is checked out exclusively via [`Pool::get`] -/// and returned to the pool when the [`PoolGuard`] is dropped, so a single -/// `lua_State` is never driven by two executions concurrently. +/// A pool of Lua VMs with a fixed steady-state size. A VM is checked out +/// exclusively via [`Pool::get`], so a single `lua_State` is never driven by two +/// executions concurrently. When the pool is empty a fresh VM is built on demand +/// instead of waiting; VMs returned while the pool is already full are torn down, +/// so the steady-state count stays at the configured size. pub struct Pool { inner: Arc>, - // `available` counts the VMs currently in `free`; acquiring a permit - // guarantees a subsequent `pop` succeeds. - available: Arc, } impl Clone for Pool { fn clone(&self) -> Self { Self { inner: self.inner.clone(), - available: self.available.clone(), } } } @@ -28,8 +36,6 @@ impl Clone for Pool { pub struct PoolGuard { vm: Option>>, inner: Arc>, - // Released only after the VM has been returned to `free` (see `Drop`). - _permit: tokio::sync::OwnedSemaphorePermit, } impl std::ops::Deref for PoolGuard { @@ -45,54 +51,113 @@ impl std::ops::Deref for PoolGuard { impl Drop for PoolGuard { fn drop(&mut self) { if let Some(vm) = self.vm.take() { - // Return the VM before `_permit` is released so the next waiter that - // wakes on the freed permit is guaranteed to find a VM in `free`. + // Return it to the pool. `push` gives the VM back when the pool is + // full -- that happens only for a VM built on demand beyond the + // configured size -- and dropping it here tears it down. let _ = self.inner.free.push(vm); } } } impl Pool { - /// Atomically checks out a VM, waiting if all VMs are currently in use. - pub async fn get(&self) -> PoolGuard { - let permit = self - .available - .clone() - .acquire_owned() - .await - .expect("vm pool semaphore is never closed"); - let vm = self - .inner - .free - .pop() - .expect("holding a permit guarantees a free vm"); - PoolGuard { + /// Checks out a VM: a pre-allocated one if the pool has any, otherwise a + /// freshly built one. Never waits on other checkouts. + pub async fn get(&self) -> anyhow::Result> { + let vm = match self.inner.free.pop() { + Some(vm) => vm, + None => { + log_warn!("vm pool empty, building a vm on demand"); + Arc::new((self.inner.factory)().await?) + } + }; + Ok(PoolGuard { vm: Some(vm), inner: self.inner.clone(), - _permit: permit, - } + }) } } -pub async fn new( - cnt: usize, - vm_supplier: impl Fn() -> F, -) -> anyhow::Result> +pub async fn new(cnt: usize, factory: Fac) -> anyhow::Result> where - F: std::future::Future>>, + E: 'static, + Fac: Fn() -> F + Send + Sync + 'static, + F: std::future::Future>> + Send + 'static, { if cnt == 0 { anyhow::bail!("vm pool size must be >= 1"); } + let factory: Box> = Box::new(move || Box::pin(factory())); + let free = crossbeam::queue::ArrayQueue::new(cnt); for _i in 0..cnt { - free.push(Arc::new(vm_supplier().await?)) + let vm = factory().await?; + free.push(Arc::new(vm)) .ok() .expect("queue capacity matches the number of VMs"); } Ok(Pool { - inner: Arc::new(Inner { free }), - available: Arc::new(tokio::sync::Semaphore::new(cnt)), + inner: Arc::new(Inner { free, factory }), }) } + +#[cfg(test)] +mod tests { + use super::*; + use genvm_common::sync::DArc; + use std::sync::atomic::{AtomicUsize, Ordering}; + + async fn trivial_vm() -> anyhow::Result> { + let cfg = crate::common::ModuleBaseConfig { + bind_address: None, + lua_script_path: String::new(), + vm_count: 1, + lua_path: String::new(), + signer_url: std::sync::Arc::from(""), + signer_headers: std::sync::Arc::new(std::collections::BTreeMap::new()), + data_dir: String::new(), + }; + crate::scripting::UserVM::create( + &cfg, + |_vm| async { Ok(()) }, + Box::new(|_vm, _table, _e: &DArc<()>| Ok(())), + ) + .await + } + + #[tokio::test] + async fn grows_on_demand_and_sheds_overflow() { + let created = Arc::new(AtomicUsize::new(0)); + let counter = created.clone(); + let pool = new(2, move || { + let counter = counter.clone(); + async move { + counter.fetch_add(1, Ordering::SeqCst); + trivial_vm().await + } + }) + .await + .unwrap(); + assert_eq!(created.load(Ordering::SeqCst), 2, "pre-allocates vm_count"); + + let g1 = pool.get().await.unwrap(); + let g2 = pool.get().await.unwrap(); + // pool is now empty: this checkout builds a VM on demand instead of waiting + let g3 = pool.get().await.unwrap(); + assert_eq!( + created.load(Ordering::SeqCst), + 3, + "grows past the pre-allocated size" + ); + + drop(g1); + drop(g2); + // the queue holds only 2, so returning a third tears it down + drop(g3); + + // the two returned VMs are reused; no new ones are built + let _g4 = pool.get().await.unwrap(); + let _g5 = pool.get().await.unwrap(); + assert_eq!(created.load(Ordering::SeqCst), 3, "reuses returned vms"); + } +} diff --git a/implementation/src/web/handler.rs b/implementation/src/web/handler.rs index c1a99931..3b445ad1 100644 --- a/implementation/src/web/handler.rs +++ b/implementation/src/web/handler.rs @@ -120,7 +120,7 @@ impl common::MessageHandlerProvider anyhow::Result< impl common::MessageHandler, > { - let user_vm = self.vm_pool.get().await; + let user_vm = self.vm_pool.get().await?; let (handler_ctx, ctx_val) = user_vm.create_ctx(&ctx)?; diff --git a/implementation/tests/overloaded.rs b/implementation/tests/overloaded.rs index 0bdca7aa..5bdeb774 100644 --- a/implementation/tests/overloaded.rs +++ b/implementation/tests/overloaded.rs @@ -96,6 +96,8 @@ async fn test_overloaded() { .unwrap() .to_owned(); extra_path.push_str("/?.lua"); + extra_path.push(';'); + extra_path.push_str(&common::tests::llm_policy_lua_path()); let config = sync::DArc::new(config::Config { base: genvm_common::BaseConfig { diff --git a/implementation/tests/policy_dispatch.rs b/implementation/tests/policy_dispatch.rs new file mode 100644 index 00000000..28ad1cb9 --- /dev/null +++ b/implementation/tests/policy_dispatch.rs @@ -0,0 +1,276 @@ +// End-to-end tests for the LLM dispatch script after backend selection and the +// retry state machine were delegated to the `llm_policy` engine. Everything here +// runs offline against fake local TCP backends; no API keys, no network. +// +// What is asserted: candidates are tried in the configured priority order, any +// provider error (overloaded or not) falls through to the next candidate, and +// capability requirements filter the catalog before the engine ever routes. + +use genvm_common::logger; +use genvm_common::*; +use genvm_modules_interfaces::llm::{self as llm_iface}; +use mlua::LuaSerdeExt; +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; + +use genvm_modules::common; +use genvm_modules::llm::*; +use genvm_modules::scripting; + +const OK_BODY: &str = concat!( + "{\"choices\":[{\"message\":{\"content\":\"ok\"}}],", + "\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":1,\"total_tokens\":2}}" +); + +/// One scripted fake backend. Each incoming connection pops the next status from +/// `responses` (the last one repeats) and appends this backend's name to the +/// shared `hits` log, so a test can assert the exact order providers were tried. +struct FakeBackend { + name: &'static str, + addr: String, +} + +fn spawn_fake( + name: &'static str, + responses: Vec, + hits: Arc>>, +) -> FakeBackend { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let addr = format!("http://{}", listener.local_addr().unwrap()); + let listener = tokio::net::TcpListener::from_std(listener).unwrap(); + + tokio::spawn(async move { + let mut idx = 0usize; + loop { + let (mut sock, _) = match listener.accept().await { + Ok(v) => v, + Err(_) => break, + }; + hits.lock().unwrap().push(name); + + // Drain the request headers so the client's write side completes. + let mut buf = [0u8; 4096]; + let _ = sock.read(&mut buf).await; + + let status = responses[idx.min(responses.len() - 1)]; + idx += 1; + + let response = if status == 200 { + format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\ + Content-Length: {}\r\nConnection: close\r\n\r\n{}", + OK_BODY.len(), + OK_BODY + ) + } else { + format!( + "HTTP/1.1 {} X\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + status + ) + }; + let _ = sock.write_all(response.as_bytes()).await; + let _ = sock.shutdown().await; + } + }); + + FakeBackend { name, addr } +} + +fn model_cfg(supports_json: bool) -> config::ModelConfig { + config::ModelConfig { + enabled: true, + supports_json, + supports_image: false, + use_max_completion_tokens: false, + meta: serde_json::Value::Null, + timeout: None, + } +} + +/// Build a config + provider map from fake backends. `priority` sets +/// `meta.priority` so the dispatch chain order is deterministic and known. +fn build_config( + backends: &[(&FakeBackend, i64, bool)], +) -> ( + sync::DArc, + Arc>>, +) { + let mut cfg_backends = BTreeMap::new(); + let mut provider_map = BTreeMap::new(); + + for (fake, priority, supports_json) in backends { + let backend = config::BackendConfig { + enabled: true, + provider: config::Provider::OpenaiCompatible, + key: "".to_owned(), + script_config: config::ScriptBackendConfig { + models: BTreeMap::from([("model".to_owned(), model_cfg(*supports_json))]), + meta: serde_json::json!({ "priority": priority }), + timeout: None, + }, + host: fake.addr.clone(), + }; + provider_map.insert(fake.name.to_owned(), backend.to_provider()); + cfg_backends.insert(fake.name.to_owned(), backend); + } + + let mut extra_path = std::path::PathBuf::from("../install/lib/genvm-lua") + .canonicalize() + .unwrap() + .to_str() + .unwrap() + .to_owned(); + extra_path.push_str("/?.lua"); + extra_path.push(';'); + extra_path.push_str(&common::tests::llm_policy_lua_path()); + + let config = sync::DArc::new(config::Config { + base: genvm_common::BaseConfig { + log_level: logger::Level::Debug, + threads: 1, + blocking_threads: 3, + log_disable: "".to_owned(), + }, + mod_base: common::ModuleBaseConfig { + vm_count: 1, + lua_script_path: "../install/config/genvm-llm-default.lua".to_string(), + bind_address: None, + lua_path: extra_path, + signer_url: Arc::from(""), + signer_headers: Arc::new(BTreeMap::new()), + data_dir: String::new(), + }, + prompt_templates: config::PromptTemplates { + eq_comparative: serde_json::Value::Null, + eq_non_comparative_leader: serde_json::Value::Null, + eq_non_comparative_validator: serde_json::Value::Null, + }, + backends: cfg_backends, + meta: serde_json::Value::Null, + timeout: None, + }); + + (config, Arc::new(provider_map)) +} + +/// Run one `ExecPrompt` through the shipped dispatch script and return the text +/// answer (or the error). Fresh `Setup` per call, as a real session does. +async fn run_prompt( + config: &sync::DArc, + providers: Arc>>, + format: llm_iface::OutputFormat, +) -> anyhow::Result { + let user_vm = create_vm(config).await.unwrap(); + + let hello = common::tests::get_hello(); + let metrics = sync::DArc::new(Metrics::default()); + let scripting_ctx = scripting::create_ctx_part( + &hello, + &config.gep(|x| &x.mod_base), + metrics.gep(|x| &x.scripting), + false, + ) + .unwrap(); + let llm_ctx = ctx::CtxPart { + providers: providers.clone(), + metrics, + }; + let sub_ctx = sync::DArc::new(genvm_modules::manager::execution_context::LlmSubContext { + scripting: scripting_ctx, + module: llm_ctx, + }); + + let (_ctx, ctx_lua) = user_vm.create_ctx(&sub_ctx).unwrap(); + + if let Some(ref setup) = user_vm.data.setup { + let _: mlua::Value = user_vm.call_fn(setup, ctx_lua.clone()).await.unwrap(); + } + + let payload = llm_iface::PromptPayload { + images: Vec::new(), + response_format: format, + prompt: "reply with ok".to_owned(), + }; + let payload = user_vm + .vm + .to_value_with(&payload, scripting::DEFAULT_LUA_SER_OPTIONS) + .unwrap(); + let fuel = user_vm + .vm + .create_userdata(scripting::rat::LuaRat(num_rational::BigRational::from( + num_bigint::BigInt::from(0), + ))) + .unwrap(); + + let res: mlua::Value = user_vm + .call_fn(&user_vm.data.exec_prompt, (ctx_lua, payload, fuel)) + .await?; + let table = res.as_table().unwrap(); + let data: llm_iface::PromptAnswerData = + user_vm.vm.from_value(table.get("data").unwrap()).unwrap(); + match data { + llm_iface::PromptAnswerData::Text(text) => Ok(text.trim().to_lowercase()), + _ => anyhow::bail!("unexpected non-text answer variant"), + } +} + +// Higher priority is tried first; an overloaded provider falls through to the +// next in priority order; the survivor's answer is returned. +#[tokio::test] +async fn chain_order_and_overload_fallthrough() { + common::tests::setup(); + let hits = Arc::new(Mutex::new(Vec::new())); + + let high = spawn_fake("high", vec![503], hits.clone()); + let mid = spawn_fake("mid", vec![503], hits.clone()); + let low = spawn_fake("low", vec![200], hits.clone()); + + let (config, providers) = + build_config(&[(&high, 100, true), (&mid, 0, true), (&low, -5, true)]); + + let answer = run_prompt(&config, providers, llm_iface::OutputFormat::Text) + .await + .unwrap(); + assert_eq!(answer, "ok"); + assert_eq!(*hits.lock().unwrap(), vec!["high", "mid", "low"]); +} + +// A non-overload user error (400) must also fall through, matching the behaviour +// of the pre-engine dispatch loop which retried on any user error. +#[tokio::test] +async fn fallthrough_on_bad_request() { + common::tests::setup(); + let hits = Arc::new(Mutex::new(Vec::new())); + + let high = spawn_fake("high", vec![400], hits.clone()); + let low = spawn_fake("low", vec![200], hits.clone()); + + let (config, providers) = build_config(&[(&high, 10, true), (&low, 0, true)]); + + let answer = run_prompt(&config, providers, llm_iface::OutputFormat::Text) + .await + .unwrap(); + assert_eq!(answer, "ok"); + assert_eq!(*hits.lock().unwrap(), vec!["high", "low"]); +} + +// A JSON prompt must never reach a backend that cannot produce JSON, even if it +// has the higher priority: capability filtering happens before routing. +#[tokio::test] +async fn json_capability_filtering() { + common::tests::setup(); + let hits = Arc::new(Mutex::new(Vec::new())); + + let no_json = spawn_fake("no_json", vec![200], hits.clone()); + let json = spawn_fake("json", vec![200], hits.clone()); + + let (config, providers) = build_config(&[(&no_json, 100, false), (&json, 0, true)]); + + let answer = run_prompt(&config, providers, llm_iface::OutputFormat::JSON) + .await + .unwrap(); + assert_eq!(answer, "ok"); + assert_eq!(*hits.lock().unwrap(), vec!["json"]); +} diff --git a/implementation/tests/providers.rs b/implementation/tests/providers.rs index 27ece56a..c41ad70a 100644 --- a/implementation/tests/providers.rs +++ b/implementation/tests/providers.rs @@ -183,7 +183,7 @@ const BIG_PROMPT: &str = r#" πŸŒ† Poem Prompt: The Shadow Citizen Task: Write a poem, approximately 16-20 lines, about the urban rat. Your goal is to move beyond the simple idea of "pest" and explore the rat as a complex, parallel inhabitant of the city. - Core Theme: Focus on the rat as a secret-keeper or a historian of the discarded. It moves through the spaces we ignoreβ€”the subway tunnels, the forgotten foundations, the labyrinth of pipes. It thrives on what we throw away. + Core Theme: Focus on the rat as a secret-keeper or a historian of the discarded. It moves through the spaces we ignore--the subway tunnels, the forgotten foundations, the labyrinth of pipes. It thrives on what we throw away. Guiding Questions & Imagery: diff --git a/implementation/tests/timeout.rs b/implementation/tests/timeout.rs index b61bf69a..45bd15a1 100644 --- a/implementation/tests/timeout.rs +++ b/implementation/tests/timeout.rs @@ -57,6 +57,8 @@ async fn test_timeout() { .unwrap() .to_owned(); extra_path.push_str("/?.lua"); + extra_path.push(';'); + extra_path.push_str(&common::tests::llm_policy_lua_path()); let config = sync::DArc::new(config::Config { base: genvm_common::BaseConfig { diff --git a/implementation/tests/web_render_webdriver.rs b/implementation/tests/web_render_webdriver.rs index 0b267b85..ae80f1a8 100644 --- a/implementation/tests/web_render_webdriver.rs +++ b/implementation/tests/web_render_webdriver.rs @@ -2,7 +2,7 @@ //! connection. `Render` talks to the operator-configured `webdriver_host`, which //! in Docker resolves to a private (RFC1918 / loopback) address the SSRF filter //! would otherwise drop. That connection must therefore go through the unfiltered -//! client β€” while contract-driven `Request` traffic stays filtered. +//! client -- while contract-driven `Request` traffic stays filtered. use std::collections::BTreeMap; use std::sync::Arc; @@ -134,7 +134,7 @@ async fn test_render_webdriver_bypasses_dns_filter() { let addr = serve_webdriver(body).await; // `localhost.direct` resolves publicly to loopback, where our stub is bound. - // With `filter_dns = true` the SSRF resolver would drop it β€” so this only + // With `filter_dns = true` the SSRF resolver would drop it -- so this only // succeeds because `Render` sends the webdriver request `unfiltered`. let webdriver_host = format!("http://localhost.direct:{}", addr.port()); let uvm = create_web_vm(webdriver_host).await; diff --git a/install/config/genvm-llm-default.lua b/install/config/genvm-llm-default.lua index 80866dc3..d49f31bd 100644 --- a/install/config/genvm-llm-default.lua +++ b/install/config/genvm-llm-default.lua @@ -1,5 +1,6 @@ local lib = require("lib-genvm") local llm = require("lib-llm") +local llm_policy = require("llm_policy") local sqlite3 = require("lsqlite3") -- There is no guarantee that different genvm executions will be executed in the same lua VM. @@ -84,116 +85,259 @@ local function compute_timeout(ctx, remaining_gen) return timeout:to_float() end -local function just_in_backend(ctx, mapped_prompt, remaining_gen) - ---@cast mapped_prompt MappedPrompt +-- llm_policy owns backend selection and the failure state machine; this script +-- keeps transport, gen accounting, stats, timeouts and the error contract. +-- +-- The catalog keys a model by "/", not the bare model name: two +-- backends may serve the same name with different capabilities, and one key +-- would let one backend's capabilities vouch for another's. The chain reproduces +-- the pre-engine ordering: priority desc, provider name desc, model name asc. +local PROFILE_NAME = "compat" + +local function build_policy_config() + local providers = {} + local models = {} + local chain = {} + + local provider_names = {} + for provider_name, _ in pairs(llm.providers) do + table.insert(provider_names, provider_name) + end - local search_in = llm.select_providers_for(mapped_prompt.prompt, mapped_prompt.format) + table.sort(provider_names, function(a, b) + local a_meta = llm.providers[a].meta + local b_meta = llm.providers[b].meta + local a_priority = a_meta and a_meta.priority or 0 + local b_priority = b_meta and b_meta.priority or 0 - lib.log { - level = "debug", - message = "executing prompt in backend", - prompt = mapped_prompt, - search_in = search_in, - } + if a_priority ~= b_priority then + return a_priority > b_priority + end + return a > b -- just compare names + end) - local timeout = compute_timeout(ctx, remaining_gen) - if timeout and timeout < 1 then - lib.log { - level = "warning", - message = "computed timeout is very low, failing immediately", - timeout = timeout, + for _, provider_name in ipairs(provider_names) do + local provider_data = llm.providers[provider_name] + + -- Placeholders: the engine validates these but never dereferences them; + -- host and key stay in Rust. + providers[provider_name] = { + base_url = "managed-by-genvm", + api_kind = "openai_compatible", + auth_env = "managed-by-genvm", + tier = "partner", + discovery = "static", } - llm.exhaust() + + local model_names = {} + for model_name, _ in pairs(provider_data.models or {}) do + table.insert(model_names, model_name) + end + table.sort(model_names) + + for _, model_name in ipairs(model_names) do + local model_data = provider_data.models[model_name] + local family = provider_name .. "/" .. model_name + + models[family] = { + served_by = { + { provider = provider_name, provider_model_id = model_name }, + }, + capabilities = { + supports_json_mode = model_data.supports_json or false, + supports_vision = model_data.supports_image or false, + supports_seed = true, + }, + } + + table.insert(chain, { provider = provider_name, model = family }) + end + end + + return { + providers = providers, + models = models, + profiles = { + [PROFILE_NAME] = { + selector = "chain", + chain = chain, + retry_policy = PROFILE_NAME, + }, + }, + -- Every failure kind falls through to the next candidate, preserving the + -- pre-engine behaviour of retrying on any user error, overloaded or not. + retry_policies = { + [PROFILE_NAME] = { + unknown = { action = "next_candidate" }, + }, + }, + } +end + +-- The engine reads `host` as a plain global. Omitting `call_provider`/`sleep_ms` +-- keeps its blocking driver unreachable: this script drives execute_step itself. +_G.host = { + now_ms = function() + return lib.rs.monotonic_ms() + end, + log = function(level, event, fields) + lib.log { level = level, message = event, fields = fields } + end, +} + +local POLICY_CONFIG = build_policy_config() + +do + -- Reject a bad catalog at module start, not on the first prompt. + local ok, err = llm_policy.init(POLICY_CONFIG) + if not ok then + error("llm_policy rejected the generated catalog: " .. tostring(err)) end +end - local provider_keys = {} - for provider_name, _ in pairs(search_in) do - table.insert(provider_keys, provider_name) +--- Describe a mapped prompt to the engine: which capabilities a candidate must +--- have to be eligible. Mirrors the checks `llm.select_providers_for` performs. +---@param mapped_prompt MappedPrompt +---@return table +local function build_contract(ctx, mapped_prompt) + local needs = {} + if lib.get_first_from_table(mapped_prompt.prompt.images) ~= nil then + table.insert(needs, "vision") end - table.sort(provider_keys, function(a, b) - local a_data = search_in[a] - local b_data = search_in[b] + local response_format = nil + if mapped_prompt.format == "json" or mapped_prompt.format == "bool" then + -- bool is parsed out of a json object, so it needs json capability too + response_format = { type = "json_object" } + end - local a_priority = a_data.meta and a_data.meta.priority or 0 - local b_priority = b_data.meta and b_data.meta.priority or 0 + return { + profile = PROFILE_NAME, + requirements = { needs = needs }, + response_format = response_format, + seed = ctx.policy_seed, + } +end - if a_priority ~= b_priority then - return a_priority > b_priority - end - return a > b -- just compare names +-- Cap on how long a policy-requested backoff may block one prompt. +local MAX_WAIT_SECONDS = 30 + +-- Run the (provider, model) the engine asked for. Returns the answer on success +-- (nil otherwise) and the feedback table to hand back to `execute_step`. A +-- non-user error is re-raised: it is a runtime fault, not a routing signal. +local function run_candidate(ctx, mapped_prompt, timeout, request) + local provider_name = request.provider_id + local model_name = request.served_model_id + local model_data = llm.providers[provider_name].models[model_name] + + mapped_prompt.prompt.use_max_completion_tokens = model_data.use_max_completion_tokens + + local call = { + provider = provider_name, + model = model_name, + prompt = mapped_prompt.prompt, + format = mapped_prompt.format, + timeout = timeout, + } + + lib.log { level = "trace", message = "calling exec_prompt_in_provider", request = call } + local success, result = pcall(exec_update_policy_data, ctx, call, function(res) + -- convert token usage into gen using the per-model rate, and report it + -- back as the gen consumed by this call (charged to the host as fuel) + local total_tokens = (res.tokens and res.tokens.total) or 0 + local consumed_gen = rat.new(total_tokens) * gen_per_token(ctx, call.provider, call.model) + res.consumed_gen = consumed_gen + return consumed_gen end) - for _, provider_name in ipairs(provider_keys) do - local provider_data = search_in[provider_name] - local model = lib.get_first_from_table(provider_data.models) + if success then + return result, { ok = true } + end - if model == nil then - goto continue - end + local as_user_error = lib.rs.as_user_error(result) + if as_user_error == nil then + lib.log { level = "warning", message = "non-user-error", original = result } + error(result) + end - mapped_prompt.prompt.use_max_completion_tokens = model.value.use_max_completion_tokens + local status = as_user_error.ctx and as_user_error.ctx.status + lib.log { level = "warning", message = "provider failed, looking for next", error = as_user_error } - local request = { - provider = provider_name, - model = model.key, - prompt = mapped_prompt.prompt, - format = mapped_prompt.format, - timeout = timeout, + return nil, + { + ok = false, + error_kind = llm.provider_http_error_description(status), + http_status = status, + -- the engine stores this as a plain string in its trace + error_message = table.concat(as_user_error.causes or {}, ","), } +end - lib.log { - level = "trace", - message = "calling exec_prompt_in_provider", - request = request, - } - local success, result = pcall(exec_update_policy_data, ctx, request, function(res) - -- convert token usage into gen using the per-model rate, and report it - -- back as the gen consumed by this call (charged to the host as fuel) - local total_tokens = (res.tokens and res.tokens.total) or 0 - local consumed_gen = rat.new(total_tokens) * gen_per_token(ctx, request.provider, request.model) - res.consumed_gen = consumed_gen - return consumed_gen - end) +-- Sleep out a policy-requested backoff, bounded by MAX_WAIT_SECONDS. `until_ms` +-- is on the same monotonic clock the engine reads through `host.now_ms`. +local function wait_until(until_ms) + local delay = ((until_ms or 0) - lib.rs.monotonic_ms()) / 1000 + if delay <= 0 then + return + end + lib.log { level = "debug", message = "policy backoff", seconds = delay } + lib.rs.sleep_seconds(math.min(delay, MAX_WAIT_SECONDS)) +end + +local function dispatch_prompt(ctx, mapped_prompt, remaining_gen) + ---@cast mapped_prompt MappedPrompt + local timeout = compute_timeout(ctx, remaining_gen) + if timeout and timeout < 1 then lib.log { - level = "debug", - message = "executed with", - type = type(result), - success = success, - result = result, + level = "warning", + message = "computed timeout is very low, failing immediately", + timeout = timeout, } + llm.exhaust() + end - if success then - return result - end - - local as_user_error = lib.rs.as_user_error(result) - if as_user_error == nil then - lib.log { level = "warning", message = "non-user-error", original = result } + local contract = build_contract(ctx, mapped_prompt) - error(result) - end + lib.log { + level = "debug", + message = "executing prompt in backend", + prompt = mapped_prompt, + contract = contract, + } - if llm.overloaded_statuses[as_user_error.ctx.status] then - lib.log { level = "warning", message = "service is overloaded, looking for next", error = as_user_error } + local step = llm_policy.execute_step(nil, contract) + local answer = nil + + while step.status ~= "done" do + if step.status == "call" then + local this_answer, feedback = run_candidate(ctx, mapped_prompt, timeout, step.request) + answer = this_answer or answer + step = llm_policy.execute_step(step.state_handle, nil, feedback) + elseif step.status == "wait" then + wait_until(step.until_ms) + step = llm_policy.execute_step(step.state_handle, nil, nil) else - lib.log { level = "error", message = "provider failed", error = as_user_error, request = request } - -- lets fall back to retry - -- as_user_error.fatal = true - -- lib.rs.user_error(as_user_error) + error("unexpected policy step status: " .. tostring(step.status)) end + end - ::continue:: + if step.result.ok then + return answer end - lib.log { level = "error", message = "no provider could handle prompt", search_in = search_in } + lib.log { + level = "error", + message = "no provider could handle prompt", + error = step.result.error, + trace = step.result.trace, + } lib.rs.user_error { causes = { "NO_PROVIDER_FOR_PROMPT" }, fatal = true, ctx = { prompt = mapped_prompt.prompt, - search_in = search_in, + trace = step.result.trace, }, } end @@ -201,6 +345,28 @@ end function Setup(ctx) ctx.stats = {} + -- The engine keeps mutable routing state (breakers, latency EMA) in module + -- scope and a VM is reused across sessions; reset per session so a decision + -- never depends on which pooled VM this session got. + local ok, err = llm_policy.init(POLICY_CONFIG) + if not ok then + error("llm_policy rejected the generated catalog: " .. tostring(err)) + end + + -- Per-session routing seed: true randomness so validators diverge, logged so + -- the decision can be replayed from the record. Under the compat chain it is + -- inert (only sampling selectors read it); wired now for auditability. + local host_data = ctx.host_data or {} + local b1, b2, b3, b4 = string.byte(lib.rs.random_bytes(4), 1, 4) + ctx.policy_seed = (((b1 * 256 + b2) * 256 + b3) * 256 + b4) % 2147483647 + lib.log { + level = "info", + message = "llm policy seed", + seed = ctx.policy_seed, + tx_id = host_data.tx_id, + node_address = host_data.node_address, + } + local gen_per_time_unit_str = ctx.gas_data and ctx.gas_data.genPerTimeUnit local gen_per_time_unit = nil if gen_per_time_unit_str then @@ -334,7 +500,7 @@ function ExecPrompt(ctx, args, remaining_gen) local mapped = llm.exec_prompt_transform(args) - return just_in_backend(ctx, mapped, remaining_gen) + return dispatch_prompt(ctx, mapped, remaining_gen) end function ExecPromptTemplate(ctx, args, remaining_gen) @@ -347,5 +513,5 @@ function ExecPromptTemplate(ctx, args, remaining_gen) local mapped = llm.exec_prompt_template_transform(args) - return just_in_backend(ctx, mapped, remaining_gen) + return dispatch_prompt(ctx, mapped, remaining_gen) end diff --git a/install/lib/genvm-lua/lib-genvm.lua b/install/lib/genvm-lua/lib-genvm.lua index 1fa76acd..73f39a70 100644 --- a/install/lib/genvm-lua/lib-genvm.lua +++ b/install/lib/genvm-lua/lib-genvm.lua @@ -24,6 +24,10 @@ local value2json = require("value2json") ---@field filter_text fun(text: string, filters: string[]): string ---@field filter_image fun(image: string, filters: any[]): string ---@field random_bytes fun(length: integer): string +---@field sha2_256 fun(data: string): string +---@field sha3_256 fun(data: string): string +---@field keccak256 fun(data: string): string +---@field monotonic_ms fun(): integer ---@field random_float fun(): number ---@field data_dir string diff --git a/install/lib/genvm-lua/lib-llm.lua b/install/lib/genvm-lua/lib-llm.lua index cf1d01c1..1c801f3c 100644 --- a/install/lib/genvm-lua/lib-llm.lua +++ b/install/lib/genvm-lua/lib-llm.lua @@ -79,6 +79,34 @@ M.overloaded_statuses = { [529] = true, } +--- Map an HTTP status onto the engine's failure vocabulary. Nil means the call +--- never reached a provider (transport failure) -> network_error. +--- Every `overloaded_statuses` entry must land on a concrete kind, not "unknown". +---@param status integer | nil +---@return string +M.provider_http_error_description = function(status) + if status == nil then + return "network_error" + elseif status == 429 then + return "rate_limit" + elseif status == 408 then + return "timeout" + elseif status == 504 then + return "gateway_timeout" + elseif status == 401 or status == 403 then + return "auth_error" + elseif status == 404 then + return "model_unavailable" + elseif status == 400 then + return "bad_request" + elseif status == 529 then + return "server_overloaded" + elseif status >= 500 then + return "server_error" + end + return "unknown" +end + --- Execute a prompt against a specific provider/model. Delegates to the runtime. ---@type fun(ctx, data: { prompt: Prompt, format: Format, model: string, provider: string }): any M.exec_prompt_in_provider = rs.exec_prompt_in_provider @@ -234,7 +262,7 @@ end M.exec_prompt_template_transform = function(args) lib.log { level = "debug", message = "exec_prompt_template_transform", args = args } - my_data = { + local my_data = { EqComparative = { template_id = "eq_comparative", format = "bool" }, EqNonComparativeValidator = { template_id = "eq_non_comparative_validator", format = "bool" }, EqNonComparativeLeader = { template_id = "eq_non_comparative_leader", format = "text" }, diff --git a/libs/unhardcoded-engine b/libs/unhardcoded-engine new file mode 160000 index 00000000..74512eb1 --- /dev/null +++ b/libs/unhardcoded-engine @@ -0,0 +1 @@ +Subproject commit 74512eb180f76ba82d84f2d836ea78b19a06cadd diff --git a/support/ci/__main__.py b/support/ci/__main__.py index f0d743ab..c1f6206b 100644 --- a/support/ci/__main__.py +++ b/support/ci/__main__.py @@ -5,7 +5,15 @@ import ci_lib from pipelines import build, checks, docs, release, tests -from tools import branches, pr_branches, script_adapters, versions +from tools import ( + branches, + genvm_merge_into_dev, + make_release_notes, + open_executor_prs, + pr_action_panel, + pr_branches, + versions, +) PIPELINES: list[ci_lib.Pipeline] = [ *build.COMMANDS, @@ -18,7 +26,10 @@ *versions.COMMANDS, *branches.COMMANDS, *pr_branches.COMMANDS, - *script_adapters.COMMANDS, + *make_release_notes.COMMANDS, + *pr_action_panel.COMMANDS, + *genvm_merge_into_dev.COMMANDS, + *open_executor_prs.COMMANDS, ] diff --git a/support/ci/gh_common.py b/support/ci/gh_common.py new file mode 100644 index 00000000..4151ea8d --- /dev/null +++ b/support/ci/gh_common.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +"""Shared GitHub context for CI tools: repos, PR number, head ref, tokens. + +Almost every GitHub-touching tool needs the same handful of inputs, resolved the +same way β€” **CLI arg > environment variable > default**. This module is the one +place that precedence lives: + +- `add_args(parser, ...)` registers the common flags (`--manager-repo`, + `--executor-repo`, `--pr`, `--head-ref`). They default to `None`; the arg + simply wins when passed. +- `Ctx.from_args(args)` folds arg β†’ env β†’ default into resolved values. A tool + reads `ctx.manager_repo`, `ctx.pr_number`, etc. regardless of which flags it + registered β€” an unregistered field is `None` on the namespace and falls + through to env/default just the same. + +Tokens are env-only (never a CLI flag) and optional: `manager_token()` / +`executor_token()` return `None` when unset, and `gh(..., token=None)` then runs +with ambient `gh` auth (a local `gh auth login`). That is what lets these tools +run outside CI without minting a PAT β€” the whole point of not hardcoding tokens. +""" + +import argparse +import dataclasses +import os +import subprocess +import sys +import time + +MANAGER_REPO_DEFAULT = 'genlayerlabs/genvm-manager' +EXECUTOR_REPO_DEFAULT = 'genlayerlabs/genvm-executor' + + +def _env(*names: str) -> str | None: + """First set, non-empty value among env `names`, else None.""" + for name in names: + val = os.environ.get(name) + if val: + return val + return None + + +def manager_token() -> str | None: + """Manager-repo token, or None for ambient `gh` auth. `MANAGER_TOKEN` is the + CI name; `GH_TOKEN` is accepted so a single ambient token also works.""" + return _env('MANAGER_TOKEN', 'GH_TOKEN') + + +def executor_token() -> str | None: + """Executor-repo token, or None for ambient `gh` auth (`EXECUTOR_TOKEN`).""" + return _env('EXECUTOR_TOKEN') + + +def current_branch() -> str: + """Current git branch β€” the local fallback for a PR head ref.""" + return subprocess.run( + ['git', 'rev-parse', '--abbrev-ref', 'HEAD'], + check=True, + text=True, + capture_output=True, + ).stdout.strip() + + +def add_args( + parser: argparse.ArgumentParser, + *, + executor_repo: bool = True, + pr: bool = True, + head_ref: bool = True, +) -> None: + """Register the common flags on `parser`. `--manager-repo` is always added; + the rest are opt-out for the few tools that don't need them. Each defaults to + None so `Ctx.from_args` can apply arg > env > default uniformly.""" + parser.add_argument( + '--manager-repo', + default=None, + help='manager repo slug (default: $MANAGER_REPO / $GITHUB_REPOSITORY / ' + f'{MANAGER_REPO_DEFAULT})', + ) + if executor_repo: + parser.add_argument( + '--executor-repo', + default=None, + help=f'executor repo slug (default: $EXECUTOR_REPO / {EXECUTOR_REPO_DEFAULT})', + ) + if pr: + parser.add_argument( + '--pr', + dest='pr_number', + default=None, + help='manager PR number (default: $PR_NUMBER)', + ) + if head_ref: + parser.add_argument( + '--head-ref', + default=None, + help='manager PR head branch ' + '(default: $HEAD_REF / $GITHUB_HEAD_REF, else the current branch)', + ) + + +@dataclasses.dataclass(frozen=True) +class Ctx: + """Resolved GitHub inputs for a tool run (arg > env > default applied).""" + + manager_repo: str + executor_repo: str + _pr_number: str | None + _head_ref: str | None + + @staticmethod + def from_args(args: argparse.Namespace) -> 'Ctx': + def arg(name: str) -> str | None: + return getattr(args, name, None) + + return Ctx( + manager_repo=arg('manager_repo') + or _env('MANAGER_REPO', 'GITHUB_REPOSITORY') + or MANAGER_REPO_DEFAULT, + executor_repo=arg('executor_repo') + or _env('EXECUTOR_REPO') + or EXECUTOR_REPO_DEFAULT, + _pr_number=arg('pr_number') or _env('PR_NUMBER'), + _head_ref=arg('head_ref') or _env('HEAD_REF', 'GITHUB_HEAD_REF'), + ) + + @property + def pr_number(self) -> str: + """The PR number, or exit with a clear message if none was provided.""" + if not self._pr_number: + raise SystemExit('no PR number: pass --pr or set $PR_NUMBER') + return self._pr_number + + @property + def pr_number_opt(self) -> str | None: + """The PR number if one was provided, else None (for no-PR no-op paths).""" + return self._pr_number or None + + @property + def head_ref(self) -> str: + """The PR head branch, falling back to the current git branch locally.""" + return self._head_ref or current_branch() + + +# Every `gh` call routed here is an idempotent read (`gh api` GET, `gh pr list`) +# or a create/force-update guarded by its caller, so a transient failure is safe +# to retry. The one we actually hit is GitHub handing back a truncated/empty body, +# which gh reports as `unexpected end of JSON input` and which killed the whole +# executor-precondition gate on a single hiccup; 5xx, rate-limit and network blips +# are retried for the same reason. A terminal 4xx (notably the 404 that +# `allow_missing` callers expect for an absent line) is not retried, so that fast +# path stays fast. +_RETRY_ATTEMPTS = 4 +_RETRY_BACKOFF_S = 2.0 + +_TRANSIENT_MARKERS = ( + 'unexpected end of json input', + 'http 500', + 'http 502', + 'http 503', + 'http 504', + 'rate limit', + 'timeout', + 'timed out', + 'connection reset', + 'connection refused', + 'eof', +) + + +def _is_transient(r: subprocess.CompletedProcess) -> bool: + blob = f'{r.stdout}\n{r.stderr}'.lower() + return any(marker in blob for marker in _TRANSIENT_MARKERS) + + +def gh( + *args: str, token: str | None = None, check: bool = True +) -> subprocess.CompletedProcess: + """`gh` with `GH_TOKEN` bound to `token`, or ambient auth when `token` is None. + + Retries transient failures (see `_TRANSIENT_MARKERS`) a few times with linear + backoff; a terminal failure is returned as-is (or raised, when `check`), + exactly as a single `subprocess.run(check=...)` would. + """ + env = os.environ if token is None else {**os.environ, 'GH_TOKEN': token} + for attempt in range(_RETRY_ATTEMPTS): + result = subprocess.run( + ['gh', *args], + env=env, + check=False, + text=True, + capture_output=True, + ) + if result.returncode == 0 or not _is_transient(result): + break + if attempt + 1 < _RETRY_ATTEMPTS: + delay = _RETRY_BACKOFF_S * (attempt + 1) + print( + f'gh {" ".join(args)}: transient failure, retrying in {delay:.0f}s ' + f'(attempt {attempt + 1}/{_RETRY_ATTEMPTS})', + file=sys.stderr, + ) + time.sleep(delay) + if check and result.returncode != 0: + raise subprocess.CalledProcessError( + result.returncode, result.args, output=result.stdout, stderr=result.stderr + ) + return result diff --git a/support/ci/pr_branches_info.py b/support/ci/pr_branches_info.py index d7040519..e696382b 100644 --- a/support/ci/pr_branches_info.py +++ b/support/ci/pr_branches_info.py @@ -30,22 +30,22 @@ pushed to the executor repo, or the line is brand new / removed at one side) leave ahead_by/behind_by as None β€” `.moved` is still meaningful. -Env (for `from_env`): PR_NUMBER, MANAGER_REPO (or GITHUB_REPOSITORY), -EXECUTOR_REPO, MANAGER_TOKEN, EXECUTOR_TOKEN. +Inputs (repo slugs, PR number, tokens) are resolved through `gh_common` +(arg > env > default); tokens are optional and fall back to ambient `gh` auth. +Use `from_ctx(gh_common.Ctx.from_args(args))` from a tool. """ import dataclasses import json -import os -import subprocess -import sys -import time +import gh_common +from gh_common import ( # re-exported: callers use pr_branches_info.gh / *_DEFAULT + EXECUTOR_REPO_DEFAULT, + MANAGER_REPO_DEFAULT, + gh, +) from tools.versions import active_versions as configured_active_versions -MANAGER_REPO_DEFAULT = 'genlayerlabs/genvm-manager' -EXECUTOR_REPO_DEFAULT = 'genlayerlabs/genvm-executor' - MANAGER_PATH = '.' @@ -105,69 +105,7 @@ def as_dict(self) -> dict: } -# Every `gh` call here is an idempotent read (`gh api` GET, `gh pr list`), so a -# transient failure is safe to retry. The one we actually hit is GitHub handing -# back a truncated/empty body, which gh reports as `unexpected end of JSON input` -# and which killed the whole executor-precondition gate on a single hiccup; 5xx, -# rate-limit and network blips are retried for the same reason. A terminal 4xx -# (notably the 404 that `allow_missing` callers expect for an absent line) is not -# retried, so that fast path stays fast. -_RETRY_ATTEMPTS = 4 -_RETRY_BACKOFF_S = 2.0 - -_TRANSIENT_MARKERS = ( - 'unexpected end of json input', - 'http 500', - 'http 502', - 'http 503', - 'http 504', - 'rate limit', - 'timeout', - 'timed out', - 'connection reset', - 'connection refused', - 'eof', -) - - -def _is_transient(r: subprocess.CompletedProcess) -> bool: - blob = f'{r.stdout}\n{r.stderr}'.lower() - return any(marker in blob for marker in _TRANSIENT_MARKERS) - - -def gh(*args: str, token: str, check: bool = True) -> subprocess.CompletedProcess: - """`gh` with GH_TOKEN bound to the given token (manager- or executor-scoped). - - Retries transient failures (see `_TRANSIENT_MARKERS`) a few times with linear - backoff; a terminal failure is returned as-is (or raised, when `check`), - exactly as a single `subprocess.run(check=...)` would. - """ - for attempt in range(_RETRY_ATTEMPTS): - result = subprocess.run( - ['gh', *args], - env={**os.environ, 'GH_TOKEN': token}, - check=False, - text=True, - capture_output=True, - ) - if result.returncode == 0 or not _is_transient(result): - break - if attempt + 1 < _RETRY_ATTEMPTS: - delay = _RETRY_BACKOFF_S * (attempt + 1) - print( - f'gh {" ".join(args)}: transient failure, retrying in {delay:.0f}s ' - f'(attempt {attempt + 1}/{_RETRY_ATTEMPTS})', - file=sys.stderr, - ) - time.sleep(delay) - if check and result.returncode != 0: - raise subprocess.CalledProcessError( - result.returncode, result.args, output=result.stdout, stderr=result.stderr - ) - return result - - -def _api(*args: str, token: str, allow_missing: bool = False): +def _api(*args: str, token: str | None = None, allow_missing: bool = False): """`gh api` returning parsed JSON, or None on a 404 when allow_missing.""" r = gh('api', *args, token=token, check=not allow_missing) if r.returncode != 0: @@ -316,8 +254,8 @@ def collect( *, manager_repo: str = MANAGER_REPO_DEFAULT, executor_repo: str = EXECUTOR_REPO_DEFAULT, - manager_token: str, - executor_token: str, + manager_token: str | None = None, + executor_token: str | None = None, ) -> dict[str, RepoInfo]: """Branch movement for every repo a manager PR touches, keyed by path. @@ -342,16 +280,12 @@ def collect( return infos -def from_env(pr_number: str | None = None) -> dict[str, RepoInfo]: - """`collect` with arguments read from the workflow environment. - - `pr_number` overrides $PR_NUMBER when given (e.g. from a CLI argument). - """ +def from_ctx(ctx: gh_common.Ctx) -> dict[str, RepoInfo]: + """`collect` with repos/PR/tokens resolved from a shared `gh_common.Ctx`.""" return collect( - pr_number or os.environ['PR_NUMBER'], - manager_repo=os.environ.get('MANAGER_REPO') - or os.environ.get('GITHUB_REPOSITORY', MANAGER_REPO_DEFAULT), - executor_repo=os.environ.get('EXECUTOR_REPO', EXECUTOR_REPO_DEFAULT), - manager_token=os.environ['MANAGER_TOKEN'], - executor_token=os.environ['EXECUTOR_TOKEN'], + ctx.pr_number, + manager_repo=ctx.manager_repo, + executor_repo=ctx.executor_repo, + manager_token=gh_common.manager_token(), + executor_token=gh_common.executor_token(), ) diff --git a/support/ci/tools/branches.py b/support/ci/tools/branches.py index ebbceeab..e907e7de 100644 --- a/support/ci/tools/branches.py +++ b/support/ci/tools/branches.py @@ -1,8 +1,8 @@ import argparse -import os import subprocess import ci_lib +import gh_common from tools.versions import active_versions @@ -17,6 +17,11 @@ class ProvisionBranches(ci_lib.Tool): def name(self) -> str: return 'provision-branches' + def add_to(self, parser: argparse.ArgumentParser) -> None: + # Only the manager repo is relevant here β€” it iterates executor lines from + # .genvm-monorepo-root, not a single PR/head. + gh_common.add_args(parser, executor_repo=False, pr=False, head_ref=False) + def remote_has(self, branch: str) -> bool: return ( ci_lib.run( @@ -27,6 +32,7 @@ def remote_has(self, branch: str) -> bool: ) def handler(self, args: argparse.Namespace) -> int: + repo = gh_common.Ctx.from_args(args).manager_repo ci_lib.run(['git', 'fetch', 'origin', '--prune']) pushrefs: list[str] = [] @@ -71,7 +77,6 @@ def handler(self, args: argparse.Namespace) -> int: else: print('All branches already exist; nothing to push') - repo = os.environ['GITHUB_REPOSITORY'] for version in active_versions(): version_branch = f'v{version}.x' dev_branch = f'v{version}-dev' diff --git a/support/ci/tools/genvm_merge_into_dev.py b/support/ci/tools/genvm_merge_into_dev.py index 9a8a092f..d6b68c96 100755 --- a/support/ci/tools/genvm_merge_into_dev.py +++ b/support/ci/tools/genvm_merge_into_dev.py @@ -17,7 +17,8 @@ Strategy: every repo ends up with ONE commit per merged PR. The manager's commits are squashed onto base (a single commit that needs no gitlink rewrite is fast-forwarded as-is, preserving its SHA); each executor line's commits are -squashed onto its own dev branch. The PR is closed afterwards. +squashed onto its own dev branch. Every linked PR is auto-closed as merged +afterwards (see below). Executor mirror: the manager release line (e.g. v0.6) is independent of the executor lines it ships (v0.2, v0.3), so a single manager commit carries a @@ -38,10 +39,12 @@ added for every other human author in the range (never tool/AI attribution β€” the commit-message hook rejects that). -After a successful merge each line's executor mirror branch -(`pr//`, opened by branch_executor_prs.yaml) is deleted β€” its -tip now lives in the executor's `-dev`, so the branch and its PR are -redundant. +After a successful merge every linked PR is auto-closed as MERGED, without deleting +any branch: each executor line's mirror branch (`pr//`, opened by +branch_executor_prs.yaml) and the manager PR's own head branch are force-moved onto +the commit that landed on their base, so head == base tip and GitHub marks the PR +merged. (Deleting a branch, or `gh pr close`, closes the PR as merely 'closed' and +drops the merged link.) (The dev -> version release-gate merge is a separate, not-yet-automated flow; on that one the executor's version branch is fast-forwarded and its @@ -51,26 +54,37 @@ not exec arbitrary PR code, though the workflow checks out this script itself from the PR head (gated on the `ci-safe` label). The dev branches are protected, so the workflow checks out with the GENVM_CI_PRIVATE_KEY deploy key -and pushes non-force -(a base that advanced is safely rejected). The active executor submodules +and pushes them non-force +(a base that advanced is safely rejected); the unprotected head/mirror branches +are instead force-moved onto the landed commit to auto-merge their PRs. The active executor submodules must be checked out with a remote `origin` that can push the executor repo (the workflow wires its own deploy key); the active lines come from .genvm-monorepo-root via tools.versions. -Env: GITHUB_REPOSITORY, PR_NUMBER, GH_TOKEN, E2E_CHECK_PATTERN. +Repo and PR number resolve arg > env > default via gh_common; the token is +optional (ambient `gh` auth when unset). Env also: E2E_CHECK_PATTERN. """ +import argparse import json import os import re import subprocess import sys -from tools.versions import active_versions as configured_active_versions +import ci_lib +import gh_common +import pr_branches_info -REPO = os.environ['GITHUB_REPOSITORY'] -PR = os.environ['PR_NUMBER'] -E2E_PATTERN = os.environ.get('E2E_CHECK_PATTERN', 'e2e') +from tools.versions import active_lines + +# Resolved GitHub context (repo, PR number) for this run; set in the handler. +_CTX: gh_common.Ctx | None = None + + +def _ctx() -> gh_common.Ctx: + assert _CTX is not None, 'gh_common.Ctx not initialized (handler must set it)' + return _CTX def run(*args, check=True, env=None): @@ -82,7 +96,12 @@ def run(*args, check=True, env=None): def gh(*args): - return run('gh', *args).stdout + """GitHub API/PR *reads* via the retrying wrapper shared with pr_branches_info, + so a transient truncated-JSON / 5xx blip on a gate query doesn't fail the whole + merge (the exact fragility that wrapper was hardened against). Writes stay on + plain `run('gh', ...)` so a retry can't double-post a comment. Token is optional + (ambient `gh` auth when unset).""" + return pr_branches_info.gh(*args, token=gh_common.manager_token()).stdout def git(*args, check=True): @@ -101,6 +120,18 @@ def is_ancestor(repo, maybe_ancestor, descendant): return egit(repo, *args, check=False).returncode == 0 +def repo(): + return _ctx().manager_repo + + +def pr_number(): + return _ctx().pr_number + + +def e2e_pattern(): + return os.environ.get('E2E_CHECK_PATTERN', 'e2e') + + # Identities the commit-message hook rejects as AI/tool attribution # (support/scripts/check-commit-message.py AI_PATTERNS). A squash must not # resurrect one as a Co-authored-by trailer β€” the hook would then reject the @@ -157,21 +188,15 @@ def coauthor_trailers(repo, rev_range, exclude): return trailers -def active_lines(): - """Active executor lines as `v` tags (e.g. ['v0.2', 'v0.3']). The manager - release line is independent of these; a manager PR ships every active line.""" - return [f'v{v}' for v in configured_active_versions()] - - def block(msg): """Comment the reason on the PR and fail the job.""" run( 'gh', 'pr', 'comment', - PR, + pr_number(), '--repo', - REPO, + repo(), '--body', f'❌ Merge blocked: {msg}', check=False, @@ -180,7 +205,7 @@ def block(msg): def pr_view(*fields): - out = gh('pr', 'view', PR, '--repo', REPO, '--json', ','.join(fields)) + out = gh('pr', 'view', pr_number(), '--repo', repo(), '--json', ','.join(fields)) return json.loads(out) @@ -209,7 +234,7 @@ def check_gates(pr): runs = json.loads( gh( 'api', - f'repos/{REPO}/actions/workflows/queue.yaml/runs?head_sha={head_sha}', + f'repos/{repo()}/actions/workflows/queue.yaml/runs?head_sha={head_sha}', ) )['workflow_runs'] if not any(r['status'] == 'completed' and r['conclusion'] == 'success' for r in runs): @@ -220,10 +245,10 @@ def check_gates(pr): checks = json.loads( gh( 'api', - f'repos/{REPO}/commits/{head_sha}/check-runs?per_page=100', + f'repos/{repo()}/commits/{head_sha}/check-runs?per_page=100', ) )['check_runs'] - e2e = [c for c in checks if re.search(E2E_PATTERN, c['name'], re.I)] + e2e = [c for c in checks if re.search(e2e_pattern(), c['name'], re.I)] if not e2e: block(f'no E2E check found on `{head_sha}` (run it on this PR first).') bad = [c for c in e2e if c['conclusion'] != 'success'] @@ -234,22 +259,75 @@ def check_gates(pr): return base, head_sha -def delete_executor_mirror_branch(submodule, line, head_ref): - """Delete the executor mirror branch `pr//` after a merge. +def land_executor_mirror_branch(submodule, line, head_ref, landed): + """Move the executor mirror branch `pr//` onto `landed` after a merge. + + The manager PR's work for this line now sits on the executor's `-dev` at + `landed` (we just pushed it there); force-moving the mirror branch to that SAME + commit makes its auto-opened PR (`pr//` -> `-dev`, + branch_executor_prs.yaml) have head == base, so GitHub auto-closes it as MERGED. + (Deleting the branch β€” the old behaviour β€” closed that PR as merely 'closed', and + dropped the merged link.) Best-effort: a failure here never unwinds an otherwise + -complete merge. The branch is unprotected, so the executor deploy key can move it. - The manager PR's work for this line lived on that namespaced branch; its tip is - now contained in the executor's `-dev` (we just fast-forwarded it there), - so the branch β€” and the auto-opened executor PR it backed (branch_executor_prs.yaml) - β€” are redundant. Deleting the branch also closes that PR. Best-effort: a failure - here never unwinds an otherwise-complete merge. The branch is unprotected, so the - executor deploy key can delete it. + Only lines this PR actually moved were provisioned a mirror branch + PR + (branch_executor_prs.yaml); an untouched line has none, and force-pushing would + CREATE a stray dangling ref (the old delete was a harmless no-op when absent) β€” so + land only a branch that already exists. `plan_executor_line` fetched every executor + head into `refs/remotes/origin/*`, so the mirror branch is present locally iff it + exists on the remote. """ branch = f'pr/{line}/{head_ref}' - r = egit(submodule, 'push', 'origin', '--delete', branch, check=False) + if ( + egit( + submodule, + 'rev-parse', + '--verify', + '--quiet', + f'refs/remotes/origin/{branch}', + check=False, + ).returncode + != 0 + ): + print(f'no executor mirror branch {branch}; nothing to auto-merge for {line}') + return + r = egit( + submodule, 'push', '--force', 'origin', f'{landed}:refs/heads/{branch}', check=False + ) if r.returncode == 0: - print(f'deleted executor mirror branch {branch}') + print( + f'landed executor mirror branch {branch} -> {landed[:12]} (auto-merges its PR)' + ) else: - print(f'note: could not delete executor mirror branch {branch}: {r.stderr.strip()}') + print(f'note: could not land executor mirror branch {branch}: {r.stderr.strip()}') + + +def close_manager_pr(pr, push_sha): + """Auto-close the manager PR as MERGED by moving its head branch onto `push_sha`. + + `push_sha` is the commit we just landed on the manager ``; force-moving the + PR head branch onto it makes head == base tip, so GitHub marks the PR merged β€” + the same trick as the executor mirrors, and unlike a bare `gh pr close` (which + reads as merely 'closed'). + + A fork PR's head branch lives in another repo, so `` is not a branch on + `origin` β€” moving it would create a stray branch and not touch the PR. Those never + reach here (the merge gate needs a same-repo run), but be explicit: cross-repo PRs + fall back to a plain close. Best-effort. + """ + head_ref = pr['headRefName'] + if pr.get('isCrossRepository'): + print(f'PR head `{head_ref}` is on a fork; closing explicitly (cannot move it)') + run('gh', 'pr', 'close', pr_number(), '--repo', repo(), check=False) + return + r = git('push', '--force', 'origin', f'{push_sha}:refs/heads/{head_ref}', check=False) + if r.returncode == 0: + print(f'moved manager PR head `{head_ref}` -> {push_sha[:12]} (auto-merges the PR)') + else: + print( + f'note: could not move PR head `{head_ref}` ({r.stderr.strip()}); closing explicitly' + ) + run('gh', 'pr', 'close', pr_number(), '--repo', repo(), check=False) def executor_gitlink(commit, submodule): @@ -292,7 +370,7 @@ def build_messages(pr): ).strip() title = pr['title'] tail = f'\n\n{body}\n' if body else '\n' - return f'{title} (#{PR}){tail}', f'{title} ({REPO}#{PR}){tail}' + return f'{title} (#{pr_number()}){tail}', f'{title} ({repo()}#{pr_number()}){tail}' def already_squashed(submodule, exec_base, exec_sha): @@ -448,7 +526,7 @@ def merge(pr, base, head_sha): 'fetch', '--no-tags', 'origin', - f'refs/pull/{PR}/head:refs/prhead', + f'refs/pull/{pr_number()}/head:refs/prhead', f'+refs/heads/{base}:refs/remotes/origin/{base}', ) @@ -533,7 +611,6 @@ def merge(pr, base, head_sha): # Push the executors first: the manager commit gitlinks into them, so the # dependencies must land before the referrer. push_executor_lines(plans) - pinned = [(submodule, line) for submodule, line, _b, _p, _l, _push in plans] # Non-force FF push; rejected if base advanced since the checks. The # executors are already advanced at this point; a rejection here leaves them @@ -551,21 +628,40 @@ def merge(pr, base, head_sha): 'gh', 'pr', 'comment', - PR, + pr_number(), '--repo', - REPO, + repo(), '--body', f'βœ… Merged into `{base}` (`{push_sha}`) via fast-forward.', check=False, ) - run('gh', 'pr', 'close', PR, '--repo', REPO, check=False) - # Clean up the now-redundant executor mirror branches (and the PRs they backed). - for submodule, line in pinned: - delete_executor_mirror_branch(submodule, line, pr['headRefName']) + # Everything landed. Now auto-close every linked PR as MERGED by moving its head + # branch onto the commit that reached its base (head == base tip). This replaces + # deleting the executor mirror branches / a bare manager `gh pr close`, both of + # which dropped the merged link. Best-effort and last: the land already succeeded. + for submodule, line, _base, _pinned, landed, _push in plans: + land_executor_mirror_branch(submodule, line, pr['headRefName'], landed) + close_manager_pr(pr, push_sha) + + +class GenvmMergeIntoDev(ci_lib.Tool): + """Gate and perform a Merge of a PR into a v-dev branch.""" + + def name(self) -> str: + return 'genvm-merge-into-dev' + + def add_to(self, parser: argparse.ArgumentParser) -> None: + gh_common.add_args(parser, executor_repo=False, head_ref=False) + + def handler(self, args: argparse.Namespace) -> int: + global _CTX + _CTX = gh_common.Ctx.from_args(args) + merge_into_dev() + return 0 -def main(): +def merge_into_dev(): pr = pr_view( 'baseRefName', 'headRefName', @@ -576,10 +672,10 @@ def main(): 'isDraft', 'title', 'body', + 'isCrossRepository', ) base, head_sha = check_gates(pr) merge(pr, base, head_sha) -if __name__ == '__main__': - main() +COMMANDS = [GenvmMergeIntoDev()] diff --git a/support/ci/tools/make_release_notes.py b/support/ci/tools/make_release_notes.py index 31281bbd..def8df88 100755 --- a/support/ci/tools/make_release_notes.py +++ b/support/ci/tools/make_release_notes.py @@ -10,6 +10,8 @@ from dataclasses import dataclass from pathlib import Path +import ci_lib + def strip_trailing_emoji(text: str) -> str: """Drop the trailing gitmoji suffix (and its variation selectors/ZWJ). @@ -146,32 +148,35 @@ def format_notes(categories: dict[str, list[str]], rev_range: str) -> str: return '\n'.join(parts).strip() + '\n' -def main() -> None: - parser = argparse.ArgumentParser( - description='Generate categorized release notes from conventional commits.', - ) - parser.add_argument( - 'rev_range', - metavar='rev-range', - help='git revision range, e.g. v0.2.7..v0.2.8', - ) - args = parser.parse_args() +class MakeReleaseNotes(ci_lib.Tool): + """Generate categorized release notes from conventional commits.""" + + def name(self) -> str: + return 'make-release-notes' + + def add_to(self, parser: argparse.ArgumentParser) -> None: + parser.add_argument( + 'rev_range', + metavar='rev-range', + help='git revision range, e.g. v0.2.7..v0.2.8', + ) - entries = parse_git_log(args.rev_range) - if not entries: - print('No commits found.', file=sys.stderr) - sys.exit(1) + def handler(self, args: argparse.Namespace) -> int: + entries = parse_git_log(args.rev_range) + if not entries: + print('No commits found.', file=sys.stderr) + return 1 - categories = categorize(entries) + categories = categorize(entries) - all_notes = [] + all_notes = [] - all_notes.append('# GenVM Manager Release Notes\n') + all_notes.append('# GenVM Manager Release Notes\n') - all_notes.append(format_notes(categories, args.rev_range)) + all_notes.append(format_notes(categories, args.rev_range)) - print(''.join(all_notes)) + print(''.join(all_notes)) + return 0 -if __name__ == '__main__': - main() +COMMANDS = [MakeReleaseNotes()] diff --git a/support/ci/tools/open_executor_prs.py b/support/ci/tools/open_executor_prs.py index 96788c4f..da760bc9 100644 --- a/support/ci/tools/open_executor_prs.py +++ b/support/ci/tools/open_executor_prs.py @@ -2,7 +2,9 @@ """Open (or reuse) the linked executor PRs for a manager PR and list them on it. Invoked by .github/workflows/branch_executor_prs.yaml when a manager PR is opened -against a dev branch. The manager release line (e.g. v0.6) is independent of the +against a dev branch or its head is pushed (`synchronize`), so mirror branches +pushed after the PR opens still get linked. The manager release line (e.g. v0.6) +is independent of the executor lines it ships (v0.2, v0.3): one manager PR bundles the work of EVERY active executor line, each on its own line-namespaced mirror branch in the shared genvm-executor repo (`pr//`, see genvm_tool.common.Repo.feature_branch). @@ -13,64 +15,45 @@ open (or reuse) a PR `pr//` -> `-dev` there, then upsert a marked comment on the manager PR listing each as an `executor: ` line. -Two tokens: the executor PRs are created with EXECUTOR_TOKEN (a PAT with +Two tokens: the executor PRs are created with the executor token (a PAT with genvm-executor access β€” the default GITHUB_TOKEN is manager-scoped and cannot touch -another repo); the manager comment uses MANAGER_TOKEN (the default GITHUB_TOKEN). -Never executes PR code β€” only `gh` API/PR calls. +another repo); the manager comment uses the manager token. Both are optional and +fall back to ambient `gh` auth for local runs. Never executes PR code β€” only `gh` +API/PR calls. -Env: MANAGER_REPO, EXECUTOR_REPO, PR_NUMBER, HEAD_REF, MANAGER_TOKEN, EXECUTOR_TOKEN. +Inputs (repos, PR number, head ref) resolve arg > env > default via gh_common. """ +import argparse import json -import os -import subprocess -from tools.versions import active_versions as configured_active_versions +import ci_lib +import gh_common +from gh_common import gh -MANAGER_REPO = os.environ['MANAGER_REPO'] -EXECUTOR_REPO = os.environ.get('EXECUTOR_REPO', 'genlayerlabs/genvm-executor') -PR = os.environ['PR_NUMBER'] -HEAD_REF = os.environ['HEAD_REF'] -MANAGER_TOKEN = os.environ['MANAGER_TOKEN'] -EXECUTOR_TOKEN = os.environ['EXECUTOR_TOKEN'] +from tools.versions import active_lines # Marker so re-runs update the same comment instead of stacking new ones. COMMENT_MARKER = '' -def gh(*args, token, check=True): - """`gh` with GH_TOKEN bound to the given token (manager- or executor-scoped).""" - return subprocess.run( - ['gh', *args], - env={**os.environ, 'GH_TOKEN': token}, - check=check, - text=True, - capture_output=True, - ) - - -def active_lines() -> list[str]: - """Active executor lines as `v` tags (e.g. ['v0.2', 'v0.3']).""" - return [f'v{v}' for v in configured_active_versions()] - - -def executor_branch_exists(branch: str) -> bool: +def executor_branch_exists(ctx: gh_common.Ctx, branch: str) -> bool: # matching-refs returns [] (not a 404) when nothing matches; it is a prefix # query, so confirm an EXACT ref before treating the branch as present. out = gh( 'api', - f'repos/{EXECUTOR_REPO}/git/matching-refs/heads/{branch}', - token=EXECUTOR_TOKEN, + f'repos/{ctx.executor_repo}/git/matching-refs/heads/{branch}', + token=gh_common.executor_token(), ).stdout return any(r.get('ref') == f'refs/heads/{branch}' for r in json.loads(out or '[]')) -def existing_pr(head: str, base: str) -> str | None: +def existing_pr(ctx: gh_common.Ctx, head: str, base: str) -> str | None: url = gh( 'pr', 'list', '--repo', - EXECUTOR_REPO, + ctx.executor_repo, '--head', head, '--base', @@ -81,34 +64,34 @@ def existing_pr(head: str, base: str) -> str | None: 'url', '--jq', '.[0].url // ""', - token=EXECUTOR_TOKEN, + token=gh_common.executor_token(), ).stdout.strip() return url or None -def open_pr(head: str, base: str) -> str: +def open_pr(ctx: gh_common.Ctx, head: str, base: str) -> str: title = gh( 'pr', 'view', - PR, + ctx.pr_number, '--repo', - MANAGER_REPO, + ctx.manager_repo, '--json', 'title', '--jq', '.title', - token=MANAGER_TOKEN, + token=gh_common.manager_token(), ).stdout.strip() body = ( - f'Auto-opened executor mirror of {MANAGER_REPO}#{PR}.\n\n' - f'Carries the executor-side work for that manager PR. Closed and its branch ' - f'`{head}` deleted automatically when the manager PR merges.' + f'Auto-opened executor mirror of {ctx.manager_repo}#{ctx.pr_number}.\n\n' + f'Carries the executor-side work for that manager PR. Auto-closed as merged ' + f'when the manager PR lands (its `{head}` branch is moved onto `{base}`).' ) r = gh( 'pr', 'create', '--repo', - EXECUTOR_REPO, + ctx.executor_repo, '--base', base, '--head', @@ -117,12 +100,12 @@ def open_pr(head: str, base: str) -> str: title, '--body', body, - token=EXECUTOR_TOKEN, + token=gh_common.executor_token(), check=False, ) if r.returncode != 0: # Lost a race against a concurrent run? Fall back to whatever is open now. - url = existing_pr(head, base) + url = existing_pr(ctx, head, base) if url: return url raise SystemExit( @@ -131,53 +114,67 @@ def open_pr(head: str, base: str) -> str: return r.stdout.strip().splitlines()[-1].strip() -def upsert_comment(lines: list[str]) -> None: +def upsert_comment(ctx: gh_common.Ctx, lines: list[str]) -> None: body = f'{COMMENT_MARKER}\n### Linked executor PR(s)\n\n' + '\n'.join(lines) + '\n' ids = gh( 'api', - f'repos/{MANAGER_REPO}/issues/{PR}/comments', + f'repos/{ctx.manager_repo}/issues/{ctx.pr_number}/comments', '--jq', f'.[] | select(.body | contains("{COMMENT_MARKER}")) | .id', - token=MANAGER_TOKEN, + token=gh_common.manager_token(), ).stdout.split() if ids: gh( 'api', '--method', 'PATCH', - f'repos/{MANAGER_REPO}/issues/comments/{ids[0]}', + f'repos/{ctx.manager_repo}/issues/comments/{ids[0]}', '-f', f'body={body}', - token=MANAGER_TOKEN, + token=gh_common.manager_token(), ) else: gh( 'api', '--method', 'POST', - f'repos/{MANAGER_REPO}/issues/{PR}/comments', + f'repos/{ctx.manager_repo}/issues/{ctx.pr_number}/comments', '-f', f'body={body}', - token=MANAGER_TOKEN, + token=gh_common.manager_token(), ) -def main() -> None: +class OpenExecutorPrs(ci_lib.Tool): + """Open (or reuse) the linked executor PRs for a manager PR and list them on it.""" + + def name(self) -> str: + return 'open-executor-prs' + + def add_to(self, parser: argparse.ArgumentParser) -> None: + gh_common.add_args(parser) + + def handler(self, args: argparse.Namespace) -> int: + open_executor_prs(gh_common.Ctx.from_args(args)) + return 0 + + +def open_executor_prs(ctx: gh_common.Ctx) -> None: + head_ref = ctx.head_ref # resolve once (may shell out to git locally) lines = [] for line in active_lines(): - head = f'pr/{line}/{HEAD_REF}' + head = f'pr/{line}/{head_ref}' base = f'{line}-dev' - if not executor_branch_exists(head): + if not executor_branch_exists(ctx, head): print(f'no executor branch `{head}`; skipping {line} (nothing pushed for it)') continue - url = existing_pr(head, base) or open_pr(head, base) + url = existing_pr(ctx, head, base) or open_pr(ctx, head, base) print(f'{line}: executor PR {url}') lines.append(f'executor: {url}') if lines: - upsert_comment(lines) + upsert_comment(ctx, lines) else: print('no executor mirror branches found for any active line; nothing to link') -if __name__ == '__main__': - main() +COMMANDS = [OpenExecutorPrs()] diff --git a/support/ci/tools/pr_action_panel.py b/support/ci/tools/pr_action_panel.py index a6003faf..ba88f94c 100644 --- a/support/ci/tools/pr_action_panel.py +++ b/support/ci/tools/pr_action_panel.py @@ -33,21 +33,45 @@ Resetting the panel re-fires issue_comment:edited, but that event is sent by the bot, so it is ignored β€” no loop. -Env: GITHUB_REPOSITORY, PR_NUMBER, COMMENT_ID, SENDER, GH_TOKEN. +Repo and PR number resolve arg > env > default via gh_common; the token is +optional (ambient `gh` auth when unset). Env also: COMMENT_ID, SENDER. """ +import argparse import os import re import subprocess -REPO = os.environ['GITHUB_REPOSITORY'] -PR = os.environ['PR_NUMBER'] -COMMENT_ID = os.environ['COMMENT_ID'] -SENDER = os.environ['SENDER'] +import ci_lib +import gh_common CI_SAFE_LABEL = 'ci-safe' RUN_FULL_TESTS_LABEL = 'run-full-tests' +# Resolved GitHub context (repo, PR number) for this run; set in the handler. +_CTX: gh_common.Ctx | None = None + + +def _ctx() -> gh_common.Ctx: + assert _CTX is not None, 'gh_common.Ctx not initialized (handler must set it)' + return _CTX + + +def repo(): + return _ctx().manager_repo + + +def pr_number(): + return _ctx().pr_number + + +def comment_id(): + return os.environ['COMMENT_ID'] + + +def sender(): + return os.environ['SENDER'] + def run(*args, check=True): return subprocess.run(args, check=check, text=True, capture_output=True) @@ -63,7 +87,9 @@ def set_output(name, value): def labels(): - out = gh('api', f'repos/{REPO}/issues/{PR}/labels', '--jq', '.[].name').stdout + out = gh( + 'api', f'repos/{repo()}/issues/{pr_number()}/labels', '--jq', '.[].name' + ).stdout return set(out.splitlines()) @@ -72,7 +98,7 @@ def add_label(name): 'api', '--method', 'POST', - f'repos/{REPO}/issues/{PR}/labels', + f'repos/{repo()}/issues/{pr_number()}/labels', '-f', f'labels[]={name}', ) @@ -80,7 +106,15 @@ def add_label(name): def head_branch(): return gh( - 'pr', 'view', PR, '--repo', REPO, '--json', 'headRefName', '--jq', '.headRefName' + 'pr', + 'view', + pr_number(), + '--repo', + repo(), + '--json', + 'headRefName', + '--jq', + '.headRefName', ).stdout.strip() @@ -92,7 +126,17 @@ def dispatch_full_tests(): if not branch: print('could not resolve PR head branch; cannot dispatch full tests') return - gh('workflow', 'run', 'queue.yaml', '--repo', REPO, '--ref', branch, '-f', f'pr={PR}') + gh( + 'workflow', + 'run', + 'queue.yaml', + '--repo', + repo(), + '--ref', + branch, + '-f', + f'pr={pr_number()}', + ) print(f'dispatched queue.yaml on `{branch}`') @@ -105,11 +149,11 @@ def dispatch_provision(): 'run', 'branch_provision_executor_prs.yaml', '--repo', - REPO, + repo(), '-f', - f'pr={PR}', + f'pr={pr_number()}', ) - print(f'dispatched executor provisioning for PR #{PR}') + print(f'dispatched executor provisioning for PR #{pr_number()}') def ticked_boxes(body): @@ -134,18 +178,34 @@ def reset(line): 'api', '--method', 'PATCH', - f'repos/{REPO}/issues/comments/{COMMENT_ID}', + f'repos/{repo()}/issues/comments/{comment_id()}', '-f', f'body={new}', ) -def main(): +class PrActionPanel(ci_lib.Tool): + """Handle a ticked box on the GenVM PR action panel.""" + + def name(self) -> str: + return 'pr-action-panel' + + def add_to(self, parser: argparse.ArgumentParser) -> None: + gh_common.add_args(parser, executor_repo=False, head_ref=False) + + def handler(self, args: argparse.Namespace) -> int: + global _CTX + _CTX = gh_common.Ctx.from_args(args) + run_panel() + return 0 + + +def run_panel(): set_output('merge', 'false') # Ignore the bot's own panel-reset edit (avoids a self-trigger loop). - if SENDER.endswith('[bot]'): - print(f'{SENDER} is a bot (panel reset echo); ignoring') + if sender().endswith('[bot]'): + print(f'{sender()} is a bot (panel reset echo); ignoring') return current = labels() @@ -153,7 +213,9 @@ def main(): print(f'PR lacks the `{CI_SAFE_LABEL}` label; ignoring panel actions') return - body = gh('api', f'repos/{REPO}/issues/comments/{COMMENT_ID}', '--jq', '.body').stdout + body = gh( + 'api', f'repos/{repo()}/issues/comments/{comment_id()}', '--jq', '.body' + ).stdout boxes = ticked_boxes(body) if not boxes: print('no ticked boxes; nothing to do') @@ -181,5 +243,4 @@ def main(): untick_momentary(body) -if __name__ == '__main__': - main() +COMMANDS = [PrActionPanel()] diff --git a/support/ci/tools/pr_branches.py b/support/ci/tools/pr_branches.py index 55d01cd6..e29f8f55 100644 --- a/support/ci/tools/pr_branches.py +++ b/support/ci/tools/pr_branches.py @@ -1,8 +1,8 @@ import argparse import json -import os import ci_lib +import gh_common import pr_branches_info from pr_branches_info import RepoInfo @@ -98,40 +98,43 @@ def name(self) -> str: def add_to(self, parser: argparse.ArgumentParser) -> None: sub = parser.add_subparsers(dest='cmd', required=True) - show = sub.add_parser( - 'show-info', - help='print each repo base/head/ahead_by/behind_by as JSON', + gh_common.add_args( + sub.add_parser( + 'show-info', + help='print each repo base/head/ahead_by/behind_by as JSON', + ) ) - show.add_argument('pr', nargs='?', help='manager PR number (default: $PR_NUMBER)') - prov = sub.add_parser( - 'provision', - help='force-push each moved executor line mirror branch and open its PR', + gh_common.add_args( + sub.add_parser( + 'provision', + help='force-push each moved executor line mirror branch and open its PR', + ) ) - prov.add_argument('pr', nargs='?', help='manager PR number (default: $PR_NUMBER)') - check = sub.add_parser( - 'check', - help='fail unless every repo is rebased and every executor pinned commit is landable', + gh_common.add_args( + sub.add_parser( + 'check', + help='fail unless every repo is rebased and every executor pinned commit is landable', + ) ) - check.add_argument('pr', nargs='?', help='manager PR number (default: $PR_NUMBER)') def show_info(self, args: argparse.Namespace) -> int: - infos = pr_branches_info.from_env(args.pr) + infos = pr_branches_info.from_ctx(gh_common.Ctx.from_args(args)) print(json.dumps({path: info.as_dict() for path, info in infos.items()}, indent=2)) return 0 def provision(self, args: argparse.Namespace) -> int: - infos = pr_branches_info.from_env(args.pr) + ctx = gh_common.Ctx.from_args(args) + infos = pr_branches_info.from_ctx(ctx) manager = infos[pr_branches_info.MANAGER_PATH] - pr_number = args.pr or os.environ['PR_NUMBER'] - manager_token = os.environ['MANAGER_TOKEN'] - executor_token = os.environ['EXECUTOR_TOKEN'] + pr_number = ctx.pr_number + executor_token = gh_common.executor_token() title = pr_branches_info.gh( 'api', f'repos/{manager.repo}/pulls/{pr_number}', '--jq', '.title', - token=manager_token, + token=gh_common.manager_token(), ).stdout.strip() for info in infos.values(): @@ -174,11 +177,11 @@ def check(self, args: argparse.Namespace) -> int: No PR number (a bare manual dispatch not tied to a PR) is a no-op pass β€” there is nothing to gate. """ - pr = args.pr or os.environ.get('PR_NUMBER') or '' - if not pr.strip(): + ctx = gh_common.Ctx.from_args(args) + if not ctx.pr_number_opt: print('no PR number provided; nothing to check') return 0 - infos = pr_branches_info.from_env(pr) + infos = pr_branches_info.from_ctx(ctx) errors: list[str] = [] diff --git a/support/ci/tools/script_adapters.py b/support/ci/tools/script_adapters.py deleted file mode 100644 index 4185ef68..00000000 --- a/support/ci/tools/script_adapters.py +++ /dev/null @@ -1,44 +0,0 @@ -import argparse -import importlib -import sys - -import ci_lib - - -class ModuleTool(ci_lib.Tool): - def __init__(self, name: str, module: str): - self._name = name - self._module = module - - def name(self) -> str: - return self._name - - def add_to(self, parser: argparse.ArgumentParser) -> None: - parser.add_argument('args', nargs=argparse.REMAINDER) - - def handler(self, args: argparse.Namespace) -> int: - module = importlib.import_module(self._module) - old_argv = sys.argv - try: - sys.argv = [self._name, *args.args] - try: - result = module.main() - except SystemExit as exc: - if exc.code is None: - return 0 - if isinstance(exc.code, int): - return exc.code - print(exc.code, file=sys.stderr) - return 1 - finally: - sys.argv = old_argv - return result or 0 - - -COMMANDS = [ - ModuleTool('genvm-merge-into-dev', 'tools.genvm_merge_into_dev'), - ModuleTool('make-release-notes', 'tools.make_release_notes'), - ModuleTool('make-relase-notes', 'tools.make_release_notes'), - ModuleTool('open-executor-prs', 'tools.open_executor_prs'), - ModuleTool('pr-action-panel', 'tools.pr_action_panel'), -] diff --git a/support/ci/tools/versions.py b/support/ci/tools/versions.py index 91dfd68a..3850a5a6 100644 --- a/support/ci/tools/versions.py +++ b/support/ci/tools/versions.py @@ -19,32 +19,57 @@ def version_key(version: str) -> tuple[int, ...]: return tuple(int(part) for part in major_minor(version).split('.')) -def active_versions() -> list[str]: +def load_root_conf() -> dict: conf_path = os.environ.get('MONOREPO_ROOT', ci_lib.ROOT_DIR / '.genvm-monorepo-root') - conf = json.loads(open(conf_path, 'rb').read()) + return json.loads(open(conf_path, 'rb').read()) + + +def active_versions() -> list[str]: + conf = load_root_conf() return sorted( (bare(version) for version in conf.get('active-versions', [])), key=version_key ) +def manager_version() -> str: + """The manager's own release train as bare major.minor (e.g. '0.6'). + + Read from the `version` field. The manager dev/release branches + (`v-dev`, `v.x`) are named after this β€” NOT after the executor lines + in `active-versions`, which move independently (currently v0.2 / v0.3). + """ + return major_minor(load_root_conf()['version']) + + +def active_lines() -> list[str]: + """Active executor lines as `v` tags (e.g. ['v0.2', 'v0.3']). + + Same data as `active_versions`, prefixed with the `v` the executor branches + (`-dev`, `pr//...`) use. The manager release line is independent of + these; a manager PR ships every active line. + """ + return [f'v{version}' for version in active_versions()] + + class BranchVersions(ci_lib.Tool): - """Read active version trains from .genvm-monorepo-root.""" + """Read version trains from .genvm-monorepo-root. + + `list` yields the active executor lines (`active-versions`); `manager` + yields the manager's own train (`version`). These differ β€” don't use one + where the other is meant. + """ def name(self) -> str: return 'branch-versions' def add_to(self, parser: argparse.ArgumentParser) -> None: - parser.add_argument('cmd', nargs='?', choices=['list', 'latest'], default='list') + parser.add_argument('cmd', nargs='?', choices=['list', 'manager'], default='list') def handler(self, args: argparse.Namespace) -> int: - versions = active_versions() - if args.cmd == 'list': - print('\n'.join(versions)) + if args.cmd == 'manager': + print(manager_version()) return 0 - if not versions: - print('no active versions configured in .genvm-monorepo-root', file=sys.stderr) - return 1 - print(versions[-1]) + print('\n'.join(active_versions())) return 0 diff --git a/support/scripts/check-source-text.py b/support/scripts/check-source-text.py new file mode 100644 index 00000000..bf975a96 --- /dev/null +++ b/support/scripts/check-source-text.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""Pre-commit guard: keep source text plain and citation-free. + +Comments should read on their own terms and stay ASCII. Rejected anywhere in a +scanned file: ADR references (`ADR-012`, `ADR 12`) and issue references +(`GVM-281`) -- they renumber and move, and that prose belongs in docs/, not a +source comment; and a blacklist of non-ASCII punctuation that each has an +obvious ASCII swap: the paragraph sign, em-dash (`--`), box-drawing horizontal +(`-`), rightwards arrow (`->`) and ellipsis (`...`). It is a blacklist, not a +full non-ASCII ban -- names and test data legitimately need other code points. + +A file opts out entirely with a comment line (a `#`, `//` or `--` leader) +containing `check-source-text: off`; one such line skips the whole file. + +Wired by the git-hooks `check-source-text` hook with the changed files as +arguments; the repo root is the working directory. Rejections go to stdout, one +per line, as `path:lineno: `. +""" + +import argparse +import re +import sys + +_OFF = re.compile(r'(?:#|//|--)[ \t]*check-source-text:[ \t]*off') + +# The non-ASCII chars are spelled with `chr` so this file stays ASCII. +_PATTERNS = [ + re.compile(r'\bADR[ \-]?\d+'), # ADR reference + re.compile(r'\bGVM-\d+'), # issue reference + re.compile(chr(0x00A7)), # paragraph sign + re.compile(chr(0x2014)), # em-dash (use two hyphens) + re.compile(chr(0x2500)), # box-drawing horizontal (use -) + re.compile(chr(0x2192)), # rightwards arrow (use ->) + re.compile(chr(0x2026)), # ellipsis (use ...) +] + + +def _violations(path: str): + try: + with open(path, encoding='utf-8') as f: + text = f.read() + except (OSError, UnicodeDecodeError) as e: + yield 0, f'could not read: {e}' + return + if _OFF.search(text): + return + for lineno, line in enumerate(text.splitlines(), 1): + if any(pat.search(line) for pat in _PATTERNS): + yield lineno, line.strip() + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument('files', nargs='*', help='files to check') + args = parser.parse_args(argv) + + found = False + for path in args.files: + for lineno, content in _violations(path): + found = True + print(f'{path}:{lineno}: {content}') + if found: + print( + '\nSource text must be plain ASCII and free of ADR/issue citations; ' + 'add `check-source-text: off` in a comment to skip a file.', + file=sys.stderr, + ) + return 1 + return 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/support/tools/genvm-tool/default.nix b/support/tools/genvm-tool/default.nix index c152cddf..4b8c487c 100644 --- a/support/tools/genvm-tool/default.nix +++ b/support/tools/genvm-tool/default.nix @@ -16,6 +16,7 @@ python.pkgs.buildPythonApplication { aiohttp jsonnet shtab + argparse-manpage ruamel-yaml cloudpickle python-dotenv @@ -24,12 +25,16 @@ python.pkgs.buildPythonApplication { nativeBuildInputs = [ pkgs.installShellFiles ]; - # Generate the shell completions from the freshly built binary (shtab walks the - # whole argparse command tree) and drop them in the standard share/ locations. + # Generate the shell completions and man page from the freshly built binary + # (shtab / argparse-manpage both walk the whole argparse command tree) and drop + # them in the standard share/ locations. postInstall = '' installShellCompletion --cmd genvm-tool \ --bash <($out/bin/genvm-tool --print-completion bash) \ --zsh <($out/bin/genvm-tool --print-completion zsh) + + $out/bin/genvm-tool --print-manpage > genvm-tool.1 + installManPage genvm-tool.1 ''; doCheck = false; diff --git a/support/tools/genvm-tool/genvm_tool/__main__.py b/support/tools/genvm-tool/genvm_tool/__main__.py index 494cfbfa..aa83b681 100644 --- a/support/tools/genvm-tool/genvm_tool/__main__.py +++ b/support/tools/genvm-tool/genvm_tool/__main__.py @@ -11,7 +11,9 @@ import argparse import asyncio +import inspect import os +import re import sys from pathlib import Path @@ -21,30 +23,120 @@ cmd_build_manifest, cmd_codegen, cmd_configure, + cmd_howto, cmd_test, common, formatter, git, ) -TOPLEVEL = [cmd_configure, cmd_test, cmd_build_manifest, cmd_codegen] +TOPLEVEL = [cmd_configure, cmd_test, cmd_build_manifest, cmd_codegen, cmd_howto] GROUPS = [git] +# One-line tagline for the man page NAME line and completion header; the fuller +# `description=` below feeds the man page DESCRIPTION section and top-level -h. +TAGLINE = 'GenVM monorepo build/test/codegen/git helper' +DESCRIPTION = """\ +Multi-repo helper for the GenVM monorepo. It configures the ninja build graph, +runs the language-agnostic test suite, generates code and manifests, and drives +git across the manager and its executor submodules. + +Commands are either top-level leaves (configure, test, ...) or a group with its +own leaves (git ls, git check-for-push, ...). Pass -h to any of them for the +per-command details, or run `genvm-tool howto` for the contributor guides.""" +EPILOG = ( + 'Guides live in docs/contributing/howto/ (rendered by `genvm-tool howto`). ' + 'Every subcommand accepts -h/--help.' +) + + +# The module docstrings double as Sphinx-style source docs, so they carry RST +# inline markup (`:class:`~pkg.Name`` cross-refs, ``literal`` code spans). Render +# it to plain text so a man page / `-h` reader doesn't see the markup verbatim. +# The optional first `:domain:` covers explicit-domain roles like `:py:class:`. +_RST_ROLE = re.compile(r':(?:[a-z]+:)?[a-z]+:`(~)?([^`]+)`') +_RST_LITERAL = re.compile(r'``([^`]+)``') + + +def _rst_to_plain(text: str) -> str: + # A leading `~` in a cross-ref means "show only the last dotted segment". + text = _RST_ROLE.sub( + lambda m: m.group(2).rsplit('.', 1)[-1] if m.group(1) else m.group(2), text + ) + return _RST_LITERAL.sub(r'`\1`', text) + + +def _long_help(mod): + """The module docstring as a subparser `description` (falls back to HELP). + + `inspect.getdoc` dedents and strips; `_rst_to_plain` drops the RST markup so + the rich per-command docstrings feed both the subcommand's `-h` and the man + page's COMMAND section as clean prose. + """ + return _rst_to_plain(inspect.getdoc(mod) or mod.HELP) + def _add_leaf(subparsers, mod) -> None: - sub = subparsers.add_parser(mod.NAME, help=mod.HELP) + sub = subparsers.add_parser( + mod.NAME, + help=mod.HELP, + description=_long_help(mod), + formatter_class=argparse.RawDescriptionHelpFormatter, + ) mod.configure(sub) sub.set_defaults(func=mod.main) -def _create_parser() -> argparse.ArgumentParser: +class _PrintManpageAction(argparse.Action): + """`--print-manpage`: emit a roff man page for the whole tree, then exit. + + The manager's `default.nix` pipes this into `installManPage` at build time, + mirroring how shtab's `--print-completion` feeds `installShellCompletion`. + `argparse_manpage` walks the same subparser tree shtab does, turning each + command's `description=` into its COMMAND section. + """ + + def __init__(self, option_strings, dest, **kwargs): + super().__init__( + option_strings, dest, nargs=0, help='print a man page and exit', **kwargs + ) + + def __call__(self, parser, namespace, values, option_string=None): + import importlib.metadata + + from argparse_manpage.manpage import Manpage + + try: + version = importlib.metadata.version('genvm-tool') + except importlib.metadata.PackageNotFoundError: + version = 'dev' + # Build a fresh, *full* tree (with the `test` runner subtree grafted in) so + # the man page documents `test run` / `test show ...`, which the dispatch + # parser hides behind argparse.REMAINDER. + full = _create_parser(full=True) + data = { + 'prog': full.prog, + 'description': TAGLINE, + 'version': version, + 'project_name': full.prog, + 'manual_section': '1', + } + sys.stdout.write(str(Manpage(full, format='pretty', _data=data))) + parser.exit() + + +def _create_parser(full: bool = False) -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog='genvm-tool', - description='GenVM monorepo git helper (build, tests, ls)', + description=DESCRIPTION, + epilog=EPILOG, + formatter_class=argparse.RawDescriptionHelpFormatter, ) # `--print-completion {bash,zsh,tcsh}` emits a shell completion script for the # whole command tree and exits (shtab walks the subparsers below at call time). shtab.add_argument_to(parser, '--print-completion') + # `--print-manpage` does the analogous thing for the man page (see the action). + parser.add_argument('--print-manpage', action=_PrintManpageAction) parser.add_argument( '-C', '--chdir', help='change working directory before doing anything' ).complete = shtab.DIRECTORY @@ -62,10 +154,25 @@ def _create_parser() -> argparse.ArgumentParser: for mod in TOPLEVEL: _add_leaf(subparsers, mod) for grp in GROUPS: - gp = subparsers.add_parser(grp.NAME, help=grp.HELP) + gp = subparsers.add_parser( + grp.NAME, + help=grp.HELP, + description=_long_help(grp), + formatter_class=argparse.RawDescriptionHelpFormatter, + ) leaves = gp.add_subparsers(dest=f'{grp.NAME}_command') for mod in grp.COMMANDS: _add_leaf(leaves, mod) + + if full: + # Dispatch keeps `test` as an argparse.REMAINDER leaf so the aiohttp/jsonnet + # runner closure stays lazy (see cmd_test). For docs we want its real + # `run` / `show ...` subtree, so import the runner's own parser (it needs no + # root or suite) and splice it into the `test` slot for the parser walk. + from .tests.cli import create_parser as _test_create_parser + + subparsers.choices['test'] = _test_create_parser().parser + return parser diff --git a/support/tools/genvm-tool/genvm_tool/cmd_build_manifest.py b/support/tools/genvm-tool/genvm_tool/cmd_build_manifest.py index c1892ecd..0058c0fe 100644 --- a/support/tools/genvm-tool/genvm_tool/cmd_build_manifest.py +++ b/support/tools/genvm-tool/genvm_tool/cmd_build_manifest.py @@ -1,9 +1,8 @@ """`genvm-tool build-manifest` β€” generate the manager's `data/manifest.yaml`. -Assembles `executor_versions` (from the active executor submodules) plus the -static base fields into the YAML manifest the manager loads at runtime. The -inputs are documented in `genvm_tool.manifest`; this is the release-packaging -entry point (dev builds call the same logic in-process from `configure`). +Writes the runtime manifest (executor versions from the active submodules plus +the static base fields) to `--output`. Used by release packaging; dev builds get +the same file from `configure`, so you rarely run this by hand. """ from pathlib import Path diff --git a/support/tools/genvm-tool/genvm_tool/cmd_codegen.py b/support/tools/genvm-tool/genvm_tool/cmd_codegen.py index 5ab8c0b9..7bbe5923 100644 --- a/support/tools/genvm-tool/genvm_tool/cmd_codegen.py +++ b/support/tools/genvm-tool/genvm_tool/cmd_codegen.py @@ -1,10 +1,9 @@ """`genvm-tool codegen` β€” generate language bindings from genvm data JSON. -Replaces the per-executor ruby templates (``rs.rb`` / ``py.rb`` / ``rst.rb`` and -the standalone go generator). One typed front-end parses the data JSON; the -``--lang`` backend renders it. The build invokes this for the rust/python/rst -outputs; ``--lang go`` is a manual/external entry point (the generated ``.go`` -lands in a separate Go module), with the package name set by ``--go-package``. +Renders a data JSON file (`-i`) to a `--lang` backend (rust/python/rst/go) at +`-o`. The build runs this for the rust/python/rst outputs; `--lang go` is a +manual entry point (its output lands in a separate Go module β€” set the package +name with `--go-package`). """ from pathlib import Path @@ -43,6 +42,12 @@ def configure(parser): default='genvm', help="package name for the go backend (default: 'genvm')", ) + parser.add_argument( + '--rst-anchor-ns', + default='', + help="namespace inserted into the rst backend's `gvm-def-*` anchors, so " + 'two data files rendering into the same appendix do not collide', + ) def main(ctx: common.Context, args) -> int: @@ -53,6 +58,7 @@ def main(ctx: common.Context, args) -> int: Path(args.input), Path(args.output), go_package=args.go_package, + rst_anchor_ns=args.rst_anchor_ns, ) ctx.printer.put( 'codegen', lang=args.lang, input=str(args.input), output=str(args.output) diff --git a/support/tools/genvm-tool/genvm_tool/cmd_configure.py b/support/tools/genvm-tool/genvm_tool/cmd_configure.py index 6a6d4854..84ac40ea 100644 --- a/support/tools/genvm-tool/genvm_tool/cmd_configure.py +++ b/support/tools/genvm-tool/genvm_tool/cmd_configure.py @@ -1,22 +1,9 @@ -"""`genvm-tool configure` β€” generate build/build.ninja and build/info.json. - -Generates the ninja build graph that drives cargo (build / clippy / clippy-fix / -fmt), the `genvm-tool codegen` outputs, the nix runner build, and the install `cp` -steps. (This was historically the manager-root `configure.rb` Ruby script.) - -The ninja DSL and the *per-line* build registrations live in the importable -``genvm_tool_plugins.ninja`` plugin. This command owns only the manager-global -graph (manager crates, shared codegen, the runner build, info.json/manifest); for -each active executor line it builds a :class:`~genvm_tool_plugins.ninja.LineContext` -and hands it to that line's ``.genvm-tool.py:configure(line)`` hook. The lines -already diverge on how they resolve their runner registry (a frozen line copies a -committed manifest, a live line derives it through nix), so each line owns its own -configuration rather than this command branching on the version. - -Every path is resolved relative to the manager root (`ctx.root`), so the graph is -identical regardless of the directory the tool is invoked from β€” the original -script assumed it was run from the source root, which is the same thing once the -root is located. +"""`genvm-tool configure` β€” generate the build graph. + +Writes `build/build.ninja` and `build/info.json`, from which `ninja -C build` +drives the cargo builds, `genvm-tool codegen` outputs, the runner build, and the +install step. Run it once after cloning, and again whenever the set of active +executor lines or their manifests changes. Works from any directory in the repo. """ import json @@ -54,25 +41,24 @@ class ExecVersion(NamedTuple): real: str -def _load_versions(source_dir: Path) -> tuple[str, list[ExecVersion]]: +def _load_versions(monorepo_cfg, source_dir: Path) -> tuple[str, list[ExecVersion]]: """Resolve every active executor line from ``.genvm-monorepo-root``. Each active line is mounted at ``executors/.x`` and its ``manifest.json`` pins the concrete ``executor-version`` that becomes the built ``out/executor/`` directory. There may be several active lines. """ - cfg = json.loads((source_dir / MONOREPO_ROOT_FILE).read_text()) versions = [] - for key in cfg['active-versions']: + for key in monorepo_cfg['active-versions']: exec_rel = f'executors/{key}.x' manifest = json.loads((source_dir / exec_rel / 'manifest.json').read_text()) versions.append( ExecVersion(key=key, exec_rel=exec_rel, real=manifest['executor-version']) ) - return cfg['version'], versions + return monorepo_cfg['version'], versions -def build_independent_info(source_dir: Path) -> dict: +def build_independent_info(monorepo_cfg, source_dir: Path) -> dict: """The parts of ``info.json`` derivable from source alone β€” no build needed. ``executor_versions`` maps each active line (e.g. ``v0.3``) to its built @@ -82,7 +68,7 @@ def build_independent_info(source_dir: Path) -> dict: ``info.json`` (see the manager-root ``.genvm-tool.py``) without first running ``configure``. ``configure`` merges this into the full, build-dependent info. """ - primary_key, versions = _load_versions(source_dir) + primary_key, versions = _load_versions(monorepo_cfg, source_dir) primary = next((v for v in versions if v.key == primary_key), versions[0]) return { 'executor_versions': {v.key: v.real for v in versions}, @@ -90,6 +76,26 @@ def build_independent_info(source_dir: Path) -> dict: } +def rust_target_dirs_info( + monorepo_cfg, source_dir: Path, rust_target_dir: Path +) -> dict: + """``rust_target_dirs``: line checkout (``executors/v0.3.x``) β†’ its cargo target dir. + + Keyed by mount so a consumer only has to ask which one contains its crate; + crates under none of them use ``rust_target_dir`` itself. Why lines cannot + share a dir: :func:`genvm_tool_plugins.ninja.target_dir_for_line`. + """ + from genvm_tool_plugins import ninja + + _, versions = _load_versions(monorepo_cfg, source_dir) + return { + 'rust_target_dirs': { + v.exec_rel: str(ninja.target_dir_for_line(rust_target_dir, v.key)) + for v in versions + } + } + + def configure(parser): parser.add_argument( '--test-always-fail-module', @@ -103,7 +109,7 @@ def configure(parser): ) -def _detect_rust_target() -> str: +def detect_rust_target() -> str: out = subprocess.run( ['rustc', '-vV'], capture_output=True, text=True, check=True ).stdout @@ -132,6 +138,8 @@ def _line_configurator(ctx: common.Context, source_dir: Path, exec_rel: str): def main(ctx: common.Context, args) -> int: source_dir = ctx.root + monorepo_cfg = json.loads(ctx.root.joinpath(MONOREPO_ROOT_FILE).read_text()) + # The ninja DSL + per-line helpers live in the importable plugin; put the # plugin search path on sys.path before importing it (and before any line's # `.genvm-tool.py:configure` hook, which may import it too). @@ -142,13 +150,13 @@ def main(ctx: common.Context, args) -> int: build_dir = source_dir / ninja.BUILD_DIR_REL - primary_key, versions = _load_versions(source_dir) + primary_key, versions = _load_versions(monorepo_cfg, source_dir) # Manager-global generated files (test fixtures, docs) have a single output, # so they are derived from the primary line's codegen data. primary = next((v for v in versions if v.key == primary_key), versions[0]) primary_exec_root = source_dir / primary.exec_rel - rust_target = _detect_rust_target() + rust_target = detect_rust_target() rust_target_dir = build_dir / 'ya-build' / 'rust-target' rust_target_dir.mkdir(parents=True, exist_ok=True) @@ -221,7 +229,10 @@ def main(ctx: common.Context, args) -> int: tool_src = source_dir / 'support' / 'tools' / 'genvm-tool' genvm_tool_sh = build_dir / 'genvm_tool.sh' build_dir.mkdir(parents=True, exist_ok=True) - pp = ':'.join(sys.path + [str(tool_src)]) + # The in-tree source must come *first*: `sys.path` carries the installed + # genvm-tool (this process was launched from it), which would otherwise shadow + # the working tree and make codegen edits silently no-ops. + pp = ':'.join([str(tool_src)] + sys.path) genvm_tool_sh.write_text( '#!/bin/sh\n' '# Generated by `genvm-tool configure`, DO NOT EDIT MANUALLY.\n' @@ -264,6 +275,7 @@ def main(ctx: common.Context, args) -> int: ninja.VAR_IN, '--out', ninja.VAR_OUT, + ninja.RawStr('$extra_flags'), ], description='Codegen $out', ) @@ -290,7 +302,12 @@ def main(ctx: common.Context, args) -> int: n.codegen(host_fns_py, 'python', p_data / 'host-fns.json') n.codegen(public_abi_py, 'python', p_data / 'public-abi.json') n.codegen(constants_rst, 'rst', p_data / 'public-abi.json') - n.codegen(constants_pending_rst, 'rst', p_data / 'public-abi-pending.json') + n.codegen( + constants_pending_rst, + 'rst', + p_data / 'public-abi-pending.json', + ['--rst-anchor-ns=pending'], + ) for out in (host_fns_py, public_abi_py, constants_rst, constants_pending_rst): codegen_phony.add_dependency(out) @@ -402,6 +419,7 @@ def main(ctx: common.Context, args) -> int: codegen_phony=codegen_phony, data_phony=data_phony, all_bin=all_exec_line, + is_support_only=v.key in monorepo_cfg.get('support-only-versions', []), ) configurator = _line_configurator(ctx, source_dir, v.exec_rel) configurator(line) @@ -487,6 +505,26 @@ def _runner_file(f: Path) -> bool: n.build('phony', 'cargo/clippy/fix').add_dependency(n.all_clippy_fix).finish() n.install('install', 'out', all_manager) + + # The LLM dispatch script requires the `llm_policy` package from the + # unhardcoded-engine submodule; fail loudly if it was not checked out. + llm_policy_dir = source_dir / 'libs' / 'unhardcoded-engine' + if not (llm_policy_dir / 'llm_policy.lua').is_file(): + raise FileNotFoundError( + f'{llm_policy_dir} is missing or incomplete; ' + 'run `git submodule update --init libs/unhardcoded-engine`' + ) + n.install( + 'libs/unhardcoded-engine/llm_policy', + 'out/lib/genvm-lua/llm_policy', + all_manager, + ) + llm_policy_entry = 'out/lib/genvm-lua/llm_policy.lua' + all_manager.add_dependency(llm_policy_entry) + n.build('cp', llm_policy_entry).add_dependency( + llm_policy_dir / 'llm_policy.lua' + ).finish() + all_manager.finish() all_bin.finish() @@ -504,10 +542,14 @@ def _runner_file(f: Path) -> bool: 'coverage_dir': str(build_dir / 'cov'), 'build_dir': str(build_dir), 'rust_target_dir': str(rust_target_dir), + # The build passes `--target` explicitly; the test harness must pass the + # same one or it gets a second, unshared unit graph in the same dir. + 'rust_target': rust_target, # executor_versions / primary_executor_version β€” derivable without a build; # the test harness synthesizes the same keys from source when info.json is # absent (see build_independent_info). - **build_independent_info(source_dir), + **build_independent_info(monorepo_cfg, source_dir), + **rust_target_dirs_info(monorepo_cfg, source_dir, rust_target_dir), } (build_dir / 'info.json').write_text(json.dumps(info, indent=2)) diff --git a/support/tools/genvm-tool/genvm_tool/cmd_howto.py b/support/tools/genvm-tool/genvm_tool/cmd_howto.py new file mode 100644 index 00000000..3d2a0fe0 --- /dev/null +++ b/support/tools/genvm-tool/genvm_tool/cmd_howto.py @@ -0,0 +1,94 @@ +"""`genvm-tool howto` β€” list the contributing how-to guides. + +Walks `docs/contributing/howto/`, reads each guide's one-line description from the +`README.md` index, and prints a grouped markdown listing of every guide. Exits +non-zero if a guide is missing from the index, or the index lists a guide that no +longer exists, so CI can gate on the index staying in sync. +""" + +import re +import sys +from pathlib import Path + +from . import common + +NAME = 'howto' +HELP = 'render the contributing how-to guides as a skill-like index' + +HOWTO_DIR = Path('docs/contributing/howto') +# Deepest directory nesting promoted to a `## header`; anything below stays in the +# guide's link text instead of spawning ever-deeper sections. +MAX_HEADER_DEPTH = 3 +# README index lines look like: `- [text](rel/target.md) β€” one-line description.` +# `rel/target.md` is relative to the howto dir; the em dash (β€”) is the separator. +_INDEX_LINE = re.compile(r'-\s*\[[^\]]*\]\(([^)]+)\)\s*β€”\s*(.+)$') + + +def configure(parser): + pass + + +def _parse_index(readme: Path) -> dict[str, str]: + """Map each README-listed target (howto-relative) to its one-line description.""" + descriptions: dict[str, str] = {} + for line in readme.read_text().splitlines(): + m = _INDEX_LINE.match(line.strip()) + if m: + descriptions[m.group(1)] = m.group(2).strip().rstrip('.') + return descriptions + + +def _h1_title(path: Path) -> str: + """First `# heading` in a markdown file, or '' if it has none.""" + for line in path.read_text().splitlines(): + s = line.strip() + if s.startswith('# '): + return s[2:].strip() + return '' + + +def main(ctx: common.Context, args) -> int: + base = ctx.root / HOWTO_DIR + if not base.is_dir(): + raise common.ToolError(f'not found: {HOWTO_DIR}') + + index = base / 'README.md' + descriptions = _parse_index(index) + + # The top-level README *is* the index we render, not a guide within it. + guides = [p for p in base.rglob('*') if p.is_file() and p != index] + on_disk = {p.relative_to(base).as_posix() for p in guides} + + rows = [] + for path in guides: + parts = path.relative_to(base).parts + cut = min(len(parts) - 1, MAX_HEADER_DEPTH) # dir segments that become a header + header = '/'.join(parts[:cut]) + link = '/'.join(parts[cut:]) + desc = descriptions.get(path.relative_to(base).as_posix()) or _h1_title(path) + rows.append((header, link, path.relative_to(ctx.root).as_posix(), desc)) + + # '' (root) sorts before any header; within a section, by link text. + current = None + started = False + for header, link, repo_path, desc in sorted(rows): + if header != current: + current = header + if header: + if started: + print() # blank line between sections, but not a leading one + print(f'## {header[:1].upper()}{header[1:]}') + print(f'- [{link}]({repo_path}) β€” {desc}') + started = True + + index_rel = index.relative_to(ctx.root).as_posix() + problems = [ + f'{index_rel}: missing entry for {rel}' + for rel in sorted(on_disk - descriptions.keys()) + ] + [ + f'{index_rel}: lists {rel}, which does not exist' + for rel in sorted(descriptions.keys() - on_disk) + ] + for problem in problems: + print(problem, file=sys.stderr) + return 1 if problems else 0 diff --git a/support/tools/genvm-tool/genvm_tool/cmd_test.py b/support/tools/genvm-tool/genvm_tool/cmd_test.py index 5888c475..f8f781b2 100644 --- a/support/tools/genvm-tool/genvm_tool/cmd_test.py +++ b/support/tools/genvm-tool/genvm_tool/cmd_test.py @@ -1,11 +1,8 @@ -"""`genvm-tool test` β€” the language-agnostic test runner. - -Everything after `test` is forwarded to the runner under `genvm_tool/tests/`: -its `run` / `show plan|test|services|tags` subcommands plus the suite-provided -filter flags. The runner reuses genvm-tool's `Context` (logger/printer) and the -top-level `-C` / `--log-format` / `--log-level`; the runner package is imported -lazily so the stdlib-only paths (`configure`, `git ls`) never pull in its -`aiohttp` / `jsonnet` dependency closure. +"""`genvm-tool test` β€” run the test suite. + +Everything after `test` is forwarded to the test runner, which has its own +`run` and `show plan|test|services|tags` subcommands plus filter flags. Run +`genvm-tool test run -h` or `genvm-tool test show -h` for those. """ import argparse diff --git a/support/tools/genvm-tool/genvm_tool/codegen/model.py b/support/tools/genvm-tool/genvm_tool/codegen/model.py index 2286f3e6..6a5306ba 100644 --- a/support/tools/genvm-tool/genvm_tool/codegen/model.py +++ b/support/tools/genvm-tool/genvm_tool/codegen/model.py @@ -49,10 +49,12 @@ class TrieNode: ``suffix`` is the CamelCase concatenation of the path heads from the root to this node (``''`` at the root); backends use it to name the generated - struct/function. ``parts`` is the space-joined path the node maps to. A node is - either a *param* node (``param`` set: consumes one typed argument) or a regular - node carrying ``leaves`` (terminal strings) and ``methods`` (child nodes), - mirroring the original ruby generators. + struct/function. ``parts`` is the path from the root to this node (``[]`` at the + root), regardless of whether the node is itself a valid string: only when + ``terminal`` is set does ``' '.join(parts)`` name a leaf. A node is either a + *param* node (``param`` set: consumes one typed argument) or a regular node + carrying ``leaves`` (terminal strings) and ``methods`` (child nodes), mirroring + the original ruby generators. """ suffix: str @@ -99,11 +101,11 @@ def _unfold(entry): return None -def _build(entries, prefix_parts: list[str], suffix: str, terminal_parts) -> TrieNode: +def _build(entries, prefix_parts: list[str], suffix: str, terminal: bool) -> TrieNode: node = TrieNode( suffix=suffix, - parts=terminal_parts if terminal_parts is not None else [], - terminal=terminal_parts is not None, + parts=prefix_parts, + terminal=terminal, param=None, leaves=[], methods=[], @@ -138,8 +140,7 @@ def _build(entries, prefix_parts: list[str], suffix: str, terminal_parts) -> Tri node.leaves.append((head, current)) node.order.append(('leaf', head, current)) else: - child_terminal = current if is_terminal else None - child = _build(non_null, current, child_suffix, child_terminal) + child = _build(non_null, current, child_suffix, is_terminal) node.methods.append((head, child)) node.order.append(('method', head, child)) return node @@ -177,7 +178,7 @@ def parse(data) -> list[Definition]: defs.append(Consts(t['name'], t['repr'], t['values'])) elif kind == 'str_trie': entries = [_unfold(e) for e in t['values']] - defs.append(StrTrie(t['name'], _build(entries, [], '', None), entries)) + defs.append(StrTrie(t['name'], _build(entries, [], '', False), entries)) else: raise ValueError(f'unknown codegen type {kind!r}') return defs diff --git a/support/tools/genvm-tool/genvm_tool/codegen/rst.py b/support/tools/genvm-tool/genvm_tool/codegen/rst.py index 07b463c7..6fd01972 100644 --- a/support/tools/genvm-tool/genvm_tool/codegen/rst.py +++ b/support/tools/genvm-tool/genvm_tool/codegen/rst.py @@ -14,51 +14,59 @@ def _dash(s: str) -> str: return s.replace('_', '-') -def render(defs: list[Definition], **_opts) -> str: +def render(defs: list[Definition], rst_anchor_ns: str = '', **_opts) -> str: + # Two data files render into the same appendix (`public-abi.json` and the + # pending one), and a `vm_error` trie in both would define the same + # `gvm-def-str-trie-vm-error` label twice. `rst_anchor_ns` namespaces one of + # them; it also keeps the pending anchors distinct from the ones the codes + # will get once they fold into the real trie. + ns = f'{_dash(rst_anchor_ns)}-' if rst_anchor_ns else '' buf: list[str] = [] buf.append('Constants\n') buf.append('=========\n') buf.append('\n') for d in defs: if isinstance(d, Enum): - buf.append(f'.. _gvm-def-enum-{_dash(d.name)}:\n\n') + buf.append(f'.. _gvm-def-{ns}enum-{_dash(d.name)}:\n\n') buf.append(f'{d.name}\n') buf.append('-' * len(d.name) + '\n\n') buf.append(f'Type: {d.repr}\n\n') for k, v in d.values.items(): - buf.append(f'.. _gvm-def-enum-value-{_dash(d.name)}-{_dash(k)}:\n\n') + buf.append(f'.. _gvm-def-{ns}enum-value-{_dash(d.name)}-{_dash(k)}:\n\n') buf.append(f'{k}\n') buf.append('~' * len(k) + '\n\n') buf.append(f'Value: ``{v}``\n\n') elif isinstance(d, Const): - buf.append(f'.. _gvm-def-const-{_dash(d.name)}:\n\n') + buf.append(f'.. _gvm-def-{ns}const-{_dash(d.name)}:\n\n') buf.append(f'{d.name}\n') buf.append('-' * len(d.name) + '\n\n') buf.append(f'Type: {d.repr}\n\n') buf.append(f'Value: ``{d.value}``\n\n') elif isinstance(d, Consts): - buf.append(f'.. _gvm-def-consts-{_dash(d.name)}:\n\n') + buf.append(f'.. _gvm-def-{ns}consts-{_dash(d.name)}:\n\n') buf.append(f'{d.name}\n') buf.append('-' * len(d.name) + '\n\n') buf.append(f'Type: {d.repr}\n\n') for k, v in d.values.items(): - buf.append(f'.. _gvm-def-consts-value-{_dash(d.name)}-{_dash(k)}:\n\n') + buf.append(f'.. _gvm-def-{ns}consts-value-{_dash(d.name)}-{_dash(k)}:\n\n') buf.append(f'{k}\n') buf.append('~' * len(k) + '\n\n') buf.append(f'Value: ``{v}``\n\n') elif isinstance(d, StrTrie): - buf.append(f'.. _gvm-def-str-trie-{_dash(d.name)}:\n\n') + buf.append(f'.. _gvm-def-{ns}str-trie-{_dash(d.name)}:\n\n') buf.append(f'{d.name}\n') buf.append('-' * len(d.name) + '\n\n') buf.append('Type: str_trie\n\n') for path, param in model.enumerate_paths(d.entries): rst_name = path.replace('_', '-').replace(' ', '-') - buf.append(f'.. _gvm-def-str-trie-value-{_dash(d.name)}-{rst_name}:\n\n') + buf.append(f'.. _gvm-def-{ns}str-trie-value-{_dash(d.name)}-{rst_name}:\n\n') buf.append(f'``{path}``\n') buf.append('~' * (len(path) + 4) + '\n\n') if param is not None: buf.append(f'Param: {param}\n\n') - return ''.join(buf) + # Sections are emitted blank-line-separated, so the last one leaves a trailing + # blank line that `end-of-file-fixer` would strip β€” permanent codegen drift. + return ''.join(buf).rstrip('\n') + '\n' __all__ = ['render'] diff --git a/support/tools/genvm-tool/genvm_tool/codegen/rust.py b/support/tools/genvm-tool/genvm_tool/codegen/rust.py index 44ed3499..dd508a96 100644 --- a/support/tools/genvm-tool/genvm_tool/codegen/rust.py +++ b/support/tools/genvm-tool/genvm_tool/codegen/rust.py @@ -1,14 +1,24 @@ """Rust backend for ``genvm-tool codegen`` (ports the old ``rs.rb`` template). -Output is byte-for-byte identical to the previous ruby generator: the build -regenerates these files in-tree and a non-empty diff would surface as drift. +Output follows the previous ruby generator, plus the later `prefix_` and +`is_valid_` additions: the build regenerates these files in-tree and a non-empty +diff would surface as drift. """ from __future__ import annotations import json -from .model import Const, Consts, Definition, Enum, StrTrie, TrieNode, to_camel +from .model import ( + Const, + Consts, + Definition, + Enum, + StrTrie, + TrieNode, + enumerate_paths, + to_camel, +) def _dump(v) -> str: @@ -96,8 +106,10 @@ def _enum(e: Enum) -> str: def _emit_struct(node: TrieNode, root_name: str, buf: list[str]) -> None: - """Emit one non-root trie struct: children first, then this struct, then its - param children (matching the original ruby buffer order).""" + """ + Emit one non-root trie struct: children first, then this struct, then its + param children (matching the original ruby buffer order). + """ for _head, child in node.methods: if child.param is None: _emit_struct(child, root_name, buf) @@ -109,6 +121,9 @@ def _emit_struct(node: TrieNode, root_name: str, buf: list[str]) -> None: f' pub const fn val(&self) -> {root_name} {{ ' f'{root_name}(Cow::Borrowed("{val}")) }}\n' ) + buf.append(" pub const fn prefix_(&self) -> &'static str {\n") + buf.append(f' "{" ".join(node.parts)}"\n') + buf.append(' }\n') for name, parts in node.leaves: val = ' '.join(parts) buf.append( @@ -183,18 +198,72 @@ def _trie(t: StrTrie, buf: list[str]) -> None: ) buf.append('}\n\n') + _emit_is_valid(t, root_name, buf) + + +def _emit_is_valid(t: StrTrie, root_name: str, buf: list[str]) -> None: + """ + Emit ``::is_valid_(&str)``: does this string name a trie path? + + Needed wherever an *externally supplied* code has to be checked against the + trie (ADR-013 validates leader-proposed `vm_error` codes this way). Generated + rather than hand-written so it cannot drift from the trie data. + """ + paths = enumerate_paths(t.entries) + plain = [p for p, param in paths if param is None] + params = [(p, param) for p, param in paths if param is not None] + + # `cargo fmt` runs over the generated files (pre-commit), and it would collapse + # the one-entry-per-line `matches!` below onto a single line β€” leaving codegen + # output and committed file permanently disagreeing, so every build dirties the + # tree and every commit re-dirties the build. + buf.append('#[rustfmt::skip]\n') + buf.append(f'impl {root_name} {{\n') + buf.append(f' /// Whether `s` is a well-formed `{t.name}` path.\n') + buf.append(' pub fn is_valid_(s: &str) -> bool {\n') + if plain: + buf.append(' if matches!(s,\n ') + buf.append(' |\n '.join(json.dumps(p) for p in plain)) + buf.append('\n ) {\n') + buf.append(' return true;\n') + buf.append(' }\n') + for path, param in params: + prefix = json.dumps(path + ' ') + if param == 'str': + # A `$str` tail accepts any non-empty remainder. + buf.append(f' if let Some(rest) = s.strip_prefix({prefix}) {{\n') + buf.append(' if !rest.is_empty() {\n') + else: + # The parameter must round-trip: `parse` alone would accept `+7` and + # `007`, making several byte-different strings name the same error. + # Codes reach consensus hashes, so only the canonical spelling is valid. + buf.append(f' if let Some(rest) = s.strip_prefix({prefix}) {{\n') + buf.append( + f' if rest.parse::<{param}>().is_ok_and(|v| v.to_string() == rest) {{\n' + ) + buf.append(' return true;\n') + buf.append(' }\n') + buf.append(' }\n') + buf.append(' false\n') + buf.append(' }\n') + buf.append('}\n\n') + def render(defs: list[Definition], **_opts) -> str: has_str_trie = any(isinstance(d, StrTrie) for d in defs) - # Only enums and str-tries derive serde; a const-only data file (e.g. - # public-abi-pending.json) must not emit an unused `use serde` import. - has_serde = any(isinstance(d, (Enum, StrTrie)) for d in defs) + # Serde imports are emitted per actual use: enums derive both halves, a trie + # root derives `Serialize` only, and a const-only data file needs neither β€” + # an unused import is a `warnings-as-errors` build failure. + has_deserialize = any(isinstance(d, Enum) for d in defs) + has_serialize = has_deserialize or has_str_trie buf: list[str] = [] buf.append('// This file is auto-generated. Do not edit!\n\n') buf.append('#![allow(dead_code, clippy::redundant_static_lifetimes)]\n') buf.append('\n') - if has_serde: + if has_deserialize: buf.append('use serde::{Deserialize, Serialize};\n\n') + elif has_serialize: + buf.append('use serde::Serialize;\n\n') if has_str_trie: buf.append('use std::borrow::Cow;\n\n') for d in defs: @@ -212,8 +281,10 @@ def render(defs: list[Definition], **_opts) -> str: elif isinstance(d, StrTrie): _trie(d, buf) - buf.append('\n// EOF\n') - return ''.join(buf) + # Definitions are emitted blank-line-terminated, so the trailer would stack a + # second blank line that `cargo fmt` (which runs over these files) strips β€” + # permanent codegen drift. + return ''.join(buf).rstrip('\n') + '\n\n// EOF\n' __all__ = ['render'] diff --git a/support/tools/genvm-tool/genvm_tool/git/check_for_push.py b/support/tools/genvm-tool/genvm_tool/git/check_for_push.py index 2d31fd51..f24ce0d5 100644 --- a/support/tools/genvm-tool/genvm_tool/git/check_for_push.py +++ b/support/tools/genvm-tool/genvm_tool/git/check_for_push.py @@ -1,39 +1,15 @@ """`genvm-tool git check-for-push` β€” pre-push status across the sub-repos. Read-only survey of the manager and every executor submodule. For each repo it -reports, folds into a per-repo `ready` verdict, and suggests the next action: - - - dirty β€” uncommitted work in the tree - - on_branch β€” on a real branch, not a detached HEAD - - committed_in_manager β€” (submodule) the submodule's HEAD is the gitlink the - manager has committed, i.e. the branch advance is - recorded in the superproject - - in_sync_with_origin β€” the branch tip vs its branch on `origin` - - action β€” the next step in the add β†’ commit β†’ push pipeline - -After all repo rows, it emits one aggregated `suggested_push_command` block: a -newline-joined shell snippet that pushes the ready repos in order, or `none` -when anything is not ready. - -By default the origin comparison is a LIVE `git ls-remote` query. The queries are -launched up front β€” one per distinct origin URL (the executor submodules share a -remote, so they batch into a single multi-ref call) and all in parallel β€” before -any local work or output, so their network latency overlaps everything else. - -`ls-remote` only yields the remote branch's existence + hash, so when the remote -holds commits we don't have locally the exact ahead/behind cannot be counted -without a fetch β€” surfaced as `fetch (cannot compare with origin)`. Pass -`--offline` to compare against the local `origin/` tracking ref (your -last fetch) with no network. - -`same_branch_as_manager` is shown for submodules as context (the line-namespaced -mirror convention: `feat/x` on the manager maps to `pr/v0.3/feat/x` in the -executor β€” see `common.Repo.feature_branch`) but does not gate `ready`. Exit -status is non-zero when any repo is not ready. An `absent` origin branch is -still push-ready because `git push` will create it. - -This NEVER fetches, resets, switches, or writes anything β€” `ls-remote` reads the -remote without updating any local ref. +reports whether it is dirty, on a branch, committed into the manager, and in sync +with `origin`, folds that into a `ready` verdict, and prints the next action in +the add β†’ commit β†’ push pipeline. After the per-repo rows it emits one +`suggested_push_command` that pushes the ready repos in order (or `none` if any +repo is not ready). Exits non-zero when any repo is not ready. + +By default it queries `origin` live (`git ls-remote`); pass `--offline` to compare +against your last-fetched `origin/` instead, with no network. This never +fetches, resets, switches, or writes anything. """ import os diff --git a/support/tools/genvm-tool/genvm_tool/git/create_branches.py b/support/tools/genvm-tool/genvm_tool/git/create_branches.py index d1a115fc..311a48cd 100644 --- a/support/tools/genvm-tool/genvm_tool/git/create_branches.py +++ b/support/tools/genvm-tool/genvm_tool/git/create_branches.py @@ -1,16 +1,10 @@ """`genvm-tool git create-branches` β€” branch the sub-repos that have new content. -Surveys the manager and every checked-out executor submodule, keeps the ones -that carry uncommitted or unpushed work, lets you tick which of those to branch, -and creates the feature branch (`git checkout -b`) across all picked repos. - -The logical branch name is shared, but every executor submodule namespaces it -under `pr//` (e.g. `pr/v0.3/`) via :meth:`common.Repo.feature_branch`, -because all executor submodules push to ONE remote (`genvm-executor`) and a bare -name would collide between two lines branched at once β€” the `` segment keeps -them apart. The manager carries the bare name. This keeps the manager and its -executor submodule in lockstep while staying collision-free, matching the -line-scoped `v-dev` release branches CI already relies on. +Surveys the manager and every checked-out executor submodule, keeps the ones with +uncommitted or unpushed work, lets you tick which of those to branch, and creates +the feature branch across the picked repos. The manager gets the bare name; each +executor submodule namespaces it under `pr//` (e.g. `pr/v0.3/`), since +they all push to one shared remote. """ import subprocess diff --git a/support/tools/genvm-tool/genvm_tool/tests/stage/filter.py b/support/tools/genvm-tool/genvm_tool/tests/stage/filter.py index 3729c84b..6076edd2 100644 --- a/support/tools/genvm-tool/genvm_tool/tests/stage/filter.py +++ b/support/tools/genvm-tool/genvm_tool/tests/stage/filter.py @@ -21,7 +21,7 @@ def add_args(parser: argparse.ArgumentParser) -> None: parser.add_argument( '--filter-tag', type=str, - help='Only run tests matching this tags, `(a|b)&!c`', + help='Only run tests matching these tags, `(a|b)&!c`', default='true', metavar='EXPR', ) diff --git a/support/tools/genvm-tool/pyproject.toml b/support/tools/genvm-tool/pyproject.toml index d2afdf6d..8dc58cc5 100644 --- a/support/tools/genvm-tool/pyproject.toml +++ b/support/tools/genvm-tool/pyproject.toml @@ -7,6 +7,7 @@ dependencies = [ "aiohttp", "jsonnet", "shtab", + "argparse-manpage", "ruamel.yaml", "cloudpickle", "python-dotenv", diff --git a/tests/runner/genvm_tool_plugins/cargo.py b/tests/runner/genvm_tool_plugins/cargo.py index ea23afac..528fcf89 100644 --- a/tests/runner/genvm_tool_plugins/cargo.py +++ b/tests/runner/genvm_tool_plugins/cargo.py @@ -19,6 +19,23 @@ TARGET_DIR = Path(build_info['rust_target_dir']) COVERAGE_DIR = Path(build_info['coverage_dir']) +if 'rust_target_dirs' not in build_info or 'rust_target' not in build_info: + raise RuntimeError( + 'build/info.json predates per-line cargo target dirs, re-run `genvm-tool configure`' + ) + +# Line checkout -> its own cargo target dir; see `rust_target_dirs_info`. Falling +# back to the shared dir on a stale info.json would resurrect the cross-line +# artifact clobbering this exists to prevent, hence the hard error above. +_LINE_TARGET_DIRS = { + Path(mount): Path(target_dir) + for mount, target_dir in build_info['rust_target_dirs'].items() +} + +# Passed to every cargo invocation, matching the ninja build: without it cargo +# builds a separate host unit graph and shares nothing with what ninja compiled. +RUST_TARGET = build_info['rust_target'] + # Add --coverage flag to run parser local_ctx.run_parser.add_argument( '--coverage', @@ -61,6 +78,12 @@ def _get_default_env() -> dict[str, str]: env['AFL_FUZZER_LOOPCOUNT'] = '20' # without it no coverage will be written! env['AFL_NO_CFG_FUZZING'] = '1' env['AFL_BENCH_UNTIL_CRASH'] = '1' + # persistent mode carries state across iterations, so calibration reports + # most of the corpus as unstable; the warning is pure noise here + env['AFL_NO_WARN_INSTABILITY'] = '1' + # drops one "attempting dry run" line per corpus entry; seeds are calibrated + # lazily instead, so broken ones surface when they are first fuzzed + env['AFL_NO_STARTUP_CALIBRATION'] = '1' return env @@ -197,6 +220,20 @@ def _coverage_post_run( _profile_objects.append(BUILD_DIR / 'out' / 'bin' / 'genvm-modules') +def _target_dir(rust_root_dir: Path) -> Path: + """Target dir for a crate: its line's, or the shared one if it is in no line.""" + rel = rust_root_dir.relative_to(local_ctx.shared.root_dir) + for mount, target_dir in _LINE_TARGET_DIRS.items(): + if rel.is_relative_to(mount): + return target_dir + return TARGET_DIR + + +def _artifact_dir(target_dir: Path) -> Path: + """Where `--target` puts built artifacts, as opposed to host build scripts.""" + return target_dir / RUST_TARGET / 'debug' + + def _load_cargo_config( rust_root_dir: Path, flags_key: str = 'cargo_test_flags' ) -> tuple[dict, dict[str, str], list[str]]: @@ -269,9 +306,11 @@ def cargo_test( cargo_toml = tomllib.loads(rust_root_dir.joinpath('Cargo.toml').read_text()) + target_dir = _target_dir(rust_root_dir) + # Track deps directory for coverage if _is_coverage_enabled(): - deps_dir = TARGET_DIR / 'debug' / 'deps' + deps_dir = _artifact_dir(target_dir) / 'deps' if deps_dir not in _profile_objects: _profile_objects.append(deps_dir) @@ -280,8 +319,10 @@ def cargo_test( 'test', '--message-format=short', '--color=always', + '--target', + RUST_TARGET, '--target-dir', - str(TARGET_DIR), + str(target_dir), ] + extra_flags # 1. Examples - verify compilation @@ -300,8 +341,10 @@ def cargo_test( 'cargo', 'check', '--color=always', + '--target', + RUST_TARGET, '--target-dir', - str(TARGET_DIR), + str(target_dir), '--example', ex_name, ] @@ -378,9 +421,21 @@ def cargo_fuzz( rust_root_dir, 'cargo_afl_build_flags' ) + target_dir = _target_dir(rust_root_dir) + + # Keyed by crate, not by target name: the lines share fuzz target stems, and + # one AFL output dir per stem would mix their corpora β€” which `cmin` then + # writes back into `fuzz/inputs-` under `--fuzz-update-corpus`. + out_dir = ( + local_ctx.shared.artifacts_dir + / 'fuzz' + / rust_root_dir.relative_to(local_ctx.shared.root_dir) + / name + ) + # Track fuzz binary for coverage if _is_coverage_enabled(): - fuzz_binary = TARGET_DIR / 'debug' / 'examples' / f'fuzz-{name}' + fuzz_binary = _artifact_dir(target_dir) / 'examples' / f'fuzz-{name}' if fuzz_binary not in _profile_objects: _profile_objects.append(fuzz_binary) @@ -397,8 +452,10 @@ def cargo_fuzz( 'cargo-afl', 'afl', 'build', + '--target', + RUST_TARGET, '--target-dir', - TARGET_DIR, + target_dir, '--example', f'fuzz-{name}', '--color=always', @@ -408,9 +465,7 @@ def cargo_fuzz( ), genvm_tool.tests.test.CommandToResultStep(), genvm_tool.tests.test.ResultStopIfErrorStep(), - genvm_tool.tests.exec.step.MkDir( - path=local_ctx.shared.artifacts_dir / 'fuzz' / name - ), + genvm_tool.tests.exec.step.MkDir(path=out_dir), genvm_tool.tests.exec.step.Run( args=[ 'cargo-afl', @@ -423,12 +478,12 @@ def cargo_fuzz( '-i', f'./fuzz/inputs-{name}', '-o', - f'{local_ctx.shared.artifacts_dir}/fuzz/{name}', + str(out_dir), '-V', str(getattr(ctx.configuration.args, 'fuzz_timeout', 30)), '-t', '5000', - f'{TARGET_DIR}/debug/examples/fuzz-{name}', + str(_artifact_dir(target_dir) / f'examples/fuzz-{name}'), ], mode=genvm_tool.tests.exec.command.RunMode.INTERACTIVE_TTY, ), @@ -438,7 +493,7 @@ def cargo_fuzz( ) if getattr(ctx.configuration.args, 'fuzz_update_corpus', False): - opt_dir = local_ctx.shared.artifacts_dir.joinpath('fuzz/', f'{name}-opt') + opt_dir = out_dir.parent / f'{name}-opt' async def remove_opt_dir(_): if opt_dir.exists(): @@ -462,11 +517,11 @@ async def remove_inputs_dir(_): '-T', 'all', '-o', - f'{local_ctx.shared.artifacts_dir}/fuzz/{name}-opt', + str(opt_dir), '-i', - f'{local_ctx.shared.artifacts_dir}/fuzz/{name}', + str(out_dir), '--', - f'{TARGET_DIR}/debug/examples/fuzz-{name}', + str(_artifact_dir(target_dir) / f'examples/fuzz-{name}'), ], mode=genvm_tool.tests.exec.command.RunMode.INTERACTIVE, ) diff --git a/tests/runner/genvm_tool_plugins/integration.py b/tests/runner/genvm_tool_plugins/integration.py index 0be01f89..565061e6 100644 --- a/tests/runner/genvm_tool_plugins/integration.py +++ b/tests/runner/genvm_tool_plugins/integration.py @@ -8,6 +8,7 @@ import base64 import difflib import gzip +import hashlib import io import itertools import json @@ -20,6 +21,7 @@ from dataclasses import dataclass from pathlib import Path +import genvm_tool.gvm32 import genvm_tool.tests import origin.base_host as base_host import origin.calldata as gvm_calldata @@ -41,7 +43,6 @@ ) BUILD_DIR = Path(build_info['build_dir']) -TARGET_DIR = Path(build_info['rust_target_dir']) local_ctx.run_parser.add_argument( '--ignore-hash', @@ -1014,7 +1015,9 @@ def integration_test_single_executor( ) ignore_hash = executor_integration_conf.get('ignore-hash', False) - jsonnet_files = list(CASES_DIR.glob('**/*.jsonnet')) + jsonnet_files = list( + x for x in CASES_DIR.glob('**/*.jsonnet') if not x.name.startswith('_') + ) jsonnet_files.sort() # pre-import @@ -1041,6 +1044,11 @@ def integration_test_single_executor( future.result() +def _gvm32_of_str(text: str) -> str: + """Hash `text` the way the executor names content: gvm32(sha3_256(utf-8)).""" + return genvm_tool.gvm32.encode(hashlib.sha3_256(text.encode('utf-8')).digest()) + + def _single_integration_test( ctx: genvm_tool.tests.stage.collection.Context, jsonnet_file: Path, @@ -1089,9 +1097,20 @@ def _single_integration_test( ) return - # Parse jsonnet at collection time + # Parse jsonnet at collection time. + # + # `native_callbacks` exposes helpers to jsonnet as `std.native('')`. + # Note the binding only marshals *primitives* β€” passing an array or object + # to a native function is a jsonnet runtime error, so anything structured + # has to cross as a JSON string. jsonnet_result = _jsonnet.evaluate_file( - str(jsonnet_file), jpathdir=[str(TEMPLATES_DIR.parent)] + str(jsonnet_file), + jpathdir=[str(TEMPLATES_DIR.parent)], + native_callbacks={ + # std.native('gvm32')('...') -> gvm32(sha3_256(utf8(s))), the id form + # used by content-addressed names such as `custom:`. + 'gvm32': (('text',), _gvm32_of_str), + }, ) jsonnet_parsed = json.loads(jsonnet_result) extra_tags = jsonnet_parsed.get('tags', []) diff --git a/tests/runner/genvm_tool_plugins/ninja.py b/tests/runner/genvm_tool_plugins/ninja.py index 9a274875..f4a750a1 100644 --- a/tests/runner/genvm_tool_plugins/ninja.py +++ b/tests/runner/genvm_tool_plugins/ninja.py @@ -35,6 +35,17 @@ ) +def target_dir_for_line(rust_target_dir: Path, key: str) -> Path: + """Cargo target dir for one executor line. + + Lines ship crates with identical names and versions; cargo's artifact hash + does not cover where a package came from, so in a shared target dir two lines + write the same ``libfoo-.rlib`` and silently overwrite each other. + Everything that runs cargo resolves through here. + """ + return rust_target_dir / key + + # --- ninja DSL ------------------------------------------------------------- @@ -258,17 +269,24 @@ def rule(self, name: str, **props) -> None: def build(self, rule: str, *outputs) -> Build: return Build(self, rule, outputs) - def codegen(self, out: Path, lang: str, data: Path) -> None: + def codegen( + self, out: Path, lang: str, data: Path, extra_flags: list[str] | None = None + ) -> None: """Emit one `codegen` edge (`genvm-tool codegen --lang `). Requires the `codegen` rule. The generated file depends on its data JSON and on the codegen backend sources (``codegen_deps``, set by the command) so an - edit to either regenerates it. + edit to either regenerates it. ``extra_flags`` are appended to the command + verbatim (e.g. ``--rst-anchor-ns``). """ b = self.build('codegen', out) b.add_dependency(data) b.add_implicit_dependency(getattr(self, 'codegen_deps', [])) b.var('lang', lang) + # `var` shell-quotes its value, so an empty one would become a literal '' + # argument on every codegen edge. + if extra_flags: + b.var('extra_flags', ' '.join(extra_flags)) b.finish() def install(self, frm: str, to: str, phony: Build) -> None: @@ -377,6 +395,8 @@ class LineContext: data_phony: Build all_bin: Build + is_support_only: bool + @property def exec_root(self) -> Path: return self.source_dir / self.exec_rel @@ -388,12 +408,14 @@ def out_exec(self) -> str: @property def line_target_dir(self) -> Path: - """Per-line cargo target dir; lines share crate names so each needs its own.""" - return self.n.rust_target_dir / self.key + """This line's cargo target dir; see :func:`target_dir_for_line`.""" + return target_dir_for_line(self.n.rust_target_dir, self.key) - def codegen(self, out: Path, lang: str, data: Path) -> None: + def codegen( + self, out: Path, lang: str, data: Path, extra_flags: list[str] | None = None + ) -> None: """Emit a codegen edge and register its output on the codegen phony.""" - self.n.codegen(out, lang, data) + self.n.codegen(out, lang, data, extra_flags) self.codegen_phony.add_dependency(out) def register_standard_codegen(self) -> None: @@ -414,12 +436,19 @@ def register_standard_codegen(self) -> None: 'rust', data / 'host-fns.json', ) + pending_abi = self.exec_root / 'executor/codegen/data/public-abi-pending.json' + if pending_abi.exists(): + self.codegen( + self.exec_root / 'executor/crates/common/src/public_abi_pending.rs', + 'rust', + self.exec_root / 'executor/codegen/data/public-abi-pending.json', + ) def register_standard_crates(self) -> None: """Build this line's `genvm` binary plus its `common`/`sdk-rs` crates. - calldata + calldata-derive are shared, manager-root crates (registered - once by the command); they are no longer per-line. + `calldata` and `calldata-derive` live in this line's tree too but get no + edge of their own β€” they are path deps of the crates listed here. """ td = self.line_target_dir self.n.register_cargo( diff --git a/tests/runner/genvm_tool_plugins/pytest.py b/tests/runner/genvm_tool_plugins/pytest.py index 09c06ae4..5cb172f0 100644 --- a/tests/runner/genvm_tool_plugins/pytest.py +++ b/tests/runner/genvm_tool_plugins/pytest.py @@ -18,10 +18,6 @@ local_ctx = genvm_tool.tests.stage.configuration.current_context() -cargo_target_dir = local_ctx.shared.root_dir.joinpath( - 'build', 'ya-build', 'rust-target' -) - # The Python unit-test interpreter is provided by a standalone, pinned nix flake # (executors/v0.3.x/support/nix/py-test) that replaced genlayer-py-std's poetry # env. It is realised lazily at run time (not collection time, so `test show` diff --git a/tests/system/permits/test.py b/tests/system/permits/test.py index 23c18b7d..db38354e 100644 --- a/tests/system/permits/test.py +++ b/tests/system/permits/test.py @@ -28,6 +28,7 @@ 'origin_address': Address('0x' + '01' * 20), 'signer_address': Address('0x' + '01' * 20), 'chain_id': 0, + 'value': 0, 'is_init': True, } diff --git a/tests/templates/util.jsonnet b/tests/templates/util.jsonnet index 6102c634..676de7b7 100644 --- a/tests/templates/util.jsonnet +++ b/tests/templates/util.jsonnet @@ -1,6 +1,6 @@ local addOutPaths(entry, data) = local parentPath = data.parentPath; - local treePath = if parentPath == null then std.toString(data.i) else parentPath + '_' + std.toString(data.i); + local treePath = (if parentPath == null then "" else parentPath + "_") + data.slug; local suff = '.' + treePath; { entry: @@ -63,7 +63,7 @@ local expandModes(entry, parentMainTreePath) = local recurse(entries, data) = std.flatMap(function(i) local res = std.foldl((function(acc, nxt) nxt(acc.entry, acc.data)), [addOutPaths], { - data: data {i: i}, + data: data { slug: if std.objectHas(entries[i], 'slug') then entries[i].slug else std.toString(i) }, entry: entries[i], }); From 64de46028afba5f36dcb62401c4c052ebf6456a1 Mon Sep 17 00:00:00 2001 From: kp2pml30 Date: Wed, 29 Jul 2026 14:08:17 +0900 Subject: [PATCH 05/45] =?UTF-8?q?chore(ci):=20rework=20branch=20automation?= =?UTF-8?q?=20and=20gate=20the=20fuzz=20examples=20=F0=9F=8F=97=EF=B8=8F?= =?UTF-8?q?=20(#10)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(implementation): gate the fuzz examples behind a feature πŸ› * chore(ci): gate merge on a maintainer approval instead of the rtm label πŸ”’οΈ * chore(ci): stage merge objects and PR branches before moving dev branches πŸ—οΈ * feat(ci): publish the docs site nightly to sdk.genlayer.com πŸš€ * fix(ci): paginate the approval review query πŸ”’οΈ * fix(ci): keep a partial merge landing repairable πŸ› * fix(ci): stop the nightly docs deploy clearing the preferred version πŸ› * chore(ci): serialise merges repository-wide πŸ—οΈ * refactor(ci): build the squash message from commit subjects ♻️ * chore: bump executor gitlinks for the fuzz example gating πŸ”’οΈ * chore(genvm-tool): drop the unwired test-always-fail-module flag 🧹 * chore(ci): harden the workflows and unify the ci helpers πŸ”’οΈβ™»οΈ * fix(ci): run the pr-scoped initial checks on dispatched runs too πŸ”’οΈ * fix(ci): nest coderabbit base_branches under auto_review πŸ› * chore: bump the v0.3 executor for the vision test rewrite βœ… * fix(ci): close the review-gate and panel gaps in pr automation πŸ› * feat(ci): add an admin-only /genvm-force-merge command ✨ --- .coderabbit.yaml | 24 +- .github/actions/get-src/action.yaml | 15 +- .github/workflows/branch_executor_prs.yaml | 7 +- .github/workflows/branch_force_merge.yaml | 116 +++ .github/workflows/branch_forward.yaml | 10 +- .github/workflows/branch_merge_into_dev.yaml | 97 +- .github/workflows/branch_pr_actions.yaml | 14 +- .github/workflows/branch_pr_checklist.yaml | 28 +- .github/workflows/branch_provision.yaml | 3 +- .../branch_provision_executor_prs.yaml | 14 +- .github/workflows/branch_rebase_watch.yaml | 71 ++ .github/workflows/branch_retarget.yaml | 3 +- .github/workflows/docs_nightly.yaml | 97 ++ .github/workflows/incl_initial.yaml | 78 +- .../workflows/incl_release_build_test.yaml | 8 + .../incl_release_build_test_cell_build.yaml | 7 + .../incl_release_build_test_cell_test.yaml | 7 + .github/workflows/queue.yaml | 40 +- .github/workflows/queue_test_cell.yaml | 7 + .github/workflows/release.yaml | 4 +- .github/workflows/webdriver-image.yaml | 1 + docs/website/generate.py | 31 +- executors/v0.2.x | 2 +- executors/v0.3.x | 2 +- implementation/.ya-test-config.json | 2 +- implementation/Cargo.toml | 6 + support/ci/__main__.py | 4 + support/ci/behind.py | 72 ++ support/ci/gh_common.py | 198 ++++- support/ci/pipelines/build.py | 7 +- support/ci/pipelines/checks.py | 147 +++- support/ci/pipelines/docs.py | 23 +- support/ci/pipelines/release.py | 15 +- support/ci/pipelines/tests.py | 16 +- support/ci/pr_branches_info.py | 78 +- support/ci/tools/branches.py | 19 +- support/ci/tools/deploy_docs.py | 142 +++ support/ci/tools/genvm_merge_into_dev.py | 825 +++++++++++++----- support/ci/tools/make_release_notes.py | 18 +- support/ci/tools/open_executor_prs.py | 7 +- support/ci/tools/pr_action_panel.py | 214 ++++- support/ci/tools/pr_branches.py | 18 +- support/ci/tools/rebase_watch.py | 338 +++++++ support/ci/tools/versions.py | 27 +- support/scripts/check-commit-message.py | 7 +- support/scripts/ci-changes.py | 25 +- .../genvm-tool/genvm_tool/cmd_configure.py | 7 - 47 files changed, 2467 insertions(+), 434 deletions(-) create mode 100644 .github/workflows/branch_force_merge.yaml create mode 100644 .github/workflows/branch_rebase_watch.yaml create mode 100644 .github/workflows/docs_nightly.yaml create mode 100644 support/ci/behind.py create mode 100644 support/ci/tools/deploy_docs.py create mode 100644 support/ci/tools/rebase_watch.py diff --git a/.coderabbit.yaml b/.coderabbit.yaml index 7c65a682..5959a757 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -1,12 +1,20 @@ +# yaml-language-server: $schema=https://coderabbit.ai/integrations/schema.v2.json reviews: - # Auto-review PRs targeting any long-lived branch: main, a release branch - # (v0.6) or its dev branch (v0.6-dev). Feature/topic branches are reviewed - # through the base they target. Patterns are regexes; keep them free of - # optional groups, an `(-dev)?` suffix here did not match `v0.6-dev`. - base_branches: - - "main" - - "v\\d+\\.\\d+" - - "v\\d+\\.\\d+-dev" + auto_review: + # `base_branches` MUST live here, under `auto_review` β€” not directly under + # `reviews`. The schema defines no `reviews.base_branches`, and `reviews` + # does not forbid extra keys, so a misplaced copy is silently ignored: + # CodeRabbit reads the file, falls back to default-branch-only, and skips + # every PR targeting a dev branch with "Auto reviews are disabled on + # base/target branches other than the default branch". + # + # Auto-review PRs targeting any long-lived branch: main, a release branch + # or its dev branch. Feature/topic branches are reviewed through the base + # they target. Patterns are regexes. + base_branches: + - "main" + - "v\\d+\\.\\d+" + - "v\\d+\\.\\d+-dev" path_filters: - "!**/*.onnx" - "!**/*.txt" diff --git a/.github/actions/get-src/action.yaml b/.github/actions/get-src/action.yaml index e0ae6c74..76ca7dba 100644 --- a/.github/actions/get-src/action.yaml +++ b/.github/actions/get-src/action.yaml @@ -12,6 +12,14 @@ inputs: github_token: description: GitHub token for nix setup required: false + pr: + description: >- + PR number to diff against, overriding `github.event.pull_request.number`. + A caller invoked by `workflow_dispatch` has no pull_request payload but + does know the number, and without this its `changes` output would come + back empty β€” silently checking nothing. + required: false + default: "" outputs: changes: description: >- @@ -28,12 +36,13 @@ runs: git config --global user.email "ci@genlayerlabs.com" git config --global user.name "CI worker" python3 support/scripts/get-all-git.py "--third-party=${{ inputs.third_party }}" - shell: bash -ex {0} + shell: bash -exo pipefail {0} - name: compute per-repo changes id: changes - shell: bash + shell: bash -exo pipefail {0} env: - GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }} + GITHUB_PR_NUMBER: ${{ inputs.pr || github.event.pull_request.number }} + GH_TOKEN: ${{ inputs.github_token }} run: | cd "$GITHUB_WORKSPACE" python3 support/scripts/ci-changes.py --github-output diff --git a/.github/workflows/branch_executor_prs.yaml b/.github/workflows/branch_executor_prs.yaml index 2c9f109d..f6f7d971 100644 --- a/.github/workflows/branch_executor_prs.yaml +++ b/.github/workflows/branch_executor_prs.yaml @@ -35,16 +35,21 @@ concurrency: cancel-in-progress: false permissions: + # Explicit, though this job only ever checks out the base repo: a `permissions` + # block sets every unlisted scope to `none`, so the checkout works today only + # because the repo is public. Naming it keeps that from being load-bearing. + contents: read pull-requests: write issues: write defaults: run: - shell: bash -x {0} + shell: bash -exo pipefail {0} jobs: open: runs-on: ubuntu-latest + timeout-minutes: 15 steps: - uses: actions/checkout@v4 - name: Open and link executor PR(s) diff --git a/.github/workflows/branch_force_merge.yaml b/.github/workflows/branch_force_merge.yaml new file mode 100644 index 00000000..842fb809 --- /dev/null +++ b/.github/workflows/branch_force_merge.yaml @@ -0,0 +1,116 @@ +name: branch / force merge PR into dev + +# `/genvm-force-merge` in a PR comment: land the PR without requiring the +# review, full-CI and E2E gates. For when those signals cannot be obtained at +# all (the E2E pipeline is down, the org's Actions budget is exhausted, ...) +# rather than for when they are red β€” a forced merge lands a commit no green run +# ever covered, and says so on the PR. +# +# Only a REPO ADMIN (or org owner, whom GitHub reports as admin here) may use +# it. The panel's Merge box stays the normal path and keeps every gate. +# +# Authorization runs HERE, in the workflow file, not in the merge script: an +# `issue_comment` workflow always runs from the default branch, so this YAML is +# trusted, whereas the merge job checks out the PR head and runs its scripts. +# The merge workflow re-checks both the admin role and the head sha in its own +# trusted guard step before any PR code is fetched. +# +# The command is bound to the head commit it was typed on: the admin authorizes +# a specific commit, not the branch. A push landing between the comment and the +# merge aborts it rather than riding in on the authorization. +# +# Written as its own workflow rather than a branch of branch_pr_actions.yaml +# because it triggers on a different event (`issue_comment: created`, not +# `edited`) and needs an authorization rule of its own. +# +# Deliberately NO workflow-level concurrency: the group would be taken before +# the job's `if` runs, so every unrelated comment on any PR would enter it and +# could evict a pending real one (a `concurrency` group keeps a single pending +# run, `cancel-in-progress: false` or not). The merge itself is already +# serialised repository-wide by branch_merge_into_dev.yaml's own group. + +on: + issue_comment: + types: [created] + +permissions: + contents: read + pull-requests: write + issues: write + checks: read + actions: read + +defaults: + run: + shell: bash -exo pipefail {0} + +jobs: + authorize: + # `startsWith` is only a cheap pre-filter (expressions have no regex); the + # step below requires the command to be the whole first line, so neither a + # quoted reply nor `/genvm-force-merge-later` fires it. The `ci-safe` gate is + # the same one the merge workflow re-checks β€” the merge runs PR code. + if: > + github.event.issue.pull_request && + startsWith(github.event.comment.body, '/genvm-force-merge') && + contains(github.event.issue.labels.*.name, 'ci-safe') + runs-on: ubuntu-latest + timeout-minutes: 15 + outputs: + ok: ${{ steps.check.outputs.ok }} + head_sha: ${{ steps.check.outputs.head_sha }} + steps: + # No checkout: nothing in this job may come from the PR. + - name: Require an exact command from a repo admin + id: check + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SENDER: ${{ github.event.comment.user.login }} + COMMENT_ID: ${{ github.event.comment.id }} + PR_NUMBER: ${{ github.event.issue.number }} + # Through the environment, never interpolated into the script: the + # body is attacker-controlled text. + BODY: ${{ github.event.comment.body }} + run: | + echo "ok=false" >> "$GITHUB_OUTPUT" + + # The command must be the entire first line. `startsWith` above would + # also accept `/genvm-force-merge-not-yet` or a longer sentence. + first="$(printf '%s' "$BODY" | head -n1 | tr -d '\r' | xargs || true)" + if [ "$first" != '/genvm-force-merge' ]; then + echo "first line is '$first', not the bare command; ignoring" + exit 0 + fi + + # A non-collaborator 403/404s here, which is "no role", not an error. + perm="$(gh api "repos/$GITHUB_REPOSITORY/collaborators/$SENDER/permission" \ + --jq '.permission' 2>/dev/null || echo none)" + if [ "$perm" != admin ]; then + echo "::warning::$SENDER has '$perm' access; /genvm-force-merge is admin-only" + # React rather than comment: the command is visible to anyone on a + # public PR, and a refusal comment per attempt is a spam vector. + gh api --method POST \ + "repos/$GITHUB_REPOSITORY/issues/comments/$COMMENT_ID/reactions" \ + -f 'content=-1' + exit 0 + fi + + # Bind the authorization to the commit it was typed on. + head_sha="$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER" --jq '.head.sha')" + { + echo "ok=true" + echo "head_sha=$head_sha" + } >> "$GITHUB_OUTPUT" + gh api --method POST \ + "repos/$GITHUB_REPOSITORY/issues/comments/$COMMENT_ID/reactions" \ + -f 'content=+1' + + merge: + needs: authorize + if: needs.authorize.outputs.ok == 'true' + uses: ./.github/workflows/branch_merge_into_dev.yaml + with: + pr_number: ${{ github.event.issue.number }} + force_actor: ${{ github.event.comment.user.login }} + force_head_sha: ${{ needs.authorize.outputs.head_sha }} + secrets: inherit diff --git a/.github/workflows/branch_forward.yaml b/.github/workflows/branch_forward.yaml index bc9c6816..63085761 100644 --- a/.github/workflows/branch_forward.yaml +++ b/.github/workflows/branch_forward.yaml @@ -29,11 +29,12 @@ concurrency: defaults: run: - shell: bash -x {0} + shell: bash -exo pipefail {0} jobs: forward: runs-on: ubuntu-latest + timeout-minutes: 15 steps: - uses: actions/checkout@v4 with: @@ -50,8 +51,11 @@ jobs: # authoritative alias), NOT the pushed branch's own copy β€” an older # dev branch carries a stale version and would otherwise try to drag # main backwards once a newer train exists. - git show origin/main:.genvm-monorepo-root > /tmp/main-monorepo-root.json - MANAGER="$(MONOREPO_ROOT=/tmp/main-monorepo-root.json ./support/ci/run.sh tool branch-versions manager)" + # $RUNNER_TEMP, not /tmp: the runner wipes it between jobs, so nothing + # is left behind on a self-hosted box. + root_json="${RUNNER_TEMP}/main-monorepo-root.json" + git show origin/main:.genvm-monorepo-root > "$root_json" + MANAGER="$(MONOREPO_ROOT="$root_json" ./support/ci/run.sh tool branch-versions manager)" if [ "$BRANCH" != "v${MANAGER}-dev" ]; then echo "Pushed branch ${BRANCH} is not the latest dev branch (v${MANAGER}-dev); nothing to do" exit 0 diff --git a/.github/workflows/branch_merge_into_dev.yaml b/.github/workflows/branch_merge_into_dev.yaml index 4f74d4fa..59b37597 100644 --- a/.github/workflows/branch_merge_into_dev.yaml +++ b/.github/workflows/branch_merge_into_dev.yaml @@ -9,11 +9,15 @@ name: branch / merge PR into dev # # Gates (all required): # 1. base branch is a v-dev branch -# 2. PR carries the `rtm` (ready-to-merge) label +# 2. a maintainer (push access) approved the head commit; pushing new commits +# revokes it, since the approval is matched against that exact sha # 3. full GenVM CI (queue.yaml) concluded success on the head commit # 4. the cross-repo E2E check concluded success on the head commit # 5. the PR is 0 commits behind base (head already contains base tip) # +# `force_actor` (set only by branch_force_merge.yaml, for `/genvm-force-merge` +# from a repo admin) waives gates 2-4; 1 and 5 always hold. +# # Merge strategy: one commit per merged PR in every repo. A manager PR that is # a single commit and rewrites no gitlink is fast-forwarded as-is (SHA # preserved); otherwise its commits are squashed onto base. Each executor line @@ -23,8 +27,10 @@ name: branch / merge PR into dev # # Executor mirror: the landed manager commit gitlinks into each active executor # line; the script squashes each line onto its SAME-NAMED -dev branch. It -# plans and fast-forward-checks every side up front, then pushes the executors -# first (the dependencies) and the manager second. +# plans and fast-forward-checks every side up front, uploads every object under a +# scratch ref, moves the PR head branches onto what is about to land, and only then +# advances the target branches: the manager first (the one push that can lose a +# race), then each -dev. # # The dev branches are protected; the default GITHUB_TOKEN cannot push to # them. We push over SSH with the GENVM_CI_PRIVATE_KEY deploy key (on the @@ -45,6 +51,25 @@ on: description: number of the PR to merge type: string required: true + force_actor: + description: >- + Login of the repo admin who ran `/genvm-force-merge`. Non-empty means + gates 2-4 (review, full CI, E2E) are skipped and the skip is announced + on the PR under this login. ONLY branch_force_merge.yaml may set it, + and only after checking the admin role in its own trusted YAML β€” the + merge script runs from the PR head and cannot authorize itself. + type: string + required: false + default: "" + force_head_sha: + description: >- + Head commit the `/genvm-force-merge` command was typed on. Required + with `force_actor`: the admin authorizes a commit, not a branch, so a + push landing in between must abort the merge rather than ride in on + the authorization. + type: string + required: false + default: "" permissions: contents: read @@ -55,16 +80,29 @@ permissions: defaults: run: - shell: bash -x {0} + shell: bash -exo pipefail {0} env: - # check-run name (case-insensitive substring) the genlayer-e2e pipeline - # posts on the head commit. Adjust if that pipeline renames its check. - E2E_CHECK_PATTERN: "e2e" + # Exact check-run name, and the slug of the app that posts it, for the E2E + # check on the head commit. Both must match: a name alone is not a protected + # namespace, so any app (or a workflow in the PR) could post a green check + # under it. Adjust if that pipeline is renamed or re-homed. + E2E_CHECK_NAME: "E2E Tests" + E2E_CHECK_APP_SLUG: "ci-core-e2e-runner" jobs: merge: runs-on: ubuntu-latest + timeout-minutes: 30 + # A constant group, so at most ONE merge runs repository-wide, whatever PR or + # dev branch it targets. Two merges racing would each pass their own + # 0-behind check and then push a base the other never saw; the loser's + # non-force push is rejected mid-way, after the executor lines have already + # been advanced. Serialising is cheaper than making that recoverable. + # Never cancel: an interrupted merge can leave the lines ahead of the manager. + concurrency: + group: merge-into-dev + cancel-in-progress: false steps: # Gate: this job checks out and runs the PR's own CI scripts while the # deploy keys are in scope, so refuse unless the PR is `ci-safe`. Uses the @@ -74,12 +112,47 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} PR_NUMBER: ${{ inputs.pr_number }} run: | - if ! gh api "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/labels" \ - --jq '.[].name' | grep -qxF 'ci-safe'; then + # Collect first, match second: under `pipefail` a `grep -q` that exits + # early can SIGPIPE the `gh` feeding it, and the resulting non-zero + # pipeline would read as "no ci-safe label" and refuse a valid merge. + names="$(gh api --paginate "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/labels?per_page=100" --jq '.[].name')" + if ! grep -qxF 'ci-safe' <<<"$names"; then echo "::error::PR #$PR_NUMBER lacks the 'ci-safe' label; refusing to merge (this job runs PR code)." exit 1 fi + # A forced merge waives the gates that prove the commit good, so its + # authorization is re-derived here β€” in the reusable workflow's own YAML, + # with the plain GITHUB_TOKEN, before any PR code is fetched. `force_actor` + # is an input like any other: a caller that is not branch_force_merge.yaml + # could pass an arbitrary login, and the merge script itself comes from the + # PR head, so neither is a place this can be trusted. + - name: Re-check the force-merge authorization + if: inputs.force_actor != '' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ inputs.pr_number }} + FORCE_ACTOR: ${{ inputs.force_actor }} + FORCE_HEAD_SHA: ${{ inputs.force_head_sha }} + run: | + perm="$(gh api "repos/$GITHUB_REPOSITORY/collaborators/$FORCE_ACTOR/permission" \ + --jq '.permission' 2>/dev/null || echo none)" + if [ "$perm" != admin ]; then + echo "::error::force merge claims '$FORCE_ACTOR', who has '$perm' access, not admin." + exit 1 + fi + if [ -z "$FORCE_HEAD_SHA" ]; then + echo "::error::force merge without force_head_sha; refusing." + exit 1 + fi + head="$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER" --jq '.head.sha')" + if [ "$head" != "$FORCE_HEAD_SHA" ]; then + gh pr comment "$PR_NUMBER" --repo "$GITHUB_REPOSITORY" --body \ + "❌ Force merge aborted: the head moved from \`${FORCE_HEAD_SHA:0:12}\` (the commit \`$FORCE_ACTOR\` authorized) to \`${head:0:12}\`. Re-issue \`/genvm-force-merge\` if that commit is also meant to land unchecked." + echo "::error::PR head moved since the force-merge command; refusing." + exit 1 + fi + # Check out the PR head (so the merge runs the PR's own scripts) with the # manager deploy key so the (protected) dev branch push succeeds. # Submodules are initialised in the next step with a SEPARATE executor key @@ -101,7 +174,12 @@ jobs: run: | install -m700 -d ~/.ssh key=~/.ssh/executor_ci + # The job-wide shell is `bash -x`, and Actions masks a multi-line secret + # line by line β€” an xtrace of the write would print PEM fragments it + # does not recognise. Trace goes off for exactly that one command. + set +x printf '%s\n' "$EXECUTOR_DEPLOY_KEY" > "$key" + set -x chmod 600 "$key" ssh="ssh -i $key -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new" # The manager release line ships every active executor line, so the @@ -119,4 +197,5 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} PR_NUMBER: ${{ inputs.pr_number }} + FORCE_ACTOR: ${{ inputs.force_actor }} run: ./support/ci/run.sh tool genvm-merge-into-dev diff --git a/.github/workflows/branch_pr_actions.yaml b/.github/workflows/branch_pr_actions.yaml index 8e05db5d..fc2476c8 100644 --- a/.github/workflows/branch_pr_actions.yaml +++ b/.github/workflows/branch_pr_actions.yaml @@ -32,17 +32,23 @@ concurrency: defaults: run: - shell: bash -x {0} + shell: bash -exo pipefail {0} jobs: dispatch: - # Only the panel comment, only on a PR, only when the PR is `ci-safe` + # Only a panel-shaped comment, only on a PR, only when the PR is `ci-safe` # (we check out and run its scripts, so an unvetted PR must not reach here). + # + # NOTE: this matches the MARKER IN THE COMMENT BODY, which anyone can put in + # a comment of their own. It is a cheap pre-filter, NOT authorization. The + # tool re-checks that the bot authored the comment and that the editor has + # write access β€” see `Who may tick a box` in pr_action_panel.py. if: > github.event.issue.pull_request && contains(github.event.comment.body, '') && contains(github.event.issue.labels.*.name, 'ci-safe') runs-on: ubuntu-latest + timeout-minutes: 15 outputs: merge: ${{ steps.act.outputs.merge }} steps: @@ -52,6 +58,10 @@ jobs: - uses: actions/checkout@v4 with: ref: refs/pull/${{ github.event.issue.number }}/head + # The handler talks to GitHub through `gh` (GH_TOKEN below) and never + # pushes, so the checkout credential is not needed β€” and leaving it in + # .git/config would hand it to the PR code this job runs. + persist-credentials: false - name: Handle ticked boxes id: act env: diff --git a/.github/workflows/branch_pr_checklist.yaml b/.github/workflows/branch_pr_checklist.yaml index 657acb19..9096660a 100644 --- a/.github/workflows/branch_pr_checklist.yaml +++ b/.github/workflows/branch_pr_checklist.yaml @@ -24,11 +24,12 @@ permissions: defaults: run: - shell: bash -x {0} + shell: bash -exo pipefail {0} jobs: post: runs-on: ubuntu-latest + timeout-minutes: 15 env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} PR: ${{ github.event.pull_request.number }} @@ -38,8 +39,14 @@ jobs: run: | # Only post once: skip if a panel (identified by the HTML marker) # already exists on the PR. `edited` can fire repeatedly. - if gh pr view "$PR" --repo "$GITHUB_REPOSITORY" --json comments \ - --jq '.comments[].body' | grep -qF ''; then + # + # Collect first, match second β€” NOT `gh ... | grep -qF`. Under + # `pipefail`, `grep -q` exiting at the first match can SIGPIPE a `gh` + # still writing a large comment list, which makes the pipeline + # non-zero and sends us down the "no panel yet" path β€” posting a + # second panel on exactly the busiest PRs. + comments="$(gh pr view "$PR" --repo "$GITHUB_REPOSITORY" --json comments --jq '.comments[].body')" + if grep -qF '' <<<"$comments"; then echo "action panel already present; skipping" exit 0 fi @@ -49,12 +56,17 @@ jobs: Tick a box to run it (the box unticks itself when handled). Actions only run while the PR has the **`ci-safe`** label. - - [ ] Force run full tests - - [ ] Rerun full tests - - [ ] Provision executor PRs - - [ ] Merge into dev + - Full GenVM CI runs only when **`rtm`** or **`run-full-tests`** is set β€” "Force run full tests" is a sticky toggle for `run-full-tests`. Adding **`rtm`** marks the PR ready-to-merge and also runs full tests. **Merge** requires: `rtm`, green full tests, green E2E, and the branch 0 commits behind. + + - [ ] Force run full tests + - [ ] Rerun full tests + - [ ] Provision executor PRs + - [ ] Merge into dev + + Full GenVM CI runs only when **`run-full-tests`** is set β€” "Force run full tests" is a sticky toggle for it. **Merge** requires: an approving review from a maintainer on the head commit (pushing revokes it), green full tests, green E2E, and the branch 0 commits behind. + + A **repo admin** can comment `/genvm-force-merge` to land the PR without the review, full-CI and E2E gates β€” for when those signals are unobtainable, not when they are red. The base, title and 0-behind checks still apply, and the skip is recorded on the PR. EOF )" diff --git a/.github/workflows/branch_provision.yaml b/.github/workflows/branch_provision.yaml index efe29277..5019f22a 100644 --- a/.github/workflows/branch_provision.yaml +++ b/.github/workflows/branch_provision.yaml @@ -20,11 +20,12 @@ permissions: defaults: run: - shell: bash -x {0} + shell: bash -exo pipefail {0} jobs: provision: runs-on: ubuntu-latest + timeout-minutes: 15 steps: - uses: actions/checkout@v4 with: diff --git a/.github/workflows/branch_provision_executor_prs.yaml b/.github/workflows/branch_provision_executor_prs.yaml index 5dd2ecae..b3f6aff2 100644 --- a/.github/workflows/branch_provision_executor_prs.yaml +++ b/.github/workflows/branch_provision_executor_prs.yaml @@ -31,11 +31,12 @@ permissions: defaults: run: - shell: bash -x {0} + shell: bash -exo pipefail {0} jobs: provision: runs-on: ubuntu-latest + timeout-minutes: 15 steps: # Gate: this job checks out and runs the PR's scripts with the executor # write PAT in scope, so refuse unless the PR is `ci-safe`. Uses the plain @@ -45,8 +46,11 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} PR_NUMBER: ${{ inputs.pr }} run: | - if ! gh api "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/labels" \ - --jq '.[].name' | grep -qxF 'ci-safe'; then + # Collect first, match second: under `pipefail` a `grep -q` that exits + # early can SIGPIPE the `gh` feeding it, and the resulting non-zero + # pipeline would read as "no ci-safe label" and refuse a valid run. + names="$(gh api "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/labels" --jq '.[].name')" + if ! grep -qxF 'ci-safe' <<<"$names"; then echo "::error::PR #$PR_NUMBER lacks the 'ci-safe' label; refusing to provision (this job runs PR code with the executor token)." exit 1 fi @@ -54,6 +58,10 @@ jobs: - uses: actions/checkout@v4 with: ref: refs/pull/${{ inputs.pr }}/head + # Provisioning goes through the executor token via `gh`, never `git + # push`, so don't leave the checkout credential in .git/config where + # the PR code this job runs could read it. + persist-credentials: false - name: Provision executor mirror PRs env: diff --git a/.github/workflows/branch_rebase_watch.yaml b/.github/workflows/branch_rebase_watch.yaml new file mode 100644 index 00000000..f18fce26 --- /dev/null +++ b/.github/workflows/branch_rebase_watch.yaml @@ -0,0 +1,71 @@ +name: branch / not-rebased watch + +# Advisory "your branch is stale" indicator. `initial / behind-check` only runs +# when the PR is pushed, so a PR that goes behind because the BASE moved stays +# green until someone ticks Merge and the fast-forward is refused. This watches +# the other edge. +# +# It gates nothing: it posts a red `not rebased` check-run (visible on the PR +# page) and a red `not rebased` label (the surface visible in the PR list), and +# clears both once the branch is 0 behind again. The enforcing checks are +# unchanged β€” `initial / behind-check` and the Merge action. +# +# Triggers: +# - push to a dev branch -> re-evaluate every open PR targeting it (this is +# the case nothing else covers); +# - pull_request_target on the PR itself -> clear the flag promptly after a +# rebase, and set it on a retarget (`edited` fires on a base change). +# +# pull_request_target, not pull_request: posting a check-run and editing labels +# needs a write-scoped token, which fork PRs do not get. Nothing here checks out +# or runs PR code β€” the checkout is the base repo's tree, and the only inputs +# are a PR number and a branch name. + +on: + push: + branches: ['v*-dev'] + # Deliberately NOT filtered on `branches`: a `branches` filter would skip the + # `edited` event that retargets a flagged PR AWAY from a dev branch, and no + # push sweep would ever see that PR again β€” its stale red flag would stick + # forever. The tool decides what a non-dev base means (clear, never set). + pull_request_target: + types: [opened, synchronize, reopened, edited] + +permissions: + contents: read + checks: write + pull-requests: write + issues: write + +# Serialise per base branch (push sweeps) or per PR: two overlapping sweeps would +# race on the same label. Never cancel β€” a cancelled sweep can leave a PR flagged +# after it was rebased. +concurrency: + group: rebase-watch-${{ github.event.pull_request.number || github.ref_name }} + cancel-in-progress: false + +defaults: + run: + shell: bash -exo pipefail {0} + +jobs: + watch: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - name: Flag PRs that are behind their base + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # On a push these are empty, so the tool sweeps every open PR on the + # branch that moved; on a PR event it evaluates just that PR. + PR_NUMBER: ${{ github.event.pull_request.number }} + BASE: ${{ github.event_name == 'push' && github.ref_name || '' }} + run: | + # `if`, not `[ ... ] && ...`: under `set -e` a false AND-list is the + # script's exit status, so the un-taken branch would abort the step. + args=() + if [ -n "$BASE" ]; then + args+=(--base "$BASE") + fi + ./support/ci/run.sh tool rebase-watch "${args[@]}" diff --git a/.github/workflows/branch_retarget.yaml b/.github/workflows/branch_retarget.yaml index 045929e6..acbba03b 100644 --- a/.github/workflows/branch_retarget.yaml +++ b/.github/workflows/branch_retarget.yaml @@ -20,7 +20,7 @@ permissions: defaults: run: - shell: bash -x {0} + shell: bash -exo pipefail {0} jobs: retarget: @@ -28,6 +28,7 @@ jobs: # version branch. if: github.event.pull_request.base.ref == 'main' runs-on: ubuntu-latest + timeout-minutes: 15 steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/docs_nightly.yaml b/.github/workflows/docs_nightly.yaml new file mode 100644 index 00000000..76505c55 --- /dev/null +++ b/.github/workflows/docs_nightly.yaml @@ -0,0 +1,97 @@ +name: docs / nightly deploy + +# Builds the docs site and publishes it to the sdk.genlayer.com Pages repo, +# which serves one directory per version out of `_site/` and lists them in +# `_site/versions.json` (the switcher every page links to). Nightly runs +# refresh `main`; `workflow_dispatch` can publish any other directory, which is +# how a release train (`v0.6.x`) gets its own copy. +# +# GitHub only schedules workflows from the DEFAULT branch, so this file has to +# reach it before the cron fires β€” merging it into a dev branch is not enough. +# +# Needs the `DOCS_DEPLOY_KEY` secret: a deploy key with write access to +# genlayerlabs/sdk.genlayer.com. Pushing there triggers that repo's own +# deploy-pages workflow, which is what actually publishes and purges the CDN. + +on: + schedule: + # 03:17 UTC β€” off the hour, where scheduling backlogs are smaller. + - cron: "17 3 * * *" + workflow_dispatch: + inputs: + version: + description: "directory to publish under (e.g. main, v0.6.x)" + required: false + default: main + preferred: + description: "make it the version the switcher highlights" + type: boolean + required: false + default: false + +# Two runs pushing to the site repo would race on the same branch; the loser +# would just fail, so serialize rather than cancel (a half-finished deploy is +# worse than a slow one). +concurrency: + group: docs-deploy + cancel-in-progress: false + +permissions: + contents: read + +env: + SITE_REPO: genlayerlabs/sdk.genlayer.com + DOCS_VERSION: ${{ inputs.version || 'main' }} + +# GitHub's stock `bash -e {0}` has no `pipefail`. +defaults: + run: + shell: bash -exo pipefail {0} + +jobs: + deploy: + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + + # The docs build spans the manager and every active executor line, each + # in its own `gen-docs` nix shell, so it needs submodules and nix. + - name: Get source + uses: ./.github/actions/get-src + with: + with_nix: "true" + github_token: ${{ secrets.GITHUB_TOKEN }} + + - name: Build docs + run: ./support/ci/run.sh pipeline docs + + - name: Checkout the site repo + uses: actions/checkout@v4 + with: + repository: ${{ env.SITE_REPO }} + path: build/site + ssh-key: ${{ secrets.DOCS_DEPLOY_KEY }} + + - name: Stage the built docs + run: | + ./support/ci/run.sh tool deploy-docs \ + --site build/site \ + --version "$DOCS_VERSION" \ + ${{ inputs.preferred && '--preferred' || '' }} + + # An unchanged nightly must not push an empty commit: the site repo's + # deploy workflow runs on every push, so that would redeploy and purge the + # CDN for nothing. + - name: Commit and push + working-directory: build/site + run: | + git config user.email "ci@genlayerlabs.com" + git config user.name "CI worker" + git add -A _site + if git diff --cached --quiet; then + echo "docs are unchanged; nothing to deploy" + exit 0 + fi + git commit -m "Deploy documentation version $DOCS_VERSION" + git push origin HEAD diff --git a/.github/workflows/incl_initial.yaml b/.github/workflows/incl_initial.yaml index 6aae6c66..800e7e3f 100644 --- a/.github/workflows/incl_initial.yaml +++ b/.github/workflows/incl_initial.yaml @@ -1,13 +1,33 @@ name: GenVM initial checks on: workflow_call: + inputs: + pr: + description: >- + PR number these checks validate. The caller resolves it from the event + (`github.event.pull_request.number` on a `pull_request`, `inputs.pr` on + the panel's `workflow_dispatch`), so the PR-scoped jobs below run for + BOTH β€” they used to key off `github.event_name == 'pull_request'` and + therefore skipped on every panel-dispatched run, while that run could + still conclude `success` and satisfy the Merge gate. Empty for a + non-PR run, which skips them. + type: string + required: false + default: "" -env: - GCS_BUCKET: "gh-af" +# A reusable workflow does not inherit the caller's `defaults`, so the shell +# is set here too: GitHub's stock `bash -e {0}` has no `pipefail`. +defaults: + run: + shell: bash -exo pipefail {0} +# Timeouts are set from measured job durations, generously (roughly 3x the +# observed maximum) so a slow-but-healthy run is never killed β€” the point is to +# cap a hung job well below the 6-hour default, not to enforce a budget. jobs: pre-commit: runs-on: ubuntu-latest + timeout-minutes: 30 steps: - uses: actions/checkout@v4 - name: Get source @@ -26,9 +46,13 @@ jobs: # `check-commit-message` is a commit-msg-stage hook, so the pre-commit run # above never fires it. Check every commit the PR adds, in the manager and in # each executor submodule that moved. + # + # This is the one initial check the Merge action does NOT re-run, so it must + # not be skippable on a run the Merge gate will accept. commit-messages: - if: github.event_name == 'pull_request' + if: inputs.pr != '' runs-on: ubuntu-latest + timeout-minutes: 15 steps: - uses: actions/checkout@v4 with: @@ -38,6 +62,11 @@ jobs: uses: ./.github/actions/get-src with: with_nix: "false" + pr: ${{ inputs.pr }} + # Needed even with with_nix=false: on a dispatched run there is no + # GITHUB_BASE_REF, so ci-changes.py resolves the base branch through + # the API and an unauthenticated `gh` exits 4. + github_token: ${{ secrets.GITHUB_TOKEN }} - name: Check commit messages env: CHANGES: ${{ steps.src.outputs.changes }} @@ -47,26 +76,39 @@ jobs: # contain the base tip (0 commits behind). We assert it here too (as part # of the always-on `initial` checks) so the gate goes red the moment the # branch falls behind, instead of surfacing only at merge time. + # + # The measurement itself lives in support/ci/behind.py, shared with + # rebase-watch and the Merge action, so the three cannot disagree about the + # same PR. behind-check: - if: github.event_name == 'pull_request' + if: inputs.pr != '' runs-on: ubuntu-latest + timeout-minutes: 15 steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Ensure 0 commits behind base env: - BASE: ${{ github.event.pull_request.base.ref }} - PR: ${{ github.event.pull_request.number }} - run: | - # Resolve base tip and PR head via explicit refs so this also - # works for fork PRs (head.sha is not fetchable by sha there). - git fetch --no-tags origin \ - "+refs/heads/${BASE}:refs/base" \ - "refs/pull/${PR}/head:refs/prhead" - if ! git merge-base --is-ancestor refs/base refs/prhead; then - BEHIND="$(git rev-list --count refs/prhead..refs/base)" - echo "::error::PR is ${BEHIND} commit(s) behind ${BASE}; update/rebase the branch so it is 0 behind before merging" - exit 1 - fi - echo "0 commits behind ${BASE}" + # BASE is resolved from the PR by the pipeline itself, so this works + # on a dispatch run where the event carries no pull_request payload. + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ inputs.pr }} + run: ./support/ci/run.sh pipeline behind-check + + # The Merge action squashes a PR into ` (#N)`, so the PR title becomes + # a commit subject verbatim and must satisfy the same rules as one. Nothing + # re-validates a generated squash message, so an invalid title would only + # surface as a commit the hook rejects β€” after it landed. + pr-title: + if: inputs.pr != '' + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - name: Check PR title is a valid commit subject + env: + # PR_TITLE likewise comes from the API, not the event payload. + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ inputs.pr }} + run: ./support/ci/run.sh pipeline pr-title diff --git a/.github/workflows/incl_release_build_test.yaml b/.github/workflows/incl_release_build_test.yaml index a705d155..c379db2a 100644 --- a/.github/workflows/incl_release_build_test.yaml +++ b/.github/workflows/incl_release_build_test.yaml @@ -39,9 +39,16 @@ on: description: the notes file's name inside artifact_notes value: ${{ jobs.plan.outputs.notes_file }} +# A reusable workflow does not inherit the caller's `defaults`, so the shell +# is set here too: GitHub's stock `bash -e {0}` has no `pipefail`. +defaults: + run: + shell: bash -exo pipefail {0} + jobs: plan: runs-on: ubuntu-latest + timeout-minutes: 15 outputs: tag: ${{ steps.plan.outputs.tag }} build_matrix: ${{ steps.plan.outputs.build_matrix }} @@ -70,6 +77,7 @@ jobs: notes: needs: plan runs-on: ubuntu-latest + timeout-minutes: 15 steps: - uses: actions/checkout@v4 with: diff --git a/.github/workflows/incl_release_build_test_cell_build.yaml b/.github/workflows/incl_release_build_test_cell_build.yaml index ed227565..3b3e3d89 100644 --- a/.github/workflows/incl_release_build_test_cell_build.yaml +++ b/.github/workflows/incl_release_build_test_cell_build.yaml @@ -27,10 +27,17 @@ on: required: true type: string +# A reusable workflow does not inherit the caller's `defaults`, so the shell +# is set here too: GitHub's stock `bash -e {0}` has no `pipefail`. +defaults: + run: + shell: bash -exo pipefail {0} + jobs: build: name: build / ${{ inputs.name }} runs-on: ${{ inputs.runs_on }} + timeout-minutes: 150 steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/incl_release_build_test_cell_test.yaml b/.github/workflows/incl_release_build_test_cell_test.yaml index 7cda7f53..e6e6dea2 100644 --- a/.github/workflows/incl_release_build_test_cell_test.yaml +++ b/.github/workflows/incl_release_build_test_cell_test.yaml @@ -35,10 +35,17 @@ on: required: true type: string +# A reusable workflow does not inherit the caller's `defaults`, so the shell +# is set here too: GitHub's stock `bash -e {0}` has no `pipefail`. +defaults: + run: + shell: bash -exo pipefail {0} + jobs: test: name: release-test / ${{ inputs.name }} runs-on: ${{ inputs.runs_on }} + timeout-minutes: 90 steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/queue.yaml b/.github/workflows/queue.yaml index bcc22f53..ad5df699 100644 --- a/.github/workflows/queue.yaml +++ b/.github/workflows/queue.yaml @@ -10,9 +10,8 @@ name: GenVM full # executor landability precondition) ALWAYS run, in parallel; # - `gate-check` aggregates them and plans the test matrix (plan-queue-matrix); # the heavy `matrix-tests` cells run only when a run-full-tests marker is set -# β€” the `rtm` (ready-to-merge) label or the `run-full-tests` label ("Force run -# full tests" sets the latter). Without a marker the plan is empty, so -# matrix-tests is skipped; +# β€” the `run-full-tests` label ("Force run full tests" sets it). Without a +# marker the plan is empty, so matrix-tests is skipped; # - without a marker `validate-end` fails, so the CI check stays red # until a full run has happened. # @@ -25,6 +24,11 @@ name: GenVM full on: pull_request: branches: ['v*-dev'] + # Deliberately NOT `edited`: this workflow's concurrency group cancels + # in-flight runs, so a typo fix in the PR body would kill a running + # matrix-tests cell (over an hour of work). The PR title can therefore be + # edited to something invalid after `initial / pr-title` went green β€” the + # Merge action re-checks it against the title it is about to land. types: [opened, synchronize, reopened, ready_for_review, labeled] workflow_dispatch: inputs: @@ -49,14 +53,18 @@ concurrency: defaults: run: - shell: bash -x {0} - -env: - GCS_BUCKET: "gh-af" + shell: bash -exo pipefail {0} jobs: initial: uses: ./.github/workflows/incl_initial.yaml + with: + # Resolved from whichever event triggered this run, so the PR-scoped + # checks run on BOTH a push and a panel dispatch. They used to test + # `github.event_name == 'pull_request'` themselves and so skipped on every + # dispatched run β€” while that run could still conclude `success` and be + # accepted by the Merge gate, which takes ANY green run on the head sha. + pr: ${{ github.event.pull_request.number || inputs.pr }} secrets: inherit # Precondition (run.sh tool pr-branches check): the PR must be landable via the @@ -82,10 +90,14 @@ jobs: (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) runs-on: ubuntu-latest + timeout-minutes: 15 steps: - uses: actions/checkout@v4 with: ref: refs/pull/${{ github.event.pull_request.number || inputs.pr }}/head + # Read-only gh API calls; nothing here pushes, so the checkout + # credential should not survive into the PR code this job runs. + persist-credentials: false - name: Executor mirror PRs are landed env: PR_NUMBER: ${{ github.event.pull_request.number || inputs.pr }} @@ -97,6 +109,7 @@ jobs: gate-check: needs: [initial, check-executor-prs] runs-on: ubuntu-latest + timeout-minutes: 15 if: ${{ always() }} outputs: matrix: ${{ steps.plan.outputs.matrix }} @@ -121,7 +134,7 @@ jobs: env: # Heavy cells run only with a marker; without one the plan is empty and # matrix-tests is skipped. - RUN_FULL_TESTS: ${{ github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'rtm') || contains(github.event.pull_request.labels.*.name, 'run-full-tests') }} + RUN_FULL_TESTS: ${{ github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run-full-tests') }} run: ./support/ci/run.sh pipeline plan-queue-matrix matrix-tests: @@ -155,13 +168,14 @@ jobs: validate-end: runs-on: ubuntu-latest + timeout-minutes: 15 if: ${{ always() }} needs: - initial - gate-check - matrix-tests env: - HAS_MARKER: ${{ github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'rtm') || contains(github.event.pull_request.labels.*.name, 'run-full-tests') }} + HAS_MARKER: ${{ github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run-full-tests') }} steps: - name: check run: | @@ -173,12 +187,16 @@ jobs: # No run-full-tests marker -> heavy tests were skipped; keep the # gate red so the PR is not mergeable until they run. if [ "$HAS_MARKER" != "true" ]; then - echo "::error::full tests have not run β€” add the 'rtm' label or check 'Force run full tests' on the action panel" + echo "::error::full tests have not run β€” check 'Force run full tests' on the action panel or add the 'run-full-tests' label" exit 1 fi results="${{ needs.matrix-tests.result }}" echo "$results" - if echo "$results" | grep -qiE '(failure|cancelled|skipped)'; then + # Herestring, not `echo "$results" |`: this is the last gate before a + # PR is mergeable, and under `pipefail` a non-zero left-hand side would + # make the condition false β€” i.e. fail OPEN, reporting green for a + # matrix that failed. Nothing should be able to swallow that. + if grep -qiE '(failure|cancelled|skipped)' <<<"$results"; then echo "::error::one or more full-test jobs failed/cancelled/skipped" exit 1 fi diff --git a/.github/workflows/queue_test_cell.yaml b/.github/workflows/queue_test_cell.yaml index 8cbda019..3b635957 100644 --- a/.github/workflows/queue_test_cell.yaml +++ b/.github/workflows/queue_test_cell.yaml @@ -29,10 +29,17 @@ on: type: boolean default: false +# A reusable workflow does not inherit the caller's `defaults`, so the shell +# is set here too: GitHub's stock `bash -e {0}` has no `pipefail`. +defaults: + run: + shell: bash -exo pipefail {0} + jobs: test: name: test / ${{ inputs.name }} runs-on: ${{ inputs.runs_on }} + timeout-minutes: 240 env: GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} OPENAIKEY: ${{ secrets.OPENAIKEY }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 5a83b27d..771fddf1 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -27,7 +27,7 @@ on: defaults: run: - shell: bash -x {0} + shell: bash -exo pipefail {0} permissions: actions: read @@ -40,6 +40,7 @@ jobs: # seconds instead of after three platform builds. check-tag: runs-on: ubuntu-latest + timeout-minutes: 15 steps: - uses: actions/checkout@v4 - name: Ensure tag is free @@ -66,6 +67,7 @@ jobs: release-publish: needs: [build-and-test] runs-on: ubuntu-latest + timeout-minutes: 30 permissions: contents: write # Needed for creating releases steps: diff --git a/.github/workflows/webdriver-image.yaml b/.github/workflows/webdriver-image.yaml index 788d7bff..09397889 100644 --- a/.github/workflows/webdriver-image.yaml +++ b/.github/workflows/webdriver-image.yaml @@ -16,6 +16,7 @@ jobs: docker-image: name: Build and push webdriver image runs-on: ubuntu-latest + timeout-minutes: 90 permissions: contents: read diff --git a/docs/website/generate.py b/docs/website/generate.py index 99c52dc9..13d5d6e0 100755 --- a/docs/website/generate.py +++ b/docs/website/generate.py @@ -20,9 +20,25 @@ # the manager's. All active lines are branches of ONE repo, so any of them sees # every executor tag; take the first. monorepo = json.loads(genvm_root_dir.joinpath('.genvm-monorepo-root').read_text()) -executor_dir = genvm_root_dir.joinpath( - 'executors', f'{monorepo["active-versions"][0]}.x' -) + + +def _newest_line() -> str: + """The newest active executor line, e.g. `v0.3`. + + Decided by version order, NOT by position: `active-versions` is hand-ordered + and `genvm-tool check-versions bump` APPENDS the new line, so after the next + bump `active-versions[0]` is no longer the newest. This must agree with + `support/ci/tools/versions.py:newest_line`, which is what decides the + sub-site this page is generated into β€” if the two disagree, this page renders + one line's runner data into another line's site. + """ + return max( + monorepo['active-versions'], + key=lambda v: tuple(int(p) for p in v.lstrip('v').split('-', 1)[0].split('.')[:2]), + ) + + +executor_dir = genvm_root_dir.joinpath('executors', f'{_newest_line()}.x') def executor_git(*args: str, check: bool = True) -> subprocess.CompletedProcess: @@ -92,10 +108,11 @@ def executor_git(*args: str, check: bool = True) -> subprocess.CompletedProcess: res = json.loads(proc.stdout) # docs.nix accumulates the runners of EVERY active line, but this page is -# per-line (it renders into active-versions[0]'s sub-site). Post-split the lines -# are independent β€” v0.2 is not "before" v0.3 β€” so scope to this line's own -# versions; otherwise the other lines' hashes leak in as bogus history. -_line = monorepo['active-versions'][0] +# per-line (it renders into the NEWEST line's sub-site β€” see `_newest_line`). +# Post-split the lines are independent β€” v0.2 is not "before" v0.3 β€” so scope to +# this line's own versions; otherwise the other lines' hashes leak in as bogus +# history. +_line = _newest_line() res = {v: r for v, r in res.items() if v.startswith(_line + '.')} # --- Post-process: sort by semver, enrich with descriptions, diffs, commits --- diff --git a/executors/v0.2.x b/executors/v0.2.x index b32cc4c9..934acbee 160000 --- a/executors/v0.2.x +++ b/executors/v0.2.x @@ -1 +1 @@ -Subproject commit b32cc4c96876fde1ec323f7cb40b9d31173bb2e5 +Subproject commit 934acbee6ed11cbe134e48fd9256a6a5651187df diff --git a/executors/v0.3.x b/executors/v0.3.x index 7c50d3d9..301d515f 160000 --- a/executors/v0.3.x +++ b/executors/v0.3.x @@ -1 +1 @@ -Subproject commit 7c50d3d9d80e7ea065f3560fbe790b6664acb233 +Subproject commit 301d515fe598113bb405e2540fa6cd0460d10845 diff --git a/implementation/.ya-test-config.json b/implementation/.ya-test-config.json index d0d712a3..e3659a87 100644 --- a/implementation/.ya-test-config.json +++ b/implementation/.ya-test-config.json @@ -3,7 +3,7 @@ "--features", "vendored-lua" ], "cargo_afl_build_flags": [ - "--features", "vendored-lua" + "--features", "vendored-lua,fuzzing" ], "keep_env": [ "OPENAIKEY", diff --git a/implementation/Cargo.toml b/implementation/Cargo.toml index deeb9636..504d7271 100644 --- a/implementation/Cargo.toml +++ b/implementation/Cargo.toml @@ -3,16 +3,22 @@ name = "genvm-modules" version = "0.6.0-rc0" edition = "2021" +# The fuzz targets link against the AFL runtime, which only `cargo-afl` supplies. +# Gate them so a plain `cargo test`/`cargo build --examples` skips them instead of +# failing to link. [[example]] name = "fuzz-genvm-modules-normalize-whitespace" path = "fuzz/genvm-modules-normalize-whitespace.rs" +required-features = ["fuzzing"] [[example]] name = "fuzz-genvm-modules-complete-json" path = "fuzz/genvm-modules-complete-json.rs" +required-features = ["fuzzing"] [features] default = [] +fuzzing = [] vendored-lua = ["mlua/vendored"] [dependencies] diff --git a/support/ci/__main__.py b/support/ci/__main__.py index c1f6206b..27cbcc78 100644 --- a/support/ci/__main__.py +++ b/support/ci/__main__.py @@ -7,11 +7,13 @@ from pipelines import build, checks, docs, release, tests from tools import ( branches, + deploy_docs, genvm_merge_into_dev, make_release_notes, open_executor_prs, pr_action_panel, pr_branches, + rebase_watch, versions, ) @@ -25,11 +27,13 @@ TOOLS: list[ci_lib.Tool] = [ *versions.COMMANDS, *branches.COMMANDS, + *deploy_docs.COMMANDS, *pr_branches.COMMANDS, *make_release_notes.COMMANDS, *pr_action_panel.COMMANDS, *genvm_merge_into_dev.COMMANDS, *open_executor_prs.COMMANDS, + *rebase_watch.COMMANDS, ] diff --git a/support/ci/behind.py b/support/ci/behind.py new file mode 100644 index 00000000..368e6727 --- /dev/null +++ b/support/ci/behind.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +""" +One definition of "how far behind its base is this branch". + +The repo merges by fast-forward only, so "0 commits behind base" is a hard +precondition in three separate places: + +- `incl_initial.yaml`'s always-on gate, which fails a PR the moment it falls + behind (`check-behind`, below); +- `tools/rebase_watch.py`, the advisory check-run/label sweep that re-evaluates + when the BASE moves; +- `tools/genvm_merge_into_dev.py`, the authoritative check at merge time. + +Each used to compute it its own way β€” two flavours of `git` plus the compare +API β€” so the three could disagree about the same PR. The measurement lives here +once, in the two flavours the callers genuinely need: + +- `behind_by_api` for callers with no checkout (a `pull_request_target` sweep); +- `behind_by_git` for callers that already have the objects local. + +Both answer the same question with the same meaning: the number of commits +reachable from `base` that `head` does not contain. Zero means the head +contains the base tip, i.e. it can be fast-forwarded. +""" + +import json +import subprocess +from pathlib import Path + +import ci_lib +import gh_common + + +def behind_by_api(repo: str, base: str, head_sha: str, token: str | None = None) -> int: + """ + Commits in `base` that `head_sha` does not contain, via the compare API. + + Compares by sha rather than by `owner:branch`, so a fork PR resolves exactly + as a branch PR does β€” GitHub keeps every PR head in the base repo. Needs no + checkout, which is what lets a `pull_request_target` job call it. + """ + return json.loads( + gh_common.gh( + 'api', + f'repos/{repo}/compare/{base}...{head_sha}', + '--jq', + '.behind_by', + token=token, + ).stdout + ) + + +def behind_by_git(base_ref: str, head_ref: str, *, cwd: Path | None = None) -> int: + """ + Commits in `base_ref` that `head_ref` does not contain, via local objects. + + Both refs must already be present locally; the caller is responsible for + fetching them (see the `behind-check` pipeline for the refspecs that work + for fork PRs). + + Defaults to the repo root rather than the process CWD, matching `ci_lib.run` + β€” a caller that fetches through `ci_lib.run` (into ROOT_DIR) and then counts + here must be looking at the same repository. + """ + out = subprocess.run( + ['git', 'rev-list', '--count', f'{head_ref}..{base_ref}'], + check=True, + text=True, + capture_output=True, + cwd=cwd or ci_lib.ROOT_DIR, + ).stdout.strip() + return int(out) diff --git a/support/ci/gh_common.py b/support/ci/gh_common.py index 4151ea8d..ae7cafc3 100644 --- a/support/ci/gh_common.py +++ b/support/ci/gh_common.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 -"""Shared GitHub context for CI tools: repos, PR number, head ref, tokens. +""" +Shared GitHub context for CI tools: repos, PR number, head ref, tokens. Almost every GitHub-touching tool needs the same handful of inputs, resolved the same way β€” **CLI arg > environment variable > default**. This module is the one @@ -17,21 +18,36 @@ `executor_token()` return `None` when unset, and `gh(..., token=None)` then runs with ambient `gh` auth (a local `gh auth login`). That is what lets these tools run outside CI without minting a PAT β€” the whole point of not hardcoding tokens. + +`gh()` traces every invocation to stderr (`+ gh ...`, and the exit code when +non-zero), so a CI log shows which API calls a tool actually made β€” including +the ones it expects to fail and handles with `check=False`. + +A tool that resolves its context once per process calls `set_ctx()` in its +handler and then reads `repo()` / `pr_number()` / `gh_manager()` as free +functions. That pattern used to be copy-pasted (a private `_CTX`, `_ctx()`, +`repo()`, `pr_number()` and a bespoke `gh()` wrapper) into every tool module; +it lives here once instead. """ import argparse import dataclasses +import json import os +import shlex import subprocess import sys import time +import urllib.parse MANAGER_REPO_DEFAULT = 'genlayerlabs/genvm-manager' EXECUTOR_REPO_DEFAULT = 'genlayerlabs/genvm-executor' def _env(*names: str) -> str | None: - """First set, non-empty value among env `names`, else None.""" + """ + First set, non-empty value among env `names`, else None. + """ for name in names: val = os.environ.get(name) if val: @@ -40,18 +56,24 @@ def _env(*names: str) -> str | None: def manager_token() -> str | None: - """Manager-repo token, or None for ambient `gh` auth. `MANAGER_TOKEN` is the - CI name; `GH_TOKEN` is accepted so a single ambient token also works.""" + """ + Manager-repo token, or None for ambient `gh` auth. `MANAGER_TOKEN` is the + CI name; `GH_TOKEN` is accepted so a single ambient token also works. + """ return _env('MANAGER_TOKEN', 'GH_TOKEN') def executor_token() -> str | None: - """Executor-repo token, or None for ambient `gh` auth (`EXECUTOR_TOKEN`).""" + """ + Executor-repo token, or None for ambient `gh` auth (`EXECUTOR_TOKEN`). + """ return _env('EXECUTOR_TOKEN') def current_branch() -> str: - """Current git branch β€” the local fallback for a PR head ref.""" + """ + Current git branch β€” the local fallback for a PR head ref. + """ return subprocess.run( ['git', 'rev-parse', '--abbrev-ref', 'HEAD'], check=True, @@ -67,9 +89,11 @@ def add_args( pr: bool = True, head_ref: bool = True, ) -> None: - """Register the common flags on `parser`. `--manager-repo` is always added; + """ + Register the common flags on `parser`. `--manager-repo` is always added; the rest are opt-out for the few tools that don't need them. Each defaults to - None so `Ctx.from_args` can apply arg > env > default uniformly.""" + None so `Ctx.from_args` can apply arg > env > default uniformly. + """ parser.add_argument( '--manager-repo', default=None, @@ -100,7 +124,9 @@ def add_args( @dataclasses.dataclass(frozen=True) class Ctx: - """Resolved GitHub inputs for a tool run (arg > env > default applied).""" + """ + Resolved GitHub inputs for a tool run (arg > env > default applied). + """ manager_repo: str executor_repo: str @@ -125,30 +151,42 @@ def arg(name: str) -> str | None: @property def pr_number(self) -> str: - """The PR number, or exit with a clear message if none was provided.""" + """ + The PR number, or exit with a clear message if none was provided. + """ if not self._pr_number: raise SystemExit('no PR number: pass --pr or set $PR_NUMBER') return self._pr_number @property def pr_number_opt(self) -> str | None: - """The PR number if one was provided, else None (for no-PR no-op paths).""" + """ + The PR number if one was provided, else None (for no-PR no-op paths). + """ return self._pr_number or None @property def head_ref(self) -> str: - """The PR head branch, falling back to the current git branch locally.""" + """ + The PR head branch, falling back to the current git branch locally. + """ return self._head_ref or current_branch() -# Every `gh` call routed here is an idempotent read (`gh api` GET, `gh pr list`) -# or a create/force-update guarded by its caller, so a transient failure is safe -# to retry. The one we actually hit is GitHub handing back a truncated/empty body, -# which gh reports as `unexpected end of JSON input` and which killed the whole -# executor-precondition gate on a single hiccup; 5xx, rate-limit and network blips -# are retried for the same reason. A terminal 4xx (notably the 404 that +# Retrying exists for one observed failure: GitHub handing back a truncated/empty +# body, which gh reports as `unexpected end of JSON input` and which killed the +# whole executor-precondition gate on a single hiccup. 5xx, rate-limit and network +# blips are retried for the same reason. A terminal 4xx (notably the 404 that # `allow_missing` callers expect for an absent line) is not retried, so that fast # path stays fast. +# +# Retrying is safe for an idempotent read (`gh api` GET, `gh pr list`) and for a +# force-update guarded by its caller. It is NOT safe for a non-idempotent write, +# because these markers are matched as substrings of the combined output and can +# misread a request that in fact succeeded β€” so a caller that creates a PR or +# posts a comment should pass `retry=False`. Not every such caller does yet +# (`tools/pr_branches.py` still retries `pr create`); tightening the markers and +# splitting reads from writes is tracked in GVM-328. _RETRY_ATTEMPTS = 4 _RETRY_BACKOFF_S = 2.0 @@ -173,16 +211,26 @@ def _is_transient(r: subprocess.CompletedProcess) -> bool: def gh( - *args: str, token: str | None = None, check: bool = True + *args: str, token: str | None = None, check: bool = True, retry: bool = True ) -> subprocess.CompletedProcess: - """`gh` with `GH_TOKEN` bound to `token`, or ambient auth when `token` is None. + """ + `gh` with `GH_TOKEN` bound to `token`, or ambient auth when `token` is None. Retries transient failures (see `_TRANSIENT_MARKERS`) a few times with linear backoff; a terminal failure is returned as-is (or raised, when `check`), exactly as a single `subprocess.run(check=...)` would. + + `retry=False` disables that entirely, for a call whose replay is not safe β€” + a non-idempotent write (posting a comment, creating a PR) would otherwise be + performed twice when a *successful* request is misread as transient. """ env = os.environ if token is None else {**os.environ, 'GH_TOKEN': token} - for attempt in range(_RETRY_ATTEMPTS): + attempts = _RETRY_ATTEMPTS if retry else 1 + # Trace every call, like ci_lib.run's `+ ...`. On stderr, so a caller that + # parses this process's stdout is unaffected. The token travels in the + # environment, never in argv, so there is nothing to redact here. + print(f'+ gh {" ".join(shlex.quote(a) for a in args)}', file=sys.stderr, flush=True) + for attempt in range(attempts): result = subprocess.run( ['gh', *args], env=env, @@ -192,16 +240,118 @@ def gh( ) if result.returncode == 0 or not _is_transient(result): break - if attempt + 1 < _RETRY_ATTEMPTS: + if attempt + 1 < attempts: delay = _RETRY_BACKOFF_S * (attempt + 1) print( - f'gh {" ".join(args)}: transient failure, retrying in {delay:.0f}s ' - f'(attempt {attempt + 1}/{_RETRY_ATTEMPTS})', + f' transient failure, retrying in {delay:.0f}s ' + f'(attempt {attempt + 1}/{attempts})', file=sys.stderr, + flush=True, ) time.sleep(delay) + if result.returncode != 0: + # Callers pass check=False for expected failures (a 404 for an absent + # branch), so this is a note, not an error β€” but without it a `gh` that + # fails silently by design leaves no trace of why the caller took the + # other path. + print( + f' gh exited {result.returncode}: {result.stderr.strip()}', + file=sys.stderr, + flush=True, + ) if check and result.returncode != 0: raise subprocess.CalledProcessError( result.returncode, result.args, output=result.stdout, stderr=result.stderr ) return result + + +def api_path(path: str, **params: str | int) -> str: + """ + An `api` path with `params` appended as a properly encoded query string. + + Never interpolate a value into a path by hand: a check-run name such as + `not rebased` contains a space, and `gh api` passes the query through + verbatim β€” the resulting request does not fail, it **hangs** until the job's + timeout. Anything that reaches a query string goes through here. + + `quote_via=quote`, so a space becomes `%20` rather than urlencode's default + `+`. Both work against GitHub today, but `+`-as-space is an HTML-form + convention a server is free not to apply, while `%20` is unambiguous. + """ + query = urllib.parse.urlencode( + {k: str(v) for k, v in params.items()}, quote_via=urllib.parse.quote + ) + if not query: + return path + sep = '&' if '?' in path else '?' + return f'{path}{sep}{query}' + + +# --- per-run context ---------------------------------------------------------- +# A tool process handles exactly one PR, so its resolved Ctx is set once in the +# handler and read as free functions from anywhere in the module. Kept here so +# the tools share one implementation instead of a private copy each. + +_CTX: 'Ctx | None' = None + + +def set_ctx(ctx: 'Ctx') -> 'Ctx': + """ + Install the resolved context for this process. Call once, from the handler. + """ + global _CTX + _CTX = ctx + return ctx + + +def ctx() -> 'Ctx': + """ + The context `set_ctx` installed. + """ + assert _CTX is not None, 'gh_common.Ctx not initialized (handler must set_ctx)' + return _CTX + + +def repo() -> str: + """ + Manager repo slug from the installed context. + """ + return ctx().manager_repo + + +def pr_number() -> str: + """ + PR number from the installed context; exits if none was provided. + """ + return ctx().pr_number + + +def gh_manager(*args: str, check: bool = True, retry: bool = True) -> str: + """ + `gh` with the manager token, returning stdout. + + The shape almost every tool wants. Pass `retry=False` for a non-idempotent + write β€” see `gh`. + """ + return gh(*args, token=manager_token(), check=check, retry=retry).stdout + + +# Repository roles that count as "trusted to drive CI". `author_association` on +# an event payload is NOT a substitute: its MEMBER value only means "in the org", +# which says nothing about push access to this repo. +WRITE_ROLES = frozenset({'admin', 'maintain', 'write'}) + + +def has_write_access(login: str) -> bool: + """ + Whether `login` has push rights on the manager repo. + + Used to authorize anything a user can trigger by interacting with a PR. A + non-collaborator yields 403/404 from this endpoint, which is a `False`, not + an error. + """ + r = gh('api', f'repos/{repo()}/collaborators/{login}/permission', check=False) + if r.returncode != 0: + return False + return json.loads(r.stdout).get('permission') in WRITE_ROLES diff --git a/support/ci/pipelines/build.py b/support/ci/pipelines/build.py index 20516847..d68f2910 100644 --- a/support/ci/pipelines/build.py +++ b/support/ci/pipelines/build.py @@ -13,7 +13,8 @@ def build_and_pack( compression_level: int = 9, show_trace: bool = False, ) -> Path: - """Realize `installable` and pack its output tree as ARTIFACTS_DIR/`artifact_name`. + """ + Realize `installable` and pack its output tree as ARTIFACTS_DIR/`artifact_name`. The out-link is keyed by artifact, not by installable: one installable may be packed into several artifacts that differ only in `globs`, and each needs its @@ -54,7 +55,9 @@ def build_and_pack( class Build(ci_lib.Pipeline): - """Build and package a GenVM target.""" + """ + Build and package a GenVM target. + """ def name(self) -> str: return 'build' diff --git a/support/ci/pipelines/checks.py b/support/ci/pipelines/checks.py index d06b4e7d..1eec7fea 100644 --- a/support/ci/pipelines/checks.py +++ b/support/ci/pipelines/checks.py @@ -5,11 +5,16 @@ import tempfile from pathlib import Path +import behind import ci_lib +import gh_common +from tools.versions import active_lines class CommitHooks(ci_lib.Pipeline): - """Run pre-commit hooks for manager and executor repos.""" + """ + Run pre-commit hooks for manager and executor repos. + """ def name(self) -> str: return 'commit-hooks' @@ -18,10 +23,12 @@ def handler(self, args: argparse.Namespace) -> int: system = ci_lib.output( ['nix', 'eval', '--impure', '--raw', '--expr', 'builtins.currentSystem'] ).strip() - repos = [ - ('.', 'manager', '.?submodules=1'), - ('executors/v0.3.x', 'executor v0.3.x', '.'), - ('executors/v0.2.x', 'executor v0.2.x', '.'), + # Derived from .genvm-monorepo-root, never listed here: a hardcoded list + # silently skips a newly added line's hooks and breaks on a removed one. + # Newest line first, so the manager's own train is exercised early. + repos = [('.', 'manager', '.?submodules=1')] + [ + (f'executors/{line}.x', f'executor {line}.x', '.') + for line in reversed(active_lines()) ] for directory, label, flakeref in repos: with ci_lib.github_group(f'pre-commit: {label}'): @@ -73,7 +80,9 @@ def handler(self, args: argparse.Namespace) -> int: class CommitMessages(ci_lib.Pipeline): - """Check commit messages for changed repos.""" + """ + Check commit messages for changed repos. + """ def name(self) -> str: return 'commit-messages' @@ -146,8 +155,130 @@ def handler(self, args: argparse.Namespace) -> int: return rc +CHECKER = Path('support') / 'scripts' / 'check-commit-message.py' + + +def pr_field(pr: str, field: str) -> str: + """ + One field of a PR, read from the API. + + The PR-scoped checks must not depend on `github.event.pull_request.*`: that + payload is absent on the panel's `workflow_dispatch`, which is how they came + to skip on exactly the runs the Merge gate accepts. A PR number is available + on every triggering event, so everything else is resolved from it here. + """ + return gh_common.gh_manager( + 'api', f'repos/{gh_common.repo()}/pulls/{pr}', '--jq', f'.{field}' + ).strip() + + +class BehindCheck(ci_lib.Pipeline): + """ + Fail unless the PR head already contains its base tip (0 commits behind). + """ + + def name(self) -> str: + return 'behind-check' + + def add_to(self, parser: argparse.ArgumentParser) -> None: + gh_common.add_args(parser, executor_repo=False, head_ref=False) + + def handler(self, args: argparse.Namespace) -> int: + ctx = gh_common.Ctx.from_args(args) + gh_common.set_ctx(ctx) + pr = ctx.pr_number_opt + if not pr: + ci_lib.github_error('PR must be set (the PR number to check)') + return 1 + base = os.environ.get('BASE', '') or pr_field(pr, 'base.ref') + if not base: + ci_lib.github_error(f'could not resolve the base branch of PR #{pr}') + return 1 + + # Resolve both sides by explicit refspec so this also works for fork PRs, + # whose head sha is not fetchable on its own. + ci_lib.run( + [ + 'git', + 'fetch', + '--no-tags', + '--no-recurse-submodules', + 'origin', + f'+refs/heads/{base}:refs/base', + f'+refs/pull/{pr}/head:refs/prhead', + ] + ) + + count = behind.behind_by_git('refs/base', 'refs/prhead') + if count: + ci_lib.github_error( + f'PR is {count} commit(s) behind {base}; update/rebase the branch so it ' + f'is 0 behind before merging' + ) + return 1 + print(f'0 commits behind {base}') + return 0 + + +class PrTitle(ci_lib.Pipeline): + """ + Fail unless the PR title is a valid conventional-commit subject. + + The Merge action squashes a PR into `<title> (#N)`, so the title becomes a + commit subject verbatim β€” and nothing re-validates a generated squash + message against the commit-message hook. A title that would not pass as a + commit subject therefore lands as a commit the hook rejects. Checking it on + the PR moves that failure to where it can still be fixed. + """ + + def name(self) -> str: + return 'pr-title' + + def add_to(self, parser: argparse.ArgumentParser) -> None: + gh_common.add_args(parser, executor_repo=False, head_ref=False) + + def handler(self, args: argparse.Namespace) -> int: + ctx = gh_common.Ctx.from_args(args) + gh_common.set_ctx(ctx) + pr = (ctx.pr_number_opt or '').strip() + if not pr: + ci_lib.github_error('PR must be set (the PR number to check)') + return 1 + # From the API, not `github.event.pull_request.title`, so this also runs + # on a panel-dispatched run β€” see `pr_field`. + title = os.environ.get('PR_TITLE', '') or pr_field(pr, 'title') + if not title.strip(): + ci_lib.github_error(f'could not resolve the title of PR #{pr}') + return 1 + + # Validate the subject that will actually LAND, not the bare title: the + # merge appends ` (#N)` (genvm_merge_into_dev.pr_reference), and the + # checker measures the raw subject length. Checking the title alone would + # pass a title just under the limit and still land an over-long subject. + subject = f'{title} (#{pr})' + + # Subject-only: the same validator the commit-msg hook runs, which treats + # a single-line message as a bare subject (format, emoji placement, + # length, AI attribution). + result = ci_lib.run( + [sys.executable, ci_lib.ROOT_DIR / CHECKER, '--message-text', subject], + check=False, + ) + if result.returncode != 0: + ci_lib.github_error( + f'PR title is not a valid commit subject: {subject!r}. The Merge action ' + f'lands exactly this as the squashed commit subject, so it must satisfy ' + f'the same rules.' + ) + return 1 + print(f'ok {subject}') + return 0 + + class CargoClippy(ci_lib.Pipeline): - """Check cargo clippy availability and enumerate crates.""" + """ + Check cargo clippy availability and enumerate crates. + """ def name(self) -> str: return 'cargo-clippy' @@ -165,4 +296,4 @@ def handler(self, args: argparse.Namespace) -> int: return 0 -COMMANDS = [CommitHooks(), CommitMessages(), CargoClippy()] +COMMANDS = [CommitHooks(), CommitMessages(), BehindCheck(), PrTitle(), CargoClippy()] diff --git a/support/ci/pipelines/docs.py b/support/ci/pipelines/docs.py index 67a1cdce..b64dce37 100644 --- a/support/ci/pipelines/docs.py +++ b/support/ci/pipelines/docs.py @@ -1,13 +1,15 @@ import argparse -import json import os import sys import ci_lib +from tools.versions import active_lines, newest_line class Docs(ci_lib.Pipeline): - """Generate and build documentation.""" + """ + Generate and build documentation. + """ def name(self) -> str: return 'docs' @@ -19,13 +21,16 @@ def handler(self, args: argparse.Namespace) -> int: ['git', 'log', '-1', '--date=format:%Y', '--format=%ad'] ).strip() - monorepo = json.loads((ci_lib.ROOT_DIR / '.genvm-monorepo-root').read_text()) - # Runner versions are per-executor-line data (the runner manifest), so the - # generated manifest + available-runners page land in the primary line's + # generated manifest + available-runners page land in the newest line's # SDK sub-site, not the manager. generate.py writes available-runners.rst # next to this json (json_path.with_name). - primary_line = monorepo['active-versions'][0] + # + # `newest_line()`, not `active-versions[0]`: that field is hand-ordered, and + # every other consumer goes through the sorted `tools.versions` helpers β€” so + # indexing it raw meant two readers of one field disagreeing on which line + # is "first". + primary_line = newest_line() runners_versions_json = ( f'executors/{primary_line}.x/docs/website/src/python-sdk/runners-versions.json' ) @@ -52,7 +57,7 @@ def handler(self, args: argparse.Namespace) -> int: # generated executors.rst page. Built after the manager site so its # objects.inv exists (intersphinx) and its html root is not clobbered. doc_root = ci_lib.ROOT_DIR / 'build' / 'doc' - for line in monorepo['active-versions']: + for line in active_lines(): exec_prefix = f'executors/{line}.x' src = (ci_lib.ROOT_DIR / exec_prefix / 'docs' / 'website' / 'src').resolve() installable = f'./{exec_prefix}#gen-docs' @@ -84,7 +89,9 @@ def handler(self, args: argparse.Namespace) -> int: class DocsSrc(ci_lib.Pipeline): - """Build docs inside the docs Nix environment.""" + """ + Build docs inside the docs Nix environment. + """ def name(self) -> str: return 'docs-src' diff --git a/support/ci/pipelines/release.py b/support/ci/pipelines/release.py index d7a703d5..aa7dc837 100644 --- a/support/ci/pipelines/release.py +++ b/support/ci/pipelines/release.py @@ -26,7 +26,8 @@ def artifact_of(cell: str) -> str: - """Artifact name a build cell uploads and a test cell downloads. + """ + Artifact name a build cell uploads and a test cell downloads. Keyed by cell rather than minted at upload time: every leg of a matrix job writes the *same* `outputs` key, so a per-leg name cannot be handed back as @@ -36,7 +37,8 @@ def artifact_of(cell: str) -> str: def asset_of(cell: str) -> str: - """The tarball a cell packs β€” already named as the published release asset. + """ + The tarball a cell packs β€” already named as the published release asset. Nothing downstream renames it: the publisher attaches what it downloads. """ @@ -92,7 +94,8 @@ def build_cells() -> list[dict]: def release_filter_tag() -> str: - """The release preset's tag expression, read at plan time. + """ + The release preset's tag expression, read at plan time. The old jobs inlined `$(cat ...)` into the shell; resolving it here keeps the steps shell-free and puts the expression in the plan, where it is visible. @@ -101,7 +104,8 @@ def release_filter_tag() -> str: def stable_test_steps(installable: str) -> GithubStep: - """configure + run the release preset inside `installable`'s shell. + """ + configure + run the release preset inside `installable`'s shell. `configure` must come first: it writes build/info.json, which is what points the runner at the extracted build/out tree. @@ -235,7 +239,8 @@ def test_cells() -> list[dict]: class PlanReleaseMatrix(ci_lib.Pipeline): - """Emit the release build/test matrices and the tag they are built at. + """ + Emit the release build/test matrices and the tag they are built at. One build cell per release artifact (runners + one per platform) and one test cell per platform, all keyed by names both sides derive from this plan. diff --git a/support/ci/pipelines/tests.py b/support/ci/pipelines/tests.py index 588ecfaa..3b3a22a6 100644 --- a/support/ci/pipelines/tests.py +++ b/support/ci/pipelines/tests.py @@ -74,7 +74,8 @@ def _check_of(step: SubStep) -> bool: def _finish(step: SubStep, result: subprocess.CompletedProcess) -> None: - """Enforce a step's `expect_exit_codes`, if it declared any. + """ + Enforce a step's `expect_exit_codes`, if it declared any. Lets a step assert an exact non-zero exit β€” e.g. a daemon smoke test killed by `timeout` must exit 124 β€” without a shell to write `|| test $? -eq 124` in. @@ -311,10 +312,11 @@ def _cell( class PlanQueueMatrix(ci_lib.Pipeline): - """Emit queue.yaml's test matrix as a GitHub `matrix` output. + """ + Emit queue.yaml's test matrix as a GitHub `matrix` output. - The heavy cells run only when a marker is set (RUN_FULL_TESTS, from the rtm / - run-full-tests labels or a manual dispatch); otherwise the matrix is empty and + The heavy cells run only when a marker is set (RUN_FULL_TESTS, from the + run-full-tests label or a manual dispatch); otherwise the matrix is empty and matrix-tests is skipped. """ @@ -335,7 +337,8 @@ def handler(self, args: argparse.Namespace) -> int: class TestMatrixCell(ci_lib.Pipeline): - """Replay one step list of a matrix cell (from plan-queue-matrix / into_commands). + """ + Replay one step list of a matrix cell (from plan-queue-matrix / into_commands). into_commands emits a list per key ('configure', 'build', 'tool'); test_cell.yaml slices it with fromJSON and hands us one of those lists, so --job-json here is a @@ -397,7 +400,8 @@ def handler(self, args: argparse.Namespace) -> int: class CheckCellKeys(ci_lib.Pipeline): - """Fail unless a cell's step mapping holds exactly the keys the caller replays. + """ + Fail unless a cell's step mapping holds exactly the keys the caller replays. into_commands' keys and test_cell.yaml's "Job:" steps must stay in sync in both directions: a key nobody replays would silently drop its steps, and a step whose diff --git a/support/ci/pr_branches_info.py b/support/ci/pr_branches_info.py index e696382b..e7873d23 100644 --- a/support/ci/pr_branches_info.py +++ b/support/ci/pr_branches_info.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 -"""Collect per-repo branch movement for a manager PR. +""" +Collect per-repo branch movement for a manager PR. A manager PR (e.g. on the v0.6 line) bundles work across several repos: the manager itself and every active executor line it ships (v0.2, v0.3, ...), each @@ -51,7 +52,8 @@ @dataclasses.dataclass(frozen=True) class RepoInfo: - """How one repo's PR head moved relative to its base branch. + """ + How one repo's PR head moved relative to its base branch. `base_sha`/`head_sha` are None when the path is absent at that side (a line added or removed by the PR). `ahead_by`/`behind_by` are None when the two @@ -72,30 +74,40 @@ class RepoInfo: @property def moved(self) -> bool: - """The head points somewhere other than the base.""" + """ + The head points somewhere other than the base. + """ return self.head_sha != self.base_sha @property def has_pr(self) -> bool: - """An open PR for this repo's head_ref -> base_ref exists.""" + """ + An open PR for this repo's head_ref -> base_ref exists. + """ return self.pr_url is not None @property def rebased(self) -> bool: - """Head contains the base tip (not behind). A positive behind_by means it - needs a rebase; unknown (None) is left to the synced check.""" + """ + Head contains the base tip (not behind). A positive behind_by means it + needs a rebase; unknown (None) is left to the synced check. + """ return not self.behind_by @property def synced(self) -> bool: - """The head gitlink is present in the executor repo (comparable to base), so + """ + The head gitlink is present in the executor repo (comparable to base), so it can land together with the manager commit. Being ahead of base is normal β€” manager and executors land almost-atomically β€” so only a head commit missing - from the executor repo (ahead_by is None) is unsynced.""" + from the executor repo (ahead_by is None) is unsynced. + """ return self.ahead_by is not None def as_dict(self) -> dict: - """Plain-dict form (all fields plus the derived flags) for JSON output.""" + """ + Plain-dict form (all fields plus the derived flags) for JSON output. + """ return { **dataclasses.asdict(self), 'moved': self.moved, @@ -106,7 +118,9 @@ def as_dict(self) -> dict: def _api(*args: str, token: str | None = None, allow_missing: bool = False): - """`gh api` returning parsed JSON, or None on a 404 when allow_missing.""" + """ + `gh api` returning parsed JSON, or None on a 404 when allow_missing. + """ r = gh('api', *args, token=token, check=not allow_missing) if r.returncode != 0: # allow_missing: treat any failure (typically HTTP 404 β€” a ref/path that @@ -116,9 +130,15 @@ def _api(*args: str, token: str | None = None, allow_missing: bool = False): def compare( - repo: str, base: str, head: str, token: str, *, allow_missing: bool = False + repo: str, + base: str, + head: str, + token: str | None, + *, + allow_missing: bool = False, ): - """GitHub `compare` of `base...head` in `repo`. + """ + GitHub `compare` of `base...head` in `repo`. Returns `(base_tip_sha, ahead_by, behind_by)`, or None when the comparison is not possible (allow_missing). `base_tip_sha` is the base ref's own commit; the @@ -134,8 +154,11 @@ def compare( return data['base_commit']['sha'], data['ahead_by'], data['behind_by'] -def submodule_sha(manager_repo: str, path: str, ref: str, token: str) -> str | None: - """The executor gitlink (submodule commit) recorded at `manager_repo`'s `ref`. +def submodule_sha( + manager_repo: str, path: str, ref: str, token: str | None +) -> str | None: + """ + The executor gitlink (submodule commit) recorded at `manager_repo`'s `ref`. Reads the submodule pointer through the contents API (`type: submodule`), so no checkout is needed. None when the path is absent at that ref (a line the PR adds @@ -149,8 +172,10 @@ def submodule_sha(manager_repo: str, path: str, ref: str, token: str) -> str | N return data['sha'] -def existing_pr(repo: str, head: str, base: str, token: str) -> str | None: - """URL of the open PR `head -> base` in `repo`, or None if none is open.""" +def existing_pr(repo: str, head: str, base: str, token: str | None) -> str | None: + """ + URL of the open PR `head -> base` in `repo`, or None if none is open. + """ url = gh( 'pr', 'list', @@ -171,8 +196,11 @@ def existing_pr(repo: str, head: str, base: str, token: str) -> str | None: return url or None -def manager_info(pr_number: str, manager_repo: str, manager_token: str) -> RepoInfo: - """RepoInfo for the manager repo itself (base branch tip vs PR head). +def manager_info( + pr_number: str, manager_repo: str, manager_token: str | None +) -> RepoInfo: + """ + RepoInfo for the manager repo itself (base branch tip vs PR head). The manager PR always exists (it is the one being inspected), so its `pr_url` is taken straight from the PR object rather than queried. @@ -205,10 +233,11 @@ def executor_info( manager_head_sha: str, manager_head_ref: str, executor_repo: str, - manager_token: str, - executor_token: str, + manager_token: str | None, + executor_token: str | None, ) -> RepoInfo: - """RepoInfo for one executor line: its gitlink at the manager base vs head. + """ + RepoInfo for one executor line: its gitlink at the manager base vs head. The executor mirror branch is deterministically named `pr/<line>/<head>` off the manager PR's head ref (genvm_tool.common.Repo.feature_branch), so its @@ -257,7 +286,8 @@ def collect( manager_token: str | None = None, executor_token: str | None = None, ) -> dict[str, RepoInfo]: - """Branch movement for every repo a manager PR touches, keyed by path. + """ + Branch movement for every repo a manager PR touches, keyed by path. "." is the manager; "executors/v<X>.x" is each active executor line (from .genvm-monorepo-root via tools.versions). @@ -281,7 +311,9 @@ def collect( def from_ctx(ctx: gh_common.Ctx) -> dict[str, RepoInfo]: - """`collect` with repos/PR/tokens resolved from a shared `gh_common.Ctx`.""" + """ + `collect` with repos/PR/tokens resolved from a shared `gh_common.Ctx`. + """ return collect( ctx.pr_number, manager_repo=ctx.manager_repo, diff --git a/support/ci/tools/branches.py b/support/ci/tools/branches.py index e907e7de..c7d88286 100644 --- a/support/ci/tools/branches.py +++ b/support/ci/tools/branches.py @@ -1,5 +1,4 @@ import argparse -import subprocess import ci_lib import gh_common @@ -7,12 +6,22 @@ from tools.versions import active_versions -def gh(*args: str, check: bool = True) -> subprocess.CompletedProcess: - return ci_lib.run(['gh', *args], check=check, capture_output=True) +def gh(*args: str, check: bool = True) -> str: + """ + `gh` with the manager token, returning stdout. + + Shares gh_common's wrapper rather than shelling out separately, so this tool + gets the same tracing, token resolution and transient-failure retries as + every other one. `retry=False`: the calls below include `pr create`, which + must not be replayed. + """ + return gh_common.gh_manager(*args, check=check, retry=False) class ProvisionBranches(ci_lib.Tool): - """Provision version/dev branches and standing release-gate PRs.""" + """ + Provision version/dev branches and standing release-gate PRs. + """ def name(self) -> str: return 'provision-branches' @@ -95,7 +104,7 @@ def handler(self, args: argparse.Namespace) -> int: 'number', '--jq', '.[0].number // ""', - ).stdout.strip() + ).strip() if existing: print( f'Standing PR {dev_branch} -> {version_branch} already open (#{existing})' diff --git a/support/ci/tools/deploy_docs.py b/support/ci/tools/deploy_docs.py new file mode 100644 index 00000000..2a78e5a5 --- /dev/null +++ b/support/ci/tools/deploy_docs.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +""" +Place a built docs tree into a checkout of the sdk.genlayer.com site repo. + +That repo publishes `_site/` to GitHub Pages, one directory per docs version +(`_site/main`, `_site/v0.3.x`, ...) plus a root `_site/versions.json` that feeds +the version switcher every built page links to. This tool only rewrites those +files; committing and pushing is left to the caller, so a nightly run and a +local `--site ../sdk.genlayer.com` do exactly the same thing. + +The version directory is replaced wholesale rather than merged, so pages deleted +upstream do not linger. + +`versions.json` keeps its existing order β€” it is hand-curated (newest train +first, point releases under it) and re-sorting it on every nightly would produce +noise. A version that is not listed yet is inserted directly after `main`. +""" + +import argparse +import json +import pathlib +import re +import shutil + +import ci_lib + +# Directory the site repo publishes; everything below is relative to it. +SITE_SUBDIR = '_site' + +VERSIONS_JSON = 'versions.json' + +SITE_URL = 'https://sdk.genlayer.com' + + +def entry_for(version: str, preferred: bool) -> dict: + name = 'latest (main)' if version == 'main' else version + return { + 'name': name, + 'version': version, + 'url': f'{SITE_URL}/{version}/', + 'preferred': preferred, + } + + +def update_versions(path, version: str, preferred: bool) -> None: + """ + Insert or refresh `version`'s entry, leaving every other entry untouched. + + `preferred` is what the switcher highlights, so at most one entry may carry it: + passing it here clears it everywhere else. NOT passing it leaves the current + highlight alone rather than clearing it β€” the nightly never passes the flag, and + it must not strip `main`'s highlight every night. + """ + versions = json.loads(path.read_text()) if path.exists() else [] + + new = entry_for(version, preferred) + for index, existing in enumerate(versions): + if existing.get('version') == version: + if not preferred: + new['preferred'] = bool(existing.get('preferred')) + versions[index] = new + break + else: + main_at = next( + (i for i, e in enumerate(versions) if e.get('version') == 'main'), + -1, + ) + versions.insert(main_at + 1, new) + print(f'{VERSIONS_JSON}: added `{version}`') + + if preferred: + for existing in versions: + if existing.get('version') != version: + existing['preferred'] = False + + path.write_text(json.dumps(versions, indent=2) + '\n') + + +class DeployDocs(ci_lib.Tool): + """ + Copy a built docs tree into an sdk.genlayer.com checkout. + """ + + def name(self) -> str: + return 'deploy-docs' + + def add_to(self, parser: argparse.ArgumentParser) -> None: + parser.add_argument( + '--site', + required=True, + help=f'checkout of the site repo (the one holding {SITE_SUBDIR}/)', + ) + parser.add_argument( + '--version', + default='main', + help='directory to publish under, e.g. `main` or `v0.6.x` (default: main)', + ) + parser.add_argument( + '--html', + default=str(ci_lib.ROOT_DIR / 'build' / 'doc' / 'html'), + help='built docs tree to publish (default: build/doc/html)', + ) + parser.add_argument( + '--preferred', + action='store_true', + help='make this the version the switcher highlights', + ) + + def handler(self, args: argparse.Namespace) -> int: + # The version becomes a path component that is then rmtree'd, and it arrives + # from a free-text workflow_dispatch input: `..` would delete the checkout. + if not re.fullmatch(r'[A-Za-z0-9._-]+', args.version) or args.version in ( + '.', + '..', + ): + print(f'`{args.version}` is not a usable version directory name') + return 1 + + html = pathlib.Path(args.html).resolve() + if not (html / 'index.html').is_file(): + print( + f'{html} is not a built docs tree (no index.html); run `pipeline docs` first' + ) + return 1 + + site = pathlib.Path(args.site).resolve() / SITE_SUBDIR + if not site.is_dir(): + print(f'{site} does not exist; is --site an sdk.genlayer.com checkout?') + return 1 + + target = site / args.version + if target.exists(): + shutil.rmtree(target) + # `.doctrees` is sphinx's incremental-build cache, not part of the site. + shutil.copytree(html, target, ignore=shutil.ignore_patterns('.doctrees')) + print(f'published {html} -> {target}') + + update_versions(site / VERSIONS_JSON, args.version, args.preferred) + return 0 + + +COMMANDS = [DeployDocs()] diff --git a/support/ci/tools/genvm_merge_into_dev.py b/support/ci/tools/genvm_merge_into_dev.py index d6b68c96..5218b836 100755 --- a/support/ci/tools/genvm_merge_into_dev.py +++ b/support/ci/tools/genvm_merge_into_dev.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 -"""Gate and perform a Merge of a PR into a v<X>-dev branch. +""" +Gate and perform a Merge of a PR into a v<X>-dev branch. Invoked by the reusable .github/workflows/branch_merge_into_dev.yaml when a maintainer ticks the "Merge" box on the PR action panel. It re-checks every @@ -9,11 +10,20 @@ Gates (all required, all on the head commit): 1. base branch is a v<X>-dev branch -2. PR carries the `rtm` (ready-to-merge) label +2. a maintainer (push access) approved the head commit β€” pushing revokes it 3. full GenVM CI (queue.yaml) concluded success -4. the cross-repo E2E check concluded success +4. the cross-repo E2E check (exact name, from the exact app) concluded success 5. the PR is 0 commits behind base +Forced merge: with `$FORCE_ACTOR` set, gates 2-4 are skipped and the skip is +announced on the PR under that login. That the login is a repo admin, and that +the head is still the commit they authorized, is settled in the workflow's own +YAML before this script is fetched β€” this script runs from the PR head, so it +can neither authorize nor refuse anything on its own. Gates 1, 1b and 5 stay +enforced: they keep the land itself well-formed, which no authority makes safe +to skip. Locally, `--force` does the same and takes the actor from the logged-in +`gh` user; it is refused under GITHUB_ACTIONS, where that user is the bot. + Strategy: every repo ends up with ONE commit per merged PR. The manager's commits are squashed onto base (a single commit that needs no gitlink rewrite is fast-forwarded as-is, preserving its SHA); each executor line's commits are @@ -29,22 +39,34 @@ Lines whose gitlink is unchanged, or that already sit at a single commit on their dev tip, are left alone. -True cross-repo atomicity is impossible, so we (1) plan and fast-forward-check -all sides up front, then (2) push the executors first β€” they are the -dependencies the manager commit points at β€” and (3) push the manager. If the -manager push then loses a race the executors are left one fast-forward ahead, -which is harmless (monotonic) and reconciled when Merge is re-ticked. - -Squashing collapses several authors into one, so `Co-authored-by:` trailers are -added for every other human author in the range (never tool/AI attribution β€” the -commit-message hook rejects that). - -After a successful merge every linked PR is auto-closed as MERGED, without deleting -any branch: each executor line's mirror branch (`pr/<line>/<feature>`, opened by +True cross-repo atomicity is impossible, so the landing is staged to keep the +window in which the repos disagree as small as it can be: + +1. plan and fast-forward-check every side up front; +2. push all the objects to both remotes under a scratch ref, so that no later +push has anything left to transfer; +3. move the PR head branches (manager, then each executor mirror) onto the +commits about to land, and pause β€” GitHub needs to see that before the base +moves to record the result as merged rather than closed; +4. move the target branches: the manager first, since it is the only push that +can lose a race, then each `<line>-dev`. + +A rejected manager push therefore lands nothing, and the branch moves from +step 3 are rolled back so the PR can be re-ticked. Only step 4's executor +pushes can leave the repos split, and they are reported with the exact ref +update to replay. + +A squashed message is the PR title plus one `* <subject>` bullet per collapsed +commit; the PR description never lands. Squashing also collapses several authors +into one, so `Co-authored-by:` trailers are added for every other human author in +the range (never tool/AI attribution β€” the commit-message hook rejects that). + +Every linked PR is auto-closed as MERGED, without deleting any branch: each +executor line's mirror branch (`pr/<line>/<feature>`, opened by branch_executor_prs.yaml) and the manager PR's own head branch are force-moved onto -the commit that landed on their base, so head == base tip and GitHub marks the PR -merged. (Deleting a branch, or `gh pr close`, closes the PR as merely 'closed' and -drops the merged link.) +the commit that is about to land on their base (step 3 above), so once the base +moves head == base tip and GitHub marks the PR merged. (Deleting a branch, or +`gh pr close`, closes the PR as merely 'closed' and drops the merged link.) (The dev -> version release-gate merge is a separate, not-yet-automated flow; on that one the executor's version branch is fast-forwarded and its @@ -62,7 +84,8 @@ .genvm-monorepo-root via tools.versions. Repo and PR number resolve arg > env > default via gh_common; the token is -optional (ambient `gh` auth when unset). Env also: E2E_CHECK_PATTERN. +optional (ambient `gh` auth when unset). Env also: E2E_CHECK_NAME, +E2E_CHECK_APP_SLUG. """ import argparse @@ -71,20 +94,23 @@ import re import subprocess import sys +import time +import behind as behind_lib import ci_lib import gh_common -import pr_branches_info +from gh_common import pr_number, repo from tools.versions import active_lines -# Resolved GitHub context (repo, PR number) for this run; set in the handler. -_CTX: gh_common.Ctx | None = None - +# Scratch ref that carries a merge's objects to both remotes before any branch +# moves; outside refs/heads so nothing watches it and no PR is opened for it. +LANDING_REF_PREFIX = 'refs/genvm-ci/landing' -def _ctx() -> gh_common.Ctx: - assert _CTX is not None, 'gh_common.Ctx not initialized (handler must set it)' - return _CTX +# Gap between moving the PR head branches and moving the branches they target. +# GitHub decides "merged" vs "closed" from the state it has indexed, and the two +# updates arriving together is enough to make it read a merge as a bare close. +SETTLE_SECONDS = 5 def run(*args, check=True, env=None): @@ -96,12 +122,15 @@ def run(*args, check=True, env=None): def gh(*args): - """GitHub API/PR *reads* via the retrying wrapper shared with pr_branches_info, - so a transient truncated-JSON / 5xx blip on a gate query doesn't fail the whole - merge (the exact fragility that wrapper was hardened against). Writes stay on - plain `run('gh', ...)` so a retry can't double-post a comment. Token is optional - (ambient `gh` auth when unset).""" - return pr_branches_info.gh(*args, token=gh_common.manager_token()).stdout + """ + GitHub API/PR *reads* via the shared retrying wrapper, so a transient + truncated-JSON / 5xx blip on a gate query doesn't fail the whole merge (the + exact fragility that wrapper was hardened against). + + Writes stay on plain `run('gh', ...)` so a retry can't double-post a comment. + Token is optional (ambient `gh` auth when unset). + """ + return gh_common.gh_manager(*args) def git(*args, check=True): @@ -109,7 +138,9 @@ def git(*args, check=True): def egit(submodule, *args, check=True, env=None): - """git inside an executor submodule checkout.""" + """ + git inside an executor submodule checkout. + """ return run('git', '-C', submodule, *args, check=check, env=env) @@ -120,16 +151,20 @@ def is_ancestor(repo, maybe_ancestor, descendant): return egit(repo, *args, check=False).returncode == 0 -def repo(): - return _ctx().manager_repo - +# Identity of the cross-repo E2E check-run on the head commit. A check-run name is +# not a protected namespace: any app installed on the repo β€” and any workflow in a +# PR β€” can post one under a name of its choosing. So the gate matches the exact +# name AND the app that posted it; matching a name alone (let alone a substring of +# it) would let an unrelated green check stand in for E2E. +E2E_CHECK_NAME = 'E2E Tests' +E2E_CHECK_APP_SLUG = 'ci-core-e2e-runner' -def pr_number(): - return _ctx().pr_number - -def e2e_pattern(): - return os.environ.get('E2E_CHECK_PATTERN', 'e2e') +def e2e_identity(): + return ( + os.environ.get('E2E_CHECK_NAME') or E2E_CHECK_NAME, + os.environ.get('E2E_CHECK_APP_SLUG') or E2E_CHECK_APP_SLUG, + ) # Identities the commit-message hook rejects as AI/tool attribution @@ -142,8 +177,10 @@ def e2e_pattern(): def commit_author(repo, sha): - """(name, email, author-date) of `sha`. NUL-separated so a name containing - `<` or spaces round-trips.""" + """ + (name, email, author-date) of `sha`. NUL-separated so a name containing + `<` or spaces round-trips. + """ out = ( egit(repo, 'log', '-1', '--format=%an%x00%ae%x00%aI', sha) if repo is not None @@ -154,7 +191,9 @@ def commit_author(repo, sha): def range_authors(repo, rev_range): - """Distinct (name, email) commit authors in `rev_range`, first-seen order.""" + """ + Distinct (name, email) commit authors in `rev_range`, first-seen order. + """ args = ('log', '--format=%an%x00%ae', rev_range) out = (git(*args) if repo is None else egit(repo, *args)).stdout @@ -169,7 +208,8 @@ def range_authors(repo, rev_range): def coauthor_trailers(repo, rev_range, exclude): - """`Co-authored-by:` for every human author in the range except `exclude`. + """ + `Co-authored-by:` for every human author in the range except `exclude`. A squash keeps only one author, so everyone else who wrote a commit in the range would otherwise lose attribution. Authors whose identity the @@ -189,7 +229,9 @@ def coauthor_trailers(repo, rev_range, exclude): def block(msg): - """Comment the reason on the PR and fail the job.""" + """ + Comment the reason on the PR and fail the job. + """ run( 'gh', 'pr', @@ -209,6 +251,176 @@ def pr_view(*fields): return json.loads(out) +def can_maintain(login): + """ + Whether `login` has push rights on the repo, i.e. may approve a merge. + + `author_association` on the review would be cheaper but says the wrong thing: + MEMBER only means "in the org", which is not push access to this repo. + """ + try: + out = gh('api', f'repos/{repo()}/collaborators/{login}/permission') + except subprocess.CalledProcessError: + # 403/404 β€” not a collaborator at all. + return False + return json.loads(out).get('permission') in ('admin', 'maintain', 'write') + + +def check_approval(head_sha): + """ + Require a maintainer's approving review of the exact commit being merged. + + This replaces the `rtm` label. A label is held by whoever can edit labels and + survives every subsequent push, so it could authorize code nobody looked at; an + approval is tied to the commit it was left on, so pushing new commits silently + revokes it here without needing branch protection to dismiss anything. + + Only the newest decision per reviewer counts, so a reviewer who approves and then + requests changes on the same commit does not still authorize it. Reviews are + returned oldest-first, and COMMENTED/PENDING ones carry no decision. An + outstanding CHANGES_REQUESTED blocks regardless of which commit it was left on. + + Paginated, because it is the LAST page that decides: on a PR past 100 reviews an + un-paginated read would keep an early APPROVED and never see the later + CHANGES_REQUESTED that overrides it. `--jq '.[]'` is what makes that safe β€” plain + `--paginate` concatenates one JSON array per page, which is not valid JSON. + """ + reviews = [ + json.loads(line) + for line in gh( + 'api', + '--paginate', + f'repos/{repo()}/pulls/{pr_number()}/reviews?per_page=100', + '--jq', + '.[]', + ).splitlines() + if line.strip() + ] + + decision = {} + latest = {} + for review in reviews: + if review['state'] not in ('APPROVED', 'CHANGES_REQUESTED', 'DISMISSED'): + continue + login = (review.get('user') or {}).get('login') + if login is None: + continue + latest[login] = review['state'] + if review.get('commit_id') == head_sha: + decision[login] = review['state'] + + # A change request is not commit-scoped the way an approval is: GitHub keeps + # it blocking until the reviewer dismisses it or reviews again, however many + # commits land after it. Judging it on `head_sha` alone would let an approval + # of a later commit merge over a still-open objection. + objectors = sorted( + login for login, state in latest.items() if state == 'CHANGES_REQUESTED' + ) + if objectors: + listed = ', '.join(f'`{login}`' for login in objectors) + block( + f'changes requested by {listed} and not withdrawn; the reviewer must ' + f'approve, re-review, or dismiss before this can merge.' + ) + + approvers = sorted(login for login, state in decision.items() if state == 'APPROVED') + if not approvers: + block( + f'no approving review on `{head_sha[:12]}`. A maintainer must approve this ' + f'exact commit β€” approvals of earlier commits do not carry over.' + ) + + maintainers = [login for login in approvers if can_maintain(login)] + if not maintainers: + listed = ', '.join(f'`{login}`' for login in approvers) + block( + f'`{head_sha[:12]}` is approved ({listed}) but by nobody with push access; ' + f'a maintainer must approve it.' + ) + + print(f'approved on {head_sha[:12]} by: {", ".join(maintainers)}') + + +def check_title(pr): + """ + Refuse a PR whose title would not survive the commit-message hook. + + `compose_message` makes the squashed subject `<title> (#N)` verbatim, and + nothing re-validates a generated message β€” so an invalid title lands as a + commit the hook would have rejected. `initial / pr-title` checks this on + every push, but the title can be edited afterwards without re-triggering CI + (queue.yaml deliberately does not run on `edited`), so the authoritative + check is here, against the exact subject about to be written. + """ + subject = f'{pr["title"]} {pr_reference(qualified=False)}' + checker = ci_lib.ROOT_DIR / 'support' / 'scripts' / 'check-commit-message.py' + result = run(sys.executable, str(checker), '--message-text', subject, check=False) + if result.returncode != 0: + detail = (result.stdout or result.stderr).strip() + block( + f'the PR title would not pass the commit-message hook, and it becomes the ' + f'squashed subject verbatim:\n\n```\n{subject}\n```\n\n```\n{detail}\n```' + ) + + +def force_actor() -> str | None: + """ + The admin who asked for a forced merge, or None on a normal merge. + + Set only after the workflow has verified, in its own YAML and before this + file was fetched, that the login is a repo admin and that the head is still + the commit they authorized. Nothing here re-derives that: this script is + checked out from the PR head, so a check it ran itself would be worth + exactly as much as the PR's own code. + """ + return (os.environ.get('FORCE_ACTOR') or '').strip() or None + + +def local_force_actor() -> str: + """ + The logged-in `gh` user, for a `--force` merge run by hand. + + `--force` is a LOCAL escape hatch: on a workstation the person at the + keyboard is the authorization, and asking them to retype a login they are + already authenticated as only invites a typo in the record. In CI the + authorization comes from the workflow instead, so `--force` is refused there + outright β€” under GITHUB_TOKEN this call answers `github-actions[bot]`, which + would make the PR's own code the actor of record. + """ + if os.environ.get('GITHUB_ACTIONS') == 'true': + raise SystemExit( + '--force is for local runs; in CI the actor comes from $FORCE_ACTOR, ' + 'set by branch_force_merge.yaml after it verifies the admin role' + ) + login = gh('api', 'user', '--jq', '.login').strip() + if not login: + raise SystemExit('could not resolve the logged-in `gh` user for --force') + return login + + +def announce_force(actor, head_sha): + """ + Record on the PR which gates were skipped and who skipped them. + + A forced merge lands a commit no green CI ever covered, so the reason it was + allowed must survive in the PR history rather than only in a job log that + expires. + """ + run( + 'gh', + 'pr', + 'comment', + pr_number(), + '--repo', + repo(), + '--body', + f'⚠️ Forced merge requested by `{actor}` (repo admin) on `{head_sha[:12]}`: ' + f'the review, full-CI and E2E gates are NOT being enforced. The base, ' + f'draft/open, title and 0-behind checks still apply.', + check=False, + ) + + def check_gates(pr): if pr['state'] != 'OPEN': block(f'PR is not open (state: {pr["state"]}).') @@ -219,12 +431,28 @@ def check_gates(pr): if not re.fullmatch(r'v.*-dev', base): block(f'base branch `{base}` is not a `v<X>-dev` branch.') - # 2. rtm label - if not any(label['name'] == 'rtm' for label in pr['labels']): - block('missing the `rtm` (ready-to-merge) label.') - head_sha = pr['headRefOid'] + # 1b. the title is about to become a commit subject verbatim. Checked even + # under --force: an invalid subject would be rejected by the commit-message + # hook on the very next push to the dev branch, so skipping it would land a + # breakage rather than accept a risk. + check_title(pr) + + # Gates 2-4 answer "is this commit known-good", and a forced merge is exactly + # the decision to land it without that answer β€” an admin said so explicitly, + # in a comment recorded on the PR. What stays enforced below is the set of + # checks that keep the LAND itself correct (base shape, fast-forwardability), + # which no amount of authority makes safe to skip. + actor = force_actor() + if actor is not None: + announce_force(actor, head_sha) + print(f'forced by {actor}: skipping the review, CI and E2E gates') + return base, head_sha + + # 2. a maintainer approved THIS commit + check_approval(head_sha) + # 3. full GenVM CI (queue.yaml) green on the head commit. The same head may # carry skipped runs (label events that didn't run full tests) alongside the # real one, and the run may have been started by either a push @@ -241,97 +469,217 @@ def check_gates(pr): got = ', '.join(f'{r["status"]}/{r["conclusion"]}' for r in runs) or 'no run' block(f'GenVM CI (queue.yaml) is not green on `{head_sha}` (runs: {got}).') - # 4. cross-repo E2E check green on the head commit - checks = json.loads( - gh( + # 4. cross-repo E2E check green on the head commit. The API defaults to + # `filter=latest`, so a re-run replaces its predecessor rather than adding a + # stale entry beside it. + name, app_slug = e2e_identity() + checks = [ + json.loads(line) + for line in gh( 'api', + '--paginate', f'repos/{repo()}/commits/{head_sha}/check-runs?per_page=100', - ) - )['check_runs'] - e2e = [c for c in checks if re.search(e2e_pattern(), c['name'], re.I)] + '--jq', + '.check_runs[]', + ).splitlines() + if line.strip() + ] + e2e = [ + c + for c in checks + if c['name'] == name and ((c.get('app') or {}).get('slug')) == app_slug + ] if not e2e: - block(f'no E2E check found on `{head_sha}` (run it on this PR first).') - bad = [c for c in e2e if c['conclusion'] != 'success'] - if bad: + seen = ', '.join(sorted({f'`{c["name"]}`' for c in checks})) or 'none' + block( + f'no `{name}` check from `{app_slug}` on `{head_sha}` (run E2E on this PR ' + f'first). Checks present: {seen}.' + ) + if any(c['conclusion'] != 'success' for c in e2e): conclusions = ' '.join(c['conclusion'] or 'pending' for c in e2e) block(f'E2E is not green on `{head_sha}` (conclusions: {conclusions}).') return base, head_sha -def land_executor_mirror_branch(submodule, line, head_ref, landed): - """Move the executor mirror branch `pr/<line>/<head_ref>` onto `landed` after a merge. +def executor_remote_sha(submodule, branch): + """ + Current `origin/<branch>` in an executor checkout, or None if there is none. + + `plan_executor_line` fetched every executor head, so a branch is present locally + iff it exists on the remote. Used to remember what a branch pointed at before we + move it, so a failed land can put it back. + """ + r = egit( + submodule, + 'rev-parse', + '--verify', + '--quiet', + f'refs/remotes/origin/{branch}', + check=False, + ) + return r.stdout.strip() if r.returncode == 0 else None + + +def landing_ref(line=None): + """ + Scratch ref carrying one repo's landing objects. + + Both executor lines are checkouts of the SAME remote, so the ref has to be keyed + by line as well as by PR β€” otherwise v0.3 force-overwrites v0.2's ref and that + line's squashed commit is left unreferenced, which is exactly the object the + recovery path assumes is still fetchable. + """ + suffix = '' if line is None else f'/{line}' + return f'{LANDING_REF_PREFIX}/pr-{pr_number()}{suffix}' + + +def stage_objects(plans, push_sha): + """ + Upload every commit that is about to land, without moving a branch anyone reads. + + Manager and executors are separate repos, so the branch moves that finish a merge + can never be one transaction β€” but they can be made short. Pushing the objects + first under a scratch ref leaves each final push a ref update with nothing to + transfer, so the window in which the manager can gitlink a commit no executor + branch contains shrinks from "however long the upload takes" to a round trip. + + The scratch refs also keep those objects reachable if the job dies mid-way, so a + repair is a ref move rather than a re-squash under a fresh sha. That is what makes + the replay instructions on a partial land runnable at all, so a failure here is + fatal rather than a warning β€” nothing has moved yet, so blocking costs nothing. + """ + for submodule, line, _base, _pinned, landed, needs_push in plans: + if not needs_push: + continue + ref = landing_ref(line) + r = egit(submodule, 'push', '--force', 'origin', f'{landed}:{ref}', check=False) + if r.returncode != 0: + block( + f'could not stage `{submodule}` objects at `{ref}`, so a partial land ' + f'could not be repaired; nothing was moved. ({r.stderr.strip()})' + ) + ref = landing_ref() + r = git('push', '--force', 'origin', f'{push_sha}:{ref}', check=False) + if r.returncode != 0: + block( + f'could not stage manager objects at `{ref}`; nothing was moved. ' + f'({r.stderr.strip()})' + ) + + +def unstage_objects(plans): + """ + Drop the scratch refs once every target branch holds the commits. Best-effort. + + Only reached on a fully successful land: after a partial one the refs are what + keeps the un-pushed commits fetchable, so leaving them is the point. + """ + for submodule, line, _base, _pinned, _landed, needs_push in plans: + if needs_push: + egit(submodule, 'push', 'origin', '--delete', landing_ref(line), check=False) + git('push', 'origin', '--delete', landing_ref(), check=False) - The manager PR's work for this line now sits on the executor's `<line>-dev` at - `landed` (we just pushed it there); force-moving the mirror branch to that SAME - commit makes its auto-opened PR (`pr/<line>/<head_ref>` -> `<line>-dev`, - branch_executor_prs.yaml) have head == base, so GitHub auto-closes it as MERGED. - (Deleting the branch β€” the old behaviour β€” closed that PR as merely 'closed', and - dropped the merged link.) Best-effort: a failure here never unwinds an otherwise - -complete merge. The branch is unprotected, so the executor deploy key can move it. - Only lines this PR actually moved were provisioned a mirror branch + PR - (branch_executor_prs.yaml); an untouched line has none, and force-pushing would - CREATE a stray dangling ref (the old delete was a harmless no-op when absent) β€” so - land only a branch that already exists. `plan_executor_line` fetched every executor - head into `refs/remotes/origin/*`, so the mirror branch is present locally iff it - exists on the remote. +def move_executor_mirror_branch(submodule, line, head_ref, landed): + """ + Move the executor mirror branch `pr/<line>/<head_ref>` onto `landed`. + + The mirror PR (`pr/<line>/<head_ref>` -> `<line>-dev`, branch_executor_prs.yaml) + is auto-closed as MERGED once its head is contained in its base, so we point the + head at the very commit `<line>-dev` is about to receive. Doing this BEFORE the + base moves is what makes GitHub read the result as a merge; doing it after races + the base update and can land the PR as merely 'closed'. (Deleting the branch β€” the + original behaviour β€” always dropped the merged link.) + + Only lines this PR actually moved were provisioned a mirror branch; an untouched + line has none, and force-pushing would CREATE a stray dangling ref, so move only a + branch that already exists. Returns the sha to restore on rollback, or None when + there was nothing to move. """ branch = f'pr/{line}/{head_ref}' - if ( - egit( - submodule, - 'rev-parse', - '--verify', - '--quiet', - f'refs/remotes/origin/{branch}', - check=False, - ).returncode - != 0 - ): + previous = executor_remote_sha(submodule, branch) + if previous is None: print(f'no executor mirror branch {branch}; nothing to auto-merge for {line}') - return + return None r = egit( submodule, 'push', '--force', 'origin', f'{landed}:refs/heads/{branch}', check=False ) - if r.returncode == 0: - print( - f'landed executor mirror branch {branch} -> {landed[:12]} (auto-merges its PR)' - ) - else: - print(f'note: could not land executor mirror branch {branch}: {r.stderr.strip()}') + if r.returncode != 0: + print(f'note: could not move executor mirror branch {branch}: {r.stderr.strip()}') + return None + print(f'moved executor mirror branch {branch} -> {landed[:12]} (auto-merges its PR)') + return previous + +def move_manager_pr_head(pr, head_sha, push_sha): + """ + Point the manager PR's head branch at `push_sha`, the commit `<base>` will get. -def close_manager_pr(pr, push_sha): - """Auto-close the manager PR as MERGED by moving its head branch onto `push_sha`. + Same trick as the executor mirrors, and again before the base moves: head contained + in base is what GitHub reads as merged, unlike a bare `gh pr close` (merely + 'closed'). - `push_sha` is the commit we just landed on the manager `<base>`; force-moving the - PR head branch onto it makes head == base tip, so GitHub marks the PR merged β€” - the same trick as the executor mirrors, and unlike a bare `gh pr close` (which - reads as merely 'closed'). + The rollback target is `head_sha` rather than whatever `origin/<head_ref>` happens + to hold locally: the gates already pinned the PR head to it and `merge()` re-checked + it, so it is exact, and it does not silently depend on the workflow checking out + with full history. A fork PR's head branch lives in another repo, so `<head_ref>` is not a branch on `origin` β€” moving it would create a stray branch and not touch the PR. Those never - reach here (the merge gate needs a same-repo run), but be explicit: cross-repo PRs - fall back to a plain close. Best-effort. + reach here (the merge gate needs a same-repo run), but be explicit and leave fork + PRs to the explicit close after the land. Returns the sha to restore on rollback, + or None when there was deliberately nothing to move. """ head_ref = pr['headRefName'] if pr.get('isCrossRepository'): - print(f'PR head `{head_ref}` is on a fork; closing explicitly (cannot move it)') - run('gh', 'pr', 'close', pr_number(), '--repo', repo(), check=False) - return + print(f'PR head `{head_ref}` is on a fork; cannot move it, will close explicitly') + return None + # A PR raised FROM one dev branch INTO another would make this force-push a + # squashed commit onto a protected branch, using a key that bypasses protection. + if re.fullmatch(r'v.*-dev', head_ref): + block(f'PR head `{head_ref}` is itself a dev branch; refusing to force-move it.') r = git('push', '--force', 'origin', f'{push_sha}:refs/heads/{head_ref}', check=False) - if r.returncode == 0: - print(f'moved manager PR head `{head_ref}` -> {push_sha[:12]} (auto-merges the PR)') - else: - print( - f'note: could not move PR head `{head_ref}` ({r.stderr.strip()}); closing explicitly' + if r.returncode != 0: + # Leaving it unmoved would land the base anyway and strand the PR open, with + # no rollback entry to undo. Nothing has moved yet, so stop here. + block(f'could not move PR head `{head_ref}`; nothing landed. ({r.stderr.strip()})') + print(f'moved manager PR head `{head_ref}` -> {push_sha[:12]} (auto-merges the PR)') + return head_sha + + +def restore_branches(moved): + """ + Put every branch we moved back where it was, after a target push was rejected. + + Without this a rejected land leaves the PR head sitting on the squashed commit: + the head sha then no longer matches the one CI and E2E ran on, so re-ticking Merge + would be refused by its own gates and the PR could not be landed at all. + """ + for submodule, branch, previous in moved: + r = ( + git('push', '--force', 'origin', f'{previous}:refs/heads/{branch}', check=False) + if submodule is None + else egit( + submodule, + 'push', + '--force', + 'origin', + f'{previous}:refs/heads/{branch}', + check=False, + ) ) - run('gh', 'pr', 'close', pr_number(), '--repo', repo(), check=False) + where = 'manager' if submodule is None else submodule + if r.returncode == 0: + print(f'restored {where} branch {branch} -> {previous[:12]}') + else: + print(f'note: could not restore {where} branch {branch}: {r.stderr.strip()}') def executor_gitlink(commit, submodule): - """Executor submodule commit (gitlink) recorded at `commit`'s tree.""" + """ + Executor submodule commit (gitlink) recorded at `commit`'s tree. + """ out = git('ls-tree', commit, submodule).stdout # "160000 commit <sha>\t<path>" m = re.match(r'\S+\s+commit\s+([0-9a-f]+)\t', out) @@ -343,51 +691,81 @@ def executor_gitlink(commit, submodule): return m.group(1) -# Provenance trailer on a squashed executor commit: the PR-side commit whose -# content it carries. A squash keeps only the TREE of that commit, so the sha the -# manager PR pins never becomes an ancestor of the dev branch β€” ancestry alone can -# no longer answer "did this line already land?". Re-ticking Merge after a failed -# manager push (the executors are pushed first) would otherwise see a tip that is -# neither ancestor nor descendant of the pinned sha and block the PR as diverged, -# forever. This trailer is how the second run recognises its own work. -SQUASHED_FROM = 'GenVM-Squashed-From' +def pr_reference(qualified=True): + """ + The `(#N)` suffix a squashed subject ends with. + + A commit landing in an executor repo has to name the manager repo for the + reference to resolve; the manager's own bare `#N` resolves in-repo. + """ + return f'({repo()}#{pr_number()})' if qualified else f'(#{pr_number()})' + +def commit_subjects(repo, rev_range): + """ + Subjects of the commits in `rev_range`, oldest first, merges excluded. + """ + args = ('log', '--reverse', '--no-merges', '--format=%s', rev_range) + out = (git(*args) if repo is None else egit(repo, *args)).stdout + return [line for line in out.split('\n') if line.strip()] -def build_messages(pr): - """Return `(manager_message, executor_message)` for this PR. - The body is the PR body with CRLF normalized (GitHub returns CRLF, and - `commit-tree` β€” unlike `git commit` β€” does no cleanup, so without this the - executor and manager commits for one PR would differ byte-for-byte) and with - any pre-existing provenance-trailer line stripped, so a body that quotes - `GenVM-Squashed-From:` (a revert PR quoting what it reverts) cannot poison the - re-tick lookup. The executor commit lands in another repo, so its PR reference - is fully qualified; the manager's bare `#N` resolves in-repo. +def compose_message(pr, submodule, rev_range, author, *, qualified=True): """ - body = (pr['body'] or '').replace('\r\n', '\n').replace('\r', '\n') - body = '\n'.join( - ln for ln in body.split('\n') if not ln.strip().startswith(f'{SQUASHED_FROM}:') - ).strip() - title = pr['title'] - tail = f'\n\n{body}\n' if body else '\n' - return f'{title} (#{pr_number()}){tail}', f'{title} ({repo()}#{pr_number()}){tail}' + The squash message for `rev_range`: the PR title, one `* <subject>` bullet + per squashed commit, then `Co-authored-by:` for every other author. + + The bullets are the whole body. They are already conventional commits (the + commit-message hook saw them) and are what `make_release_notes` promotes into + the changelog, whereas a PR description is written for reviewers and only rots + on a dev branch. A range whose one commit already says the title gets no + bullets β€” the umbrella would just repeat it. + """ + subjects = commit_subjects(submodule, rev_range) + message = f'{pr["title"]} {pr_reference(qualified)}\n' + if subjects != [pr['title']]: + message += '\n' + ''.join(f'* {subject}\n' for subject in subjects) + trailers = coauthor_trailers(submodule, rev_range, author) + if trailers: + message += '\n' + '\n'.join(trailers) + '\n' + return message -def already_squashed(submodule, exec_base, exec_sha): - """The commit on `<exec_base>` that already carries `exec_sha`'s content, if any.""" +def already_squashed(submodule, exec_base, tree): + """ + The commit on `<exec_base>` that this run already landed for `tree`, if any. + + A squash keeps only the TREE of the pinned commit, so that commit never becomes + an ancestor of the dev branch and ancestry alone cannot answer "did this line + already land?" β€” a re-tick after a partial land would see a tip that is neither + ancestor nor descendant and block the PR as diverged, forever. + + A commit is this PR's own work when BOTH halves of its identity match: its + subject references this PR, and its tree is the one being landed. Neither alone + is enough β€” the subject alone would mistake an earlier landing for the current + one after the author pushed more executor commits (silently dropping them), and + the tree alone would mistake a revert that restored an older tree. + """ out = egit( submodule, 'log', f'refs/remotes/origin/{exec_base}', - f'--grep=^{SQUASHED_FROM}: {exec_sha}$', - '--format=%H', - '-1', - ).stdout.strip() - return out or None + '-F', + f'--grep={pr_reference()}', + '--format=%H%x00%T%x00%s', + ).stdout + for line in out.split('\n'): + if line.count('\x00') != 2: + continue + sha, commit_tree, subject = line.split('\x00') + if commit_tree == tree and subject.endswith(pr_reference()): + return sha + return None -def plan_executor_line(submodule, exec_base, exec_sha, message): - """Work out what this line should land on its `<line>-dev`. +def plan_executor_line(submodule, exec_base, exec_sha, pr): + """ + Work out what this line should land on its `<line>-dev`. Returns `(sha, needs_push)`. The PR's commits on this line are squashed into one commit on top of the line's dev tip, so the executor history mirrors the @@ -420,7 +798,9 @@ def plan_executor_line(submodule, exec_base, exec_sha, message): if tip == exec_sha: return exec_sha, False # already there - landed = already_squashed(submodule, exec_base, exec_sha) + tree = egit(submodule, 'rev-parse', f'{exec_sha}^{{tree}}').stdout.strip() + + landed = already_squashed(submodule, exec_base, tree) if landed is not None: # A previous run squashed this line and pushed it; only the manager push # is left. Reuse that commit rather than squashing a second one. @@ -432,11 +812,10 @@ def plan_executor_line(submodule, exec_base, exec_sha, message): # already contained, so there is nothing to push. return exec_sha, False - tree = egit(submodule, 'rev-parse', f'{exec_sha}^{{tree}}').stdout.strip() if not is_ancestor(submodule, tip, exec_sha): - # A maintainer may have merged the auto-opened executor PR by hand: the - # tree is already on the tip under a different sha and without our trailer. - # Same content already landed, so mirror the tip rather than block. + # A maintainer may have merged the auto-opened executor PR by hand: the tree + # is already on the tip under a different sha and a subject of their own + # choosing. Same content already landed, so mirror the tip rather than block. if egit(submodule, 'rev-parse', f'{tip}^{{tree}}').stdout.strip() == tree: print( f'Executor {exec_base} already carries {exec_sha[:12]} as its tip {tip[:12]}' @@ -454,11 +833,7 @@ def plan_executor_line(submodule, exec_base, exec_sha, message): return exec_sha, True name, email, date = commit_author(submodule, exec_sha) - - trailers = coauthor_trailers(submodule, f'{tip}..{exec_sha}', (name, email)) - # The provenance trailer must be there for a re-tick to recognise this commit. - trailers.append(f'{SQUASHED_FROM}: {exec_sha}') - full_message = message + '\n' + '\n'.join(trailers) + '\n' + full_message = compose_message(pr, submodule, f'{tip}..{exec_sha}', (name, email)) # A squash is just the PR's tree on top of the tip; commit-tree builds it # without touching the working tree or index. @@ -482,8 +857,9 @@ def plan_executor_line(submodule, exec_base, exec_sha, message): return squashed, True -def plan_executor_lines(head_sha, message): - """Plan every active line off the PR head's gitlinks. +def plan_executor_lines(head_sha, pr): + """ + Plan every active line off the PR head's gitlinks. Returns a list of `(submodule, line, exec_base, pinned_sha, landed_sha, needs_push)`. `landed_sha` differs from `pinned_sha` when the line's commits @@ -495,29 +871,43 @@ def plan_executor_lines(head_sha, message): submodule = f'executors/{line}.x' exec_base = f'{line}-dev' pinned = executor_gitlink(head_sha, submodule) - landed, needs_push = plan_executor_line(submodule, exec_base, pinned, message) + landed, needs_push = plan_executor_line(submodule, exec_base, pinned, pr) plans.append((submodule, line, exec_base, pinned, landed, needs_push)) return plans def push_executor_lines(plans): - """Push each line's landed commit onto its `<line>-dev`, BEFORE the manager - push (the manager commit gitlinks into them, so the dependencies land first).""" + """ + Advance each line's `<line>-dev` onto its landed commit, after the manager push. + + Every line is attempted even if an earlier one fails, so one rejected line does + not strand the rest: the manager commit already gitlinks all of them, and each + push is independent. A failure is reported at the end with the exact ref update + to replay, because by this point the PR is landed and closed β€” re-ticking Merge + is no longer an option. + """ + failed = [] for submodule, _line, exec_base, _pinned, landed, needs_push in plans: if not needs_push: print(f'Executor {exec_base} already contains {landed}; nothing to mirror') continue print(f'Mirroring executor {exec_base} -> {landed}') - if ( - egit( - submodule, 'push', 'origin', f'{landed}:refs/heads/{exec_base}', check=False - ).returncode - != 0 - ): - block( - f'fast-forward push of executor `{exec_base}` was rejected (it advanced); ' - f're-tick Merge.' - ) + r = egit( + submodule, 'push', 'origin', f'{landed}:refs/heads/{exec_base}', check=False + ) + if r.returncode != 0: + failed.append((submodule, exec_base, landed, r.stderr.strip())) + + if failed: + details = '\n'.join( + f'- `{submodule}`: `git push origin {landed}:refs/heads/{exec_base}` ({err})' + for submodule, exec_base, landed, err in failed + ) + block( + 'the manager side landed but these executor branches did not advance, so the ' + 'new gitlinks point at commits missing from their dev branches. Replay:\n' + f'{details}' + ) def merge(pr, base, head_sha): @@ -536,23 +926,22 @@ def merge(pr, base, head_sha): f'head moved during merge (expected `{head_sha}`, got `{fetched}`); re-tick Merge.' ) - # 5. authoritative 0-commits-behind check at merge time. - if not is_ancestor(None, f'origin/{base}', 'refs/prhead'): - behind = git('rev-list', '--count', f'refs/prhead..origin/{base}').stdout.strip() + # 5. authoritative 0-commits-behind check at merge time. Same measurement as + # `initial / behind-check` and rebase-watch, so the three cannot disagree. + count = behind_lib.behind_by_git(f'origin/{base}', 'refs/prhead') + if count: block( - f'PR is {behind} commit(s) behind `{base}`; update the branch and re-tick Merge.' + f'PR is {count} commit(s) behind `{base}`; update the branch and re-tick Merge.' ) git('config', 'user.name', 'genvm-ci') git('config', 'user.email', 'genvm-ci@genlayer.com') - manager_message, executor_message = build_messages(pr) - # Plan every active executor line first: each line's commits are squashed into # one commit on its `<line>-dev`, so a line that was squashed lands under a # NEW sha and the manager commit has to gitlink that one instead of the sha the # PR pinned (which never lands on the dev branch). - plans = plan_executor_lines(head_sha, executor_message) + plans = plan_executor_lines(head_sha, pr) # Refuse to move any line's gitlink backward. The 0-behind gate is on manager # commits, not the gitlink, so a PR can be current on commits yet stale on the @@ -592,10 +981,9 @@ def merge(pr, base, head_sha): for submodule, landed in rewritten.items(): git('update-index', '--cacheinfo', f'160000,{landed},{submodule}') - trailers = coauthor_trailers(None, f'origin/{base}..{head_sha}', (name, email)) - full_message = manager_message - if trailers: - full_message += '\n' + '\n'.join(trailers) + '\n' + full_message = compose_message( + pr, None, f'origin/{base}..{head_sha}', (name, email), qualified=False + ) git( 'commit', @@ -608,22 +996,46 @@ def merge(pr, base, head_sha): ) push_sha = git('rev-parse', 'HEAD').stdout.strip() - # Push the executors first: the manager commit gitlinks into them, so the - # dependencies must land before the referrer. - push_executor_lines(plans) + # 1. Get every object onto both remotes while nothing is watching, so the two + # branch moves below are ref updates with nothing left to transfer. + stage_objects(plans, push_sha) - # Non-force FF push; rejected if base advanced since the checks. The - # executors are already advanced at this point; a rejection here leaves them - # one fast-forward ahead (harmless, monotonic) and re-ticking reconciles. + # 2. Point the PR head branches at what is about to land. This has to precede + # the base moves for GitHub to read the result as a merge rather than a close, + # and it is undone if the land is then rejected β€” a PR head left on the squashed + # commit no longer matches the sha CI ran on, which would lock the PR out of its + # own gates. + moved = [] + head_ref = pr['headRefName'] + previous = move_manager_pr_head(pr, head_sha, push_sha) + if previous is not None: + moved.append((None, head_ref, previous)) + for submodule, line, _base, _pinned, landed, _push in plans: + previous = move_executor_mirror_branch(submodule, line, head_ref, landed) + if previous is not None: + moved.append((submodule, f'pr/{line}/{head_ref}', previous)) + + time.sleep(SETTLE_SECONDS) + + # 3. Move the target branches. The manager goes first: it is the only push that + # can lose a race (the executor dev branches only move through this same tool, + # under the same PR gates), so a rejection here has nothing to unwind but the + # branch moves above. Its commit gitlinks executor commits that are already on + # the remote β€” reachable from the scratch ref and the mirror branches β€” so the + # gitlinks resolve even in the window before the executor branches advance. if ( git('push', 'origin', f'{push_sha}:refs/heads/{base}', check=False).returncode != 0 ): + restore_branches(moved) block( - f'fast-forward push to manager `{base}` was rejected (base advanced) β€” the ' - f'executor lines were already fast-forwarded; re-tick Merge to land the ' - f'manager side.' + f'fast-forward push to manager `{base}` was rejected (base advanced); ' + f'nothing landed. Update the branch and re-tick Merge.' ) + push_executor_lines(plans) + + unstage_objects(plans) + run( 'gh', 'pr', @@ -632,31 +1044,44 @@ def merge(pr, base, head_sha): '--repo', repo(), '--body', - f'βœ… Merged into `{base}` (`{push_sha}`) via fast-forward.', + f'βœ… Merged into `{base}` (`{push_sha}`) via fast-forward.' + + ( + f' Forced by `{force_actor()}` β€” the review, CI and E2E gates were not enforced.' + if force_actor() + else '' + ), check=False, ) - # Everything landed. Now auto-close every linked PR as MERGED by moving its head - # branch onto the commit that reached its base (head == base tip). This replaces - # deleting the executor mirror branches / a bare manager `gh pr close`, both of - # which dropped the merged link. Best-effort and last: the land already succeeded. - for submodule, line, _base, _pinned, landed, _push in plans: - land_executor_mirror_branch(submodule, line, pr['headRefName'], landed) - close_manager_pr(pr, push_sha) + # A fork PR's head lives in another repo, so it was never moved and GitHub has + # nothing to detect; close it explicitly (as merely 'closed' β€” unavoidable). + if pr.get('isCrossRepository'): + run('gh', 'pr', 'close', pr_number(), '--repo', repo(), check=False) class GenvmMergeIntoDev(ci_lib.Tool): - """Gate and perform a Merge of a PR into a v<X>-dev branch.""" + """ + Gate and perform a Merge of a PR into a v<X>-dev branch. + """ def name(self) -> str: return 'genvm-merge-into-dev' def add_to(self, parser: argparse.ArgumentParser) -> None: gh_common.add_args(parser, executor_repo=False, head_ref=False) + parser.add_argument( + '--force', + action='store_true', + help='merge without requiring an approving review, green CI or green ' + 'E2E; local runs only, and recorded on the PR under your `gh` login', + ) def handler(self, args: argparse.Namespace) -> int: - global _CTX - _CTX = gh_common.Ctx.from_args(args) + gh_common.set_ctx(gh_common.Ctx.from_args(args)) + if args.force: + # One channel for "who forced this", whatever set it: the announcement + # and the merge comment both read it back through `force_actor()`. + os.environ['FORCE_ACTOR'] = local_force_actor() merge_into_dev() return 0 @@ -667,11 +1092,9 @@ def merge_into_dev(): 'headRefName', 'headRefOid', 'commits', - 'labels', 'state', 'isDraft', 'title', - 'body', 'isCrossRepository', ) base, head_sha = check_gates(pr) diff --git a/support/ci/tools/make_release_notes.py b/support/ci/tools/make_release_notes.py index def8df88..602d2d4d 100755 --- a/support/ci/tools/make_release_notes.py +++ b/support/ci/tools/make_release_notes.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 -"""Generate categorized release notes from conventional commits.""" +""" +Generate categorized release notes from conventional commits. +""" import argparse import re @@ -14,7 +16,8 @@ def strip_trailing_emoji(text: str) -> str: - """Drop the trailing gitmoji suffix (and its variation selectors/ZWJ). + """ + Drop the trailing gitmoji suffix (and its variation selectors/ZWJ). Commit subjects follow ``type(scope): summary <emoji>``; the category heading already conveys the type, so the emoji is redundant noise here. @@ -35,7 +38,9 @@ def strip_trailing_emoji(text: str) -> str: def format_author(email: str) -> str: - """Collapse GitHub noreply addresses to ``@handle``; keep real emails as-is.""" + """ + Collapse GitHub noreply addresses to ``@handle``; keep real emails as-is. + """ m = NOREPLY_RE.fullmatch(email) return f'@{m.group("handle")}' if m else email @@ -62,7 +67,8 @@ class CommitEntry: def parse_git_log(rev_range: str, *, dir: Path | None = None) -> list[CommitEntry]: - """Run git log and return parsed commit entries. + """ + Run git log and return parsed commit entries. Body lines that look like ``* type: description`` are promoted to top-level entries so they get categorised on their own (inheriting the commit's @@ -149,7 +155,9 @@ def format_notes(categories: dict[str, list[str]], rev_range: str) -> str: class MakeReleaseNotes(ci_lib.Tool): - """Generate categorized release notes from conventional commits.""" + """ + Generate categorized release notes from conventional commits. + """ def name(self) -> str: return 'make-release-notes' diff --git a/support/ci/tools/open_executor_prs.py b/support/ci/tools/open_executor_prs.py index da760bc9..68b7fe7c 100644 --- a/support/ci/tools/open_executor_prs.py +++ b/support/ci/tools/open_executor_prs.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 -"""Open (or reuse) the linked executor PRs for a manager PR and list them on it. +""" +Open (or reuse) the linked executor PRs for a manager PR and list them on it. Invoked by .github/workflows/branch_executor_prs.yaml when a manager PR is opened against a dev branch or its head is pushed (`synchronize`), so mirror branches @@ -146,7 +147,9 @@ def upsert_comment(ctx: gh_common.Ctx, lines: list[str]) -> None: class OpenExecutorPrs(ci_lib.Tool): - """Open (or reuse) the linked executor PRs for a manager PR and list them on it.""" + """ + Open (or reuse) the linked executor PRs for a manager PR and list them on it. + """ def name(self) -> str: return 'open-executor-prs' diff --git a/support/ci/tools/pr_action_panel.py b/support/ci/tools/pr_action_panel.py index ba88f94c..fd95817c 100644 --- a/support/ci/tools/pr_action_panel.py +++ b/support/ci/tools/pr_action_panel.py @@ -1,13 +1,14 @@ #!/usr/bin/env python3 -"""Handle a ticked box on the GenVM PR action panel. +""" +Handle a ticked box on the GenVM PR action panel. Invoked by .github/workflows/branch_pr_actions.yaml on issue_comment:edited when the edited comment is the panel (it carries the `<!-- genvm-actions -->` marker). Steps: 1. ignore edits made by a bot (the panel's own reset echo); -2. do nothing unless the PR carries the `ci-safe` label (the gate that lets -any of these actions run; auto-added for write-access authors); +2. do nothing unless the PR carries the `ci-safe` label (see "Who may tick a +box" below); 3. parse which boxes are ticked and act: - "Force run full tests" is a STICKY toggle: on the edit that newly ticks it we set the `run-full-tests` label (so queue.yaml runs on every future push) @@ -22,6 +23,31 @@ reusable merge workflow, then untick. 4. untick the momentary boxes (not the sticky Force one). +Which action a ticked box maps to is decided by the stable `<!-- action:<id> -->` +marker the panel embeds in each line, NOT by the box's prose. Matching prose +substrings meant rewording a checkbox silently disabled its action, and a new box +whose wording happened to contain another action's keyword would fire that one. +Panels posted before the markers existed fall back to prose matching. + +Who may tick a box +------------------ +Three independent conditions, all required: + +1. the PR carries `ci-safe` β€” the same label the merge and provision workflows +check before running PR code with credentials in scope, so a target branch +cannot be moved unless the PR is vetted; +2. the edited comment was authored by the bot, i.e. it is the panel we posted; +3. the user who edited it has write access to the repo. + +(2) and (3) are both needed, and neither is implied by GitHub's permissions. +It is tempting to reason that "only the comment's author or a write-access user +may edit a comment, so an edit is already authorized" β€” but the workflow selects +the comment by a MARKER IN ITS BODY, not by who wrote it. Any user may post +their own comment carrying that marker on a public PR and freely edit their own +copy, which without (2) would let a non-collaborator force-push executor mirror +branches and request merges. `SENDER` drives (3), and the bot check on it is +only the loop-breaker for the panel's own reset edit. + Runs are started with `gh workflow run queue.yaml --ref <PR head branch>` (a `workflow_dispatch`), NOT by toggling a label: a label applied by the bot's GITHUB_TOKEN does not emit a `labeled` event (GitHub blocks recursive runs), @@ -38,31 +64,39 @@ """ import argparse +import json import os import re -import subprocess import ci_lib import gh_common +from gh_common import pr_number, repo CI_SAFE_LABEL = 'ci-safe' RUN_FULL_TESTS_LABEL = 'run-full-tests' -# Resolved GitHub context (repo, PR number) for this run; set in the handler. -_CTX: gh_common.Ctx | None = None - - -def _ctx() -> gh_common.Ctx: - assert _CTX is not None, 'gh_common.Ctx not initialized (handler must set it)' - return _CTX +# Stable ids for the panel's boxes. branch_pr_checklist.yaml embeds one per line +# as `<!-- action:<id> -->`; nothing here depends on the prose beside it, so the +# wording can change freely. +ACTION_FORCE = 'force' +ACTION_RERUN = 'rerun' +ACTION_PROVISION = 'provision' +ACTION_MERGE = 'merge' +ACTION_MARKER_RE = re.compile(r'<!--\s*action:([a-z-]+)\s*-->') -def repo(): - return _ctx().manager_repo +# Prose fallback for panels posted before the markers existed. Ordered so the +# most specific wins; a line matching none of them maps to no action. +LEGACY_ACTION_KEYWORDS = ( + (ACTION_FORCE, 'force'), + (ACTION_RERUN, 'rerun'), + (ACTION_PROVISION, 'provision'), + (ACTION_MERGE, 'merge'), +) - -def pr_number(): - return _ctx().pr_number +# The one box that is a toggle rather than a button: its ticked state mirrors the +# run-full-tests label, so the reset pass must leave it alone. +STICKY_ACTIONS = frozenset({ACTION_FORCE}) def comment_id(): @@ -73,12 +107,16 @@ def sender(): return os.environ['SENDER'] -def run(*args, check=True): - return subprocess.run(args, check=check, text=True, capture_output=True) - +def gh(*args: str, check: bool = True) -> str: + """ + `gh` with the manager token, returning stdout. -def gh(*args, check=True): - return run('gh', *args, check=check) + `retry=False`: this tool mixes reads with non-idempotent writes (adding a + label, PATCHing the panel body, dispatching a run) and the retry heuristic + cannot tell them apart, so replaying one could act twice. Keeps today's + behaviour; separating the two is GVM-328. + """ + return gh_common.gh_manager(*args, check=check, retry=False) def set_output(name, value): @@ -88,8 +126,12 @@ def set_output(name, value): def labels(): out = gh( - 'api', f'repos/{repo()}/issues/{pr_number()}/labels', '--jq', '.[].name' - ).stdout + 'api', + '--paginate', + f'repos/{repo()}/issues/{pr_number()}/labels?per_page=100', + '--jq', + '.[].name', + ) return set(out.splitlines()) @@ -115,7 +157,7 @@ def head_branch(): 'headRefName', '--jq', '.headRefName', - ).stdout.strip() + ).strip() def dispatch_full_tests(): @@ -156,19 +198,65 @@ def dispatch_provision(): print(f'dispatched executor provisioning for PR #{pr_number()}') -def ticked_boxes(body): - return [ - m.group(1).strip().lower() - for m in re.finditer(r'(?m)^\s*-\s*\[[xX]\]\s*(.+?)\s*$', body) - ] +def action_of(line: str, *, legacy: bool = False) -> str | None: + """ + The action a panel line drives, from its `<!-- action:<id> -->` marker. + + `legacy` enables the prose fallback, for panels posted before the markers + existed β€” those comments are already on open PRs and cannot be rewritten + retroactively. Returns None for a line that names no action. + """ + m = ACTION_MARKER_RE.search(line) + if m: + return m.group(1) + if not legacy: + return None + lowered = line.lower() + for action, keyword in LEGACY_ACTION_KEYWORDS: + if keyword in lowered: + return action + return None + + +def is_legacy_panel(body: str) -> bool: + """ + Whether this panel predates the `action:` markers. + + Decided per PANEL, not per line. Judging line by line would let the prose + fallback run on a modern panel β€” so a checkbox a reviewer adds by hand + (`- [x] merge conflicts resolved`) would fire `merge`, which is precisely + what the markers exist to prevent. + """ + return ACTION_MARKER_RE.search(body) is None + + +TICKED_BOX_RE = re.compile(r'(?m)^\s*-\s*\[[xX]\]\s*(.+?)\s*$') + + +def ticked_actions(body: str) -> set[str]: + """ + The set of action ids whose boxes are ticked. + """ + legacy = is_legacy_panel(body) + found = set() + for m in TICKED_BOX_RE.finditer(body): + action = action_of(m.group(1), legacy=legacy) + if action is not None: + found.add(action) + return found def untick_momentary(body): - # Untick every ticked box EXCEPT the sticky "Force run full tests" one. + # Untick every ticked box except the sticky ones, whose ticked state mirrors a + # label rather than requesting a one-off action. + legacy = is_legacy_panel(body) + def reset(line): - if re.match(r'\s*-\s*\[[xX]\]', line) and 'force' not in line.lower(): - return re.sub(r'\[[xX]\]', '[ ]', line, count=1) - return line + if not re.match(r'\s*-\s*\[[xX]\]', line): + return line + if action_of(line, legacy=legacy) in STICKY_ACTIONS: + return line + return re.sub(r'\[[xX]\]', '[ ]', line, count=1) new = '\n'.join(reset(line) for line in body.splitlines()) if body.endswith('\n'): @@ -185,7 +273,9 @@ def reset(line): class PrActionPanel(ci_lib.Tool): - """Handle a ticked box on the GenVM PR action panel.""" + """ + Handle a ticked box on the GenVM PR action panel. + """ def name(self) -> str: return 'pr-action-panel' @@ -194,8 +284,7 @@ def add_to(self, parser: argparse.ArgumentParser) -> None: gh_common.add_args(parser, executor_repo=False, head_ref=False) def handler(self, args: argparse.Namespace) -> int: - global _CTX - _CTX = gh_common.Ctx.from_args(args) + gh_common.set_ctx(gh_common.Ctx.from_args(args)) run_panel() return 0 @@ -213,31 +302,64 @@ def run_panel(): print(f'PR lacks the `{CI_SAFE_LABEL}` label; ignoring panel actions') return - body = gh( - 'api', f'repos/{repo()}/issues/comments/{comment_id()}', '--jq', '.body' - ).stdout - boxes = ticked_boxes(body) - if not boxes: + comment = json.loads( + gh( + 'api', + f'repos/{repo()}/issues/comments/{comment_id()}', + '--jq', + '{body, author: .user.login}', + ) + ) + + # The panel must be the one WE posted. The workflow only matches the marker + # on the comment body, and anyone may post a comment containing that marker + # on a public PR and then edit their own copy β€” which would let a + # non-collaborator drive these actions. Authorship is what distinguishes the + # real panel from a forgery. + author = comment['author'] + if not author.endswith('[bot]'): + print(f'comment {comment_id()} was authored by `{author}`, not the panel bot;') + print('ignoring (only the bot-posted action panel drives these actions)') + return + + # And the editor must be allowed to drive CI. Ticking a box force-pushes + # executor mirror branches and can request a merge, so it is a privileged + # action even though the merge itself re-checks its own gates. + if not gh_common.has_write_access(sender()): + print(f'`{sender()}` has no write access to {repo()}; ignoring panel actions') + return + + body = comment['body'] + actions = ticked_actions(body) + if not actions: print('no ticked boxes; nothing to do') return + print(f'ticked actions: {", ".join(sorted(actions))}') # Force: sticky enable of the run-full-tests marker so every future push # runs full tests. Only on the edit that newly sets it do we also dispatch # a run now (an already-set label means this is an unrelated edit -> no # duplicate run). - if any('force' in b for b in boxes) and RUN_FULL_TESTS_LABEL not in current: + dispatch = False + if ACTION_FORCE in actions and RUN_FULL_TESTS_LABEL not in current: add_label(RUN_FULL_TESTS_LABEL) - dispatch_full_tests() + dispatch = True # Rerun: momentary -> always dispatch a fresh run on the current head. - if any('rerun' in b for b in boxes): + if ACTION_RERUN in actions: + dispatch = True + + # One dispatch at most: ticking Force and Rerun in the same edit would + # otherwise queue two runs for the same head, and the second cancels the + # first through the `queue.yaml` concurrency group. + if dispatch: dispatch_full_tests() # Provision: momentary -> force-push executor mirror branches and open their PRs. - if any('provision' in b for b in boxes): + if ACTION_PROVISION in actions: dispatch_provision() - if any('merge' in b for b in boxes): + if ACTION_MERGE in actions: set_output('merge', 'true') untick_momentary(body) diff --git a/support/ci/tools/pr_branches.py b/support/ci/tools/pr_branches.py index e29f8f55..30488d37 100644 --- a/support/ci/tools/pr_branches.py +++ b/support/ci/tools/pr_branches.py @@ -8,7 +8,8 @@ def force_push_head(repo: str, branch: str, sha: str, token: str) -> str: - """Point `refs/heads/<branch>` in `repo` at `sha`, creating or force-moving it. + """ + Point `refs/heads/<branch>` in `repo` at `sha`, creating or force-moving it. `sha` must already be an object in `repo` (the executor commit the manager gitlink pins). Returns 'created' or 'updated'. @@ -59,7 +60,9 @@ def open_pr( manager_pr: str, token: str, ) -> str: - """Open the executor mirror PR `head -> base`, or reuse one that races in.""" + """ + Open the executor mirror PR `head -> base`, or reuse one that races in. + """ body = ( f'Auto-opened executor mirror of {manager_repo}#{manager_pr}.\n\n' f'Carries the executor-side work for that manager PR.' @@ -91,7 +94,9 @@ def open_pr( class PrBranches(ci_lib.Tool): - """Inspect and provision per-repo branch movement for a manager PR.""" + """ + Inspect and provision per-repo branch movement for a manager PR. + """ def name(self) -> str: return 'pr-branches' @@ -160,7 +165,8 @@ def provision(self, args: argparse.Namespace) -> int: return 0 def check(self, args: argparse.Namespace) -> int: - """Gate for the manager PR's CI. Records every problem, then fails if any. + """ + Gate for the manager PR's CI. Records every problem, then fails if any. The manager PR and its executor PRs land almost-atomically via the panel (Provision then Merge), so this checks that the land is POSSIBLE, not that the @@ -221,7 +227,9 @@ def _label(self, info: RepoInfo) -> str: return 'manager' if info.line is None else f'executor {info.line}' def _provisionable(self, info: RepoInfo) -> bool: - """Whether an executor line has a mirror branch + PR to provision.""" + """ + Whether an executor line has a mirror branch + PR to provision. + """ if info.base_sha is None: print(f'{info.line}: base sha is null (line absent at base); skipping') return False diff --git a/support/ci/tools/rebase_watch.py b/support/ci/tools/rebase_watch.py new file mode 100644 index 00000000..24859f45 --- /dev/null +++ b/support/ci/tools/rebase_watch.py @@ -0,0 +1,338 @@ +#!/usr/bin/env python3 +""" +Flag open PRs whose branch has fallen behind its `v<X>-dev` base. + +`initial / behind-check` (incl_initial.yaml) already asserts 0-behind, but it only +runs when the PR is pushed β€” a PR goes stale when the BASE moves, and nothing +re-evaluates it then. So a PR can sit green for days and only discover at Merge +time that it is unmergeable (the Merge action fast-forwards, so behind > 0 is +fatal there). + +This tool re-evaluates on the other edge: every push to a dev branch sweeps the +PRs targeting it. It is deliberately advisory β€” it posts a check-run and a label, +and both are cosmetic. Nothing here gates a merge: the authoritative 0-behind +check stays in `initial` and in genvm_merge_into_dev. The point is that the PR +page turns red the moment the branch goes stale rather than at merge time. + +Two surfaces, because they are visible in different places: + +- a `not rebased` check-run on the PR head β€” red on the PR page, with the exact + `git` command to fix it; +- a red `not rebased` label β€” the only one of the two visible in the PR *list*. + +Both are cleared as soon as the PR is 0 behind again. "Behind" comes from +`behind.behind_by_api` β€” the same measurement the enforcing checks use, via the +compare API, so nothing is checked out and fork PRs work the same as branch PRs. + +Repo and PR number resolve arg > env > default via gh_common; the token is +optional (ambient `gh` auth when unset). +""" + +import argparse +import json +import re +import tempfile + +import ci_lib +import gh_common + +# The function, not the module: `sweep` has a local named `behind`, and a module +# of that name would be shadowed by it. +from behind import behind_by_api +from gh_common import gh_manager as gh +from gh_common import repo + +LABEL = 'not rebased' +LABEL_COLOR = 'b60205' +LABEL_DESCRIPTION = 'branch is behind its base; rebase before it can be merged' + +CHECK_NAME = 'not rebased' +# Stamped on every check-run this tool creates, so it can find its own again +# without trusting the name (which any app may also use). +CHECK_EXTERNAL_ID = 'genvm-ci/rebase-watch' + +# Bases this sweeps. The Merge action only ever fast-forwards onto a dev branch, +# so a PR targeting anything else has no 0-behind requirement to report on. +DEV_BASE_RE = re.compile(r'v.*-dev') + + +def pr_state(number: int | str) -> dict: + """ + Current number/base/head/state of one PR. + """ + return json.loads( + gh( + 'api', + f'repos/{repo()}/pulls/{number}', + '--jq', + '{number, base: .base.ref, head: .head.sha, state}', + ) + ) + + +def open_prs(base: str | None, pr_number: str | None) -> list[dict]: + """ + The PRs to evaluate: one explicit PR, or every open PR on a dev base. + + `--pr` wins so the PR-side triggers (a push to the PR, a retarget) can refresh + just that PR instead of re-sweeping the repo. An explicit PR is returned even + when its base is not a dev branch: `sweep` still has to CLEAR a flag it set + before the PR was retargeted off one. + """ + if pr_number is not None: + pr = pr_state(pr_number) + return [pr] if pr['state'] == 'open' else [] + + prs = [ + json.loads(line) + for line in gh( + 'api', + '--paginate', + f'repos/{repo()}/pulls?state=open&per_page=100', + '--jq', + '.[] | {number, base: .base.ref, head: .head.sha}', + ).splitlines() + if line.strip() + ] + if base is not None: + return [pr for pr in prs if pr['base'] == base] + return [pr for pr in prs if DEV_BASE_RE.fullmatch(pr['base'])] + + +def behind_by(base: str, head_sha: str) -> int: + """ + Commits in `base` that `head_sha` does not contain. + """ + return behind_by_api(repo(), base, head_sha, token=gh_common.manager_token()) + + +def ensure_label() -> None: + """ + Create the label if it is missing. Already-exists (422) is the normal case. + """ + gh_common.gh( + 'api', + '--method', + 'POST', + f'repos/{repo()}/labels', + '-f', + f'name={LABEL}', + '-f', + f'color={LABEL_COLOR}', + '-f', + f'description={LABEL_DESCRIPTION}', + token=gh_common.manager_token(), + check=False, + ) + + +def set_label( + pr: int, present: bool, had: bool, expect_head: str, expect_base: str +) -> None: + """ + Add or remove the label, but only when it actually changes. + + A no-op add still writes a timeline event and notifies subscribers, and this + tool runs on every push to a dev branch β€” so re-asserting the current state on + every sweep would spam every open PR. + + The label is the one surface NOT keyed by commit (a check-run lives on its own + sha, so a stale one is invisible), which makes it the only thing a base-push + sweep and a PR-push sweep can fight over. Both coordinates are re-read before + writing, because either can move underneath a sweep: + + - the head: a base sweep reads head H0, the author rebases to H1, the PR + sweep clears the label, and the base sweep would re-add it for a commit that + no longer exists; + - the base: a base sweep computes "behind", the PR is retargeted off that dev + branch (its head unchanged, so a head-only guard passes), the retarget run + clears the flag, and the base sweep would re-add it β€” permanently, since no + later sweep of that base will ever see this PR again. + + In both cases the run for the NEW state has already flagged it correctly. + """ + if present == had: + return + current = pr_state(pr) + if current['head'] != expect_head: + print(f'#{pr}: head moved while evaluating; leaving the label to that run') + return + if current['base'] != expect_base: + print( + f'#{pr}: base changed to `{current["base"]}` while evaluating; ' + f'leaving the label to that run' + ) + return + if present: + gh_common.gh( + 'api', + '--method', + 'POST', + f'repos/{repo()}/issues/{pr}/labels', + '-f', + f'labels[]={LABEL}', + token=gh_common.manager_token(), + check=False, + ) + else: + # 404 when the label is already gone (a concurrent sweep, a manual removal). + gh_common.gh( + 'api', + '--method', + 'DELETE', + f'repos/{repo()}/issues/{pr}/labels/{LABEL}', + token=gh_common.manager_token(), + check=False, + ) + + +def has_label(pr: int) -> bool: + names = gh( + 'api', '--paginate', f'repos/{repo()}/issues/{pr}/labels', '--jq', '.[].name' + ).splitlines() + return LABEL in [n.strip() for n in names] + + +def existing_check_id(head_sha: str) -> int | None: + """ + Our own check-run on `head_sha`, if we already posted one. + + Matched on `external_id`, not on the name: a name is not a namespace, so + another app could hold a check-run called `not rebased` and we would PATCH + (or, lacking its app's token, fail to PATCH) a check that is not ours. The + API defaults to `filter=latest`, so this returns the one we would replace β€” + updating in place keeps the PR's checks list to a single entry rather than one + per dev-branch push. + + `api_path`, not an f-string: `CHECK_NAME` contains a space, and `gh api` + forwards a query string verbatim β€” an unencoded space makes the request + **hang** rather than fail, which would wedge this job until its timeout. + """ + ids = gh( + 'api', + gh_common.api_path( + f'repos/{repo()}/commits/{head_sha}/check-runs', + check_name=CHECK_NAME, + per_page=100, + ), + '--jq', + f'.check_runs[] | select(.external_id == "{CHECK_EXTERNAL_ID}") | .id', + ).splitlines() + return int(ids[0]) if ids else None + + +def post_check(head_sha: str, base: str, behind: int, *, stale: bool = False) -> None: + """ + Create or update the advisory check-run on the PR head. + + `stale` means the PR no longer targets a dev branch, so the check has nothing + to say β€” but it is only cleared, never freshly posted, so PRs that never had + one stay untouched. + """ + check_id = existing_check_id(head_sha) + if stale: + if check_id is None: + return + conclusion = 'neutral' + title = f'not applicable β€” base `{base}` is not a dev branch' + summary = f'`{base}` is not merged by fast-forward, so being behind it is fine.' + elif behind: + conclusion = 'failure' + title = f'β›” {behind} commit(s) behind {base} β€” rebase' + summary = ( + f'This branch is **{behind} commit(s) behind `{base}`**.\n\n' + f'The Merge action only fast-forwards, so it will refuse this branch until ' + f'it contains the base tip:\n\n' + f'```sh\ngit fetch origin\ngit rebase origin/{base}\ngit push --force-with-lease\n```\n\n' + f'(Advisory: this check does not block anything. `initial / behind-check` ' + f'and the Merge action are what actually enforce it.)' + ) + else: + conclusion = 'success' + title = f'up to date with {base}' + summary = f'This branch contains the tip of `{base}`.' + + payload = { + 'name': CHECK_NAME, + 'head_sha': head_sha, + 'external_id': CHECK_EXTERNAL_ID, + 'status': 'completed', + 'conclusion': conclusion, + 'output': {'title': title, 'summary': summary}, + } + if check_id is None: + method, path = 'POST', f'repos/{repo()}/check-runs' + else: + method, path = 'PATCH', f'repos/{repo()}/check-runs/{check_id}' + payload.pop('head_sha') + + # Via a file rather than `-f` fields: the payload nests (`output.title`), which + # `gh api -f` cannot express, and `--input -` would need stdin that gh_common.gh + # does not plumb. + with tempfile.NamedTemporaryFile('w', suffix='.json') as body: + json.dump(payload, body) + body.flush() + gh_common.gh( + 'api', + '--method', + method, + path, + '--input', + body.name, + token=gh_common.manager_token(), + ) + + +def sweep(base: str | None, pr_number: str | None, dry_run: bool) -> None: + prs = open_prs(base, pr_number) + if not prs: + print('no open PRs to evaluate') + return + if not dry_run: + ensure_label() + for pr in prs: + # A PR retargeted off a dev branch keeps whatever flag it was last given, + # and no later push sweep will ever see it again β€” so clear it here. Only + # clearing, never setting: a PR that never carried the flag stays untouched. + stale = not DEV_BASE_RE.fullmatch(pr['base']) + behind = 0 if stale else behind_by(pr['base'], pr['head']) + if stale: + state = f'base `{pr["base"]}` is not a dev branch; clearing' + else: + state = f'{behind} behind' if behind else 'up to date' + print(f'#{pr["number"]} ({pr["base"]}): {state}') + if dry_run: + continue + had = has_label(pr['number']) + post_check(pr['head'], pr['base'], behind, stale=stale) + set_label(pr['number'], behind > 0, had, pr['head'], pr['base']) + + +class RebaseWatch(ci_lib.Tool): + """ + Flag open PRs that have fallen behind their dev base (advisory only). + """ + + def name(self) -> str: + return 'rebase-watch' + + def add_to(self, parser: argparse.ArgumentParser) -> None: + gh_common.add_args(parser, executor_repo=False, head_ref=False) + parser.add_argument( + '--base', + default=None, + help='only sweep PRs targeting this base branch (default: every v<X>-dev base)', + ) + parser.add_argument( + '--dry-run', + action='store_true', + help='report how far behind each PR is without posting a check or label', + ) + + def handler(self, args: argparse.Namespace) -> int: + ctx = gh_common.set_ctx(gh_common.Ctx.from_args(args)) + sweep(args.base, ctx.pr_number_opt, args.dry_run) + return 0 + + +COMMANDS = [RebaseWatch()] diff --git a/support/ci/tools/versions.py b/support/ci/tools/versions.py index 3850a5a6..4797da4f 100644 --- a/support/ci/tools/versions.py +++ b/support/ci/tools/versions.py @@ -3,6 +3,7 @@ import os import re import sys +from pathlib import Path import ci_lib @@ -21,7 +22,7 @@ def version_key(version: str) -> tuple[int, ...]: def load_root_conf() -> dict: conf_path = os.environ.get('MONOREPO_ROOT', ci_lib.ROOT_DIR / '.genvm-monorepo-root') - return json.loads(open(conf_path, 'rb').read()) + return json.loads(Path(conf_path).read_bytes()) def active_versions() -> list[str]: @@ -32,7 +33,8 @@ def active_versions() -> list[str]: def manager_version() -> str: - """The manager's own release train as bare major.minor (e.g. '0.6'). + """ + The manager's own release train as bare major.minor (e.g. '0.6'). Read from the `version` field. The manager dev/release branches (`v<X>-dev`, `v<X>.x`) are named after this β€” NOT after the executor lines @@ -42,7 +44,8 @@ def manager_version() -> str: def active_lines() -> list[str]: - """Active executor lines as `v<X>` tags (e.g. ['v0.2', 'v0.3']). + """ + Active executor lines as `v<X>` tags (e.g. ['v0.2', 'v0.3']). Same data as `active_versions`, prefixed with the `v` the executor branches (`<line>-dev`, `pr/<line>/...`) use. The manager release line is independent of @@ -51,8 +54,20 @@ def active_lines() -> list[str]: return [f'v{version}' for version in active_versions()] +def newest_line() -> str: + """ + The newest active executor line as `v<X>` (e.g. 'v0.3'). + + `active-versions` is hand-ordered in .genvm-monorepo-root, so "newest" must be + decided by `version_key`, not by position β€” reading `active-versions[0]` + directly is what let two consumers disagree on which line is primary. + """ + return active_lines()[-1] + + class BranchVersions(ci_lib.Tool): - """Read version trains from .genvm-monorepo-root. + """ + Read version trains from .genvm-monorepo-root. `list` yields the active executor lines (`active-versions`); `manager` yields the manager's own train (`version`). These differ β€” don't use one @@ -74,7 +89,9 @@ def handler(self, args: argparse.Namespace) -> int: class CheckVersions(ci_lib.Tool): - """Keep manager-owned Rust crate versions in sync with .genvm-monorepo-root.""" + """ + Keep manager-owned Rust crate versions in sync with .genvm-monorepo-root. + """ CARGO_FILES = ['implementation/Cargo.toml'] diff --git a/support/scripts/check-commit-message.py b/support/scripts/check-commit-message.py index 01f1bce4..e7e70db0 100755 --- a/support/scripts/check-commit-message.py +++ b/support/scripts/check-commit-message.py @@ -52,7 +52,12 @@ # Matches git trailer lines: "Token: value" (Co-authored-by:, Signed-off-by:, etc.) GIT_TRAILER_RE = re.compile(r'^[A-Za-z][A-Za-z0-9-]*:\s*.+$') -PR_REF_RE = re.compile(r'\s*\(#\d+\)\s*$') +# A trailing PR reference, either bare (`(#42)`) or repo-qualified +# (`(genlayerlabs/genvm-manager#42)`). The qualified form is what a squash +# landing in the EXECUTOR repo carries, because a bare `#42` would not resolve +# there β€” see genvm_merge_into_dev.pr_reference. Both must be recognised, or a +# perfectly good subject is read as having its emoji in the middle. +PR_REF_RE = re.compile(r'\s*\((?:[\w.-]+/[\w.-]+)?#\d+\)\s*$') SQUASH_BULLET_RE = re.compile(r'^\*\s*') diff --git a/support/scripts/ci-changes.py b/support/scripts/ci-changes.py index 92d4581b..402f588d 100755 --- a/support/scripts/ci-changes.py +++ b/support/scripts/ci-changes.py @@ -91,6 +91,23 @@ def fetch_pr_refs(base_ref: str, pr_number: str) -> tuple[str, str]: return merge_base, head +def gh(*args: str) -> str: + proc = subprocess.run(['gh', *args], check=True, capture_output=True, text=True) + return proc.stdout.strip() + + +def base_ref_of(pr_number: str) -> str: + """The PR's base branch, read from the API. + + `GITHUB_BASE_REF` is only set on `pull_request` events, but these checks also + run on the panel's `workflow_dispatch`, where the PR number is known and the + event payload is not. Falling back to "not a pull request" there would report + no changes at all β€” a silent pass. + """ + repo = os.environ.get('GITHUB_REPOSITORY', '') + return gh('api', f'repos/{repo}/pulls/{pr_number}', '--jq', '.base.ref') + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -100,13 +117,17 @@ def main() -> int: ) args = parser.parse_args() - event = os.environ.get('GITHUB_EVENT_NAME', '') base_ref = os.environ.get('GITHUB_BASE_REF', '') pr_number = os.environ.get('GITHUB_PR_NUMBER', '') paths = [MANAGER, *submodule_paths()] - if event == 'pull_request' and base_ref and pr_number: + # Keyed off the PR NUMBER, not the event name: the number is available on + # every event that validates a PR, and keying off `pull_request` made this + # silently report "no changes" on panel-dispatched runs. + if pr_number: + if not base_ref: + base_ref = base_ref_of(pr_number) base, head = fetch_pr_refs(base_ref, pr_number) else: # Not a pull request: nothing to diff against. diff --git a/support/tools/genvm-tool/genvm_tool/cmd_configure.py b/support/tools/genvm-tool/genvm_tool/cmd_configure.py index 84ac40ea..b6814b48 100644 --- a/support/tools/genvm-tool/genvm_tool/cmd_configure.py +++ b/support/tools/genvm-tool/genvm_tool/cmd_configure.py @@ -97,11 +97,6 @@ def rust_target_dirs_info( def configure(parser): - parser.add_argument( - '--test-always-fail-module', - action='store_true', - help='enable the test_always_fail_module feature on the executor', - ) parser.add_argument( '--ci', action='store_true', @@ -244,8 +239,6 @@ def main(ctx: common.Context, args) -> int: # Regenerate build.ninja when this script changes (re-runs the tool via the # launcher above). Preserve the configure flags so a regen keeps mode. regen_args = [] - if args.test_always_fail_module: - regen_args.append('--test-always-fail-module') if args.ci: regen_args.append('--ci') regen = n.build('CUSTOM_COMMAND', 'build.ninja') From 98148e99f82d4e9150c31a1ffec6704e7dfc3cd1 Mon Sep 17 00:00:00 2001 From: kp2pml30 <kp2pml30@gmail.com> Date: Fri, 31 Jul 2026 15:39:48 +0900 Subject: [PATCH 06/45] chore: test improvements (#17) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: add pre-push hook * chore: restructure contributor docs and rework the test harness πŸ“β™»οΈ * fix(git): ask the remote for the gitlink commit, not for a matching ref πŸ› * chore(ci): push locally built store paths to the nix cache πŸ—οΈ * docs: fix wording and the versions.json path flagged in review πŸ“ * chore(ci): make the whole nix cache push fail-soft πŸ’š * fix(build): keep a user pre-push hook and serialize the submodule cache πŸ› * chore(codegen): assert a non-empty description in $str constructors βœ… --- .claude/settings.fuzzing.json | 18 - .claude/skills/agentic-fuzzing/SKILL.md | 139 +++++++ .genvm-tool.py | 39 +- .../genlayerlabs-nix-cache-push/action.yaml | 88 ++++ .../genlayerlabs-nix-setup/action.yaml | 79 ++++ .github/actions/get-src/action.yaml | 13 +- .github/workflows/incl_initial.yaml | 8 + .../incl_release_build_test_cell_build.yaml | 8 + .../incl_release_build_test_cell_test.yaml | 8 + .github/workflows/queue_test_cell.yaml | 6 + CLAUDE.md | 58 ++- docs/README.md | 15 + docs/contributing/README.md | 26 +- docs/contributing/explanation/README.md | 9 + docs/contributing/explanation/docs-layout.md | 37 ++ .../explanation/executor-lines.md | 30 ++ docs/contributing/explanation/merge-model.md | 31 ++ .../explanation/vendored-trees.md | 21 + docs/contributing/howto/README.md | 48 ++- docs/contributing/howto/building/build.md | 73 ++-- docs/contributing/howto/building/docs.md | 75 ++-- docs/contributing/howto/building/runners.md | 19 +- .../howto/committing/git-third-party.md | 91 ++--- docs/contributing/howto/committing/runners.md | 40 +- .../howto/committing/submodules.md | 111 +++-- docs/contributing/howto/docs/style.md | 43 ++ .../howto/extending/add-host-function.md | 21 +- .../howto/extending/add-llm-provider.md | 22 +- .../howto/extending/add-wasi-function.md | 29 +- .../howto/extending/modify-runner.md | 25 +- .../howto/extending/modify-wasmtime.md | 41 +- .../howto/extending/write-a-script.md | 36 ++ docs/contributing/howto/genvm-tool.md | 64 ++- docs/contributing/howto/pr.md | 54 +++ .../howto/releasing/release-build.md | 54 +-- .../howto/releasing/versioning.md | 56 ++- docs/contributing/howto/setup.md | 33 +- docs/contributing/howto/testing/README.md | 56 ++- docs/contributing/howto/testing/fuzzing.md | 24 +- .../contributing/howto/testing/integration.md | 105 ++--- docs/contributing/howto/testing/python.md | 12 +- docs/contributing/howto/testing/rust.md | 30 +- docs/contributing/howto/write-a-script.md | 43 -- docs/contributing/tutorial/README.md | 6 + .../tutorial/first-contribution.md | 109 ++--- executors/v0.2.x | 2 +- executors/v0.3.x | 2 +- flake.nix | 59 ++- support/scripts/check-source-text.py | 4 +- support/scripts/get-all-git.py | 36 +- support/tools/genvm-tool/default.nix | 9 +- .../tools/genvm-tool/genvm_tool/__main__.py | 8 +- .../genvm-tool/genvm_tool/cmd_configure.py | 37 +- .../tools/genvm-tool/genvm_tool/cmd_docs.py | 114 ++++++ .../tools/genvm-tool/genvm_tool/cmd_howto.py | 94 ----- .../genvm-tool/genvm_tool/codegen/model.py | 10 +- .../genvm-tool/genvm_tool/codegen/rst.py | 2 + .../genvm-tool/genvm_tool/codegen/rust.py | 9 + support/tools/genvm-tool/genvm_tool/common.py | 57 ++- .../genvm-tool/genvm_tool/git/__init__.py | 16 +- .../genvm_tool/git/check_for_push.py | 18 +- .../genvm_tool/git/create_branches.py | 6 +- .../genvm_tool/git/verify_pushed_executors.py | 235 +++++++++++ .../genvm-tool/genvm_tool/tests/__init__.py | 3 +- .../genvm_tool/tests/util/watchdog.py | 97 +++-- .../genvm_tool/tests/util/watchdog_main.py | 127 +++--- .../test_verify_pushed_executors.py | 378 ++++++++++++++++++ tests/runner/genvm_tool_plugins/genvm.py | 26 +- .../runner/genvm_tool_plugins/integration.py | 74 +++- tests/runner/genvm_tool_plugins/ninja.py | 14 +- tests/runner/origin/log_asserts.py | 2 +- tests/runner/origin/logger.py | 48 ++- tests/templates/util.jsonnet | 19 +- 73 files changed, 2475 insertions(+), 984 deletions(-) delete mode 100644 .claude/settings.fuzzing.json create mode 100644 .claude/skills/agentic-fuzzing/SKILL.md create mode 100644 .github/actions/genlayerlabs-nix-cache-push/action.yaml create mode 100644 .github/actions/genlayerlabs-nix-setup/action.yaml create mode 100644 docs/README.md create mode 100644 docs/contributing/explanation/README.md create mode 100644 docs/contributing/explanation/docs-layout.md create mode 100644 docs/contributing/explanation/executor-lines.md create mode 100644 docs/contributing/explanation/merge-model.md create mode 100644 docs/contributing/explanation/vendored-trees.md create mode 100644 docs/contributing/howto/docs/style.md create mode 100644 docs/contributing/howto/extending/write-a-script.md create mode 100644 docs/contributing/howto/pr.md delete mode 100644 docs/contributing/howto/write-a-script.md create mode 100644 docs/contributing/tutorial/README.md create mode 100644 support/tools/genvm-tool/genvm_tool/cmd_docs.py delete mode 100644 support/tools/genvm-tool/genvm_tool/cmd_howto.py create mode 100644 support/tools/genvm-tool/genvm_tool/git/verify_pushed_executors.py create mode 100644 support/tools/genvm-tool/unit_tests/test_verify_pushed_executors.py diff --git a/.claude/settings.fuzzing.json b/.claude/settings.fuzzing.json deleted file mode 100644 index eac0945d..00000000 --- a/.claude/settings.fuzzing.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "permissions": { - "allow": [ - "Bash(./tests/cases/claude/run_test.py:*)", - "Edit(tests/cases/claude/agent/**)", - "Write(tests/cases/claude/agent/**)", - "Edit(tests/cases/claude/intelligence/**)", - "Write(tests/cases/claude/intelligence/**)" - ] - }, - "sandbox": { - "enabled": true, - "autoAllowBashIfSandboxed": true, - "excludedCommands": [ - "./tests/cases/claude/run_test.py" - ] - } -} diff --git a/.claude/skills/agentic-fuzzing/SKILL.md b/.claude/skills/agentic-fuzzing/SKILL.md new file mode 100644 index 00000000..4c03387a --- /dev/null +++ b/.claude/skills/agentic-fuzzing/SKILL.md @@ -0,0 +1,139 @@ +--- +name: agentic-fuzzing +description: Hunts for determinism violations and internal errors in a GenVM executor by writing throwaway probe contracts and running them across leader/validator/sync. Use when asked to fuzz the VM, look for nondeterminism, or turn a PR into a failing test. +--- + +# Agentic fuzzing + +Target the two `SECURITY.md` severities a Python contract can reach: + +- **2 β€” determinism violation.** Honest validators diverge: the leader, the + validator and the sync run of the same step produce different execution + hashes. +- **4 β€” crash / internal error.** A contract triggers `INTERNAL_ERROR` β€” a + panic or unhandled error where a canonical `VMError` belongs. + +Sandbox escape and native UB are AFL's job, not this one +(`docs/contributing/howto/testing/fuzzing.md`). + +## What is and is not a finding + +| Outcome | Finding? | +|---|---| +| leader / validator / sync hashes disagree | **yes**, severity 2 | +| `INTERNAL_ERROR` | **yes**, severity 4 | +| WASM trap | no β€” that is the sandbox working | +| `UserError`, any other `VMError` | no | +| timeout | no, and they are flaky | +| mock-host error | no β€” that is the harness, not the VM | + +## Before you can find anything + +Nothing. The v0.3 line returns `save-hashes: False` from +`executors/v0.3.x/.genvm-tool.py`, but that only stops the committed `.hash` +sidecars from being tracked β€” the leader-vs-validator/sync comparison, which is +what a severity-2 finding consists of, runs regardless. Leave the setting alone. + +(It used to be `ignore-hash: True` and it did switch off every comparison, +including that one. That was a footgun and it is gone.) + +## Where to work + +Probes are throwaway. Write them to +`executors/v0.3.x/tests/integration/claude/_scratch/<slug>/`, which is +gitignored, and delete them at the end of the session. Collection is a glob over +`tests/integration/**/*.jsonnet`, so nothing has to be registered β€” but it drops +files whose *name* starts with `_`, so name the jsonnet after the slug and leave +the underscore to the directory. + +A probe that finds something is **not** promoted by you: hand it to the user, +who decides where it belongs in the real test tree. A probe that finds nothing +dies, leaving a paragraph in +`executors/v0.3.x/tests/integration/claude/intelligence/EXPLORED_PATHS.md` +saying what was ruled out and how β€” that file is the only thing that survives +between sessions, so it is worth writing well. + +## Writing a probe + +Copy the shape from `tests/integration/claude/example/`: one or more `.py` +contracts and a `.jsonnet` scenario. Templates live in `tests/templates/` β€” +`util.jsonnet` (`addPaths`, `chain`) structures multi-step cases, +`simple_deploy.jsonnet` covers deploy-and-call, `message.json` is the base +message. + +On every step: + +```jsonnet +expected_semantics_components: [], // stdout is not what you are checking +modes: 'lvs', // the three runs whose hashes must agree +stable_hash: false, // compare against the leader's runtime hash +``` + +and on the top-level object `tags: ['stable']`, which tells the harness the case +needs neither LLM keys nor a webdriver. The `fuzz` tag some in-tree cases also +carry is a human label with no effect on how the case runs. + +Two things about contract sources: + +- The runner header is the **first line**, and the parser concatenates *every* + leading `#` line into one JSON document (`executor/src/runners/parse.rs`). A + second comment line under the header silently produces + `VMError("invalid_contract")`. Put explanations below the imports. +- `# { "Depends": "py-genlayer:test" }` is the normal header. The `:test` alias + resolves only from debug mode `unsafe` up. + +## Debug mode + +Cases run at `unsafe` by default. A case can lower that with a top-level +`debug_mode` in its jsonnet β€” one of `safe`, `safe-unbounded`, `unsafe`, +`unsafe-tracing` (`disabled` is rejected: below `safe` the case stops being +routed to its own line's executor and silently runs another one). + +Lower it to `safe` when you need to be sure a divergence is the contract's and +not a debug facility's β€” `safe` refuses both wall-clock exposure and the `:test` +alias. Once `:test` no longer resolves the contract must name its runner by +hash, read out of `build/out/executor/<version>/data/latest.json`, e.g. +`# { "Depends": "py-genlayer:9b8kjy…" }`. That hash changes whenever the SDK +does, which is exactly why in-tree cases keep the alias and only probes give it +up. + +## Running one + +```bash +genvm-tool test run --filter-name 'claude/_scratch/<slug>' +``` + +`--filter-name` is an unanchored regex over the test name and does isolate a +single case. Artifacts land in +`build/test-artifacts/cases/<test name>/<tree_path><mode>/`, with leading +underscores stripped from the path β€” a probe in `_scratch/` reports under +`claude/scratch/`. `genvm.log.gz` is the executor's own log, `config.json` what +the step was given, `hash` its base64 execution hash, `stderr.txt` the guest's +traceback. There is no `semantics.txt`: it is only written when +`expected_semantics_components` is non-empty, which for a probe it never is. + +### A green probe proves nothing on its own + +`expected_semantics_components: []` means no output is compared, and the +leader-vs-validator comparison does not catch a load failure either: it is +identical in all three modes, so the hashes agree and the case still passes β€” a +probe whose contract never loaded reports `βœ“` in 50ms. Before believing a pass, open +`genvm.log.gz` and check the run reached the contract, and decode the `hash` +artifact β€” `base64 -d < hash` on a failed load reads `invalid_contract +malformed_runner`. + +## Reviewing a PR for a failing test + +When the ask is "find me a failing test for this branch" rather than open-ended +fuzzing: + +1. Read the diff first β€” `git diff <base>...HEAD`. Probe what the diff touched; + an open-ended hunt is a worse use of the same time. +2. Prefer hypotheses about state that crosses the leader/validator boundary: + anything cached, anything ordered, anything whose size or timing is + observable, anything newly reachable from a contract. +3. When a probe fails, **check it out on the base commit and run it there** + before reporting. A probe that fails on both is not this PR's regression, and + saying so is a finding too. This step is what makes the report worth reading. +4. Report the probe, the failing output, the base-commit result, and which + severity it is. Do not fix the code. diff --git a/.genvm-tool.py b/.genvm-tool.py index 681d1429..979bef33 100644 --- a/.genvm-tool.py +++ b/.genvm-tool.py @@ -3,13 +3,13 @@ Loaded once by genvm-tool (``common.load_project``) before any subcommand runs; subcommands ask it for what they need: -- ``tests(ctx)`` β€” the umbrella test suite (was the top-level ``.ya-test.py``): +- ``tests(ctx)`` -- the umbrella test suite (was the top-level ``.ya-test.py``): imports the plugins it needs and registers collectors + CLI args on the runner's configuration ``Context``. This is the plugins' "initial run" hook; plugins themselves are plain importable modules (no test-runner-specific registry). Per-suite test definitions live next to the tests in ``tests/system/<name>/test.py`` and are pulled in via ``ctx.collect_dir``. - Heavy imports stay inside this function β€” they need the plugin search path + Heavy imports stay inside this function -- they need the plugin search path (``extra_python_paths`` in ``.genvm-monorepo-root``), which the test command applies before calling us. @@ -34,31 +34,23 @@ def tests(ctx): _info_path = ctx.shared.root_dir / 'build' / 'info.json' if not _info_path.exists(): + # CI runs tests without configuring first. `configure` writes this same + # base plus what the build teaches it, and both call base_info, so the + # two writers cannot disagree about the tree. ctx.shared.logger.warning('build/info.json not found, generating default') _build_dir = ctx.shared.root_dir / 'build' _build_dir.mkdir(parents=True, exist_ok=True) _monorepo_cfg = json.loads( (ctx.shared.root_dir / genvm_tool.cmd_configure.MONOREPO_ROOT_FILE).read_text() ) - _rust_target_dir = _build_dir / 'ya-build' / 'rust-target' _info_path.write_text( json.dumps( - { - 'coverage_dir': str(_build_dir / 'cov'), - 'build_dir': str(_build_dir), - 'rust_target_dir': str(_rust_target_dir), - 'rust_target': genvm_tool.cmd_configure.detect_rust_target(), - # executor_versions / primary_executor_version: resolved from - # source (.genvm-monorepo-root + committed manifests), so a - # build-less test run still knows which out/executor/<real> dir - # to reroute to. configure emits the same keys. - **genvm_tool.cmd_configure.build_independent_info( - _monorepo_cfg, ctx.shared.root_dir - ), - **genvm_tool.cmd_configure.rust_target_dirs_info( - _monorepo_cfg, ctx.shared.root_dir, _rust_target_dir - ), - }, + genvm_tool.cmd_configure.base_info( + _monorepo_cfg, + ctx.shared.root_dir, + _build_dir, + _build_dir / 'ya-build' / 'rust-target', + ), indent=2, ) + '\n' @@ -195,7 +187,9 @@ def collect_integration(ctx: genvm_tool.tests.stage.collection.Context): manager_port = genvm.get_manager_port(ctx.configuration) if no_manager: - manager_impl = genvm.ExternalManagerService(port=manager_port) + manager_impl = genvm.ExternalManagerService( + port=manager_port, + ) webdriver_impl = genvm.NoOpService() modules_impl = genvm.NoOpService() else: @@ -221,7 +215,10 @@ def collect_integration(ctx: genvm_tool.tests.stage.collection.Context): name='manager', manager=manager_impl, ) - manager_service.meta = {'port': manager_port, 'reroute_to': reroute_to} + manager_service.meta = { + 'port': manager_port, + 'reroute_to': reroute_to, + } webdriver_service = ctx.new_service( name='webdriver', diff --git a/.github/actions/genlayerlabs-nix-cache-push/action.yaml b/.github/actions/genlayerlabs-nix-cache-push/action.yaml new file mode 100644 index 00000000..d5d64e79 --- /dev/null +++ b/.github/actions/genlayerlabs-nix-cache-push/action.yaml @@ -0,0 +1,88 @@ +name: GenLayer nix cache push +description: >- + Upload everything this job built locally to the attic binary cache, so the + next run substitutes it instead of rebuilding. +inputs: + token: + description: >- + push token for the cache. Empty disables the step entirely: a pull request + from a fork gets no secrets, and such a run must still pass. + required: false + default: "" + cache: + description: cache name on the server + required: false + default: genlayer + endpoint: + description: attic server the cache lives on + required: false + default: https://nix-cache.ygr.ai + include: + description: >- + Regex matched against the store path name with its hash stripped. Narrows + the locally-built set to our own derivations: a build also realizes + third-party intermediates that upstream caches already serve. + required: false + default: genlayer|genvm + exclude: + description: regex matched against the full store path; empty disables + required: false + default: "" +runs: + using: composite + steps: + # No `-x`: the token is passed to `attic login` as an argument. + - shell: bash -eo pipefail {0} + if: ${{ inputs.token != '' }} + env: + NIX_CACHE_PUSH_TOKEN: ${{ inputs.token }} + CACHE: ${{ inputs.cache }} + ENDPOINT: ${{ inputs.endpoint }} + INCLUDE: ${{ inputs.include }} + EXCLUDE: ${{ inputs.exclude }} + run: | + cd "$GITHUB_WORKSPACE" + + # A cache that refuses uploads must not turn a green build red, so every + # step of this script warns and gives up instead of failing the job. + attic_link="${RUNNER_TEMP:-/tmp}/attic-client" + # An out-link rather than `nix run`: it is a gcroot, and a second call + # in the same job skips both evaluation and substitution. + # `--inputs-from .` resolves `nixpkgs` through our flake lock, so the + # client comes from the pinned revision instead of the mutable registry. + if ! nix build --out-link "$attic_link" --inputs-from . nixpkgs#attic-client; then + echo "::warning::could not build the attic client; nothing pushed" + exit 0 + fi + + # `ultimate` is nix's own flag for "built on this machine", which is + # exactly the set worth uploading: everything else arrived from a + # substituter that already has it. It covers the intermediates the + # final artifact is assembled from, not just the artifact. + paths="${RUNNER_TEMP:-/tmp}/attic-paths" + nix path-info --all --json --json-format 1 \ + | jq -r --arg inc "$INCLUDE" --arg exc "$EXCLUDE" ' + to_entries[] + | select(.value.ultimate) + | select($inc == "" or (.key | sub("^/nix/store/[^-]+-"; "") | test($inc))) + | select(.key | test("(-|\\.)(source|socket|service|env|pam|drv)$") | not) + | select($exc == "" or (.key | test($exc) | not)) + | .key' > "$paths" || echo "::error::failed to enumerate locally-built paths" + + echo "::group::artifacts" + cat "$paths" + echo "::endgroup::" + + if [ ! -s "$paths" ]; then + echo "nothing was built locally; nothing to push" + exit 0 + fi + echo "pushing $(wc -l < "$paths") path(s) to $CACHE" + + if ! "$attic_link/bin/attic" login ci "$ENDPOINT" "$NIX_CACHE_PUSH_TOKEN" --set-default; then + echo "::warning::attic login failed; nothing pushed" + exit 0 + fi + + xargs -a "$paths" "$attic_link/bin/attic" push --jobs 4 "$CACHE" \ + || echo "::warning::attic push failed" diff --git a/.github/actions/genlayerlabs-nix-setup/action.yaml b/.github/actions/genlayerlabs-nix-setup/action.yaml new file mode 100644 index 00000000..099df2fc --- /dev/null +++ b/.github/actions/genlayerlabs-nix-setup/action.yaml @@ -0,0 +1,79 @@ +name: GenLayer nix setup +description: Install nix, configured against the caches we substitute from. +inputs: + github_token: + description: token nix authenticates GitHub flake inputs with; they are rate-limited without one + required: false + enable_genlayerlabs_cache: + description: >- + substitute from our own cache. Turn off to measure a cold build, or to + keep a job from consuming paths a broken push may have poisoned. + required: false + default: "true" + extra_substituters: + description: >- + additional substituters, whitespace-separated. Each needs its key in + `extra_trusted_public_keys`, or nix refuses its paths and silently rebuilds. + required: false + default: "" + extra_trusted_public_keys: + description: additional trusted public keys, whitespace-separated + required: false + default: "" + extra_nix_config: + description: >- + lines appended verbatim to nix.conf, for settings this action has no input + for. Later lines win, so this can also override one it does. + required: false + default: "" +runs: + using: composite + steps: + - id: config + shell: bash -eo pipefail {0} + env: + GENLAYERLABS_CACHE: ${{ inputs.enable_genlayerlabs_cache }} + EXTRA_SUBSTITUTERS: ${{ inputs.extra_substituters }} + EXTRA_KEYS: ${{ inputs.extra_trusted_public_keys }} + EXTRA_CONFIG: ${{ inputs.extra_nix_config }} + run: | + substituters="https://cache.nixos.org/ https://nix-community.cachix.org" + keys="cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" + keys="$keys nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=" + + # The key stays trusted either way: disabling the cache stops us pulling + # from it, it does not make paths already in this store untrustworthy. + keys="$keys genlayer:hMvP8BkI2v8E3BtzMB9HQyIEZz/ahNdBZo/JEQ5hpVA=" + if [ "$GENLAYERLABS_CACHE" != "false" ]; then + substituters="$substituters https://nix-cache.ygr.ai/genlayer" + fi + + # nix.conf is line-based, so a multi-line input has to collapse to one. + append() { + local extra + extra="$(printf '%s' "$2" | tr -s '[:space:]' ' ' | sed 's/^ *//; s/ *$//')" + if [ -n "$extra" ]; then printf '%s %s' "$1" "$extra"; else printf '%s' "$1"; fi + } + substituters="$(append "$substituters" "$EXTRA_SUBSTITUTERS")" + keys="$(append "$keys" "$EXTRA_KEYS")" + + { + echo 'nix_config<<EOF_NIX_CONFIG' + echo "experimental-features = nix-command flakes recursive-nix" + echo "substituters = $substituters" + echo "trusted-public-keys = $keys" + echo "download-attempts = 10" + echo "connect-timeout = 15" + echo "stalled-download-timeout = 60" + echo "fallback = true" + if [ -n "$EXTRA_CONFIG" ]; then printf '%s\n' "$EXTRA_CONFIG"; fi + echo 'EOF_NIX_CONFIG' + } >> "$GITHUB_OUTPUT" + + # Pinned by commit: a tag can be moved, and this action runs with the + # workflow's token. + - uses: cachix/install-nix-action@a809471b5c7c913aa67bec8f459a11a0decc3fce # v31.6.2 + with: + extra_nix_config: ${{ steps.config.outputs.nix_config }} + github_access_token: ${{ inputs.github_token }} + install_url: https://releases.nixos.org/nix/nix-2.34.7/install diff --git a/.github/actions/get-src/action.yaml b/.github/actions/get-src/action.yaml index 76ca7dba..c52c9ef4 100644 --- a/.github/actions/get-src/action.yaml +++ b/.github/actions/get-src/action.yaml @@ -48,15 +48,6 @@ runs: python3 support/scripts/ci-changes.py --github-output - name: setup nix if: ${{ inputs.with_nix == 'true' }} - uses: cachix/install-nix-action@v31.6.2 + uses: ./.github/actions/genlayerlabs-nix-setup with: - extra_nix_config: | - experimental-features = nix-command flakes recursive-nix - substituters = https://cache.nixos.org/ https://nix-community.cachix.org https://cache.nix.kp2pml30.moe - trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs= cache.nix.kp2pml30.moe:9izVvysdKza7+bjw2gI/etkDVfEQy+kne2aU/TZpA5o= - download-attempts = 10 - connect-timeout = 15 - stalled-download-timeout = 60 - fallback = true - github_access_token: ${{ inputs.github_token }} - install_url: https://releases.nixos.org/nix/nix-2.24.14/install + github_token: ${{ inputs.github_token }} diff --git a/.github/workflows/incl_initial.yaml b/.github/workflows/incl_initial.yaml index 800e7e3f..6fbc75d9 100644 --- a/.github/workflows/incl_initial.yaml +++ b/.github/workflows/incl_initial.yaml @@ -43,6 +43,14 @@ jobs: - name: Run hooks run: ./support/ci/run.sh pipeline commit-hooks + # The hook toolchains are per-repo flake outputs, so this is the job that + # first realizes them after a `git-hooks.nix` bump. + - name: Push to nix cache + if: ${{ !cancelled() }} + uses: ./.github/actions/genlayerlabs-nix-cache-push + with: + token: ${{ secrets.NIX_CACHE_PUSH_TOKEN }} + # `check-commit-message` is a commit-msg-stage hook, so the pre-commit run # above never fires it. Check every commit the PR adds, in the manager and in # each executor submodule that moved. diff --git a/.github/workflows/incl_release_build_test_cell_build.yaml b/.github/workflows/incl_release_build_test_cell_build.yaml index 3b3e3d89..8f9212de 100644 --- a/.github/workflows/incl_release_build_test_cell_build.yaml +++ b/.github/workflows/incl_release_build_test_cell_build.yaml @@ -68,6 +68,14 @@ jobs: job_json: ${{ inputs.job_json }} key: build + # Not `success()`: a cell that failed late still built most of the + # closure, and that is exactly the work the retry should not repeat. + - name: Push to nix cache + if: ${{ !cancelled() }} + uses: ./.github/actions/genlayerlabs-nix-cache-push + with: + token: ${{ secrets.NIX_CACHE_PUSH_TOKEN }} + - name: Upload artifacts uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/incl_release_build_test_cell_test.yaml b/.github/workflows/incl_release_build_test_cell_test.yaml index e6e6dea2..43205f93 100644 --- a/.github/workflows/incl_release_build_test_cell_test.yaml +++ b/.github/workflows/incl_release_build_test_cell_test.yaml @@ -108,6 +108,14 @@ jobs: job_json: ${{ inputs.job_json }} key: test + # A cell without nix has no store to push, and the macOS cells have no + # nix at all. + - name: Push to nix cache + if: ${{ !cancelled() && inputs.with_nix }} + uses: ./.github/actions/genlayerlabs-nix-cache-push + with: + token: ${{ secrets.NIX_CACHE_PUSH_TOKEN }} + # Postmortem on the same runner the cell just used; see the action for # what it probes and why it is not an `incl_*` reusable workflow. # `failure()` covers any earlier step failing and is already false on a diff --git a/.github/workflows/queue_test_cell.yaml b/.github/workflows/queue_test_cell.yaml index 3b635957..aaf24dd4 100644 --- a/.github/workflows/queue_test_cell.yaml +++ b/.github/workflows/queue_test_cell.yaml @@ -112,6 +112,12 @@ jobs: job_json: ${{ inputs.job_json }} key: tool + - name: Push to nix cache + if: ${{ !cancelled() }} + uses: ./.github/actions/genlayerlabs-nix-cache-push + with: + token: ${{ secrets.NIX_CACHE_PUSH_TOKEN }} + # Postmortem on the same runner the cell just used; see the action for # what it probes and why it is not an `incl_*` reusable workflow. # `failure()` covers any earlier step failing and is already false on a diff --git a/CLAUDE.md b/CLAUDE.md index f3eaaa09..ad279be7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,36 +1,52 @@ # GenVM manager -Before doing build, test, submodule, release/versioning, or codegen work, -read the matching guide in `docs/contributing/howto/`: +Before changing this repository, read the matching page in `docs/contributing/`: +a task has a how-to, a design question has an explanation -- [genvm-tool.md](docs/contributing/howto/genvm-tool.md) β€” command overview, codegen data β†’ generated files, plugin/test-definition layout -- [setup.md](docs/contributing/howto/setup.md) β€” first-time clone: submodule init, nix/direnv dev shell, third-party materialization -- [write-a-script.md](docs/contributing/howto/write-a-script.md) β€” conventions for new helper scripts: python + argparse, integrate into genvm-tool / support/ci +<!-- below is generated with `genvm-tool docs` --> -## Building -- [build.md](docs/contributing/howto/building/build.md) β€” debug build: configure + ninja, outputs, cargo quirks -- [docs.md](docs/contributing/howto/building/docs.md) β€” building the website, spec vs impl-spec, ADRs -- [runners.md](docs/contributing/howto/building/runners.md) β€” where runners come from (build vs download) +## Tutorial +- [first-contribution.md](docs/contributing/tutorial/first-contribution.md) β€” patch an executor, branch, commit, push, open the PR -## Committing -- [git-third-party.md](docs/contributing/howto/committing/git-third-party.md) β€” how vendored trees (wasmtime, …) are tracked, edited, and committed as patches -- [runners.md](docs/contributing/howto/committing/runners.md) β€” runner dev-mode and hash hygiene before committing an executor line -- [submodules.md](docs/contributing/howto/committing/submodules.md) β€” manager + executor lines, gitlink bumps, pre-commit hook, push order +## Howto +- [genvm-tool.md](docs/contributing/howto/genvm-tool.md) β€” the umbrella tool: man page, test and git subcommands, codegen +- [pr.md](docs/contributing/howto/pr.md) β€” branch model, PR action panel, `ci-safe` / `run-full-tests`, merge gates +- [setup.md](docs/contributing/howto/setup.md) β€” first-time clone: submodules, vendored trees, nix/direnv dev shell -## Extending +## Howto/Building +- [build.md](docs/contributing/howto/building/build.md) β€” debug build: configure + ninja, targets, outputs, cargo quirks +- [docs.md](docs/contributing/howto/building/docs.md) β€” building and publishing the website, spec vs impl-spec, ADRs +- [runners.md](docs/contributing/howto/building/runners.md) β€” where runners come from: build on Linux, download elsewhere + +## Howto/Committing +- [git-third-party.md](docs/contributing/howto/committing/git-third-party.md) β€” how vendored trees (wasmtime, …) are pinned and patched +- [runners.md](docs/contributing/howto/committing/runners.md) β€” clearing runner dev-mode and refreshing hashes before a commit +- [submodules.md](docs/contributing/howto/committing/submodules.md) β€” repo topology, gitlink bumps, pre-commit hooks, push order + +## Howto/Docs +- [style.md](docs/contributing/howto/docs/style.md) β€” prose conventions for guides, specs, ADRs and commit bodies + +## Howto/Extending - [add-host-function.md](docs/contributing/howto/extending/add-host-function.md) β€” new executor↔host protocol method - [add-llm-provider.md](docs/contributing/howto/extending/add-llm-provider.md) β€” new LLM backend in the manager -- [add-wasi-function.md](docs/contributing/howto/extending/add-wasi-function.md) β€” new gl_call method or raw WASI function in an executor +- [add-wasi-function.md](docs/contributing/howto/extending/add-wasi-function.md) β€” new gl_call method or raw WASI function - [modify-runner.md](docs/contributing/howto/extending/modify-runner.md) β€” runner dev-mode and hash refresh - [modify-wasmtime.md](docs/contributing/howto/extending/modify-wasmtime.md) β€” patching vendored wasmtime, trap plumbing +- [write-a-script.md](docs/contributing/howto/extending/write-a-script.md) β€” conventions for helper scripts and pre-commit hooks -## Releasing +## Howto/Releasing - [release-build.md](docs/contributing/howto/releasing/release-build.md) β€” nix packages, platforms, release assets -- [versioning.md](docs/contributing/howto/releasing/versioning.md) β€” release trains, `.genvm-monorepo-root`, check-versions/branch-versions scripts +- [versioning.md](docs/contributing/howto/releasing/versioning.md) β€” release trains, `.genvm-monorepo-root`, version tools -## Testing -- [README.md](docs/contributing/howto/testing/README.md) β€” `genvm-tool test` itself: filters, presets, continue files +## Howto/Testing +- [README.md](docs/contributing/howto/testing/README.md) β€” `genvm-tool test`: filters, presets, continue files - [fuzzing.md](docs/contributing/howto/testing/fuzzing.md) β€” AFL fuzz targets, host sysctl prep -- [integration.md](docs/contributing/howto/testing/integration.md) β€” running one executor line, `stable` (offline) subset, golden `.stdout`/`.hash` files, `--ignore-hash`, services +- [integration.md](docs/contributing/howto/testing/integration.md) β€” jsonnet cases, tags, golden `.stdout`/`.hash` files, services - [python.md](docs/contributing/howto/testing/python.md) β€” Python tests, direct pytest for genlayer-py-std -- [rust.md](docs/contributing/howto/testing/rust.md) β€” Rust tests, `#rust-test` shell, coverage +- [rust.md](docs/contributing/howto/testing/rust.md) β€” Rust tests: where they go, how to run them, coverage + +## Explanation +- [docs-layout.md](docs/contributing/explanation/docs-layout.md) β€” the 4 kinds of page and where each belongs +- [executor-lines.md](docs/contributing/explanation/executor-lines.md) β€” why several executor lines ship side by side, and what it costs +- [merge-model.md](docs/contributing/explanation/merge-model.md) β€” why merging is a maintainer panel rather than a merge queue +- [vendored-trees.md](docs/contributing/explanation/vendored-trees.md) β€” why third-party sources are patch series, not forks diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 00000000..7d18e92c --- /dev/null +++ b/docs/README.md @@ -0,0 +1,15 @@ +# Documentation + +| Directory | Holds | For | +|---|---|---| +| [`website/`](website/src/) | the published site: `spec/`, `impl-spec/`, `overview/` | people using GenVM | +| [`contributing/`](contributing/README.md) | tutorial, how-to guides, explanation | people changing this repository | +| [`adr/`](adr/) | dated decision records, an archive | anyone asking what was decided and when | +| [`schemas/`](schemas/) | JSON Schema for the shipped config files and `runner.json` | config authors, referenced from the spec | +| [`internal/`](internal/) | design comparisons written while choosing an approach | us, not published | + +Which of these a new page belongs in, and why the line falls where it does: +[docs-layout.md](contributing/explanation/docs-layout.md) + +Prose conventions apply to every one of them: +[style.md](contributing/howto/docs/style.md) diff --git a/docs/contributing/README.md b/docs/contributing/README.md index d48e8269..c1d65b1e 100644 --- a/docs/contributing/README.md +++ b/docs/contributing/README.md @@ -1,14 +1,20 @@ -# Contributing to GenVM development +# Contributing to GenVM -## Sub pages -- [first-contribution tutorial](./tutorial/first-contribution.md) β€” end-to-end walkthrough: patch an executor, branch, commit, push, open the PR. -- [how-to guides](./howto/README.md) β€” setup, build, test, submodules, versioning, extension recipes, … +1. [First Contribution](tutorial/first-contribution.md) β€” end-to-end + walkthrough: patch an executor, branch, commit, push, open the PR +2. [How-To Guides](howto/README.md) β€” setup, build, test, submodules, + versioning, extension recipes +3. [Explanation](explanation/docs-layout.md) β€” why things are the way they are, + starting with the documentation layout itself +4. [Text Style](howto/docs/style.md) β€” how prose is written here -## PR requirements -All checks must pass: [pre-commit](https://pre-commit.com) hooks and the test -suites. PRs land through a merge queue that runs the full test set and merges -*iff* everything passes β€” this keeps `HEAD` of `main` always stable. +## Requirements + +A PR must pass the [pre-commit](https://pre-commit.com) hooks and the full test +suite, and lands when a maintainer ticks *Merge into dev* on the PR action panel +β€” see [pr.md](howto/pr.md) ## Security -GenVM is consensus-critical. Do not open public issues for vulnerabilities β€” -follow [SECURITY.md](../../SECURITY.md) (private reporting). + +GenVM is consensus-critical. Do not open a public issue for a vulnerability. +Follow [SECURITY.md](../../SECURITY.md) instead diff --git a/docs/contributing/explanation/README.md b/docs/contributing/explanation/README.md new file mode 100644 index 00000000..be68f9a2 --- /dev/null +++ b/docs/contributing/explanation/README.md @@ -0,0 +1,9 @@ +# Explanation + +Why the repository is shaped the way it is. Nothing here is normative β€” a rule +lives in the spec, and these pages only motivate it and link out + +- [docs-layout.md](docs-layout.md) β€” the 4 kinds of page and where each belongs +- [executor-lines.md](executor-lines.md) β€” why several executor lines ship side by side, and what it costs +- [merge-model.md](merge-model.md) β€” why merging is a maintainer panel rather than a merge queue +- [vendored-trees.md](vendored-trees.md) β€” why third-party sources are patch series, not forks diff --git a/docs/contributing/explanation/docs-layout.md b/docs/contributing/explanation/docs-layout.md new file mode 100644 index 00000000..d1d1d5ff --- /dev/null +++ b/docs/contributing/explanation/docs-layout.md @@ -0,0 +1,37 @@ +# Documentation Layout + +Where a new page goes, and why the tree is cut this way. The split follows +[DiΓ‘taxis](https://diataxis.fr): 4 kinds of page, each answering a different +question, never mixed on one page + +| Kind | Question | Lives in | +|---|---|---| +| Tutorial | How do I get my first change landed? | `docs/contributing/tutorial/` | +| How-To | How do I do this one task? | `docs/contributing/howto/` | +| Reference | What exactly does it do? | `docs/website/src/spec`, `impl-spec` | +| Explanation | Why is it like this? | `docs/contributing/explanation/` | + +The cut that matters in practice: **anything normative is reference**. A +statement constraining what an implementation must do belongs in the spec, even +when its reasoning is interesting. Explanation may motivate a rule and link to +it, never restate it β€” 2 copies drift, and the copy a reader finds first wins + +## Why Spec and Impl-Spec Are Separate + +GenVM is consensus-critical, so `spec/` is the contract between independent +implementations, readable without this repository open. `impl-spec/` describes +how this codebase happens to satisfy it, and is free to change in any release +that keeps `spec/` true + +## Why ADRs Are Not Explanation + +An ADR is a dated record of one decision, superseded rather than rewritten; an +explanation page describes the present and is edited freely. So +[`docs/adr/`](../../adr/) stays an archive, off the website, and an accepted +ADR leaves 2 traces: its rule in the spec, its reasoning here + +## Audience + +`docs/contributing/` is for people changing this repository, `docs/website/` +for people using GenVM. Both are read by humans and AI agents, so +[style.md](../howto/docs/style.md) applies to both diff --git a/docs/contributing/explanation/executor-lines.md b/docs/contributing/explanation/executor-lines.md new file mode 100644 index 00000000..98e6fd03 --- /dev/null +++ b/docs/contributing/explanation/executor-lines.md @@ -0,0 +1,30 @@ +# Why Several Executor Lines Ship At Once + +A deployed contract is bytes on a chain that nobody can migrate, so the +semantics it was validated against must stay reachable for as long as it does. A +breaking change is therefore a new executor beside the old one, with the +contract naming the one it wants β€” normatively +[`03-versioning.rst`](../../website/src/spec/01-core-architecture/03-versioning.rst), +decided in [ADR-010](<../../adr/010. versions.md>) + +That constraint explains most of the repository's shape + +## What It Costs + +1. A release ships one binary per active line, so the artifact layout is indexed + by version rather than flat +2. The lines are branches of one executor repository, checked out as separate + submodules. They share crate names and versions, so cargo needs a target + directory per line ([build.md](../howto/building/build.md)) +3. Anything manager-global is owned by exactly one line, the first in + `active-versions` ([versioning.md](../howto/releasing/versioning.md)) +4. Every doc, test filter and config key naming a version is a place a new line + must be added + +This is the price of the guarantee, not complexity to be simplified away + +## Lines Are Not Peers + +`active-versions` is ordered: the first entry is the current line, the rest are +kept so existing work keeps running. `support-only-versions` goes further β€” still +shipped, no longer built from full sources diff --git a/docs/contributing/explanation/merge-model.md b/docs/contributing/explanation/merge-model.md new file mode 100644 index 00000000..cf779db4 --- /dev/null +++ b/docs/contributing/explanation/merge-model.md @@ -0,0 +1,31 @@ +# Why Merging Is a Panel, Not a Queue + +A change spans repositories: the executor lines carry the edit, the manager +carries the gitlinks pinning them. GitHub's merge queue orders pull requests in +one repository and has no opinion about a second, so the landing step is ours β€” +a maintainer ticks *Merge into dev* and the automation re-checks every gate +before writing anything ([pr.md](../howto/pr.md)) + +## Why the Gates Re-Run at Merge Time + +A green check describes the commit that was tested, not the branch it sits on, +and between review and merge the head, the base and the pinned executor branch +can all move + +The heavy matrix is opt-in for a related reason: a check that runs on every push +to every draft gets ignored + +## Why Only One Merge Runs at a Time + +Checking the gates and advancing the branches is not atomic, and the executor +lines advance before the manager, so a lost race leaves a half-landed change. +Hence one merge repository-wide, and never a cancelled one + +That costs a maintainer something: GitHub keeps 1 pending run per concurrency +group, so a third tick cancels the waiting one + +## Why `ci-safe` Is a Human Decision + +These jobs run the pull request's own `support/ci/` scripts with deploy keys in +scope. Approving a run approves code execution with credentials, which is not +inferable from the diff diff --git a/docs/contributing/explanation/vendored-trees.md b/docs/contributing/explanation/vendored-trees.md new file mode 100644 index 00000000..e317c5c5 --- /dev/null +++ b/docs/contributing/explanation/vendored-trees.md @@ -0,0 +1,21 @@ +# Why Vendored Trees Are Patches, Not Forks + +Some third-party sources, wasmtime above all, need changes no upstream release +exposes. What is tracked is a pinned upstream commit plus numbered +`git format-patch` files; the tree itself is git-ignored and materialized on +demand ([git-third-party.md](../howto/committing/git-third-party.md)) + +A fork would hide our delta in a second history and drift silently; as a patch +series, the delta *is* the tracked file, it shows up in an ordinary pull request +diff, and an upstream bump fails loudly instead of quietly diverging + +Committing the modified source instead would bury the same delta in a few +hundred thousand lines, where a change to upstream code and a change to ours +look identical + +## What It Costs + +A checkout is not buildable until the trees are materialized, and edits live in +a nested repository whose history must be turned back into patches to survive. +Nix reads the pinned commit and patches directly, so an unsaved edit is +invisible to it and a build can silently use the previous state diff --git a/docs/contributing/howto/README.md b/docs/contributing/howto/README.md index af91dc13..7988ef81 100644 --- a/docs/contributing/howto/README.md +++ b/docs/contributing/howto/README.md @@ -1,33 +1,37 @@ -# How-to guides +# How-To Guides -- [setup.md](setup.md) β€” first-time clone: submodule init, nix/direnv dev shell, third-party materialization. -- [genvm-tool.md](genvm-tool.md) β€” command overview, codegen data β†’ generated files, plugin/test-definition layout. -- [write-a-script.md](write-a-script.md) β€” conventions for new helper scripts: python + argparse, integrate into genvm-tool / support/ci. +- [setup.md](setup.md) β€” first-time clone: submodules, vendored trees, nix/direnv dev shell +- [genvm-tool.md](genvm-tool.md) β€” the umbrella tool: man page, test and git subcommands, codegen +- [pr.md](pr.md) β€” branch model, PR action panel, `ci-safe` / `run-full-tests`, merge gates ## building/ -- [build.md](building/build.md) β€” debug build: configure + ninja, outputs, cargo quirks. -- [runners.md](building/runners.md) β€” where runners come from (build vs download). -- [docs.md](building/docs.md) β€” building the website, spec vs impl-spec, ADRs. +- [build.md](building/build.md) β€” debug build: configure + ninja, targets, outputs, cargo quirks +- [runners.md](building/runners.md) β€” where runners come from: build on Linux, download elsewhere +- [docs.md](building/docs.md) β€” building and publishing the website, spec vs impl-spec, ADRs + +## docs/ +- [style.md](docs/style.md) β€” prose conventions for guides, specs, ADRs and commit bodies ## testing/ -- [testing/README.md](testing/README.md) β€” `genvm-tool test` itself: filters, presets, continue files. -- [rust.md](testing/rust.md) β€” Rust tests, `#rust-test` shell, coverage. -- [python.md](testing/python.md) β€” Python tests, direct pytest for genlayer-py-std. -- [integration.md](testing/integration.md) β€” running one executor line, `stable` (offline) subset, golden `.stdout`/`.hash` files, `--ignore-hash`, services. -- [fuzzing.md](testing/fuzzing.md) β€” AFL fuzz targets, host sysctl prep. +- [testing/README.md](testing/README.md) β€” `genvm-tool test`: filters, presets, continue files +- [rust.md](testing/rust.md) β€” Rust tests: where they go, how to run them, coverage +- [python.md](testing/python.md) β€” Python tests, direct pytest for genlayer-py-std +- [integration.md](testing/integration.md) β€” jsonnet cases, tags, golden `.stdout`/`.hash` files, services +- [fuzzing.md](testing/fuzzing.md) β€” AFL fuzz targets, host sysctl prep ## committing/ -- [submodules.md](committing/submodules.md) β€” manager + executor lines, gitlink bumps, pre-commit hook, push order. -- [runners.md](committing/runners.md) β€” runner dev-mode and hash hygiene before committing an executor line. -- [git-third-party.md](committing/git-third-party.md) β€” how vendored trees (wasmtime, …) are tracked, edited, and committed as patches. +- [submodules.md](committing/submodules.md) β€” repo topology, gitlink bumps, pre-commit hooks, push order +- [runners.md](committing/runners.md) β€” clearing runner dev-mode and refreshing hashes before a commit +- [git-third-party.md](committing/git-third-party.md) β€” how vendored trees (wasmtime, …) are pinned and patched ## releasing/ -- [versioning.md](releasing/versioning.md) β€” release trains, `.genvm-monorepo-root`, check-versions/branch-versions scripts. -- [release-build.md](releasing/release-build.md) β€” nix packages, platforms, release assets. +- [versioning.md](releasing/versioning.md) β€” release trains, `.genvm-monorepo-root`, version tools +- [release-build.md](releasing/release-build.md) β€” nix packages, platforms, release assets ## extending/ -- [add-llm-provider.md](extending/add-llm-provider.md) β€” new LLM backend in the manager. -- [add-wasi-function.md](extending/add-wasi-function.md) β€” new gl_call method or raw WASI function in an executor. -- [add-host-function.md](extending/add-host-function.md) β€” new executor↔host protocol method. -- [modify-runner.md](extending/modify-runner.md) β€” runner dev-mode and hash refresh. -- [modify-wasmtime.md](extending/modify-wasmtime.md) β€” patching vendored wasmtime, trap plumbing. +- [add-llm-provider.md](extending/add-llm-provider.md) β€” new LLM backend in the manager +- [add-wasi-function.md](extending/add-wasi-function.md) β€” new gl_call method or raw WASI function +- [add-host-function.md](extending/add-host-function.md) β€” new executor↔host protocol method +- [modify-runner.md](extending/modify-runner.md) β€” runner dev-mode and hash refresh +- [modify-wasmtime.md](extending/modify-wasmtime.md) β€” patching vendored wasmtime, trap plumbing +- [write-a-script.md](extending/write-a-script.md) β€” conventions for helper scripts and pre-commit hooks diff --git a/docs/contributing/howto/building/build.md b/docs/contributing/howto/building/build.md index 2e24df26..38c91739 100644 --- a/docs/contributing/howto/building/build.md +++ b/docs/contributing/howto/building/build.md @@ -1,48 +1,45 @@ -# Build (debug) +# Debug Build -All commands assume the dev shell (`nix develop '.?submodules=1#full'`, usually -via direnv) β€” it provides `genvm-tool`, `ninja`, `cargo`, `python3`. -First-time clone: [setup.md](../setup.md). +Everything assumes the dev shell, which provides `genvm-tool`, `ninja`, `cargo` +and `python3`. First clone: [setup.md](../setup.md) ```bash -genvm-tool configure # generates build/build.ninja + build/info.json (like CMake) +genvm-tool configure # writes build/build.ninja and build/info.json, like CMake ninja -C build all/bin ``` -Outside the dev shell call `support/tools/genvm-tool/genvm-tool configure`. - -Re-run `genvm-tool configure` after adding/removing/renaming source files β€” -symptom if you forget: ninja `missing and no known rule to make it`. +Re-run `configure` after adding, removing or renaming a source file; forgetting +shows up as ninja's `missing and no known rule to make it` | Ninja target | Builds | |---|---| -| `all` | everything (incl. runners; x86_64 Linux only) | -| `all/bin` | all Rust binaries | +| `all` | `all/bin` + `all/data` | +| `all/bin` | every Rust binary: the manager and each executor line | +| `all/manager` | the manager binary only | +| `all/executor/<line>` | one executor line | | `all/data` | runner data via nix | -| `codegen` | code generation (see [genvm-tool.md](../genvm-tool.md)) | - -Outputs under `build/out`: - -- `bin/genvm-modules` β€” modules binary -- `executor/<version>/bin/genvm` β€” one dir per built executor version; the - version comes from `executors/<line>.x/manifest.json` and is recorded in - `build/info.json` - -Release/nix packages: [release-build.md](../releasing/release-build.md). -Getting or changing runners: [runners.md](runners.md), [modify-runner.md](../extending/modify-runner.md). - -## Cargo notes - -- Plain `cargo` inside a submodule needs the dev-shell env (lua, pkg-config, …); - prefer the ninja build. -- In this nix environment, `cargo check` inside an executor's `executor/` dir - may need `LD_LIBRARY_PATH="$(nix eval --raw nixpkgs#zlib)/lib"` so rustc - finds libz. -- Each executor line gets its own cargo target dir - (`build/ya-build/rust-target/<line>`, listed in `build/info.json` under - `rust_target_dirs`); crates outside a line use the parent dir. Lines ship - crates with identical names and versions, and cargo's artifact hash ignores - where a package came from, so a shared dir lets them overwrite each other. -- Build and tests both pass `--target <host triple>` (`rust_target` in - `build/info.json`). Omitting it builds a second, unshared unit graph in the - same dir, so running cargo by hand without it recompiles everything. +| `all/runners` | the runners themselves via nix, x86_64 Linux only | +| `codegen` | generated sources ([genvm-tool.md](../genvm-tool.md)) | + +Outputs land in `build/out`: `bin/genvm-modules`, `bin/genvm-manager`, +`bin/genvm-post-install`, and `executor/<version>/bin/genvm` per built line, +where the version comes from `executors/<line>.x/manifest.json` and is recorded +in `build/info.json` + +Release packages: [release-build.md](../releasing/release-build.md). Runners: +[runners.md](runners.md) + +## Cargo Notes + +1. Prefer the ninja build β€” plain `cargo` in a submodule needs the dev-shell + environment (lua, pkg-config, …) +2. `cargo check` in an executor's `executor/` may need + `LD_LIBRARY_PATH="$(nix eval --raw nixpkgs#zlib)/lib"` so rustc finds libz +3. Each line has its own target directory + (`build/ya-build/rust-target/<line>`, in `build/info.json` under + `rust_target_dirs`); crates outside a line use the parent. Lines ship crates + with identical names and versions, and cargo's artifact hash ignores where a + package came from, so a shared directory would let them overwrite each other +4. Build and tests pass `--target <host triple>` (`rust_target` in + `build/info.json`). Without it cargo builds a second, unshared unit graph in + the same directory and recompiles everything diff --git a/docs/contributing/howto/building/docs.md b/docs/contributing/howto/building/docs.md index 49280c0d..b90d54fc 100644 --- a/docs/contributing/howto/building/docs.md +++ b/docs/contributing/howto/building/docs.md @@ -1,56 +1,63 @@ # Docs -## Website (`docs/website`) +## Website -Sphinx site. The toolchain (sphinx + extensions) comes from the `gen-docs` Nix -shell β€” no poetry/venv. Build everything with: +Sphinx, with its toolchain from the `gen-docs` nix shell β€” no poetry, no venv: ```bash ./support/ci/run.sh pipeline docs ``` -That regenerates the derived pages, then enters `.#gen-docs` and runs -`sphinx-build` for `-b html` and `-b text` (merging the text output into -`_static/ai/*.txt`). Finally, for every active executor line it enters that -line's own `.#gen-docs` shell (`executors/<line>.x/flake.nix`) and builds the -line's standalone docs sub-site into `build/doc/html/executors/<line>/`; the -manager's generated `executors.rst` page links to them. To iterate on just the -sphinx build: +That regenerates the derived pages, builds the manager site (`html` plus `text` +merged into `_static/ai/*.txt`), then each active line's sub-site in that +line's own `gen-docs` shell, into `build/doc/html/executors/<line>/`. Details: +`support/ci/pipelines/docs.py`. To iterate on sphinx alone: ```bash nix develop '.?submodules=1#gen-docs' --command \ sphinx-build -q -b dirhtml docs/website/src <out> ``` -See `support/ci/pipelines/docs.py` for the full pipeline. +## Source Layout -Source layout under `docs/website/src/`: +Under `docs/website/src/`: `spec/` specifies observable behavior, `impl-spec/` +how this codebase does it, `overview/` is intro material. The Python SDK reference and +the runner-version pages live in the line that ships them, +`executors/<line>.x/docs/website/src/python-sdk/`. Cross-linking is one-way: a +sub-site references the manager through intersphinx, the manager only links out -- `spec/` β€” the protocol specification (observable behavior). - `spec/appendix/constants.rst` is **generated** from codegen data β€” do not - edit by hand, see [genvm-tool.md](../genvm-tool.md). -- `impl-spec/` β€” the implementation specification (how this codebase does it). -- `overview/` β€” intro material. +Generated, never edited by hand: -The Python SDK reference and the per-line runner-version pages are **not** in the -manager tree β€” they live in the executor line that ships them, under -`executors/<line>.x/docs/website/src/python-sdk/` (see that line's `conf.py`), -and build as sub-sites under `build/doc/html/executors/<line>/`. Cross-linking is -one-way: the sub-sites reference the manager via intersphinx (the `genvm` -inventory); the manager only links out to them from the generated -`executors.rst` index page. +| Page | Generator | +|---|---| +| `spec/appendix/constants.rst` | codegen ([genvm-tool.md](../genvm-tool.md)) | +| `available-runners.rst`, `runners-versions.json`, SDK `changelog.rst`, in the primary line | `docs/website/generate.py` | +| `overview/executor-lines_generated.rst` | `docs/website/generate.py` | -`docs/website/generate.py` regenerates derived pages β€” the runner-version tables -(`available-runners.rst` + `runners-versions.json`) and the SDK `changelog.rst` -into the primary line's sub-site, and the `executors.rst` index into the manager -β€” from the repo state; the `docs` pipeline runs it before the sphinx build. +## Publishing -## ADRs +The site is `sdk.genlayer.com` β€” GitHub Pages out of +`genlayerlabs/sdk.genlayer.com`, one directory per version under `_site/` plus +`_site/versions.json` driving the version switcher; pushing that repo deploys +it + +`.github/workflows/docs_nightly.yaml` republishes `main` nightly and takes a +version/preferred pair on `workflow_dispatch` for a release train. It needs the +`DOCS_DEPLOY_KEY` secret, and β€” since GitHub schedules only from the default +branch β€” the workflow file must reach that branch before the cron can fire + +By hand, into a checkout of the site repo: + +```bash +./support/ci/run.sh pipeline docs +./support/ci/run.sh tool deploy-docs --site ../sdk.genlayer.com --version main +``` -Architecture decision records live in [`docs/adr/`](../../../adr/) as numbered markdown files -(`000. WASI.md`, …); start new ones from [`docs/adr/_template.md`](../../../adr/_template.md). +The version directory is replaced wholesale, so deleted pages do not linger; +`_site/versions.json` keeps its curated order, a missing version is inserted +after `main`. Committing and pushing the site repo is left to you -## Contributor docs +## ADRs -[`docs/contributing/`](../..) β€” PR requirements and workflows; how-to guides are in -this directory. +Numbered markdown in [`docs/adr/`](../../../adr/); start from +[`_template.md`](../../../adr/_template.md) diff --git a/docs/contributing/howto/building/runners.md b/docs/contributing/howto/building/runners.md index d326fe0f..ddf10b30 100644 --- a/docs/contributing/howto/building/runners.md +++ b/docs/contributing/howto/building/runners.md @@ -1,17 +1,16 @@ -# Runners +# Getting Runners -Runners build only on x86_64 Linux (part of the `all` ninja target / -`#runners-all` nix attr). On other hosts, download them: +Runners build only on x86_64 Linux β€” `ninja -C build all/runners`, or the +`#runners-all` nix attribute. Never build them natively on macOS β€” deterministic +runner artifacts require a Linux builder; use a remote nix builder or download: ```bash -python3 build/out/bin/genvm-post-install --create-venv false --default-step false --runners-download true --error-on-missing-executor false +python3 build/out/bin/genvm-post-install \ + --default-steps false --runners-download true --error-on-missing-executor false ``` -or fetch `genvm-universal.tar.xz` from a +Alternatively fetch `genvm-universal.tar.xz` from a [genvm-manager release](https://github.com/genlayerlabs/genvm-manager/releases) -and extract it over `build/out`. +and extract it over `build/out` -Do **not** build runners natively on macOS β€” deterministic runner artifacts -require a Linux builder (remote nix builder or download). - -Changing a runner: [modify-runner.md](../extending/modify-runner.md). +Changing a runner: [modify-runner.md](../extending/modify-runner.md) diff --git a/docs/contributing/howto/committing/git-third-party.md b/docs/contributing/howto/committing/git-third-party.md index 020e8035..e38b2954 100644 --- a/docs/contributing/howto/committing/git-third-party.md +++ b/docs/contributing/howto/committing/git-third-party.md @@ -1,54 +1,49 @@ -# git-third-party (vendored trees) +# Vendored Trees (git-third-party) -Vendored third-party sources (e.g. `executors/v0.3.x/executor/third-party/wasmtime`, -`.../wasm-tools`) are **not** tracked by the executor repo β€” they are -git-ignored (`executor/.gitignore: third-party`) and materialized on demand. -What *is* tracked, per executor line, is `.git-third-party/`: +Vendored sources such as `executors/v0.3.x/executor/third-party/wasmtime` are +git-ignored and materialized on demand. Why patches rather than a fork: +[vendored-trees.md](../../explanation/vendored-trees.md). What is tracked, per +line, is `.git-third-party/`: -- `.git-third-party/config.json` β€” per repo: upstream `url`, pinned base - `commit`, `patches` count, optional `submodules` list (absent = update all - submodules, `[]` = none, list = only those). -- `.git-third-party/patches/<repo-path>/<n>` β€” numbered `git format-patch` - files (1-based, mbox format) applied on top of the base commit. +1. `config.json` β€” per repo: upstream `url`, pinned base `commit`, `patches` + count, optional `submodules` list +2. `patches/<repo-path>/<n>` β€” numbered `git format-patch` files, 1-based, mbox + format, applied on top of the base commit -The tool is `support/tools/git-third-party/git-third-party` (single Python -script; on PATH via `env.sh` and the dev shell; invoked as `git third-party`). -It has **no `--help`** β€” running with no subcommand prints usage. Subcommands: +The tool is `support/tools/git-third-party/git-third-party`, invoked as +`git third-party`, on `PATH` in the dev shell and through `env.sh`. It has no +`--help`; running it bare prints usage | Command | Effect | |---|---| -| `git third-party add <PATH> <REPO_URL> <COMMIT>` | register a new vendored repo (path must already be git-ignored) and materialize it | -| `git third-party update {--all \| <path>...}` | materialize: nested `git init`+fetch of the pinned commit, submodule update, then `git am` the patches | -| `git third-party save {--all \| <path>...}` | regenerate patch files from commits above the base commit; updates the count in `config.json` | - -`update` refuses to run on a dirty vendored tree. The materialized tree is a -real nested git repo whose history is `base commit` + one commit per patch. - -## Editing a vendored tree (e.g. a wasmtime patch) - -1. Edit files in `executors/<line>.x/executor/third-party/wasmtime` and - **commit inside that nested repo** (normal `git commit`; patches are derived - from history above the base commit). -2. `git third-party save executor/third-party/wasmtime` (path relative to cwd) β€” - rewrites `.git-third-party/patches/...` deterministically - (`format-patch --zero-commit --no-signature --numbered-files`). -3. In the executor repo: `git add .git-third-party/`, commit, push. -4. In the manager: bump the gitlink ([submodules.md](submodules.md)). - -The nested repo itself is never pushed β€” only the patch files persist. - -To bump the pinned upstream version: edit `commit` in `config.json`, then -`git third-party update <path>` (patches are re-applied with `git am`; resolve -conflicts by fixing up the nested repo's commits and running `save`). - -## Build integration - -- Local/CI checkouts run `git third-party update --all` **inside each executor - submodule** β€” the config is resolved from the current git toplevel, so running - it once at the manager root materializes nothing (CI: - `.github/actions/get-src/action.yaml`; local: [setup.md](../setup.md)). -- Nix does **not** call the tool: `support/nix/git-third-party.nix` re-reads - `config.json` and uses `builtins.fetchGit` + `pkgs.applyPatches`, mounted into - the source layout by `support/default.nix`. So nix builds see exactly the - committed base commit + patches β€” another reason to `save` and commit before - building flake packages. +| `add <PATH> <REPO_URL> <COMMIT>` | register a vendored repo (the path must already be git-ignored) and materialize it | +| `update {--all \| <path>...}` | materialize: nested `git init` + fetch of the pinned commit, submodule update, `git am` of the patches | +| `save {--all \| <path>...}` | regenerate the patches from the commits above the base commit, updating the count | + +`update` refuses a dirty tree. The result is a real nested git repo: base commit +plus one commit per patch. It always runs +`git submodule update --init --recursive --depth 1` first, so `submodules: []` +only skips the second pass and a list only adds a targeted one + +## Editing a Vendored Tree + +1. Edit, and **commit inside the nested repo** β€” patches come from the history + above the base commit +2. `git third-party save executor/third-party/wasmtime`, path relative to the + current directory +3. In the executor repo: `git add .git-third-party/`, commit, push +4. Bump the gitlink in the manager ([submodules.md](submodules.md)) + +The nested repo is never pushed, only the patches persist. To bump upstream, +edit `commit` in `config.json` and run `update`; on an `am` conflict fix up the +nested repo's commits and `save` + +## Build Integration + +1. Local and CI checkouts run `git third-party update --all` **inside each + executor submodule** β€” the config resolves from the current git toplevel, so + running it at the manager root materializes nothing +2. Nix never calls the tool: `support/nix/git-third-party.nix` re-reads + `config.json` and uses `builtins.fetchGit` plus `pkgs.applyPatches`, ignoring + `submodules`. So nix sees the committed base plus patches β€” `save` and commit + before building any flake package diff --git a/docs/contributing/howto/committing/runners.md b/docs/contributing/howto/committing/runners.md index ad98c207..55a298ab 100644 --- a/docs/contributing/howto/committing/runners.md +++ b/docs/contributing/howto/committing/runners.md @@ -1,28 +1,28 @@ -# Committing runner changes +# Committing Runner Changes -Runner sources and version pins live inside the executor submodule -(`executors/<line>.x/`). All paths below are relative to that submodule root. - -## Dev mode +Runner sources and version pins live in the forward-rolling executor line, +`executors/v0.3.x/`; every path below is relative to it. Frozen v0.2.x has no +`runners/support/versions` tree β€” it ships a committed registry instead During development `runners/support/versions/dev-mode.nix` may be `true` and -runner hashes in `runners/support/versions/current.nix` may be `"test"`. -**Neither must reach a commit.** +hashes in `runners/support/versions/current.nix` may be `"test"`. **Neither may +reach a commit**; a pre-commit guard rejects dev-mode Before committing: -1. Set `dev-mode.nix` back to `false`. -2. Set "test" hash to `null` in `current.nix`. -3. Run `runners/support/versions/hash-updater.py` (from anywhere inside the - repo) to recompute real hashes. It builds `#runners-all` with - `--keep-going`, writes every reported `got:` value back into `current.nix`, - and repeats until fixed point. Hashes are content-addressed and depend on - resolved dependency uids, so they must be discovered bottom-up β€” the script - handles that automatically. -4. Commit once all hashes are real `sha256-…` values. +1. Set `dev-mode.nix` back to `false` +2. Set every `"test"` hash in `current.nix` to `null` +3. Run `runners/support/versions/hash-updater.py`, from anywhere inside the + repo. It builds the umbrella's `#runners-all` with `--keep-going`, writes + every reported `got:` value back into `current.nix`, and repeats until a + fixed point. Hashes are content-addressed and depend on resolved dependency + uids, so they must be discovered bottom-up, which the script handles. It + needs the executor nested under the manager umbrella; a standalone checkout + cannot build `runners-all` +4. Commit once every hash is a real `sha256-…` value -**Never restore an old hash value manually.** It becomes wrong the moment any -source or dependency changes; only `hash-updater.py` produces a correct value. +**Never restore an old hash by hand.** It becomes wrong the moment any source or +dependency changes, and only `hash-updater.py` produces a correct one -See [modify-runner.md](../extending/modify-runner.md) for the full -runner-modification workflow. +The full modification workflow: +[modify-runner.md](../extending/modify-runner.md) diff --git a/docs/contributing/howto/committing/submodules.md b/docs/contributing/howto/committing/submodules.md index a8ab1fb4..77b1c7cd 100644 --- a/docs/contributing/howto/committing/submodules.md +++ b/docs/contributing/howto/committing/submodules.md @@ -1,86 +1,71 @@ -# Submodules (multi-repo workflow) +# Submodules ## Topology -- **Manager (umbrella)** β€” this repo (`genlayerlabs/genvm-manager`): `flake.nix`, - `implementation/`, `runners/`, `support/tools/genvm-tool`, `install/`. -- **Executor submodules** β€” `executors/v0.2.x`, `executors/v0.3.x`: checkouts of - the **same** repo (`genlayerlabs/genvm-executor`) on different branches (see - `.gitmodules`). Each line's `manifest.json` owns its `executor-version`. -- `.genvm-monorepo-root` lists the active lines and the manager's `version` β€” - see [versioning.md](../releasing/versioning.md). +1. **Manager**, the umbrella β€” this repo: `flake.nix`, `implementation/`, + `runners/`, `support/`, `install/` +2. **Executor lines** β€” `executors/v0.2.x`, `executors/v0.3.x`: checkouts of + the **same** repo, `genlayerlabs/genvm-executor`, on different branches. + Each line's `manifest.json` owns its `executor-version` +3. **`libs/unhardcoded-engine`** β€” the LLM policy engine used by + `implementation/`; the build fails loudly when it is missing -Branch model: manager `feat/<name>` (base usually `v<X>-dev`); each executor -submodule `pr/<line>/feat/<name>`. +Active lines and the manager version live in `.genvm-monorepo-root` +([versioning.md](../releasing/versioning.md)); `genvm-tool git` and the +`commit-hooks` pipeline model the manager and the active lines only. Branches: +manager `feat/<name>` on `v<X>-dev`, each line `pr/<line>/feat/<name>` -## The critical nix flag +## The Critical Nix Flag -Every flake ref needs `?submodules=1` (`nix develop '.?submodules=1#full'`, -`nix build '.?submodules=1#genvm'`). Flake packages only see **committed** -submodule files β€” commit inside the submodule before building packages. +Every manager flake ref needs `?submodules=1`, as `.envrc` does. Flake packages +see **committed** submodule content only, so commit inside a submodule before +building packages. An executor's own flake is a standalone dev shell and takes +no such flag -## Committing across repos (order matters) +## Committing Across Repos -The manager pins each submodule by a **gitlink** (a pinned commit): +The manager pins each submodule by a gitlink, so the order matters: -1. Commit **inside** the submodule: `cd executors/<line>.x && git add … && git commit`. -2. In the manager, bump the gitlink: `git add executors/<line>.x`. -3. Commit the manager (code + gitlink bumps together). +1. Commit **inside** the submodule +2. `git add executors/<line>.x` in the manager +3. Commit the manager: code and gitlink bumps together -Verify consistency: +Verify with `git ls-tree HEAD executors/v0.3.x` against +`git -C executors/v0.3.x rev-parse HEAD`. To reshape unpushed history: +`git reset --mixed <base>`, re-stage per commit, re-bump the gitlink -```bash -git ls-tree HEAD executors/v0.2.x executors/v0.3.x # pinned gitlinks -git -C executors/v0.3.x rev-parse HEAD # must match -``` +## Pre-Commit Hooks + +Each repo declares its hooks in its `flake.nix` +([git-hooks.nix](https://github.com/cachix/git-hooks.nix)); entering a repo's +dev shell installs them, so `git commit` runs the hooks of the repo you commit +in. Never `git commit --no-verify`, not even for a pure gitlink bump -To reshape unpushed history: `git reset --mixed <base>`, re-stage per commit, -re-bump the gitlink to the new submodule HEAD. - -## Pre-commit hook (per repo) - -Each repo β€” the manager and every executor submodule β€” carries its own -[git-hooks.nix](https://github.com/cachix/git-hooks.nix) config in its -`flake.nix`, pinned to that repo's own toolchain. Entering the dev shell -(`.envrc` β†’ `nix develop .#full` for the manager; `nix develop` for a -standalone executor checkout) installs the `pre-commit` / `commit-msg` stubs -into that repo's `.git/hooks`, so `git commit` runs the hooks for the repo you -are committing in. - -- Run everything by hand: `nix develop -c pre-commit run --all-files` in the - repo (or `./support/ci/run.sh pipeline commit-hooks` from the manager root to - check the manager + every executor at once, as CI does). -- Do not `git commit --no-verify` β€” let the hooks run, even for a pure gitlink - bump (they are cheap when nothing needs formatting). -- The executor crates reach the manager's shared `crates/` via relative paths, - so executor hooks must run against the **nested working tree** (submodule - checked out under the manager), not an isolated flake sandbox β€” this is why - CI uses the `commit-hooks` pipeline (in-tree) instead of `nix flake check`. +1. Manager: `nix develop '.?submodules=1#full' --command pre-commit run --all-files` +2. Everything, as CI does: `./support/ci/run.sh pipeline commit-hooks` + +Executor crates reach the manager's shared `crates/` by relative paths, so +executor hooks must run against the **nested working tree** β€” hence the in-tree +pipeline rather than `nix flake check` ## Pushing -Check first: `genvm-tool git check-for-push` (`--offline` compares against the -last fetch). After the repo rows it prints one aggregated -`suggested_push_command` block; when anything is not ready, that block is -`none`. Push **submodules before the manager** so gitlinks resolve: +Push **submodules before the manager**, so the gitlinks resolve. +`genvm-tool git check-for-push` reports readiness per repo (`--offline` +compares against the last fetch) and prints one aggregated +`suggested_push_command`, `none` while anything is not ready ```bash -git -C executors/v0.2.x push origin pr/v0.2/feat/<name> git -C executors/v0.3.x push origin pr/v0.3/feat/<name> git push origin feat/<name> ``` -After rebasing the manager onto its base (e.g. `origin/v0.6-dev`), it diverges: -use `git push --force-with-lease origin feat/<name>`. Executor feature branches -are normally ahead-only (plain fast-forward push). - -For runner hash hygiene before committing (dev-mode, `hash-updater.py`), see -[runners.md](runners.md). +A rebased manager branch needs `--force-with-lease`; executor feature branches +are normally ahead-only. Runner hash hygiene first: [runners.md](runners.md) -## Runner trees (why there are two) +## Why There Are Two Runner Trees -Forward-rolling lines (v0.3.x) share the top-level `runners/<id>/<aa>/<rest>.tar` -tree (Crockford base32 of `sha256(tar)`). Frozen v0.2.x keeps runners under -`executor/<version>/legacy-runners/` with **Nix base32** hashes β€” a different -scheme, hence the separate trees. `flake.nix` packaging splits the runner list -per line; `genvm check` verifies each with its own scheme. +Forward-rolling lines share the top-level `runners/<id>/<aa>/<rest>.tar` tree, +named by Crockford base32 of `sha256(tar)`; frozen v0.2.x keeps +`executor/<version>/legacy-runners/` with Nix base32 hashes. `flake.nix` splits +the runner list per line, and `genvm check` verifies each with its own scheme diff --git a/docs/contributing/howto/docs/style.md b/docs/contributing/howto/docs/style.md new file mode 100644 index 00000000..52857685 --- /dev/null +++ b/docs/contributing/howto/docs/style.md @@ -0,0 +1,43 @@ +# Text Style + +Conventions for prose in this repository: guides, specs, ADRs, commit bodies, +and code comments as far as it makes sense + +## Language + +English, whatever language the thinking behind it happened in + +## Titles + +Title Case + +## Text + +1. Keep source lines under 128 columns, so the source reads as well as the + output +2. Italicize unlinked terms and variables, such as *iff* +3. A trailing dot seems rude, so neither a paragraph nor a list item ends with + one +4. Start sentences with a capital letter +5. Prefer digits to words: `2` over `two`, excluding plain English phrases like + `no one` + +## Politeness + +1. Refer to anyone whose gender you do not know as **they**. This includes AI +2. For generic AI say **AI** or **AI agent**; reserve LLM, neural network and + agent for a specific approach or implementation +3. Write abbreviations in all capitals +4. Write names in their native form, with proper capitalization: + Jane Doe (English, first name first), Yamada Hanako (Japanese, surname + first), Vasya Pupkin or Ivanov Ivan Ivanovich (Russian), Ludwig van + Beethoven (German) + +## Lists + +1. Start items with a capital letter; a sub-list or less important information + may start lower, but every item of a given list must agree +2. Again, no trailing periods +3. Prefer numbered lists, they are easier to grasp and to reference. Use an + unnumbered one when explicit order would mislead: *I like berries and + fruits* numbered would suggest a preference for berries diff --git a/docs/contributing/howto/extending/add-host-function.md b/docs/contributing/howto/extending/add-host-function.md index d7595427..f123f9ef 100644 --- a/docs/contributing/howto/extending/add-host-function.md +++ b/docs/contributing/howto/extending/add-host-function.md @@ -1,11 +1,12 @@ -# Adding a host function +# Adding a Host Function -1. `executors/<line>.x/executor/codegen/data/host-fns.json` β€” add the method - id, then regenerate (`ninja -C build codegen`); this updates - `executor/crates/common/src/host_fns.rs` (per line) and the manager's - `tests/runner/origin/host_fns.py` β€” see [genvm-tool.md](../genvm-tool.md). -2. `tests/runner/origin/base_host.py` β€” handle the new case in the read loop - and add the method to the `IHost` protocol. NOTE: `origin/` is mirrored - into the node (`backend/node/genvm/origin/`). -3. `tests/runner/gvm_extra/mock_host.py` β€” add the test implementation. -4. Update the node (and simulator) host implementations. +1. `executors/<line>.x/executor/codegen/data/host-fns.json` β€” add the method id, + then `ninja -C build codegen` ([genvm-tool.md](../genvm-tool.md)). That + regenerates the line's `executor/crates/common/src/host_fns.rs` and the + manager's `tests/runner/origin/host_fns.py` +2. `executors/<line>.x/executor/src/host/mod.rs` β€” wire the executor-side client +3. `tests/runner/origin/base_host.py` β€” handle the new case in the read loop and + add the method to the `IHost` protocol. `origin/` is mirrored into the node, + so keep it self-contained +4. `tests/runner/gvm_extra/mock_host.py` β€” add the test implementation +5. Update the node and simulator host implementations, in their own repos diff --git a/docs/contributing/howto/extending/add-llm-provider.md b/docs/contributing/howto/extending/add-llm-provider.md index ede8518a..9a826a4b 100644 --- a/docs/contributing/howto/extending/add-llm-provider.md +++ b/docs/contributing/howto/extending/add-llm-provider.md @@ -1,14 +1,16 @@ -# Adding an LLM provider +# Adding an LLM Provider -If the provider is OpenAI-API-compatible, no work is needed. +An OpenAI-API-compatible provider needs no work at all β€” configure it as +`openai-compatible` -Otherwise (all in the manager's `implementation/src/llm/`): +Otherwise: -1. `providers.rs` β€” declare and implement the provider (`impl Provider`); - a separate json mode is recommended. -2. `config.rs` β€” add a `Provider` enum value and a `BackendConfig::to_provider` - case. +1. `implementation/src/llm/providers.rs` β€” declare the provider and + `impl Provider` for it; a separate json mode is recommended +2. `implementation/src/llm/config.rs` β€” add a `Provider` enum value and the + matching `BackendConfig::to_provider` arm +3. `docs/schemas/default-config.json` β€” add the id to the provider enum -Test (optional): add a case to `implementation/src/llm/handler.rs`, patch -`.github/workflows/queue.yaml` to pass the API-key secret, and provide the key -to the repository owners. +Testing is optional: add a case to `implementation/src/llm/handler.rs`, pass the +API-key secret in `.github/workflows/queue_test_cell.yaml`, and hand the key to +the repository owners diff --git a/docs/contributing/howto/extending/add-wasi-function.md b/docs/contributing/howto/extending/add-wasi-function.md index ce69c2d6..1901c18b 100644 --- a/docs/contributing/howto/extending/add-wasi-function.md +++ b/docs/contributing/howto/extending/add-wasi-function.md @@ -1,21 +1,22 @@ -# Adding a WASI function to an executor +# Adding a WASI Function -Paths are relative to `executors/<line>.x/`. +Paths are relative to `executors/<line>.x/` -## Via gl_call (preferred β€” no ABI surface change) +## Through gl_call β€” Preferred -1. `executor/crates/sdk-rs/src/abi/gl_call.rs` β€” add the message shape. -2. `executor/src/wasi/genlayer_sdk.rs` β€” implement the handler; add a version - check so older contracts don't observe the new method. +A new gl_call method is a minor-compatible ABI addition, see the +[versioning spec](../../../website/src/spec/01-core-architecture/03-versioning.rst) -New gl_call methods are a minor-compatible ABI addition β€” see the -[versioning spec](../../../website/src/spec/01-core-architecture/03-versioning.rst). +1. `executor/crates/sdk-rs/src/abi/gl_call.rs` β€” add the message shape +2. `executor/src/wasi/genlayer_sdk/` β€” implement the handler, with a version + check so an older contract does not observe the new method -## Raw WASI function (new ABI surface) +## Raw WASI Function -1. `executor/src/wasi/witx/genlayer_sdk.witx` β€” add the declaration. -2. `executor/src/wasi/genlayer_sdk.rs` β€” add the implementation (under the - generated `impl` trait). +This changes the ABI surface itself + +1. `executor/src/wasi/witx/genlayer_sdk.witx` β€” add the declaration +2. `executor/src/wasi/genlayer_sdk/` β€” implement it under the generated trait 3. `runners/cpython/modules/_genlayer_wasi/genlayer.c` β€” add the Python proxy. - This changes the cpython runner hash β€” refresh it per - [modify-runner.md](modify-runner.md). + This changes the cpython runner hash, so refresh it per + [modify-runner.md](modify-runner.md) diff --git a/docs/contributing/howto/extending/modify-runner.md b/docs/contributing/howto/extending/modify-runner.md index f0f3b96b..da7f39f5 100644 --- a/docs/contributing/howto/extending/modify-runner.md +++ b/docs/contributing/howto/extending/modify-runner.md @@ -1,18 +1,15 @@ -# Modifying a runner +# Modifying a Runner -Runner sources and version pins live per executor line; paths below are -relative to `executors/<line>.x/`. +Paths are relative to `executors/v0.3.x/`, the line that owns runner sources 1. Set `runners/support/versions/dev-mode.nix` to `true` and the runner's hash - in `runners/support/versions/current.nix` to `"test"`; develop and test. -2. Set dev-mode back to `false`; set the changed runner's hash (and its - dependents') to `null`. -3. Refresh hashes with `runners/support/versions/hash-updater.py` (run from - anywhere inside the repo): it builds `#runners-all` with `--keep-going`, - writes every reported `hash mismatch … got:` value back into `current.nix`, - and repeats until fixed point. Hashes depend on dependencies' resolved uids, - so they must be discovered in dependency order β€” the script handles that. + in `runners/support/versions/current.nix` to `"test"`, then develop and test. + Hashes move, so integration tests need `--ignore-hash` + ([integration.md](../testing/integration.md)) +2. Set dev-mode back to `false`, and the changed runner's hash, plus its + dependents', to `null` +3. Refresh the hashes with `runners/support/versions/hash-updater.py` +4. Rebuild to confirm no mismatch remains - Manual alternative: `ninja -C build all`, copy the `got:` hash from the - error into `current.nix`, repeat per dependent runner. -4. Rebuild to verify no mismatches remain. +Step 3 in detail, and why a hash is never restored by hand: +[committing/runners.md](../committing/runners.md) diff --git a/docs/contributing/howto/extending/modify-wasmtime.md b/docs/contributing/howto/extending/modify-wasmtime.md index 09ebfeaf..f20d77dd 100644 --- a/docs/contributing/howto/extending/modify-wasmtime.md +++ b/docs/contributing/howto/extending/modify-wasmtime.md @@ -1,27 +1,22 @@ -# Modifying vendored wasmtime +# Modifying Vendored Wasmtime -The tree at `executors/<line>.x/executor/third-party/wasmtime` is a -git-third-party clone (upstream pin + genvm patches) β€” mechanism and commit -workflow: [git-third-party.md](../committing/git-third-party.md). +`executors/<line>.x/executor/third-party/wasmtime` is a git-third-party clone: +an upstream pin plus genvm patches. Mechanism and commit flow: +[git-third-party.md](../committing/git-third-party.md) -Edit the tree directly; `cargo check`/ninja build it in-tree, so iteration is -normal Rust development. Two caveats: +Edit the tree directly β€” `cargo check` and ninja build it in place, so iteration +is normal Rust development. Nix builds do **not** see working-tree edits, they +apply the saved patches, so run `git third-party save` before any nix build or +release must pick a change up -- **Nix builds don't see working-tree edits** β€” they apply the saved patches - (`support/nix/git-third-party.nix`). Run `git third-party save` before any - nix build/release must pick the change up. -- Commit flow: commit inside the nested wasmtime repo β†’ - `git third-party save executor/third-party/wasmtime` β†’ commit - `.git-third-party/` in the executor repo β†’ gitlink bump in the manager. +## genvm Patch Points -## genvm patch points - -- Custom trap variant `Trap::NondetInstruction` - (`crates/environ/src/trap_encoding.rs`) β€” raised for instructions disallowed - in deterministic mode (e.g. unmapped float/SIMD ops); emitted via - `trap_nondet_instruction` (`crates/cranelift/src/func_environ.rs`) from - `float_op_unreachable_check` (`crates/cranelift/src/translate/code_translator.rs`). -- When adding a `Trap` variant: update the `check!` macro in `from_u8` and its - `Display` arm (see the comment at the enum), the `TRAP_*` const in - `crates/cranelift/src/lib.rs`, and the executor-side remap to a public VM - error code in `executor/src/rt/errors.rs` (`trap_to_vm_error`). +1. The custom trap `Trap::NondetInstruction` + (`crates/environ/src/trap_encoding.rs`), raised for instructions disallowed + in deterministic mode, such as unmapped float and SIMD ops. It is emitted + through `trap_nondet_instruction` + (`crates/cranelift/src/func_environ.rs`) from `float_op_unreachable_check` + (`crates/cranelift/src/translate/code_translator.rs`) +2. A new `Trap` variant needs: the `check!` macro in `from_u8` and its `Display` + arm, the `TRAP_*` const in `crates/cranelift/src/lib.rs`, and the + executor-side remap in `executor/src/rt/errors.rs` (`trap_to_vm_error`) diff --git a/docs/contributing/howto/extending/write-a-script.md b/docs/contributing/howto/extending/write-a-script.md new file mode 100644 index 00000000..1211ae39 --- /dev/null +++ b/docs/contributing/howto/extending/write-a-script.md @@ -0,0 +1,36 @@ +# Writing a Script + +1. Prefer Python over Bash, with `argparse` for the CLI +2. Integrate into `genvm-tool` or `support/ci` when it fits, rather than adding + a loose entry point. A standalone script goes to `support/scripts/`, + executable and pure-stdlib, so it runs without the dev shell +3. If the script checks something, print every rejection reason to `stdout`, so + a failure needs no further digging + +## Wiring a Check as a Pre-Commit Hook + +Hooks are declared in `flake.nix`; the committed `.pre-commit-config.yaml` is +generated from it by `git-hooks.nix`, so never edit it by hand. Add an entry +under the local-guards block: + +```nix +check-source-text = { + enable = true; + name = "check-source-text"; + entry = "${hooks-python}/bin/python3 support/scripts/check-source-text.py"; + files = "\\.rs$"; # narrow to the file types you check +}; +``` + +pre-commit passes the changed files as arguments, so take them as an `argparse` +positional; exit non-zero to reject the commit, printing each rejection in a +greppable form, ideally `path:lineno: <offending line>`. +`support/scripts/check-source-text.py` is a worked example, including a +per-file `check-source-text: off` opt-out + +## Guards Shared With a Submodule + +A commit inside an executor submodule runs that repo's own hooks from its own +checkout β€” it cannot reach into the umbrella. A guard needed in both places +keeps a copy under the submodule's own `support/scripts/`, wired from its +`flake.nix`; keep the copies in sync, and diff them when you touch either diff --git a/docs/contributing/howto/genvm-tool.md b/docs/contributing/howto/genvm-tool.md index 2209bd9b..e7b3742b 100644 --- a/docs/contributing/howto/genvm-tool.md +++ b/docs/contributing/howto/genvm-tool.md @@ -1,46 +1,34 @@ # genvm-tool -Lives at `support/tools/genvm-tool` (on `PATH` as `genvm-tool` in the dev shell; -outside it, call `support/tools/genvm-tool/genvm-tool`). +The umbrella tool, at `support/tools/genvm-tool`, on `PATH` in the dev shell. +Full reference: `genvm-tool --print-manpage | man -l -`. Global options go +before the subcommand: `genvm-tool --log-level debug test run …`. Outside the +dev shell run `build/genvm_tool.sh`, which `configure` writes β€” the source tree +has no wrapper -## Command reference - -The full man page is generated from the parser itself: - -```bash -genvm-tool --print-manpage | man -l - -``` - -## genvm-tool test - -```sh -genvm-tool test run [--filter-name REGEX] [--filter-tag EXPR] [--filter-continue FILE] [--fail-fast] -``` - -All given filters must match (they are AND-ed). `--filter-name` is an -unanchored regex over the test name; `--filter-tag` is a boolean tag -expression like `(a|b)&!c`. - -To list every test a filter selects (several hundred entries β€” pipe it, or use -the json log format for machine consumption): - -```sh -genvm-tool --log-format=json test show test -``` - -Full testing workflow: [testing/README.md](testing/README.md). +| Command | Effect | +|---|---| +| `configure` | write `build/build.ninja` and `build/info.json` | +| `test` | run the suite ([testing/README.md](testing/README.md)) | +| `codegen` | render a data JSON into rust/python/rst/go | +| `build-manifest` | write the manager's `data/manifest.yaml` | +| `docs` | render the `docs/contributing/` indexes, the source of the list in `CLAUDE.md` | +| `git` | `ls`, `list-repo`, `create-branches`, `check-for-push` across the manager and every executor submodule ([submodules.md](committing/submodules.md)) | -## genvm-tool git +## Codegen -Multi-repo helpers over the manager and every executor submodule: +`ninja -C build codegen` regenerates from data JSON; never edit an output by +hand. Host protocol data is per line, public ABI data comes from the primary one -| Command | Effect | +| Data | Generated | |---|---| -| `genvm-tool git ls` | list tracked files across all repos | -| `genvm-tool git list-repo` | list the manager and every executor submodule repo | -| `genvm-tool git create-branches` | create a feature branch across the sub-repos that have new content | -| `genvm-tool git check-for-push` | read-only pre-push status; prints the suggested push command | +| `executors/<line>.x/executor/codegen/data/host-fns.json` | that line's `executor/crates/common/src/host_fns.rs`, `tests/runner/origin/host_fns.py` | +| `executors/<primary>.x/executor/codegen/data/public-abi.json` | `tests/runner/origin/public_abi.py`, `docs/website/src/spec/appendix/constants.rst` | + +## Suite Layout -Where they fit in the workflow: -[first-contribution tutorial](../tutorial/first-contribution.md), -[submodules.md](committing/submodules.md). +The suite is `tests(ctx)` in the root `.genvm-tool.py`, which registers +collectors. Plugins in `tests/runner/genvm_tool_plugins/` are plain importable +modules, with no runner-specific registry. Per-suite definitions live next to +the tests in `tests/system/<name>/test.py`, pulled in with `ctx.collect_dir`. +Each executor line carries its own `.genvm-tool.py` diff --git a/docs/contributing/howto/pr.md b/docs/contributing/howto/pr.md new file mode 100644 index 00000000..9a7037bf --- /dev/null +++ b/docs/contributing/howto/pr.md @@ -0,0 +1,54 @@ +# Pull Requests + +There is no GitHub merge queue. A PR lands when a maintainer ticks a box on the +PR action panel, which re-checks every gate. Automation: +`.github/workflows/branch_*.yaml` and `queue.yaml`; why: +[merge-model.md](../explanation/merge-model.md) + +## Branches + +1. `main` is an alias of the latest `v<X>-dev` and is never developed on +2. A feature PR lands on `v<X>-dev`, which reaches `v<X>.x` through a standing + release-gate PR +3. `genvm-tool git create-branches` makes `feat/<name>` in the manager and + `pr/<line>/feat/<name>` per executor line +4. Push the submodules first, then open **one PR, in the manager only**, against + `v<X>-dev`; a PR targeting `main` is retargeted for you +5. Never open an executor PR by hand β€” automation opens and links one per active + line that has a pushed `pr/<line>/…` branch, and they land together + +## Panel + +Boxes: *Force / Rerun full tests*, *Provision executor PRs* (force-push each +mirror branch to the pinned gitlink, after a rebase or an amend), *Merge into +dev*. Nothing works without the **`ci-safe`** label, since these jobs run the +PR's own `support/ci/` scripts with deploy keys in scope. Authors with write +access get it automatically. Everyone else needs a maintainer to add it after +reading the diff + +## Gates + +`queue.yaml` always runs the pre-commit hooks. On a `pull_request` event it also +requires 0 commits behind base and the executor precondition β€” every repo +rebased, every pinned executor commit pushed β€” which is skipped for fork PRs. +The heavy matrix runs only with the `run-full-tests` marker; without it the +check stays red on purpose + +Merging re-verifies against the **exact head sha**: maintainer approval (any +push revokes it), green full CI, green cross-repo E2E, 0 behind. Merges are +serialised repository-wide, so a tick may wait for another PR to land first + +Each repo gains one commit per PR. Executor lines are always squashed onto +`<line>-dev`, so the manager commit is rewritten to gitlink the new commit. A +manager PR that is a single commit and rewrites no gitlink keeps its sha + +## Troubleshooting + +| Symptom | Fix | +| --- | --- | +| Panel does nothing | missing `ci-safe` | +| `full tests have not run` | tick *Force run full tests* | +| `... commit(s) behind base` | rebase, `push --force-with-lease` | +| `pinned commit ... is not on the executor repo` | push it, or tick *Provision executor PRs* | +| Approval revoked | you pushed after the review, re-request it | +| Merge run cancelled before it started | another merge was already queued, tick *Merge into dev* again | diff --git a/docs/contributing/howto/releasing/release-build.md b/docs/contributing/howto/releasing/release-build.md index 5c0edd82..09e546b3 100644 --- a/docs/contributing/howto/releasing/release-build.md +++ b/docs/contributing/howto/releasing/release-build.md @@ -1,44 +1,44 @@ -# Release / nix build +# Release Build -Nix flake packages see only **committed** submodule content β€” commit inside -submodules first ([submodules.md](../committing/submodules.md)), and always pass -`?submodules=1`. - -Platform suffixes: none (native), `-amd64-linux`, `-arm64-linux`, `-arm64-macos`. - -| Package | Contents | -|---|---| -| `genvm[-<platform>]` | everything: manager + all executor lines + runners | -| `manager[-<platform>]` | manager bundle (`bin/`, `lib/`, `config/`, `data/`) | -| `executor[-<platform>]` | all active executor lines merged (`executor/<version>/…`) | -| `executor-<version>[-<platform>]` | one executor line | -| `artifact-prepack-genvm-<platform>` | a platform's release asset: manager + every active executor line | -| `artifact-prepack-genvm-universal` | the universal release asset: runners (`runners/` prefix + legacy-line overlay) | +Nix flake packages see **committed** submodule content only, so commit inside +the submodules first ([submodules.md](../committing/submodules.md)), and always +pass `?submodules=1`: ```bash nix build -v -L '.?submodules=1#genvm-amd64-linux' ``` -## Release assets +Platform suffixes: none for native, `-amd64-linux`, `-arm64-linux`, +`-arm64-macos` + +| Package | Contents | +|---|---| +| `genvm[-<platform>]` | everything: manager, every executor line, runners | +| `manager[-<platform>]` | the manager bundle: `bin/`, `lib/`, `config/`, `data/` | +| `executor[-<platform>]` | every active line merged, as `executor/<version>/…` | +| `executor-v0_3[-<platform>]` | one line; the dot in the line tag becomes an underscore | +| `artifact-prepack-genvm-<platform>` | a platform's release asset | +| `artifact-prepack-genvm-universal` | the universal asset: runners under a `runners/` prefix, plus the legacy-line overlay | + +## Release Assets -`.github/workflows/release.yaml` is `incl_release_build_test.yaml` (plan β†’ build -every asset β†’ test every platform) plus a publisher. Everything is built here β€” -the executor is not a standalone build entry point. +`.github/workflows/release.yaml` is `incl_release_build_test.yaml` β€” plan, build +every asset, test every platform β€” plus a publisher. Everything is built there; +the executor is not a standalone build entry point -Assets are named with the same platform token as the nix targets -(`amd64-linux`, …), not a reversed one. Each is one prepacked tree, and the -tarball is already named as the asset, so nothing renames it on the way out: +An asset is the prepacked tree itself, already named β€” nothing renames it: | Asset | Package | |---|---| | `genvm-<platform>.tar.xz` | `artifact-prepack-genvm-<platform>` | | `genvm-universal.tar.xz` | `artifact-prepack-genvm-universal` | -All four go onto one **genvm-manager** release, tagged with `version` from -`.genvm-monorepo-root` (see [versioning.md](versioning.md)). They overlay onto a -single install root, so a full install is one platform asset + `genvm-universal`. +All of them go onto a single **genvm-manager** release, tagged with `version` +from `.genvm-monorepo-root` ([versioning.md](versioning.md)). They overlay onto +one install root, so a full install is one platform asset plus +`genvm-universal` To exercise the whole pipeline on a PR without releasing anything, add the `test-release-pipeline` label: `queue.yaml` then runs the same -`incl_release_build_test.yaml`. It also generates the release notes and prints -them, so they can be checked before the release. +`incl_release_build_test.yaml`, and prints the generated release notes so they +can be checked beforehand diff --git a/docs/contributing/howto/releasing/versioning.md b/docs/contributing/howto/releasing/versioning.md index 90b4c109..1263e16c 100644 --- a/docs/contributing/howto/releasing/versioning.md +++ b/docs/contributing/howto/releasing/versioning.md @@ -1,40 +1,36 @@ -# Versioning and release trains +# Versioning and Release Trains -`.genvm-monorepo-root` (repo root, JSON) is the source of truth: +`.genvm-monorepo-root` is the source of truth: -- `version` β€” the manager's own release version (e.g. `v0.6.0-rc0`). -- `active-versions` β€” the live release trains (e.g. `["v0.3", "v0.2"]`); each - train maps to an `executors/v<X>.x` submodule line. +1. `version` β€” the manager's own release version, e.g. `v0.6.0-rc1` +2. `active-versions` β€” the live executor lines, each an `executors/v<X>.x` + submodule. The first is the primary line, which owns the manager-global + generated files +3. `support-only-versions` β€” lines still shipped but no longer built from full + sources -Executor crates are versioned **independently** in their submodules (each -line's `manifest.json` holds `executor-version`); manager scripts never rewrite -them. +Executor crates are versioned **independently**: a line's `manifest.json` holds +`executor-version` and no manager script rewrites it -## Scripts +Manager branches are named after the *manager* train, not an executor line: +`v<X>.x` is the release branch, `v<X>-dev` the integration branch -`support/ci/check-versions.py` β€” keeps manager-owned crate versions -(`implementation/Cargo.toml`) in sync with `version`: +## Tools + +Both are `support/ci/tools/versions.py`, run through `./support/ci/run.sh tool`. +`MONOREPO_ROOT=<path>` points them at a different `.genvm-monorepo-root`, e.g. +one extracted from `origin/main` | Command | Effect | |---|---| -| `check-versions.py sync` (default) | pre-commit fixer: rewrite crate `[package]` major.minor to match `version` (patch kept); non-zero exit if it changed anything | -| `check-versions.py bump minor` | cut next train, e.g. 0.3 β†’ 0.4 (default level) | -| `check-versions.py bump major` | 0.3 β†’ 1.0 | -| `check-versions.py bump --set 0.6` | explicit major.minor | - -`bump` updates `version`, appends the new train to `active-versions`, sets -manager crates to `<new>.0`, and prints the bare major.minor (callers build -branch names from it). - -`support/ci/branch-versions.py list|manager` β€” `list` prints the active -executor lines from `active-versions`; `manager` prints the manager's own train -from `version` (both as bare major.minor). The manager dev/release branches are -named after `manager`, NOT the executor lines. Branch naming per train `<X>`: - -- `v<X>.x` β€” release branch (release-ready) -- `v<X>-dev` β€” integration branch +| `check-versions sync` (default) | pre-commit fixer: rewrite `implementation/Cargo.toml` major.minor to match `version`, keeping the patch; non-zero exit if it changed anything | +| `check-versions bump [minor\|major]` | cut the next train, default `minor` | +| `check-versions bump --set 0.6` | explicit major.minor | +| `branch-versions list\|manager` | the active executor lines, or the manager's train, as bare major.minor | -`MONOREPO_ROOT=<path>` points it at a different `.genvm-monorepo-root` (e.g. -extracted from `origin/main`). +`bump` updates `version`, appends the new train to `active-versions`, sets the +manager crates to `<new>.0`, and prints the bare major.minor for callers to +build branch names from -See also: [submodules.md](../committing/submodules.md) for the branch model of feature work. +Branch model for feature work: [submodules.md](../committing/submodules.md). Why +the lines exist at all: [executor-lines.md](../../explanation/executor-lines.md) diff --git a/docs/contributing/howto/setup.md b/docs/contributing/howto/setup.md index 1be964e4..65cf4f34 100644 --- a/docs/contributing/howto/setup.md +++ b/docs/contributing/howto/setup.md @@ -1,31 +1,26 @@ -# First-time setup +# First-Time Setup -1. Get the executor submodules and vendored third-party trees **before** entering - the dev shell (the nix flake reads them): +1. Materialize the submodules and the vendored third-party trees **before** + entering the dev shell β€” the nix flake reads them: ```bash python3 support/scripts/get-all-git.py ``` - The default options are sufficient for normal setup: the command resolves - duplicate submodule remotes, fetches each unique remote once into a local - cache, initializes submodules from that cache, and materializes all - third-party trees. - -2. Enter the dev shell: + The defaults fetch every unique submodule remote once into a local cache, + initialize the submodules from it, and materialize all third-party trees +2. Enter the dev shell β€” normally direnv does this, `.envrc` is + `use flake '.?submodules=1#full'`: ```bash nix develop '.?submodules=1#full' ``` - `?submodules=1` is required on **every** flake ref β€” without it: - `Path 'executors/v0.3.x' … is not tracked by Git`. Usually auto-loaded via - direnv (`.envrc` is `use flake '.?submodules=1#full'`). - -3. `source env.sh` β€” adds `support/tools/git-third-party` to PATH and sources - `.env` if present. Only needed outside the dev shell for manual - `git third-party` use. - -See [git-third-party.md](committing/git-third-party.md) for how vendoring works. + `?submodules=1` is mandatory on every manager flake ref; without it nix + fails with `Path 'executors/v0.3.x' … is not tracked by Git` +3. Only outside the dev shell, and only for manual `git third-party` calls: + `source env.sh` puts `support/tools/git-third-party` on `PATH` and sources + `.env` if present -Next: [build.md](building/build.md). +Next: [build.md](building/build.md). Vendoring mechanics: +[git-third-party.md](committing/git-third-party.md) diff --git a/docs/contributing/howto/testing/README.md b/docs/contributing/howto/testing/README.md index c7ae16c6..0e5886f3 100644 --- a/docs/contributing/howto/testing/README.md +++ b/docs/contributing/howto/testing/README.md @@ -1,48 +1,42 @@ -# Test +# Testing -All tests run through `genvm-tool test`. Build first -([build.md](../building/build.md)). Commands assume the `#full` dev shell. - -Per-kind guides: [rust.md](rust.md) Β· [python.md](python.md) Β· -[integration.md](integration.md) Β· [fuzzing.md](fuzzing.md). - -## Quick reference +Everything runs through `genvm-tool test`, from the `#full` dev shell, after a +build ([build.md](../building/build.md)). Per-kind: [rust.md](rust.md) Β· +[python.md](python.md) Β· [integration.md](integration.md) Β· +[fuzzing.md](fuzzing.md) | What | Command | |---|---| | All tests | `genvm-tool test run` | -| Release preset | `genvm-tool test run --filter-tag "$(cat tests/presets/release.txt)"` | +| Preset | `genvm-tool test run --filter-tag "$(cat tests/presets/release.txt)"` | | One test | `genvm-tool test run --filter-name 'test_name'` | | Re-run failed | `genvm-tool test run --filter-continue <file>` | -| List tests / plan / tags / services | `genvm-tool test show test\|plan\|tags\|services` | +| Inspect | `genvm-tool test show test\|plan\|tags\|services` | -Options: `--filter-name REGEX`, `--filter-tag EXPR` (e.g. `stable & !slow`), -`--filter-continue FILE`, `--fail-fast`, `--coverage`, -`--log-level {trace,debug,info,warning,error}`, `--ignore-hash`. +`run` options: `--filter-name REGEX`, `--filter-tag EXPR`, +`--filter-continue FILE`, `--fail-fast`, `--max-concurrent N`, `--coverage`, +`--ignore-hash`, `--junit-xml`. `--log-level` is global, so it goes before the +subcommand -Presets in `tests/presets/`: `release.txt` (`integration & stable & !bench`), -`rust.txt` (`(rust | integration) & !bench & !fuzz`), `rust-fuzz.txt` -(`rust & fuzz`), `python.txt` (`python`). +Presets in `tests/presets/`: `release.txt` is `integration & stable & !bench`, +`rust.txt` is `(rust | integration) & !bench & !fuzz`, `rust-fuzz.txt` is +`rust & fuzz`, `python.txt` is `python`. `genvm-tool test show tags` lists every +tag, the integration ones are in [integration.md](integration.md) -Common tags are documented in [integration.md](integration.md); -`genvm-tool test show tags` lists everything. +## Fix–Rerun Loop -## Fix–rerun loop: use continue files - -Every failed run writes the failing test names to -`build/test-artifacts/continue/<timestamp>-<random>` (the name is printed in -the failure summary). **After fixing something, rerun just those tests first** -β€” much faster than a full rerun: +A failed run writes the failing names to +`build/test-artifacts/continue/<timestamp>-<random>` and prints that name in the +summary. After a fix rerun only those, then do a full run once they pass: ```bash genvm-tool test run --filter-continue <name-or-path> ``` -Do a full run only once the continue set passes. - -## Configuration +## Where Tests Come From -Test-suite configuration (artifacts dir, python paths) comes from -`.genvm-monorepo-root`; the suite itself is the `tests` function in -`.genvm-tool.py`. Test definitions live in `tests/system/<name>/test.py` β€” -see [genvm-tool.md](../genvm-tool.md). +Rust from every tracked `Cargo.toml` and its `fuzz/*.rs`, python from +`executors/v0.3.x/runners/genlayer-py-std`, integration from each line's +`tests/integration/`, system from `tests/system/<name>/test.py`. Collection is +wired in the root `.genvm-tool.py`, runner configuration in +`.genvm-monorepo-root` ([genvm-tool.md](../genvm-tool.md)) diff --git a/docs/contributing/howto/testing/fuzzing.md b/docs/contributing/howto/testing/fuzzing.md index 64c13908..5901182f 100644 --- a/docs/contributing/howto/testing/fuzzing.md +++ b/docs/contributing/howto/testing/fuzzing.md @@ -1,24 +1,32 @@ # Fuzzing -Rust fuzz targets run under AFL (`cargo-afl`): +Rust fuzz targets run under AFL (`cargo-afl`); a target is any `fuzz/*.rs` next +to a tracked `Cargo.toml`: ```bash genvm-tool test run --filter-tag "$(cat tests/presets/rust-fuzz.txt)" # rust & fuzz ``` -Python fuzz targets exist but their collection is currently disabled in -`.genvm-tool.py`. Some integration cases also carry the `fuzz` tag. +Python fuzz targets exist, but their collection is currently disabled in +`.genvm-tool.py`. Some integration cases carry the `fuzz` tag too -Options: `--fuzz-timeout N` (seconds per run, default 30), -`--fuzz-update-corpus` (write back interesting inputs). +Options: `--fuzz-timeout N` (seconds per run, default 30) and +`--fuzz-update-corpus` (write interesting inputs back) -## Host preparation +## Host Preparation -AFL needs these before a session (crashes must land as files, CPU frequency -must be stable, and the fuzzer must be able to ptrace): +AFL needs crashes to land as files, a stable CPU frequency, and permission to +ptrace: ```bash echo core | sudo tee /proc/sys/kernel/core_pattern echo performance | sudo tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +cat /proc/sys/kernel/yama/ptrace_scope # note it down, to restore afterwards echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope ``` + +The last one is machine-wide and lasts until reboot: with `ptrace_scope` at `0` +any process of yours can attach to any other one of yours, browser and ssh-agent +included. Fuzz on a throwaway machine or a container, and put the noted value +back (`echo <value> | sudo tee /proc/sys/kernel/yama/ptrace_scope`) when you are +done diff --git a/docs/contributing/howto/testing/integration.md b/docs/contributing/howto/testing/integration.md index e6a3ced8..97e36183 100644 --- a/docs/contributing/howto/testing/integration.md +++ b/docs/contributing/howto/testing/integration.md @@ -1,61 +1,70 @@ -# Integration tests +# Integration Tests ```bash genvm-tool test run --filter-tag integration ``` Cases are `.jsonnet` files in `executors/<line>.x/tests/integration/`; each -produces a `<name>/prepare` case plus one case per step (`/0l`, `/0v`, `/0s`, …). -A sibling `.skip` file marks a case skipped. The suite iterates every line in -`active-versions` (`.genvm-monorepo-root`) and runs each line's cases against -that line's own built executor (`build/info.json` maps `v0.3` β†’ the concrete -built version). +yields a `<name>/prepare` case plus one per step (`/0l`, `/0v`, `/0s`, …), and +a sibling `.skip` file marks a case skipped. Every line in `active-versions` +runs its own cases against its own built executor, via `build/info.json` -## Common tags +## Tags | Tag | Meaning | |---|---| -| `stable` | needs **no LLM keys and no webdriver** (manager only) β€” safe to run offline | -| `unstable`, `semi-stable` | need modules + webdriver; `unstable` is retried (3 attempts), `semi-stable` is not | -| `bench` | benchmarks β€” excluded from the presets | - -## Running a subset - -- **One executor line** β€” test names are repo-relative paths and - `--filter-name` is an unanchored regex, so: - `genvm-tool test run --filter-name 'executors/v0.3.x/'` -- **Offline / no LLMs / no web** β€” the `stable` tag marks tests needing only - the manager (no modules, no webdriver; the executor is told `no_modules`): - `--filter-tag 'integration & stable'` (`tests/presets/release.txt` - additionally excludes `bench`). Stability is the second argument of - `util.features(paths, stability)` in the case's jsonnet `tags` - (`'stable'` / `'semi-stable'` / `'unstable'`). (Legacy: v0.2.x instead uses a - `stable/` top-level directory.) -- **After a fix** β€” rerun only the failed cases via the continue file: - `--filter-continue <file>` (see [README.md](README.md)). - -## Golden files and hashes - -Goldens live next to the cases (`*.N.stdout`). A missing golden is -auto-created on the first run; after that a mismatch **fails** the test β€” -delete the sidecar to regenerate it. - -With `stable_hash: true` on a step entry (the default) the leader's execution -hash is likewise compared against an auto-created `.N.hash` sidecar; with -`stable_hash: false` there is no sidecar and validators compare to the -leader's runtime hash instead. - -`--ignore-hash` skips hash comparison *and* sidecar creation (a new case does -NOT need it β€” its hash is auto-created). Use it when hashes legitimately -differ, e.g. **runner dev mode** -([modify-runner.md](../extending/modify-runner.md)): a modified runner changes -runner hashes and therefore every execution hash. An executor line can also -disable hash checks wholesale via `integration()` in its `.genvm-tool.py` -returning `{'ignore-hash': True}` β€” currently the case for v0.3.x. +| `stable` | no LLM keys and no webdriver, runs offline; the executor is told `no_modules` | +| `semi-stable`, `unstable` | need modules and webdriver; `unstable` is retried up to 3 times | +| `bench` | benchmarks, excluded from the presets | +| `feature-*` | generated from the case's feature paths | + +Stability is the second argument of `util.features(paths, stability)` in the +case's jsonnet `tags`; legacy v0.2.x uses a `stable/` top-level directory +instead. `--filter-tag` tokenizes alphabetic words only, so a hyphenated tag β€” +`semi-stable`, any `feature-*` β€” is unmatchable there; use `--filter-name` + +## Running a Subset + +1. One line: `--filter-name 'executors/v0.3.x/'`, an unanchored regex over the + repo-relative test name +2. Offline: `--filter-tag 'integration & stable'`, or the `release.txt` preset, + which also drops `bench` +3. After a fix: `--filter-continue <file>`, see [README.md](README.md) + +## Golden Files and Hashes + +Goldens are `*.N.stdout` next to the case: created on the first run, a mismatch +**fails** afterwards, so delete the sidecar to regenerate. `stable_hash: true` +on a step, the default, likewise compares the leader's execution hash against a +`.N.hash` sidecar; with `false` there is no sidecar and validators compare +against the leader's runtime hash + +`--ignore-hash` skips both comparison and sidecar creation β€” needed in runner +dev mode, where every execution hash moves +([modify-runner.md](../extending/modify-runner.md)), not for a new case. A line +can stop tracking sidecars altogether by returning `{'save-hashes': False}` from +`integration()` in its `.genvm-tool.py`, as v0.3.x currently does while its +hashes still move + +Neither switch turns off the leader-vs-validator/sync comparison: with sidecars +disabled a non-main mode is compared against the main mode's hash from the same +run instead. Only that comparison makes a determinism regression visible, and a +non-main step has no semantics goldens of its own, so without it the step would +assert nothing beyond "did not crash" + +## Debug Mode + +Cases run at `unsafe`, where `py-genlayer:test` resolves and wall-clock is +exposed. A top-level `debug_mode` in the jsonnet lowers that to `safe`, +`safe-unbounded` or raises it to `unsafe-tracing`; `disabled` is rejected, +because `reroute_to` β€” what points a case at its own line's executor β€” is +honored only from `safe` up. Below `unsafe` a contract must name its runner by +hash instead of `:test`, read from +`build/out/executor/<version>/data/latest.json` ## Services -The manager, modules (LLM/web), and webdriver services start automatically; -`--no-manager` / `--no-webdriver` use externally running ones instead. Manual -webdriver: `bash webdriver/build-and-run.sh`. If WASM files changed, -`./build/out/executor/<version>/bin/genvm precompile` saves test time. +The manager, the modules and the webdriver start automatically; `--no-manager` +and `--no-webdriver` reuse externally running ones (a manual webdriver is +`bash webdriver/build-and-run.sh`). After WASM files change, +`./build/out/executor/<version>/bin/genvm precompile` saves test time diff --git a/docs/contributing/howto/testing/python.md b/docs/contributing/howto/testing/python.md index 61e56e81..74cb6b51 100644 --- a/docs/contributing/howto/testing/python.md +++ b/docs/contributing/howto/testing/python.md @@ -1,16 +1,18 @@ -# Python tests +# Python Tests ```bash genvm-tool test run --filter-tag python ``` -## Direct pytest (genlayer-py-std, no dev shell) +The suite is `genlayer-py-std`, which lives in the primary executor line. The +interpreter comes from a pinned standalone flake, so pytest can be run directly, +without the dev shell: ```bash -cd runners/genlayer-py-std +cd executors/v0.3.x/runners/genlayer-py-std env_dir="$(nix build --no-link --print-out-paths path:../../support/nix/py-test)" PYTHONPATH="$PWD/src:$PWD/src-emb" "$env_dir/bin/pytest" tests/ ``` -Coverage is enforced (`--cov-fail-under=75`); numpy-dependent tests need the -nix env. +Coverage is enforced (`--cov-fail-under=75`), and the numpy-dependent tests need +that nix environment diff --git a/docs/contributing/howto/testing/rust.md b/docs/contributing/howto/testing/rust.md index b8aff9f6..0d5aa57f 100644 --- a/docs/contributing/howto/testing/rust.md +++ b/docs/contributing/howto/testing/rust.md @@ -1,25 +1,27 @@ -# Rust tests +# Rust Tests -## Where to put tests +Collection scans every tracked `Cargo.toml` in the umbrella, so manager crates, +shared `crates/` and executor crates are all picked up -1. If it is trivial, put it as an integration test into `<crate-dir>/tests/*.rs` -2. If the entire `mod tests` is below 30 lines, you can put it inline -3. Tests for `mod.rs` go in a `tests.rs` alongside it, declared with `mod tests;` -4. Tests for `abc.rs` go in `abc_test.rs`, referenced with `#[path = "abc_test.rs"] mod tests;` +## Where To Put a Test -## How to run +1. If it is trivial, make it an integration test in `<crate-dir>/tests/*.rs` +2. If the whole `mod tests` stays below 30 lines, inline it +3. Tests for `mod.rs` go into a sibling `tests.rs`, declared as `mod tests;` +4. Tests for `abc.rs` go into `abc_test.rs`, referenced with + `#[path = "abc_test.rs"] mod tests;` + +## How To Run ```bash -genvm-tool test run --filter-tag rust # add --coverage for coverage +genvm-tool test run --filter-tag rust # --coverage for coverage ``` -If a Rust test fails to load a shared library under the `#full` shell, use the -dedicated shell: +The `rust.txt` preset is `(rust | integration) & !bench & !fuzz` + +If a test fails to load a shared library under the `#full` shell, use the +dedicated one: ```bash nix develop '.?submodules=1#rust-test' --command genvm-tool test run --filter-tag rust ``` - -Tests live in the executor crates (inline `#[cfg(test)] mod tests` and each -crate's `tests/`); `genvm-tool test` discovers them. Preset: -`tests/presets/rust.txt` (`rust | integration`). diff --git a/docs/contributing/howto/write-a-script.md b/docs/contributing/howto/write-a-script.md deleted file mode 100644 index ce343e5f..00000000 --- a/docs/contributing/howto/write-a-script.md +++ /dev/null @@ -1,43 +0,0 @@ -# Writing a script - -1. Prefer python over bash. -2. Use `argparse` for the CLI. -3. Integrate into `genvm-tool` or `support/ci` when it fits, rather than adding a loose entry point. -4. If the script checks something, print each rejection reason to `stdout`, so a failure needs no further digging. - -## Where it lives - -Standalone scripts go in `support/scripts/`. Keep them executable -(`chmod +x`) and pure-stdlib unless there is a strong reason otherwise, so they -run without the dev shell. - -## Wiring a check as a pre-commit hook - -Hooks are declared in `flake.nix` (the committed `.pre-commit-config.yaml` is -generated from it by `git-hooks.nix` on `nix develop` / `nix flake check` -- do -not edit it by hand). Add an entry under the local-guards block: - -```nix -check-source-text = { - enable = true; - name = "check-source-text"; - entry = "${hooks-python}/bin/python3 support/scripts/check-source-text.py"; - files = "\\.rs$"; # narrow to the file types you check -}; -``` - -pre-commit passes the changed files as arguments (`argparse` positional -`files`). Exit non-zero to reject the commit; per point 4, print each rejection -to stdout in a greppable form, ideally `path:lineno: <offending line>`. See -`support/scripts/check-source-text.py` for a worked example (it also supports a -per-file `check-source-text: off` opt-out comment). - -## Scripts shared with a submodule - -Each executor submodule (`executors/vD.D.x`) is a separate git repo with its own -pre-commit config, and a commit made inside it runs those hooks from its own -checkout -- it cannot reach into this umbrella. (The submodule is not built -standalone; the umbrella drives its build. This is only about its independent -git hooks.) So a guard that must run in both places keeps an in-tree copy under -the submodule's own `support/scripts/`, wired from the submodule's `flake.nix`. -Keep the two copies identical. diff --git a/docs/contributing/tutorial/README.md b/docs/contributing/tutorial/README.md new file mode 100644 index 00000000..bbe77535 --- /dev/null +++ b/docs/contributing/tutorial/README.md @@ -0,0 +1,6 @@ +# Tutorial + +Learning-oriented walkthroughs: follow them start to finish, on a change that +does not matter, before you need the how-to guides + +- [first-contribution.md](first-contribution.md) β€” patch an executor, branch, commit, push, open the PR diff --git a/docs/contributing/tutorial/first-contribution.md b/docs/contributing/tutorial/first-contribution.md index 481b7ba8..aaf26bea 100644 --- a/docs/contributing/tutorial/first-contribution.md +++ b/docs/contributing/tutorial/first-contribution.md @@ -1,73 +1,36 @@ -# Your first contribution: patching an executor - -End-to-end walkthrough for the most common change β€” fixing something inside an -executor line (`executors/<line>.x`). Each step links to the how-to guide with -the full details; read those when a step surprises you. - -## 1. Set up the checkout - -Follow [setup.md](../howto/setup.md): materialize submodules and vendored trees -(`python3 support/scripts/get-all-git.py`), then enter the dev shell -(`nix develop '.?submodules=1#full'`, usually auto-loaded via direnv). - -## 2. Make and verify the change - -Edit inside `executors/<line>.x/`, then build and test: - -```bash -genvm-tool configure && ninja -C build all/bin # building/build.md -genvm-tool test run --filter-name 'executors/<line>.x/' # testing/README.md -``` - -## 3. Create the branches - -```bash -genvm-tool git create-branches feat/<name> -``` - -It surveys which repos carry new content and creates the branch in the ones -you tick: the manager gets `feat/<name>` verbatim, each executor gets -`pr/<line>/feat/<name>` (all lines push to one shared executor remote, hence -the namespace). Branch model: -[submodules.md](../howto/committing/submodules.md). - -## 4. Commit, then update the gitlink - -The manager pins each submodule by a gitlink (a pinned commit), so the order -matters ([submodules.md](../howto/committing/submodules.md)): - -1. Commit **inside** the executor: `cd executors/<line>.x && git add … && git commit`. -2. In the manager, stage the moved gitlink: `git add executors/<line>.x`. -3. Commit the manager (gitlink bump + any manager-side changes together). - -If you touched runners, do the hash hygiene first: -[committing/runners.md](../howto/committing/runners.md). - -## 5. Pass pre-commit - -The dev shell installs each repo's pre-commit hooks, so `git commit` already -runs them where you commit β€” do **not** bypass with `--no-verify`. To run them -ahead of time: - -- manager only: `nix fmt` (runs the whole generated pre-commit config over the - working tree); -- everything at once, as CI does: `./support/ci/run.sh pipeline commit-hooks`. - -## 6. Push β€” executor first, then manager - -```bash -genvm-tool git check-for-push # readiness per repo + suggested push command -git -C executors/<line>.x push origin pr/<line>/feat/<name> -git push origin feat/<name> -``` - -Submodules must be pushed before the manager so its pinned gitlinks resolve. - -## 7. Open one PR β€” in the manager only - -Open the manager PR against its dev base (e.g. `v0.6-dev`). Do **not** open -executor PRs yourself: CI automatically opens the matching executor PR for -every active line and links each on the manager PR as an `executor: <url>` -line. - -Merge requirements (tests, merge queue): [../README.md](../README.md). +# Your First Contribution: Patching an Executor + +End-to-end walkthrough of the most common change, a fix inside an executor line. +Each step links to the guide with the full details + +1. **Set up** ([setup.md](../howto/setup.md)) β€” + `python3 support/scripts/get-all-git.py`, then enter the dev shell +2. **Change and verify** ([build.md](../howto/building/build.md), + [testing/README.md](../howto/testing/README.md)): + + ```bash + genvm-tool configure && ninja -C build all/bin + genvm-tool test run --filter-name 'executors/<line>.x/' + ``` +3. **Branch** β€” `genvm-tool git create-branches feat/<name>` surveys which repos + carry new content and branches the ones you tick: the manager gets + `feat/<name>`, each executor gets `pr/<line>/feat/<name>`, since all lines + push to one shared executor remote +4. **Commit** ([submodules.md](../howto/committing/submodules.md)) β€” inside the + executor first, then `git add executors/<line>.x` in the manager and commit + the gitlink bump together with any manager-side change. If you touched + runners, do the hash hygiene first + ([committing/runners.md](../howto/committing/runners.md)) +5. **Pass pre-commit** β€” the hooks run where you commit, so never + `--no-verify`. Ahead of time: `nix fmt` for the manager, or + `./support/ci/run.sh pipeline commit-hooks` for everything, as CI does +6. **Push, submodules first**, so the gitlinks resolve: + + ```bash + genvm-tool git check-for-push # readiness per repo + suggested command + git -C executors/<line>.x push origin pr/<line>/feat/<name> + git push origin feat/<name> + ``` +7. **Open one PR, in the manager only** ([pr.md](../howto/pr.md)), against the + dev base. CI opens and links the matching executor PR for every line you + pushed a branch for diff --git a/executors/v0.2.x b/executors/v0.2.x index 934acbee..1c8126d6 160000 --- a/executors/v0.2.x +++ b/executors/v0.2.x @@ -1 +1 @@ -Subproject commit 934acbee6ed11cbe134e48fd9256a6a5651187df +Subproject commit 1c8126d6efb6cab9fdce02b1745217fc571a343b diff --git a/executors/v0.3.x b/executors/v0.3.x index 301d515f..9d2db0dd 160000 --- a/executors/v0.3.x +++ b/executors/v0.3.x @@ -1 +1 @@ -Subproject commit 301d515fe598113bb405e2540fa6cd0460d10845 +Subproject commit 9d2db0dd98506cca712c18af4cde7b5dd926bf0d diff --git a/flake.nix b/flake.nix index 38f158f0..fce2b678 100644 --- a/flake.nix +++ b/flake.nix @@ -190,6 +190,59 @@ done < <(git ls-files '*Cargo.toml' ':(exclude)executors/') exit $rc ''; + # pre-commit 4.5 exposes only the first non-deletion update from + # pre-push stdin, and can skip hook stages when the outgoing commit is + # already on another remote ref. Its supported legacy-hook chaining + # preserves stdin, so validate every outgoing SHA there directly. + verify-manager-push-refs = pkgs.writeShellScript "verify-manager-push-refs" '' + set -euo pipefail + input="$(${pkgs.coreutils}/bin/cat)" + if [[ -z "$input" ]]; then + exit 0 + fi + declare -A manager_refs=() + while read -r local_ref local_sha remote_ref remote_sha extra; do + if [[ -z "$local_ref" || -z "$local_sha" || -z "$remote_ref" || -z "$remote_sha" || -n "$extra" ]]; then + echo "verify-pushed-executors: malformed pre-push input; refusing push" + exit 1 + fi + if [[ "$local_sha" != 0000000000000000000000000000000000000000 ]]; then + manager_refs["$local_sha"]=1 + fi + done <<< "$input" + rc=0 + for manager_ref in "''${!manager_refs[@]}"; do + ${genvm-tool}/bin/genvm-tool git verify-pushed-executors --manager-ref "$manager_ref" || rc=1 + done + if ((rc)); then + exit 1 + fi + user_legacy="$(dirname -- "$0")/pre-push.user-legacy" + if [[ -x "$user_legacy" ]]; then + printf '%s\n' "$input" | "$user_legacy" "$@" + fi + ''; + install-manager-push-refs = '' + hooks_dir="$(git rev-parse --git-path hooks)" + legacy="$hooks_dir/pre-push.legacy" + user_legacy="$hooks_dir/pre-push.user-legacy" + installed="$(${pkgs.coreutils}/bin/readlink "$legacy" 2>/dev/null || true)" + install_hook=1 + # `-e` alone would call a dangling symlink absent and overwrite it. + if [[ ( -e "$legacy" || -L "$legacy" ) && "$installed" != *-verify-manager-push-refs && "$installed" != *-verify-single-manager-push-ref ]]; then + if [[ ! -e "$user_legacy" && ! -L "$user_legacy" ]]; then + mv "$legacy" "$user_legacy" + else + # Overwriting would lose a hook we cannot put anywhere, so the + # shell comes up without ours rather than without theirs. + echo "cannot preserve existing $legacy: $user_legacy already exists; leaving it in place" >&2 + install_hook=0 + fi + fi + if ((install_hook)); then + ln -sfn ${verify-manager-push-refs} "$legacy" + fi + ''; pre-commit-check = git-hooks.lib.${system}.run { src = ./.; excludes = [ @@ -699,8 +752,10 @@ # custom-rust toolchain is not shadowed.) ++ pre-commit-check.enabledPackages; # The interactive dev shell (.envrc β†’ `.#full`) also installs the - # git-hooks pre-commit/commit-msg stubs into .git/hooks. - shellHook = shell-hook-base + pre-commit-check.shellHook; + # git-hooks pre-commit/commit-msg/pre-push stubs into .git/hooks. + # The final fragment adds the stdin-aware outgoing-ref guard chained + # by pre-commit's supported `.legacy` hook mechanism. + shellHook = shell-hook-base + pre-commit-check.shellHook + install-manager-push-refs; }; devShells.check-qemu = pkgs.mkShell { packages = packages-0 ++ [ pkgs.qemu ]; diff --git a/support/scripts/check-source-text.py b/support/scripts/check-source-text.py index bf975a96..0046e8a5 100644 --- a/support/scripts/check-source-text.py +++ b/support/scripts/check-source-text.py @@ -2,8 +2,8 @@ """Pre-commit guard: keep source text plain and citation-free. Comments should read on their own terms and stay ASCII. Rejected anywhere in a -scanned file: ADR references (`ADR-012`, `ADR 12`) and issue references -(`GVM-281`) -- they renumber and move, and that prose belongs in docs/, not a +scanned file: architecture-decision and issue-ticket references -- they +renumber and move, and that prose belongs in docs/, not a source comment; and a blacklist of non-ASCII punctuation that each has an obvious ASCII swap: the paragraph sign, em-dash (`--`), box-drawing horizontal (`-`), rightwards arrow (`->`) and ellipsis (`...`). It is a blacklist, not a diff --git a/support/scripts/get-all-git.py b/support/scripts/get-all-git.py index 89104875..c43a7b7d 100755 --- a/support/scripts/get-all-git.py +++ b/support/scripts/get-all-git.py @@ -1,12 +1,15 @@ #!/usr/bin/env python3 import argparse +import contextlib +import fcntl import hashlib import os import posixpath import re import shlex import subprocess +from collections.abc import Iterator from dataclasses import dataclass from pathlib import Path from urllib.parse import urlparse, urlunparse @@ -146,6 +149,12 @@ def get_all_repo_git_info(root: Path) -> list[RepoGitInfo]: return result +def git_common_dir(root: Path) -> Path: + return Path( + git_output(['git', 'rev-parse', '--path-format=absolute', '--git-common-dir'], root) + ) + + def cache_path_for_url(root: Path, url: str) -> Path: parsed = urlparse(url) url_path = parsed.path if parsed.scheme else url.rsplit(':', 1)[-1] @@ -153,7 +162,27 @@ def cache_path_for_url(root: Path, url: str) -> Path: name = re.sub(r'\.git$', '', name) name = re.sub(r'[^A-Za-z0-9._-]+', '-', name).strip('-') or 'repo' digest = hashlib.sha256(url.encode()).hexdigest()[:16] - return root / '.git' / 'genvm-submodule-cache' / f'{name}-{digest}.git' + return git_common_dir(root) / 'genvm-submodule-cache' / f'{name}-{digest}.git' + + +@contextlib.contextmanager +def cache_lock(root: Path) -> Iterator[None]: + """ + Serialize the shared submodule cache against another copy of this script. + + CI runs it from several jobs against the same checkout, and both the bare + cache and the worktrees that borrow it with ``--reference`` are mutated in + place: an interleaved init/fetch loses to git's own lockfiles, or leaves a + cache another job is already reading from half-updated. + """ + lock = git_common_dir(root) / 'genvm-submodule-cache.lock' + with open(lock, 'w') as handle: + try: + fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + print(f'+ waiting for {lock}', flush=True) + fcntl.flock(handle, fcntl.LOCK_EX) + yield def ensure_cache(root: Path, url: str, hashes: list[str], env: dict[str, str]) -> Path: @@ -199,6 +228,11 @@ def set_remote(repo: Path, name: str, url: str, env: dict[str, str]) -> None: def update_submodules_with_cache(root: Path, env: dict[str, str]) -> None: + with cache_lock(root): + _update_submodules_with_cache(root, env) + + +def _update_submodules_with_cache(root: Path, env: dict[str, str]) -> None: submodules = get_submodule_git_info(root) by_url: dict[str, list[SubmoduleGitInfo]] = {} for submodule in submodules: diff --git a/support/tools/genvm-tool/default.nix b/support/tools/genvm-tool/default.nix index 4b8c487c..c0cc1d19 100644 --- a/support/tools/genvm-tool/default.nix +++ b/support/tools/genvm-tool/default.nix @@ -23,7 +23,12 @@ python.pkgs.buildPythonApplication { questionary ]; - nativeBuildInputs = [ pkgs.installShellFiles ]; + nativeBuildInputs = [ + pkgs.git + pkgs.installShellFiles + pkgs.pre-commit + python.pkgs.pytestCheckHook + ]; # Generate the shell completions and man page from the freshly built binary # (shtab / argparse-manpage both walk the whole argparse command tree) and drop @@ -37,7 +42,7 @@ python.pkgs.buildPythonApplication { installManPage genvm-tool.1 ''; - doCheck = false; + pytestFlags = [ "unit_tests" ]; meta = { description = "GenVM monorepo git helper (build, tests, hooks, ls)"; diff --git a/support/tools/genvm-tool/genvm_tool/__main__.py b/support/tools/genvm-tool/genvm_tool/__main__.py index aa83b681..9cd56b92 100644 --- a/support/tools/genvm-tool/genvm_tool/__main__.py +++ b/support/tools/genvm-tool/genvm_tool/__main__.py @@ -23,14 +23,14 @@ cmd_build_manifest, cmd_codegen, cmd_configure, - cmd_howto, + cmd_docs, cmd_test, common, formatter, git, ) -TOPLEVEL = [cmd_configure, cmd_test, cmd_build_manifest, cmd_codegen, cmd_howto] +TOPLEVEL = [cmd_configure, cmd_test, cmd_build_manifest, cmd_codegen, cmd_docs] GROUPS = [git] # One-line tagline for the man page NAME line and completion header; the fuller @@ -43,9 +43,9 @@ Commands are either top-level leaves (configure, test, ...) or a group with its own leaves (git ls, git check-for-push, ...). Pass -h to any of them for the -per-command details, or run `genvm-tool howto` for the contributor guides.""" +per-command details, or run `genvm-tool docs` for the contributor documentation.""" EPILOG = ( - 'Guides live in docs/contributing/howto/ (rendered by `genvm-tool howto`). ' + 'Guides live in docs/contributing/ (rendered by `genvm-tool docs`). ' 'Every subcommand accepts -h/--help.' ) diff --git a/support/tools/genvm-tool/genvm_tool/cmd_configure.py b/support/tools/genvm-tool/genvm_tool/cmd_configure.py index b6814b48..41fef014 100644 --- a/support/tools/genvm-tool/genvm_tool/cmd_configure.py +++ b/support/tools/genvm-tool/genvm_tool/cmd_configure.py @@ -76,6 +76,29 @@ def build_independent_info(monorepo_cfg, source_dir: Path) -> dict: } +def base_info( + monorepo_cfg, source_dir: Path, build_dir: Path, rust_target_dir: Path +) -> dict: + """Every ``info.json`` key that does not depend on having built anything. + + Two commands write ``info.json``: ``configure``, and the test harness when it + finds none (so CI can run tests without configuring first, see the + manager-root ``.genvm-tool.py``). ``configure`` writes this plus whatever the + build teaches it; the harness writes exactly this. They share the function so + the two files cannot describe the same tree differently. + """ + return { + 'coverage_dir': str(build_dir / 'cov'), + 'build_dir': str(build_dir), + 'rust_target_dir': str(rust_target_dir), + # The build passes `--target` explicitly; the test harness must pass the + # same one or it gets a second, unshared unit graph in the same dir. + 'rust_target': detect_rust_target(), + **build_independent_info(monorepo_cfg, source_dir), + **rust_target_dirs_info(monorepo_cfg, source_dir, rust_target_dir), + } + + def rust_target_dirs_info( monorepo_cfg, source_dir: Path, rust_target_dir: Path ) -> dict: @@ -531,19 +554,7 @@ def _runner_file(f: Path) -> bool: (build_dir / 'cov').mkdir(parents=True, exist_ok=True) (build_dir / 'build.ninja').write_text(''.join(n.buf)) - info = { - 'coverage_dir': str(build_dir / 'cov'), - 'build_dir': str(build_dir), - 'rust_target_dir': str(rust_target_dir), - # The build passes `--target` explicitly; the test harness must pass the - # same one or it gets a second, unshared unit graph in the same dir. - 'rust_target': rust_target, - # executor_versions / primary_executor_version β€” derivable without a build; - # the test harness synthesizes the same keys from source when info.json is - # absent (see build_independent_info). - **build_independent_info(monorepo_cfg, source_dir), - **rust_target_dirs_info(monorepo_cfg, source_dir, rust_target_dir), - } + info = base_info(monorepo_cfg, source_dir, build_dir, rust_target_dir) (build_dir / 'info.json').write_text(json.dumps(info, indent=2)) # The manager reads out/data/manifest.yaml at runtime; assemble it from the diff --git a/support/tools/genvm-tool/genvm_tool/cmd_docs.py b/support/tools/genvm-tool/genvm_tool/cmd_docs.py new file mode 100644 index 00000000..1cd05782 --- /dev/null +++ b/support/tools/genvm-tool/genvm_tool/cmd_docs.py @@ -0,0 +1,114 @@ +"""`genvm-tool docs` β€” list the contributor documentation. + +Walks each quadrant of `docs/contributing/` (tutorial, how-to, explanation), reads +every page's one-line description from that quadrant's `README.md` index, and prints +a grouped markdown listing. Exits non-zero if a page is missing from its index, or an +index lists a page that no longer exists, so CI can gate on the indexes staying in +sync. +""" + +import re +import sys +from pathlib import Path + +from . import common + +NAME = 'docs' +HELP = 'render the contributor documentation as a skill-like index' + +CONTRIBUTING_DIR = Path('docs/contributing') +# Rendered in this order: learn it, do it, understand it. +QUADRANTS = ['tutorial', 'howto', 'explanation'] +# Deepest directory nesting promoted to a `## header`; anything below stays in the +# page's link text instead of spawning ever-deeper sections. +MAX_HEADER_DEPTH = 3 +# README index lines look like: `- [text](rel/target.md) β€” one-line description.` +# `rel/target.md` is relative to the quadrant dir; the em dash (β€”) is the separator. +_INDEX_LINE = re.compile(r'-\s*\[[^\]]*\]\(([^)]+)\)\s*β€”\s*(.+)$') + + +def configure(parser): + pass + + +def _parse_index(readme: Path) -> dict[str, str]: + """Map each README-listed target (quadrant-relative) to its one-line description.""" + descriptions: dict[str, str] = {} + for line in readme.read_text().splitlines(): + m = _INDEX_LINE.match(line.strip()) + if m: + descriptions[m.group(1)] = m.group(2).strip().rstrip('.') + return descriptions + + +def _h1_title(path: Path) -> str: + """First `# heading` in a markdown file, or '' if it has none.""" + for line in path.read_text().splitlines(): + s = line.strip() + if s.startswith('# '): + return s[2:].strip() + return '' + + +def _rows(ctx: common.Context, base: Path) -> tuple[list, list[str]]: + """Listing rows for one quadrant, plus its index-vs-disk complaints.""" + index = base / 'README.md' + if not index.is_file(): + raise common.ToolError(f'not found: {index.relative_to(ctx.root).as_posix()}') + descriptions = _parse_index(index) + + # The quadrant README *is* the index being rendered, not a page within it. + pages = [p for p in base.rglob('*') if p.is_file() and p != index] + on_disk = {p.relative_to(base).as_posix() for p in pages} + + rows = [] + for path in pages: + parts = path.relative_to(base).parts + cut = min(len(parts) - 1, MAX_HEADER_DEPTH) # dir segments that become a header + header = '/'.join(parts[:cut]) + link = '/'.join(parts[cut:]) + desc = descriptions.get(path.relative_to(base).as_posix()) or _h1_title(path) + rows.append((header, link, path.relative_to(ctx.root).as_posix(), desc)) + + index_rel = index.relative_to(ctx.root).as_posix() + problems = [ + f'{index_rel}: missing entry for {rel}' + for rel in sorted(on_disk - descriptions.keys()) + ] + [ + f'{index_rel}: lists {rel}, which does not exist' + for rel in sorted(descriptions.keys() - on_disk) + ] + return rows, problems + + +def _section_title(quadrant: str, header: str) -> str: + """`howto`, `building` -> `Howto/Building`; the empty header is the quadrant root.""" + parts = [quadrant] + ([header] if header else []) + return '/'.join(f'{p[:1].upper()}{p[1:]}' for p in '/'.join(parts).split('/')) + + +def main(ctx: common.Context, args) -> int: + problems = [] + started = False + for quadrant in QUADRANTS: + base = ctx.root / CONTRIBUTING_DIR / quadrant + if not base.is_dir(): + raise common.ToolError(f'not found: {CONTRIBUTING_DIR / quadrant}') + + rows, quadrant_problems = _rows(ctx, base) + problems += quadrant_problems + + # '' (the quadrant root) sorts before any header; within a section, by link text. + current = None + for header, link, repo_path, desc in sorted(rows): + if header != current: + current = header + if started: + print() # blank line between sections, but not a leading one + print(f'## {_section_title(quadrant, header)}') + print(f'- [{link}]({repo_path}) β€” {desc}') + started = True + + for problem in problems: + print(problem, file=sys.stderr) + return 1 if problems else 0 diff --git a/support/tools/genvm-tool/genvm_tool/cmd_howto.py b/support/tools/genvm-tool/genvm_tool/cmd_howto.py deleted file mode 100644 index 3d2a0fe0..00000000 --- a/support/tools/genvm-tool/genvm_tool/cmd_howto.py +++ /dev/null @@ -1,94 +0,0 @@ -"""`genvm-tool howto` β€” list the contributing how-to guides. - -Walks `docs/contributing/howto/`, reads each guide's one-line description from the -`README.md` index, and prints a grouped markdown listing of every guide. Exits -non-zero if a guide is missing from the index, or the index lists a guide that no -longer exists, so CI can gate on the index staying in sync. -""" - -import re -import sys -from pathlib import Path - -from . import common - -NAME = 'howto' -HELP = 'render the contributing how-to guides as a skill-like index' - -HOWTO_DIR = Path('docs/contributing/howto') -# Deepest directory nesting promoted to a `## header`; anything below stays in the -# guide's link text instead of spawning ever-deeper sections. -MAX_HEADER_DEPTH = 3 -# README index lines look like: `- [text](rel/target.md) β€” one-line description.` -# `rel/target.md` is relative to the howto dir; the em dash (β€”) is the separator. -_INDEX_LINE = re.compile(r'-\s*\[[^\]]*\]\(([^)]+)\)\s*β€”\s*(.+)$') - - -def configure(parser): - pass - - -def _parse_index(readme: Path) -> dict[str, str]: - """Map each README-listed target (howto-relative) to its one-line description.""" - descriptions: dict[str, str] = {} - for line in readme.read_text().splitlines(): - m = _INDEX_LINE.match(line.strip()) - if m: - descriptions[m.group(1)] = m.group(2).strip().rstrip('.') - return descriptions - - -def _h1_title(path: Path) -> str: - """First `# heading` in a markdown file, or '' if it has none.""" - for line in path.read_text().splitlines(): - s = line.strip() - if s.startswith('# '): - return s[2:].strip() - return '' - - -def main(ctx: common.Context, args) -> int: - base = ctx.root / HOWTO_DIR - if not base.is_dir(): - raise common.ToolError(f'not found: {HOWTO_DIR}') - - index = base / 'README.md' - descriptions = _parse_index(index) - - # The top-level README *is* the index we render, not a guide within it. - guides = [p for p in base.rglob('*') if p.is_file() and p != index] - on_disk = {p.relative_to(base).as_posix() for p in guides} - - rows = [] - for path in guides: - parts = path.relative_to(base).parts - cut = min(len(parts) - 1, MAX_HEADER_DEPTH) # dir segments that become a header - header = '/'.join(parts[:cut]) - link = '/'.join(parts[cut:]) - desc = descriptions.get(path.relative_to(base).as_posix()) or _h1_title(path) - rows.append((header, link, path.relative_to(ctx.root).as_posix(), desc)) - - # '' (root) sorts before any header; within a section, by link text. - current = None - started = False - for header, link, repo_path, desc in sorted(rows): - if header != current: - current = header - if header: - if started: - print() # blank line between sections, but not a leading one - print(f'## {header[:1].upper()}{header[1:]}') - print(f'- [{link}]({repo_path}) β€” {desc}') - started = True - - index_rel = index.relative_to(ctx.root).as_posix() - problems = [ - f'{index_rel}: missing entry for {rel}' - for rel in sorted(on_disk - descriptions.keys()) - ] + [ - f'{index_rel}: lists {rel}, which does not exist' - for rel in sorted(descriptions.keys() - on_disk) - ] - for problem in problems: - print(problem, file=sys.stderr) - return 1 if problems else 0 diff --git a/support/tools/genvm-tool/genvm_tool/codegen/model.py b/support/tools/genvm-tool/genvm_tool/codegen/model.py index 6a5306ba..a7912d99 100644 --- a/support/tools/genvm-tool/genvm_tool/codegen/model.py +++ b/support/tools/genvm-tool/genvm_tool/codegen/model.py @@ -75,6 +75,7 @@ class StrTrie: name: str root: TrieNode entries: list # unfolded entries, kept for flat path enumeration (rst) + docs: dict[str, str] Definition = Enum | Const | Consts | StrTrie @@ -178,7 +179,14 @@ def parse(data) -> list[Definition]: defs.append(Consts(t['name'], t['repr'], t['values'])) elif kind == 'str_trie': entries = [_unfold(e) for e in t['values']] - defs.append(StrTrie(t['name'], _build(entries, [], '', False), entries)) + defs.append( + StrTrie( + t['name'], + _build(entries, [], '', False), + entries, + t.get('docs', {}), + ) + ) else: raise ValueError(f'unknown codegen type {kind!r}') return defs diff --git a/support/tools/genvm-tool/genvm_tool/codegen/rst.py b/support/tools/genvm-tool/genvm_tool/codegen/rst.py index 6fd01972..e21e48a2 100644 --- a/support/tools/genvm-tool/genvm_tool/codegen/rst.py +++ b/support/tools/genvm-tool/genvm_tool/codegen/rst.py @@ -64,6 +64,8 @@ def render(defs: list[Definition], rst_anchor_ns: str = '', **_opts) -> str: buf.append('~' * (len(path) + 4) + '\n\n') if param is not None: buf.append(f'Param: {param}\n\n') + if path in d.docs: + buf.append(d.docs[path].rstrip() + '\n\n') # Sections are emitted blank-line-separated, so the last one leaves a trailing # blank line that `end-of-file-fixer` would strip β€” permanent codegen drift. return ''.join(buf).rstrip('\n') + '\n' diff --git a/support/tools/genvm-tool/genvm_tool/codegen/rust.py b/support/tools/genvm-tool/genvm_tool/codegen/rust.py index dd508a96..68e52f03 100644 --- a/support/tools/genvm-tool/genvm_tool/codegen/rust.py +++ b/support/tools/genvm-tool/genvm_tool/codegen/rust.py @@ -149,6 +149,15 @@ def _emit_param(node: TrieNode, root_name: str, buf: list[str]) -> None: f' pub fn val_{param_type}(&self, v: {_rust_param_type(param_type)}) ' f'-> {root_name} {{\n' ) + if param_type == 'str': + # `is_valid_` rejects an empty tail, so an empty argument would build a + # code the validator refuses. Callers pass a fixed description, never + # user input, which makes an empty one a bug in the caller. + # The message is a `format!` string, so braces in the path have to be + # doubled on top of the usual string-literal escaping. + msg = json.dumps(f'{" ".join(parts)} needs a non-empty description') + msg = msg.replace('{', '{{').replace('}', '}}') + buf.append(f' debug_assert!(!v.is_empty(), {msg});\n') buf.append(f' {root_name}(Cow::Owned(format!("{fmt_str}")))\n') buf.append(' }\n') buf.append('}\n\n') diff --git a/support/tools/genvm-tool/genvm_tool/common.py b/support/tools/genvm-tool/genvm_tool/common.py index 900c3bfe..35c1b1c7 100644 --- a/support/tools/genvm-tool/genvm_tool/common.py +++ b/support/tools/genvm-tool/genvm_tool/common.py @@ -19,6 +19,28 @@ # Per-repo project file (manager + every executor). Exec'd once before any # subcommand runs; on the manager it also provides `tests(ctx)` (the test suite). PROJECT_FILE = '.genvm-tool.py' +GIT_REMOTE_TIMEOUT = 20 +GIT_NETWORK_ENV = {'GIT_TERMINAL_PROMPT': '0'} +# Git exports these to hooks to describe the repository the hook fires for, and +# they outrank `-C`. Left in place, every query below would silently answer for +# the manager instead of the submodule it names. +GIT_SCOPE_ENV = ( + 'GIT_DIR', + 'GIT_COMMON_DIR', + 'GIT_WORK_TREE', + 'GIT_INDEX_FILE', + 'GIT_OBJECT_DIRECTORY', + 'GIT_ALTERNATE_OBJECT_DIRECTORIES', + 'GIT_NAMESPACE', + 'GIT_PREFIX', +) + + +def git_env(**extra: str) -> dict[str, str]: + """The ambient environment with git's repo-scoping variables removed.""" + env = {k: v for k, v in os.environ.items() if k not in GIT_SCOPE_ENV} + env.update(extra) + return env @functools.lru_cache(maxsize=None) @@ -94,10 +116,20 @@ def monorepo_config(root: Path) -> dict: def active_versions(root: Path) -> list[str]: """Active executor trains from .genvm-monorepo-root, ascending, bare.""" - vers = (_bare(v) for v in monorepo_config(root).get('active-versions', [])) + return active_versions_from_config(monorepo_config(root)) + + +def active_versions_from_config(config: dict) -> list[str]: + """Active executor trains from parsed monorepo config, ascending, bare.""" + vers = (_bare(v) for v in config.get('active-versions', [])) return sorted(vers, key=_ver_key) +def executor_rel(version: str) -> str: + """Manager-relative checkout/gitlink path for an executor version line.""" + return f'executors/v{_bare(version)}.x' + + # Release/branch-model branches (`main`, a version branch `v0.3.x`, or a dev # branch `v0.3-dev`) β€” already line-scoped, so they are never feature-namespaced. _MODEL_BRANCH = re.compile(r'main|v[\d.]+\.x|v[\d.]+-dev') @@ -144,10 +176,28 @@ def feature_branch(self, name: str) -> str: return f'pr/{line}/{name}' +def git(repo: Repo | Path, *args: str, **kwargs) -> subprocess.CompletedProcess: + """Run Git in a repo, capturing text output unless the caller overrides it.""" + path = repo.path if isinstance(repo, Repo) else repo + kwargs.setdefault('env', git_env()) + return subprocess.run( + ['git', '-C', str(path), *args], + capture_output=True, + text=True, + **kwargs, + ) + + +def remote_url(repo: Repo | Path, remote: str = 'origin') -> str: + """Configured URL for ``remote`` or an empty string when it is absent.""" + result = git(repo, 'config', '--get', f'remote.{remote}.url') + return result.stdout.strip() if result.returncode == 0 else '' + + def discover_repos(root: Path, which: str = 'all') -> list[Repo]: repos = [Repo('manager', root, '')] for v in active_versions(root): - rel = f'executors/v{v}.x' + rel = executor_rel(v) repos.append(Repo(rel, root / rel, rel)) if which == 'all': return repos @@ -160,7 +210,7 @@ def discover_repos(root: Path, which: str = 'all') -> list[Repo]: def executor_rels(root: Path) -> set[str]: - return {f'executors/v{v}.x' for v in active_versions(root)} + return {executor_rel(v) for v in active_versions(root)} # --- git ------------------------------------------------------------------- @@ -172,6 +222,7 @@ def _git_z(repo: Repo, *args: str) -> list[str]: capture_output=True, text=True, check=True, + env=git_env(), ).stdout return [f for f in out.split('\0') if f] diff --git a/support/tools/genvm-tool/genvm_tool/git/__init__.py b/support/tools/genvm-tool/genvm_tool/git/__init__.py index ff6c3dda..ca836116 100644 --- a/support/tools/genvm-tool/genvm_tool/git/__init__.py +++ b/support/tools/genvm-tool/genvm_tool/git/__init__.py @@ -1,7 +1,19 @@ """`git` command group: git helpers spanning the manager + executor submodules.""" -from . import check_for_push, create_branches, list_repo, ls +from . import ( + check_for_push, + create_branches, + list_repo, + ls, + verify_pushed_executors, +) NAME = 'git' HELP = 'git helpers across the manager and executor submodules' -COMMANDS = [ls, list_repo, create_branches, check_for_push] +COMMANDS = [ + ls, + list_repo, + create_branches, + check_for_push, + verify_pushed_executors, +] diff --git a/support/tools/genvm-tool/genvm_tool/git/check_for_push.py b/support/tools/genvm-tool/genvm_tool/git/check_for_push.py index f24ce0d5..53695a9d 100644 --- a/support/tools/genvm-tool/genvm_tool/git/check_for_push.py +++ b/support/tools/genvm-tool/genvm_tool/git/check_for_push.py @@ -12,7 +12,6 @@ fetches, resets, switches, or writes anything. """ -import os import shlex import subprocess @@ -22,10 +21,6 @@ HELP = 'read-only pre-push status + suggested action across manager and submodules' REMOTE = 'origin' -# `ls-remote` is a network round-trip; cap it so an unreachable remote can't hang -# the survey, and never let git pop an interactive auth prompt. -_LS_REMOTE_TIMEOUT = 20 -_NET_ENV = {'GIT_TERMINAL_PROMPT': '0'} def configure(parser): @@ -43,11 +38,7 @@ def configure(parser): def _git(repo: common.Repo, *args: str) -> subprocess.CompletedProcess: - return subprocess.run( - ['git', '-C', str(repo.path), *args], - capture_output=True, - text=True, - ) + return common.git(repo, *args) def _current_branch(repo: common.Repo) -> str: @@ -56,8 +47,7 @@ def _current_branch(repo: common.Repo) -> str: def _origin_url(repo: common.Repo) -> str: - r = _git(repo, 'config', '--get', f'remote.{REMOTE}.url') - return r.stdout.strip() if r.returncode == 0 else '' + return common.remote_url(repo, REMOTE) def _worktree_state( @@ -114,7 +104,7 @@ def _launch_origin_queries(targets: list[tuple[common.Repo, str]]): g['branches'].add(branch) g['members'].append((repo, branch)) - env = {**os.environ, **_NET_ENV} + env = common.git_env(**common.GIT_NETWORK_ENV) procs = [] for g in groups.values(): refs = [f'refs/heads/{b}' for b in sorted(g['branches'])] @@ -136,7 +126,7 @@ def _collect_origin_queries(launched) -> dict[str, tuple[str | None, bool]]: procs, result = launched for g, proc in procs: try: - out, _ = proc.communicate(timeout=_LS_REMOTE_TIMEOUT) + out, _ = proc.communicate(timeout=common.GIT_REMOTE_TIMEOUT) ok = proc.returncode == 0 except subprocess.TimeoutExpired: proc.kill() diff --git a/support/tools/genvm-tool/genvm_tool/git/create_branches.py b/support/tools/genvm-tool/genvm_tool/git/create_branches.py index 311a48cd..ba1c0f63 100644 --- a/support/tools/genvm-tool/genvm_tool/git/create_branches.py +++ b/support/tools/genvm-tool/genvm_tool/git/create_branches.py @@ -31,11 +31,7 @@ def configure(parser): def _git(repo: common.Repo, *args: str) -> subprocess.CompletedProcess: - return subprocess.run( - ['git', '-C', str(repo.path), *args], - capture_output=True, - text=True, - ) + return common.git(repo, *args) def _new_content(repo: common.Repo) -> str | None: diff --git a/support/tools/genvm-tool/genvm_tool/git/verify_pushed_executors.py b/support/tools/genvm-tool/genvm_tool/git/verify_pushed_executors.py new file mode 100644 index 00000000..ba689a79 --- /dev/null +++ b/support/tools/genvm-tool/genvm_tool/git/verify_pushed_executors.py @@ -0,0 +1,235 @@ +"""Verify that an outgoing manager commit pins published executor commits. + +The manager ref defaults to ``PRE_COMMIT_TO_REF`` in a pre-push hook and to +``HEAD`` for manual runs. The active executor set and gitlinks are read from +that commit, never from the working tree. Remote queries are read-only, +non-interactive, and bounded to 20 seconds. + +Publication is a question about a commit, not about a ref: a gitlink counts as +published when the remote holds that object, whatever branch carries it and +whether or not it is a branch tip. ``fetch --negotiate-only`` asks exactly +that -- it reports which of our commits the remote already has and transfers +nothing -- so a stale, shallow or single-branch local clone cannot skew the +answer. Listing refs cannot: ``ls-remote`` matches ref names, so it can only +recognize a gitlink that happens to be a tip right now. + +Only a proven unpublished gitlink is rejected. Missing local state, unreachable +remotes and timeouts warn but allow the push. +""" + +import json +import os +import subprocess +from dataclasses import dataclass +from pathlib import Path + +from .. import common + +NAME = 'verify-pushed-executors' +HELP = 'reject manager refs that pin executor commits absent from their origin' + + +@dataclass(frozen=True) +class _Target: + rel: str + path: Path + sha: str + + +def configure(parser): + parser.add_argument( + '--manager-ref', + help='manager commit to verify (default: PRE_COMMIT_TO_REF, then HEAD)', + ) + parser.add_argument( + '--executor-remote', + default='origin', + help='executor remote name (default: origin)', + ) + + +def _git(root: Path, *args: str) -> subprocess.CompletedProcess: + return common.git(root, *args) + + +def _failure(ctx: common.Context, rel: str, sha: str, problem: str) -> None: + ctx.printer.put( + 'verify-pushed-executors', + executor=rel, + gitlink=sha[:12] if sha else 'unknown', + status='REJECTED', + problem=problem, + ) + + +def _warning(ctx: common.Context, rel: str, sha: str, problem: str) -> None: + ctx.printer.put( + 'verify-pushed-executors', + executor=rel, + gitlink=sha[:12] if sha else 'unknown', + status='WARNING (push allowed)', + problem=problem, + ) + + +def _recovery(ctx: common.Context) -> None: + ctx.printer.put( + 'recovery', + command=( + 'genvm-tool git check-for-push\n' + 'git -C executors/<line>.x push origin <executor-branch>\n' + 'git push origin <manager-branch>' + ), + ) + + +def _resolve_manager_commit(root: Path, manager_ref: str) -> tuple[str, str | None]: + result = _git( + root, + 'rev-parse', + '--verify', + '--end-of-options', + f'{manager_ref}^{{commit}}', + ) + if result.returncode: + return '', f'cannot resolve manager ref {manager_ref!r} to a commit' + return result.stdout.strip(), None + + +def _targets_from_commit( + root: Path, commit: str, remote: str +) -> tuple[ + list[_Target], + list[tuple[str, str, str]], + list[tuple[str, str, str]], +]: + failures: list[tuple[str, str, str]] = [] + warnings: list[tuple[str, str, str]] = [] + shown = _git(root, 'show', f'{commit}:{common.MONOREPO_MARKER}') + if shown.returncode: + failures.append( + ( + common.MONOREPO_MARKER, + '', + f'cannot read {common.MONOREPO_MARKER} from outgoing manager commit', + ) + ) + return [], failures, warnings + try: + config = json.loads(shown.stdout) + versions = common.active_versions_from_config(config) + except (AttributeError, json.JSONDecodeError, TypeError, ValueError): + failures.append( + ( + common.MONOREPO_MARKER, + '', + f'malformed {common.MONOREPO_MARKER} in outgoing manager commit', + ) + ) + return [], failures, warnings + + targets = [] + for version in versions: + rel = common.executor_rel(version) + entry = _git(root, 'ls-tree', commit, '--', rel) + fields = entry.stdout.strip().split(maxsplit=3) + if ( + entry.returncode + or len(fields) != 4 + or fields[0] != '160000' + or fields[1] != 'commit' + or fields[3] != rel + ): + failures.append((rel, '', 'missing or malformed gitlink tree entry')) + continue + sha = fields[2] + path = root / rel + if not (path / '.git').exists(): + warnings.append((rel, sha, 'executor checkout is missing; not verified')) + continue + if not common.remote_url(path, remote): + warnings.append( + (rel, sha, f'executor remote {remote!r} is missing; not verified') + ) + continue + targets.append(_Target(rel, path, sha)) + return targets, failures, warnings + + +def _published(target: _Target, remote: str) -> tuple[bool | None, str]: + """Whether ``remote`` already holds the gitlink commit. + + ``--negotiate-only`` runs the fetch negotiation and stops before any + transfer, printing the commits both sides have. Our own sha comes back only + if the remote has that object, which is the property we care about; no ref + has to point at it and no part of the remote history has to be local. + """ + if _git(target.path, 'cat-file', '-e', f'{target.sha}^{{commit}}').returncode: + return None, 'gitlink is absent from this executor checkout; not verified' + try: + result = subprocess.run( + [ + 'git', + '-C', + str(target.path), + '-c', + 'protocol.version=2', + 'fetch', + '--negotiate-only', + f'--negotiation-tip={target.sha}', + remote, + ], + capture_output=True, + text=True, + timeout=common.GIT_REMOTE_TIMEOUT, + env=common.git_env(**common.GIT_NETWORK_ENV), + ) + except subprocess.TimeoutExpired: + return None, f'executor remote {remote!r} timed out' + if result.returncode: + if 'negotiate-only' in result.stderr: + return None, 'git is too old for --negotiate-only; not verified' + return None, f'executor remote {remote!r} is unreachable' + if target.sha in result.stdout.split(): + return True, '' + return False, f'gitlink is absent from executor remote {remote!r}' + + +def main(ctx: common.Context, args) -> int: + manager_ref = args.manager_ref or os.environ.get('PRE_COMMIT_TO_REF') or 'HEAD' + commit, problem = _resolve_manager_commit(ctx.root, manager_ref) + if problem: + _failure(ctx, 'manager', '', problem) + _recovery(ctx) + return 1 + + targets, failures, warnings = _targets_from_commit( + ctx.root, commit, args.executor_remote + ) + # Asked per executor, not per remote URL: the lines share one repository but + # not one object store, so each question must be put from the clone that + # holds the commit being asked about. + for target in targets: + ok, problem = _published(target, args.executor_remote) + if ok is False: + failures.append((target.rel, target.sha, problem)) + elif ok is None: + warnings.append((target.rel, target.sha, problem)) + + for rel, sha, warning in warnings: + _warning(ctx, rel, sha, warning) + + if failures: + for rel, sha, failure in failures: + _failure(ctx, rel, sha, failure) + _recovery(ctx) + return 1 + + ctx.printer.put( + 'verify-pushed-executors', + manager_ref=manager_ref, + manager_commit=commit[:12], + status='ok-with-warnings' if warnings else 'ok', + warnings=len(warnings), + ) + return 0 diff --git a/support/tools/genvm-tool/genvm_tool/tests/__init__.py b/support/tools/genvm-tool/genvm_tool/tests/__init__.py index 5709aaf4..12545693 100644 --- a/support/tools/genvm-tool/genvm_tool/tests/__init__.py +++ b/support/tools/genvm-tool/genvm_tool/tests/__init__.py @@ -20,7 +20,7 @@ from pathlib import Path from typing import Any -from genvm_tool import formatter +from genvm_tool import common, formatter from genvm_tool.formatter import Formatter, Sink from .util.watchdog import Watchdog @@ -120,6 +120,7 @@ def git_files(self) -> list[Path]: check=True, capture_output=True, text=True, + env=common.git_env(), ) r = [ diff --git a/support/tools/genvm-tool/genvm_tool/tests/util/watchdog.py b/support/tools/genvm-tool/genvm_tool/tests/util/watchdog.py index 1a6a0744..a6796753 100644 --- a/support/tools/genvm-tool/genvm_tool/tests/util/watchdog.py +++ b/support/tools/genvm-tool/genvm_tool/tests/util/watchdog.py @@ -16,12 +16,20 @@ class Watchdog: exits normally (:meth:`stop`), and runs them too if this process dies unexpectedly (detected via reparenting). That way a container is always torn down through the watchdog and nothing is left dangling at the end of a run. + + Cleanups are also kept here, and are run in-process whenever the watchdog is + not available to run them (it stopped responding, or died before exiting + cleanly). Otherwise a broken watchdog would silently leak every container. """ def __init__(self, process: subprocess.Popen): self._process = process self._lock = threading.Lock() self._next_token = 0 + self._cleanups: dict[int, typing.Callable[[], typing.Any]] = {} + # Cleared as soon as the watchdog misses anything we asked of it: from + # then on its exit status says nothing about what it actually ran. + self._healthy = True @staticmethod def start() -> 'Watchdog': @@ -31,33 +39,40 @@ def start() -> 'Watchdog': stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + # Own session: Ctrl+C (and any other signal aimed at our process + # group) must not reach the watchdog, it has cleanups to run. + start_new_session=True, ) return Watchdog(process) - def _send(self, msg: dict) -> None: - assert self._process.stdin is not None + def _send(self, msg: dict) -> bool: + """Send one frame; ``False`` if the watchdog is gone.""" + if self._process.stdin is None or self._process.poll() is not None: + self._healthy = False + return False payload = cloudpickle.dumps(msg) - self._process.stdin.write(len(payload).to_bytes(4, 'big') + payload) - self._process.stdin.flush() + try: + self._process.stdin.write(len(payload).to_bytes(4, 'big') + payload) + self._process.stdin.flush() + except (OSError, ValueError): + self._healthy = False + return False + return True def register(self, cleanup: typing.Callable[[], typing.Any]) -> int: """Register a cleanup callable; return a token to :meth:`kill` it later.""" with self._lock: token = self._next_token self._next_token += 1 - try: - self._send({'action': 'add', 'token': token, 'cleanup': cleanup}) - except (BrokenPipeError, OSError): - pass + self._cleanups[token] = cleanup + self._send({'action': 'add', 'token': token, 'cleanup': cleanup}) return token def unregister(self, token: int) -> None: """Forget a cleanup without running it.""" with self._lock: - try: - self._send({'action': 'remove', 'token': token}) - except (BrokenPipeError, OSError): - pass + self._cleanups.pop(token, None) + self._send({'action': 'remove', 'token': token}) def kill(self, token: int) -> None: """Run a registered cleanup now (via the watchdog) and forget it. @@ -67,24 +82,42 @@ def kill(self, token: int) -> None: call from an event loop via ``asyncio.to_thread``. """ with self._lock: - if self._process.poll() is not None: - return - try: - self._send({'action': 'kill', 'token': token}) - except (BrokenPipeError, OSError): + cleanup = self._cleanups.pop(token, None) + if self._send({'action': 'kill', 'token': token}) and self._await_ack(token): return - assert self._process.stdout is not None + # The watchdog never acknowledged the request: it is dead, or died + # while running the cleanup. Run it here instead - doing it twice is + # harmless, not doing it at all leaks the resource. + self._healthy = False + if cleanup is not None: + _run_cleanup(cleanup) + + def _await_ack(self, token: int) -> bool: + """Wait for this kill's ack, skipping stale ones. ``False`` on EOF.""" + assert self._process.stdout is not None + while True: try: - # One ack line per kill; requests are serialized by the lock. - self._process.stdout.readline() + line = self._process.stdout.readline() except (OSError, ValueError): - pass + return False + if not line: + return False + try: + if int(line) == token: + return True + except ValueError: + return False def stop(self) -> None: """Stop the watchdog, running any cleanups still registered. - Closing stdin signals a normal exit: the subprocess tears down every - dangling cleanup before exiting, so we wait for it to finish. + Closing stdin signals the subprocess to exit: it tears down every dangling + cleanup first, so we wait for it to finish. It exits 0 only after they all + ran, so any other status means we have to run them. + + Deliberately not under ``_lock``: a :meth:`kill` may be blocked waiting + for an ack, and closing stdin is what unwedges it (the watchdog acks a + pending kill before it ever sees the EOF). """ if self._process.stdin: try: @@ -92,7 +125,21 @@ def stop(self) -> None: except OSError: pass try: - self._process.wait(timeout=60) + code = self._process.wait(timeout=60) except subprocess.TimeoutExpired: self._process.kill() - self._process.wait() + code = self._process.wait() + with self._lock: + cleanups, self._cleanups = self._cleanups, {} + healthy = self._healthy + self._healthy = False + if code != 0 or not healthy: + for cleanup in cleanups.values(): + _run_cleanup(cleanup) + + +def _run_cleanup(cleanup: typing.Callable[[], typing.Any]) -> None: + try: + cleanup() + except Exception: + pass diff --git a/support/tools/genvm-tool/genvm_tool/tests/util/watchdog_main.py b/support/tools/genvm-tool/genvm_tool/tests/util/watchdog_main.py index 1b392c93..205443a7 100644 --- a/support/tools/genvm-tool/genvm_tool/tests/util/watchdog_main.py +++ b/support/tools/genvm-tool/genvm_tool/tests/util/watchdog_main.py @@ -2,15 +2,16 @@ """ Watchdog process for genvm-tool test. -Started by the test runner as a child process. It owns the cleanups (e.g. -``docker stop``) registered by the runner and is the single place that runs -them, so a container is always torn down through the watchdog: +Started by the test runner as a child process, in its own session and with +terminating signals ignored, so that Ctrl+C does not take it down before it had +a chance to clean up. It owns the cleanups (e.g. ``docker stop``) registered by +the runner and is the single place that runs them, so a container is always torn +down through the watchdog: - on a ``kill`` request: runs that one cleanup now and acknowledges it; -- on normal exit (stdin closed): runs every still-registered cleanup, killing - whatever the runner left dangling; -- on unexpected death (detected via reparenting): kills the original process - group, then runs every still-registered cleanup. +- when the runner is gone - stdin closed, or reparenting for the cases where no + EOF ever comes - runs every still-registered cleanup, whether the runner + exited normally or died on us. Communication protocol (length-prefixed cloudpickle frames on stdin, a 4-byte big-endian length followed by the pickled message dict): @@ -35,73 +36,75 @@ def _run_cleanup(cleanup) -> None: pass +def _read(fd: int, timeout: float) -> bytes | None: + """Pending stdin bytes, ``b''`` if none within ``timeout``, ``None`` on EOF.""" + try: + ready, _, _ = select.select([fd], [], [], timeout) + if not ready: + return b'' + return os.read(fd, 65536) or None + except (ValueError, OSError): + return None + + +def _consume(buffer: bytes, cleanups: dict[int, object]) -> bytes: + """Handle every complete frame in ``buffer``; return the incomplete tail.""" + while len(buffer) >= 4: + length = int.from_bytes(buffer[:4], 'big') + if len(buffer) < 4 + length: + break + frame, buffer = buffer[4 : 4 + length], buffer[4 + length :] + try: + msg = cloudpickle.loads(frame) + action = msg.get('action') + token = msg.get('token') + except Exception: + continue + if action == 'add': + cleanups[token] = msg.get('cleanup') + elif action == 'remove': + cleanups.pop(token, None) + elif action == 'kill': + cleanup = cleanups.pop(token, None) + if cleanup is not None: + _run_cleanup(cleanup) + try: + os.write(1, f'{token}\n'.encode()) + except OSError: + pass + return buffer + + def main(): + # Dying on Ctrl+C, or on a SIGTERM aimed at the whole tree, is exactly the + # case cleanups exist for. Own session (see the parent) keeps group-wide + # signals away; this also covers the ones addressed to us directly. The + # parent closing stdin is the only exit signal we honour. + for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP, signal.SIGQUIT): + signal.signal(sig, signal.SIG_IGN) + original_ppid = os.getppid() - original_pgrp = os.getpgrp() cleanups: dict[int, object] = {} buffer = b'' stdin_fd = sys.stdin.fileno() - reparented = False while True: + # A dying parent closes stdin, so EOF is the usual way out; the ppid + # check only covers a parent that somehow leaked the write end. if os.getppid() != original_ppid: - reparented = True + # Drain whatever it managed to send before dying. + while True: + data = _read(stdin_fd, 0) + if not data: + break + buffer = _consume(buffer + data, cleanups) break - try: - ready, _, _ = select.select([stdin_fd], [], [], 1.0) - except (ValueError, OSError): + data = _read(stdin_fd, 1.0) + if data is None: break - - if not ready: - continue - - try: - data = os.read(stdin_fd, 65536) - except OSError: - break - if not data: - # stdin closed - parent finished normally - break - - buffer += data - while len(buffer) >= 4: - length = int.from_bytes(buffer[:4], 'big') - if len(buffer) < 4 + length: - break - payload = buffer[4 + length :] - frame, buffer = buffer[4 : 4 + length], payload - try: - msg = cloudpickle.loads(frame) - action = msg.get('action') - token = msg.get('token') - except Exception: - continue - if action == 'add': - cleanups[token] = msg.get('cleanup') - elif action == 'remove': - cleanups.pop(token, None) - elif action == 'kill': - cleanup = cleanups.pop(token, None) - if cleanup is not None: - _run_cleanup(cleanup) - try: - os.write(1, f'{token}\n'.encode()) - except OSError: - pass - - if reparented: - # Parent died. Move to our own group so we survive the group kill, then - # take down the original group before running cleanups. - try: - os.setpgrp() - except OSError: - pass - try: - os.killpg(original_pgrp, signal.SIGKILL) - except (ProcessLookupError, PermissionError, OSError): - pass + buffer = _consume(buffer + data, cleanups) # Tear down everything still registered (dangling cleanups). for cleanup in cleanups.values(): diff --git a/support/tools/genvm-tool/unit_tests/test_verify_pushed_executors.py b/support/tools/genvm-tool/unit_tests/test_verify_pushed_executors.py new file mode 100644 index 00000000..2a442576 --- /dev/null +++ b/support/tools/genvm-tool/unit_tests/test_verify_pushed_executors.py @@ -0,0 +1,378 @@ +import json +import os +import subprocess +from pathlib import Path +from types import SimpleNamespace + +from genvm_tool.git import verify_pushed_executors as verify + + +class _Printer: + def __init__(self): + self.records = [] + + def put(self, topic, **fields): + self.records.append((topic, fields)) + + @property + def text(self): + return '\n'.join( + ' '.join([topic, *(str(value) for value in fields.values())]) + for topic, fields in self.records + ) + + +def _git(repo: Path, *args: str, input: str | None = None) -> str: + result = subprocess.run( + ['git', '-C', str(repo), *args], + input=input, + capture_output=True, + text=True, + check=True, + ) + return result.stdout.strip() + + +def _init_repo(path: Path, bare: bool = False) -> None: + path.mkdir(parents=True) + args = ['init'] + if bare: + args.append('--bare') + _git(path, *args) + if not bare: + _git(path, 'config', 'user.email', 'tests@example.com') + _git(path, 'config', 'user.name', 'Tests') + + +def _commit(repo: Path, name: str) -> str: + (repo / 'content').write_text(name) + _git(repo, 'add', 'content') + _git(repo, 'commit', '-m', name) + return _git(repo, 'rev-parse', 'HEAD') + + +def _remote(tmp_path: Path) -> tuple[Path, Path, str]: + bare = tmp_path / 'executor.git' + seed = tmp_path / 'seed' + _init_repo(bare, bare=True) + _init_repo(seed) + base = _commit(seed, 'base') + _git(seed, 'remote', 'add', 'origin', str(bare)) + _git(seed, 'push', '-u', 'origin', 'HEAD:main') + return bare, seed, base + + +def _manager(tmp_path: Path) -> Path: + manager = tmp_path / 'manager' + _init_repo(manager) + return manager + + +def _clone_line(manager: Path, remote: Path, version: str) -> Path: + path = manager / f'executors/v{version}.x' + path.parent.mkdir(parents=True, exist_ok=True) + subprocess.run( + ['git', 'clone', '--branch', 'main', str(remote), str(path)], + capture_output=True, + text=True, + check=True, + ) + _git(path, 'config', 'user.email', 'tests@example.com') + _git(path, 'config', 'user.name', 'Tests') + return path + + +def _manager_commit( + manager: Path, versions: list[str], gitlinks: dict[str, str] +) -> str: + (manager / '.genvm-monorepo-root').write_text( + json.dumps({'active-versions': [f'v{version}' for version in versions]}) + ) + _git(manager, 'add', '.genvm-monorepo-root') + for version, sha in gitlinks.items(): + _git( + manager, + 'update-index', + '--add', + '--cacheinfo', + f'160000,{sha},executors/v{version}.x', + ) + _git(manager, 'commit', '-m', 'manager') + return _git(manager, 'rev-parse', 'HEAD') + + +def _run(manager: Path, manager_ref: str | None = None): + printer = _Printer() + args = SimpleNamespace( + manager_ref=manager_ref, + executor_remote='origin', + ) + rc = verify.main(SimpleNamespace(root=manager, printer=printer), args) + return rc, printer + + +def test_no_active_executor_lines_pass(tmp_path): + manager = _manager(tmp_path) + _manager_commit(manager, [], {}) + rc, printer = _run(manager) + assert rc == 0 + assert printer.records[-1][1]['status'] == 'ok' + + +def test_tip_and_ancestor_are_published(tmp_path): + remote, _, base = _remote(tmp_path) + manager = _manager(tmp_path) + line = _clone_line(manager, remote, '0.3') + newer = _commit(line, 'newer') + _git(line, 'push', 'origin', 'HEAD:main') + + _manager_commit(manager, ['0.2', '0.3'], {'0.2': newer, '0.3': base}) + _clone_line(manager, remote, '0.2') + rc, _ = _run(manager) + assert rc == 0 + + +def test_local_only_gitlink_is_rejected(tmp_path): + remote, _, _ = _remote(tmp_path) + manager = _manager(tmp_path) + line = _clone_line(manager, remote, '0.3') + local = _commit(line, 'local-only') + _manager_commit(manager, ['0.3'], {'0.3': local}) + + rc, printer = _run(manager) + assert rc == 1 + assert 'executors/v0.3.x' in printer.text + assert 'absent from executor remote' in printer.text + + +def test_each_line_is_asked_from_its_own_clone(tmp_path, monkeypatch): + remote, _, base = _remote(tmp_path) + manager = _manager(tmp_path) + line2 = _clone_line(manager, remote, '0.2') + line3 = _clone_line(manager, remote, '0.3') + local2 = _commit(line2, 'local-2') + local3 = _commit(line3, 'local-3') + _manager_commit( + manager, + ['0.2', '0.3'], + {'0.2': local2, '0.3': local3}, + ) + + # Both lines are clones of one repository, so they share a remote URL while + # holding disjoint objects. The question has to be put once per line, from + # the clone that holds the commit being asked about. + asked = [] + original = verify._published + + def counted(target, remote): + asked.append(target.path) + return original(target, remote) + + monkeypatch.setattr(verify, '_published', counted) + rc, printer = _run(manager) + assert rc == 1 + assert asked == [line2, line3] + assert 'executors/v0.2.x' in printer.text + assert 'executors/v0.3.x' in printer.text + assert base not in printer.text + + +def test_only_unpublished_line_is_named(tmp_path): + remote, _, base = _remote(tmp_path) + manager = _manager(tmp_path) + _clone_line(manager, remote, '0.2') + line3 = _clone_line(manager, remote, '0.3') + local = _commit(line3, 'local') + _manager_commit(manager, ['0.2', '0.3'], {'0.2': base, '0.3': local}) + + rc, printer = _run(manager) + assert rc == 1 + assert 'executors/v0.2.x' not in printer.text + assert 'executors/v0.3.x' in printer.text + + +def test_supplied_ref_controls_active_set_and_gitlinks(tmp_path): + remote, _, base = _remote(tmp_path) + manager = _manager(tmp_path) + line = _clone_line(manager, remote, '0.3') + published_ref = _manager_commit(manager, ['0.3'], {'0.3': base}) + local = _commit(line, 'local') + _manager_commit(manager, ['0.3'], {'0.3': local}) + + assert _run(manager)[0] == 1 + assert _run(manager, published_ref)[0] == 0 + + _git(manager, 'checkout', '--detach', published_ref) + removed_ref = _manager_commit(manager, [], {}) + assert _run(manager, removed_ref)[0] == 0 + + +def test_missing_gitlink_rejects_but_unverifiable_local_state_warns(tmp_path): + remote, _, base = _remote(tmp_path) + + manager = _manager(tmp_path / 'missing-gitlink') + _manager_commit(manager, ['0.3'], {}) + rc, printer = _run(manager) + assert rc == 1 + assert 'malformed gitlink' in printer.text + + manager = _manager(tmp_path / 'missing-checkout') + _manager_commit(manager, ['0.3'], {'0.3': base}) + rc, printer = _run(manager) + assert rc == 0 + assert 'checkout is missing' in printer.text + assert 'WARNING (push allowed)' in printer.text + + manager = _manager(tmp_path / 'missing-remote') + line = _clone_line(manager, remote, '0.3') + _git(line, 'remote', 'remove', 'origin') + _manager_commit(manager, ['0.3'], {'0.3': base}) + rc, printer = _run(manager) + assert rc == 0 + assert "remote 'origin' is missing" in printer.text + assert 'WARNING (push allowed)' in printer.text + + +def test_remote_failure_and_timeout_warn_without_rejecting(tmp_path, monkeypatch): + remote, _, base = _remote(tmp_path) + manager = _manager(tmp_path) + line = _clone_line(manager, remote, '0.3') + _manager_commit(manager, ['0.3'], {'0.3': base}) + _git(line, 'remote', 'set-url', 'origin', str(tmp_path / 'absent.git')) + rc, printer = _run(manager) + assert rc == 0 + assert 'unreachable' in printer.text + assert 'WARNING (push allowed)' in printer.text + + original = subprocess.run + + def timed_out(*args, **kwargs): + # Only the negotiation may hang; the local object probe still has to run. + if 'fetch' in args[0]: + raise subprocess.TimeoutExpired(args[0], verify.common.GIT_REMOTE_TIMEOUT) + return original(*args, **kwargs) + + with monkeypatch.context() as patch: + patch.setattr(verify.subprocess, 'run', timed_out) + published, problem = verify._published( + verify._Target('executors/v0.3.x', line, base), 'origin' + ) + assert published is None + assert 'timed out' in problem + + +def test_gitlink_missing_from_the_checkout_warns_without_rejecting(tmp_path): + remote, seed, _ = _remote(tmp_path) + manager = _manager(tmp_path) + _clone_line(manager, remote, '0.3') + unfetched = _commit(seed, 'remote-only') + _git(seed, 'push', 'origin', 'HEAD:main') + _manager_commit(manager, ['0.3'], {'0.3': unfetched}) + + rc, printer = _run(manager) + assert rc == 0 + assert 'absent from this executor checkout' in printer.text + assert 'WARNING (push allowed)' in printer.text + + +def test_a_hooks_git_dir_does_not_capture_submodule_queries(tmp_path, monkeypatch): + # Git exports GIT_DIR (and friends) to hooks, and they outrank `-C`. Left in + # the environment, every executor query would answer for the manager instead. + remote, _, base = _remote(tmp_path) + manager = _manager(tmp_path) + _clone_line(manager, remote, '0.3') + _manager_commit(manager, ['0.3'], {'0.3': base}) + + monkeypatch.setenv('GIT_DIR', str(manager / '.git')) + monkeypatch.setenv('GIT_WORK_TREE', str(manager)) + rc, printer = _run(manager) + assert rc == 0 + assert printer.records[-1][1]['status'] == 'ok' + + +def test_a_stale_clone_still_recognizes_a_published_gitlink(tmp_path): + remote, seed, base = _remote(tmp_path) + manager = _manager(tmp_path) + _clone_line(manager, remote, '0.3') + # The remote moves on and the clone never fetches: the gitlink is no longer a + # tip anywhere the clone knows of, but the remote still holds the commit. + _commit(seed, 'remote-only') + _git(seed, 'push', 'origin', 'HEAD:main') + _manager_commit(manager, ['0.3'], {'0.3': base}) + + rc, printer = _run(manager) + assert rc == 0 + assert printer.records[-1][1]['status'] == 'ok' + + +def test_defaults_to_hook_ref_then_head(tmp_path, monkeypatch): + remote, _, base = _remote(tmp_path) + manager = _manager(tmp_path) + line = _clone_line(manager, remote, '0.3') + published_ref = _manager_commit(manager, ['0.3'], {'0.3': base}) + local = _commit(line, 'local') + _manager_commit(manager, ['0.3'], {'0.3': local}) + + monkeypatch.setenv('PRE_COMMIT_TO_REF', published_ref) + assert _run(manager)[0] == 0 + monkeypatch.delenv('PRE_COMMIT_TO_REF') + assert _run(manager)[0] == 1 + + +_ZERO = '0' * 40 + + +def test_pinned_pre_commit_exposes_only_first_push_update(tmp_path): + repo = tmp_path / 'pre-commit' + _init_repo(repo) + base = _commit(repo, 'base') + first = _commit(repo, 'first') + second = _commit(repo, 'second') + _git(repo, 'update-ref', 'refs/remotes/origin/main', base) + capture = repo / 'captured' + (repo / 'capture.py').write_text( + 'import os\n' + 'from pathlib import Path\n' + "Path(os.environ['CAPTURE_FILE']).write_text(" + "os.environ.get('PRE_COMMIT_TO_REF', 'missing'))\n" + ) + (repo / '.pre-commit-config.yaml').write_text( + 'repos:\n' + '- repo: local\n' + ' hooks:\n' + ' - id: capture-ref\n' + ' name: capture-ref\n' + ' entry: python capture.py\n' + ' language: system\n' + ' stages: [pre-push]\n' + ' always_run: true\n' + ' pass_filenames: false\n' + ) + env = { + **os.environ, + 'CAPTURE_FILE': str(capture), + 'PRE_COMMIT_HOME': str(tmp_path / 'pre-commit-cache'), + } + subprocess.run( + ['pre-commit', 'install', '--hook-type', 'pre-push'], + cwd=repo, + capture_output=True, + text=True, + env=env, + check=True, + ) + hook = Path(_git(repo, 'rev-parse', '--git-path', 'hooks/pre-push')) + stdin = ( + f'refs/heads/first {first} refs/heads/first {_ZERO}\n' + f'refs/heads/second {second} refs/heads/second {_ZERO}\n' + ) + subprocess.run( + [hook, 'origin', 'unused'], + cwd=repo, + input=stdin, + capture_output=True, + text=True, + env=env, + check=True, + ) + assert capture.read_text() == first diff --git a/tests/runner/genvm_tool_plugins/genvm.py b/tests/runner/genvm_tool_plugins/genvm.py index 4965ff56..1bfc5c5c 100644 --- a/tests/runner/genvm_tool_plugins/genvm.py +++ b/tests/runner/genvm_tool_plugins/genvm.py @@ -157,6 +157,23 @@ async def start(self) -> ManagerHandle: log_file = open(self._log_path, 'w') port = get_manager_port(self._env) + config_dir = self._log_path.parent if self._log_path is not None else Path('/tmp') + config_dir.mkdir(parents=True, exist_ok=True) + config_path = config_dir / 'genvm-manager.yaml' + manifest_path = self._bin_path.parent.parent / 'data' / 'manifest.yaml' + config_path.write_text( + '\n'.join( + [ + 'threads: 4', + 'blocking_threads: 48', + 'log_disable: tracing*,polling*,tungstenite*,tokio_tungstenite*', + 'log_level: info', + f'manifest_path: {json.dumps(str(manifest_path))}', + 'permits: null', + '', + ] + ) + ) process = await asyncio.subprocess.create_subprocess_exec( str(self._bin_path), @@ -164,6 +181,8 @@ async def start(self) -> ManagerHandle: '--port', str(port), '--die-with-parent', + '--config', + str(config_path), stdin=asyncio.subprocess.DEVNULL, stdout=asyncio.subprocess.PIPE if use_pipe else asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.PIPE if use_pipe else asyncio.subprocess.DEVNULL, @@ -247,4 +266,9 @@ def __init__(self, *, port: int): self._port = port async def start(self) -> ManagerHandle: - return ManagerHandle(self._port, process=None, log_file=None, log_tasks=None) + return ManagerHandle( + self._port, + process=None, + log_file=None, + log_tasks=None, + ) diff --git a/tests/runner/genvm_tool_plugins/integration.py b/tests/runner/genvm_tool_plugins/integration.py index 565061e6..910e5b7a 100644 --- a/tests/runner/genvm_tool_plugins/integration.py +++ b/tests/runner/genvm_tool_plugins/integration.py @@ -48,7 +48,7 @@ '--ignore-hash', action='store_true', default=False, - help='Ignore .hash files entirely (skip hash comparison and do not write missing ones)', + help='Skip the committed .hash golden sidecars (no comparison, no creation). Leader-vs-validator/sync comparison still runs.', ) @@ -233,7 +233,13 @@ def _flatten_tree( return result -_TOP_LEVEL_METADATA_KEYS = frozenset({'entry', 'tags'}) +_TOP_LEVEL_METADATA_KEYS = frozenset({'entry', 'tags', 'debug_mode'}) + +# Ordered as in `genvm_common::DebugMode`; each level adds to the previous one. +# `unsafe` is what every case has always run with: contracts name their runner +# as `py-genlayer:test`, and that alias only resolves from `unsafe` up. +_DEBUG_MODES = ('disabled', 'safe', 'safe-unbounded', 'unsafe', 'unsafe-tracing') +_DEBUG_MODE_DEFAULT = 'unsafe' @dataclass @@ -267,7 +273,8 @@ class IntegrationSingleCase(genvm_tool.tests.test.Case): total_steps: int tmp_dir: Path max_attempts: int - ignore_hash: bool = False + debug_mode: str = _DEBUG_MODE_DEFAULT + save_hashes: bool = True is_benchmark: bool = False ci: bool = False @@ -476,7 +483,8 @@ def __init__(self, case: IntegrationSingleCase): self._total_steps = case.total_steps self._tmp_dir = case.tmp_dir self._max_attempts = case.max_attempts - self._ignore_hash = case.ignore_hash + self._debug_mode = case.debug_mode + self._save_hashes = case.save_hashes self._ci = case.ci def to_str(self) -> str: @@ -737,7 +745,7 @@ async def _run_single_step( host_data=host_data, ctx=ctx, host='unix://' + mock_host.path, - debug_mode='unsafe', + debug_mode=self._debug_mode, code=code, calldata=calldata_bytes, bucket_totals=bucket_totals, @@ -913,13 +921,23 @@ async def _run_single_step( expected_hash_path = single_conf['expected_hash_path'] + # A golden sidecar pins the hash across commits, and a line whose hashes + # are still moving may legitimately stop tracking those. It may not stop + # comparing this run's own modes against each other -- that is the + # determinism property itself, and dropping it would leave a validator + # step asserting only that it did not crash, since `expandModes` also + # empties the semantics of every non-main mode. So when goldens are off, + # fall back to the main mode's artifact rather than checking nothing. + compares_golden = single_conf.get('expected_hash_is_golden', True) + if compares_golden and (_is_ignore_hash_enabled() or not self._save_hashes): + expected_hash_path = single_conf.get('runtime_hash_path') + compares_golden = False + check_expected_hash = expected_hash_path is not None if ( res.result_kind == public_abi.ResultCode.VM_ERROR and res.result_data == 'timeout' ): check_expected_hash = False # Don't check hash for timeouts, as they can be flaky - if _is_ignore_hash_enabled() or self._ignore_hash: - check_expected_hash = False if check_expected_hash: expected_hash_path = Path(expected_hash_path) @@ -933,12 +951,28 @@ async def _run_single_step( return { 'passed': False, 'context': { - 'reason': 'hash mismatch', + 'reason': 'hash mismatch' + if compares_golden + else 'modes of one run disagree', + 'compared_against': 'committed golden' + if compares_golden + else 'this run of the main mode', 'hash_path': str(expected_hash_path), 'tree_path': tree_path, **diff, }, } + elif not compares_golden: + # The main mode runs first and always writes its artifact, so a + # missing one is a harness bug, not a golden waiting to be born. + return { + 'passed': False, + 'context': { + 'reason': 'main mode produced no hash artifact', + 'hash_path': str(expected_hash_path), + 'tree_path': tree_path, + }, + } else: expected_hash_path.parent.mkdir(exist_ok=True, parents=True) expected_hash_path.write_text( @@ -1013,7 +1047,11 @@ def integration_test_single_executor( executor_integration_conf: dict = ( executor_integration_fn() if executor_integration_fn else {} ) - ignore_hash = executor_integration_conf.get('ignore-hash', False) + # `save-hashes` replaces the inverted `ignore-hash`; a line that has not been + # renamed yet still gets what it asked for. + save_hashes = executor_integration_conf.get( + 'save-hashes', not executor_integration_conf.get('ignore-hash', False) + ) jsonnet_files = list( x for x in CASES_DIR.glob('**/*.jsonnet') if not x.name.startswith('_') @@ -1032,7 +1070,7 @@ def integration_test_single_executor( jsonnet_file, cases_dir=CASES_DIR, reroute_to=reroute_to, - ignore_hash=ignore_hash, + save_hashes=save_hashes, manager_service=manager_service, modules_service=modules_service, webdriver_service=webdriver_service, @@ -1055,7 +1093,7 @@ def _single_integration_test( *, cases_dir: Path, reroute_to: str, - ignore_hash: bool, + save_hashes: bool, manager_service: genvm_tool.tests.stage.collection.Service, modules_service: genvm_tool.tests.stage.collection.Service, webdriver_service: genvm_tool.tests.stage.collection.Service, @@ -1117,6 +1155,17 @@ def _single_integration_test( if extra_tags: tags.update(extra_tags) + debug_mode = jsonnet_parsed.get('debug_mode', _DEBUG_MODE_DEFAULT) + if debug_mode not in _DEBUG_MODES: + raise ValueError( + f'{jsonnet_file}: unknown debug_mode {debug_mode!r}, expected one of {_DEBUG_MODES}' + ) + if debug_mode == 'disabled': + # `reroute_to` is honored only from `safe` up, and that is what points a + # case at its own line's executor. Below it the case would silently run + # whatever the manifest resolves to. + raise ValueError(f'{jsonnet_file}: debug_mode "disabled" cannot reroute') + # Recompute needed_services after tags update needed_services = {manager_service} if 'stable' not in tags: @@ -1204,7 +1253,8 @@ def _single_integration_test( total_steps=total_steps, tmp_dir=tmp_dir, max_attempts=max_attempts, - ignore_hash=ignore_hash, + debug_mode=debug_mode, + save_hashes=save_hashes, is_benchmark=is_benchmark, ci=ci, ) diff --git a/tests/runner/genvm_tool_plugins/ninja.py b/tests/runner/genvm_tool_plugins/ninja.py index f4a750a1..8ed58707 100644 --- a/tests/runner/genvm_tool_plugins/ninja.py +++ b/tests/runner/genvm_tool_plugins/ninja.py @@ -57,8 +57,12 @@ class RawStr(str): VAR_IN = RawStr('$in') VAR_OUT = RawStr('$out') -# clippy/build runs are instrumented for coverage but discard the profile here. -COVERAGE_ENV = RawStr('RUSTFLAGS="-C instrument-coverage" LLVM_PROFILE_FILE=/dev/null') +# Env for every cargo edge. Coverage instrumentation is deliberately absent: +# only `genvm-tool test run --coverage` builds instrumented, and it uses its own +# RUSTFLAGS. Instrumenting here cost ~2x in every wasm compilation the debug +# executor performs (`precompile`, first run of a contract) while the profile it +# produced went to /dev/null. +CARGO_ENV = RawStr('LLVM_PROFILE_FILE=/dev/null') # Ruby's Shellwords.escape leaves this set unescaped; everything else (including # spaces, quotes, `$`, `?`, `#`) gets a leading backslash. @@ -332,7 +336,7 @@ def register_cargo( clippy.var('subcommand', 'clippy') clippy.var('wd', crate_dir) clippy.var('extra_args', extra_args + clippy_lints) - clippy.var('env', COVERAGE_ENV) + clippy.var('env', CARGO_ENV) if target_dir is not None: clippy.var('target_dir', str(td)) clippy.description('Run cargo clippy for ' + rel_path) @@ -346,7 +350,7 @@ def register_cargo( 'extra_args', extra_args + ['--fix', '--allow-dirty', '--allow-staged'] + clippy_lints, ) - fix.var('env', COVERAGE_ENV) + fix.var('env', CARGO_ENV) if target_dir is not None: fix.var('target_dir', str(td)) fix.finish() @@ -364,7 +368,7 @@ def register_cargo( build.add_dependency(files_trg) build.var('wd', crate_dir) build.var('extra_args', extra_args) - build.var('env', COVERAGE_ENV) + build.var('env', CARGO_ENV) if target_dir is not None: build.var('target_dir', str(td)) build.finish() diff --git a/tests/runner/origin/log_asserts.py b/tests/runner/origin/log_asserts.py index 828a8d5a..a46507a6 100644 --- a/tests/runner/origin/log_asserts.py +++ b/tests/runner/origin/log_asserts.py @@ -14,7 +14,7 @@ (``public_abi::memory_limiter_consts::RUNNER_LOAD_COST``); - ``size`` β€” the charged content size (archive ``total_size``); - ``status`` β€” ``"charged"`` (first load in this VM) or ``"cached"`` - (already in the VM's loaded set β€” free). + (already in the VM's loaded set β€” free). This module is deliberately message-agnostic: a matcher's ``message`` defaults to ``"runner load"`` but may name any message, so the same machinery serves diff --git a/tests/runner/origin/logger.py b/tests/runner/origin/logger.py index 8d2016a5..95cfd1c6 100644 --- a/tests/runner/origin/logger.py +++ b/tests/runner/origin/logger.py @@ -1,6 +1,8 @@ import abc +import collections.abc import json import sys +import traceback class Logger(metaclass=abc.ABCMeta): @@ -26,6 +28,37 @@ def with_keys(self, keys: dict) -> 'Logger': return _WithKeysLogger(self, keys) +def _log_unwrap(x, seen: set[int]): + if isinstance(x, bytes): + return x.hex() + if x is None: + return None + if isinstance(x, (str, int, float, bool)): + return x + if isinstance(x, BaseException): + return _log_unwrap( + { + 'message': x.args[0] if len(x.args) == 1 else x.args, + 'type': x.__class__.__name__, + 'notes': getattr(x, '__notes__', []), + 'traceback': traceback.format_exception(x), + }, + seen, + ) + x_id = id(x) + if x_id in seen: + return f'<{x_id}>' + seen.add(x_id) + if isinstance(x, dict): + res = {k: _log_unwrap(v, seen) for k, v in x.items()} + elif isinstance(x, collections.abc.Sequence): + res = [_log_unwrap(v, seen) for v in x] + else: + res = repr(x) + seen.remove(x_id) + return res + + class _WithKeysLogger(Logger): keys: dict @@ -69,10 +102,15 @@ def log(self, level: str, msg: str, **kwargs) -> None: if _level_to_num[level] < self.min_level: return json.dump( - { - 'message': msg, - 'level': level, - **kwargs, - }, + _log_unwrap( + { + 'message': msg, + 'level': level, + **kwargs, + }, + set(), + ), sys.stderr, ) + sys.stderr.write('\n') + sys.stderr.flush() diff --git a/tests/templates/util.jsonnet b/tests/templates/util.jsonnet index 676de7b7..fd087482 100644 --- a/tests/templates/util.jsonnet +++ b/tests/templates/util.jsonnet @@ -30,10 +30,10 @@ local expandModes(entry, parentMainTreePath) = entries: [ local result_path = '${tmpDir}/' + entry.tree_path + m + '/result.pickle'; local isLeader = m == 'l'; - local expected_hash_path = - if is_stable_hash - then '${jsonnetDir}/${fileBaseName}.' + entry.tree_path + '.hash' - else if isLeader + // This run's own main-mode artifact: what a non-main mode is + // compared against when there is no golden to compare to. + local runtime_hash_path = + if isLeader then null else if hasLeaderMode then '${tmpDir}/' + entry.tree_path + 'l/hash' @@ -41,6 +41,11 @@ local expandModes(entry, parentMainTreePath) = then '${tmpDir}/' + entry.tree_path + 'v/hash' else null ; + local expected_hash_path = + if is_stable_hash + then '${jsonnetDir}/${fileBaseName}.' + entry.tree_path + '.hash' + else runtime_hash_path + ; local base = if isLeader then entry @@ -49,6 +54,12 @@ local expandModes(entry, parentMainTreePath) = mode: m, result_path: result_path, expected_hash_path: expected_hash_path, + // Whether that path is a committed golden sidecar, and where to + // fall back when a line does not track goldens. A line may stop + // pinning hashes across commits; it may not stop comparing the + // modes of one run against each other. + expected_hash_is_golden: is_stable_hash, + runtime_hash_path: runtime_hash_path, tree_path: entry.tree_path + m, depends_on: if m == mainMode then parentMainTreePath else mainTreePath, } + From 70fdf7823ec2caa56e259b2b9c8cf20ed9562f66 Mon Sep 17 00:00:00 2001 From: kp2pml30 <kp2pml30@gmail.com> Date: Tue, 4 Aug 2026 19:14:23 +0900 Subject: [PATCH 07/45] chore: migrate to a separate cache repo (#18) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: migrate to a separate cache repo * chore(git): check submodules out as worktrees of the shared cache πŸ—οΈ * chore(ci): accept the rtm label in place of an approval πŸ—οΈ * chore(ci): carry commit bodies through a squash, flatten bullets ✨ Squashing has to be associative: a collapsed commit that was itself a squash contributes its bullets to the same flat list, so re-squashing lands what squashing the originals would have. * feat: add caching to jsonnet eval * fix: address the first branch-review pass πŸ› * fix(tests): key the jsonnet cache on the files each case really imports πŸ› The digest covered only `_*.jsonnet` and the template `*.jsonnet` files, so editing `tests/templates/message.json` or an `importstr`-ed contract source kept returning yesterday's evaluation. Jsonnet's own importer now reports what it reads and the entry carries those digests, which also lets the case path into the key: two byte-identical cases in different directories import different files under the same relative names. * fix(git): refuse to delete a staging path that is not ours πŸ› Conversion cleared `.genvm-convert-<name>` with an unconditional `rmtree`, which is only ever right when the directory is what an interrupted conversion left there. * fix(ci): keep a squashed body's own paragraph breaks πŸ› * fix(ci): reject control characters in a commit message πŸ”’οΈπŸ› A squash lays a range out as one text and splits it back apart, so a control character is invisible where the message is written and splits it somewhere else where it is parsed: `fix: a<SOH>b` passed the hook and landed truncated at the SOH, with the tail moved into the body. The hook refuses them now, and the range is read with `-z` so nothing in-band delimits the records at all. * fix: address the second branch-review pass πŸ›πŸ”’οΈ * fix(git): repair the worktree entry before the staging path goes πŸ› Converting a second submodule prunes the first one's entry, which still named a staging directory by then deleted, and a checkout left on a deleted gitdir is past what `worktree repair` can mend. * fix(git): replace the fetch refspec rather than add to it πŸ› * fix(tests): resolve imports the way jsonnet reports them πŸ› An import was recorded at its symlink target, so a nested import of its own resolved from the wrong directory; the two library search paths every vm is born with were missing as well. * fix(ci): pin the shared actions to a sha πŸ”’οΈ They run with the cache and model tokens, and a job re-run months later has to run the action it first ran. * fix(ci): agree with the commit hook on what a bullet is πŸ› * fix(tests): lock metric bumps, which collector threads share πŸ› * docs: name both cases a conversion refuses, and what an empty pull token means πŸ“ * perf(tests): let the jsonnet cache commit without an fsync ⚑ Its entries are all re-derivable, so losing the tail of one to a power cut costs a re-evaluation. Measured over 314 committed inserts on ext4: 1.035s at `synchronous=FULL`, 0.026s at `NORMAL`. * chore(tests): keep the jsonnet cache with the test artifacts ♻️ `artifacts_dir` is where a run's own scratch belongs, and it honours the config override that `build/` does not. * fix: address the third branch-review pass πŸ› * fix(tests): re-resolve a cached case's imports, not just their bytes πŸ› A file dropped in beside a case takes over an import that used to fall through to the search path, and the file it used to land on never changed. * fix(git): refuse a gitlink the index has left unmerged πŸ› `ls-files --stage` prints one line per conflict side there, and the first one was taken as the pin. * chore: cut comments back to what they say ♻️ * chore: fix tests --- .genvm-tool.py | 11 + .../genlayerlabs-nix-cache-push/action.yaml | 88 ---- .../genlayerlabs-nix-setup/action.yaml | 79 ---- .github/actions/get-src/action.yaml | 13 +- .github/workflows/branch_merge_into_dev.yaml | 3 +- .github/workflows/branch_pr_checklist.yaml | 18 +- .github/workflows/docs_nightly.yaml | 1 + .github/workflows/incl_initial.yaml | 3 +- .../incl_release_build_test_cell_build.yaml | 4 +- .../incl_release_build_test_cell_test.yaml | 3 +- .github/workflows/queue.yaml | 16 +- .github/workflows/queue_test_cell.yaml | 3 +- CLAUDE.md | 1 + docs/contributing/explanation/README.md | 1 + .../explanation/shared-submodule-cache.md | 37 ++ .../howto/committing/submodules.md | 14 + docs/contributing/howto/pr.md | 10 +- docs/contributing/howto/setup.md | 6 +- docs/contributing/howto/testing/python.md | 9 +- support/ci/pipelines/tests.py | 10 +- support/ci/tools/genvm_merge_into_dev.py | 123 ++++-- support/ci/tools/make_release_notes.py | 6 +- support/ci/unit_tests/conftest.py | 6 + support/ci/unit_tests/test_compose_message.py | 291 +++++++++++++ support/scripts/check-commit-message.py | 51 ++- support/scripts/get-all-git.py | 393 +++++++++++++++--- .../genvm-tool/genvm_tool/tests/__init__.py | 7 + .../tools/genvm-tool/genvm_tool/tests/cli.py | 1 + .../genvm_tool/tests/stage/pipeline.py | 1 + .../runner/genvm_tool_plugins/integration.py | 237 ++++++++++- tests/system/parse_version/test.py | 4 +- tests/system/permits/test.py | 2 +- 32 files changed, 1160 insertions(+), 292 deletions(-) delete mode 100644 .github/actions/genlayerlabs-nix-cache-push/action.yaml delete mode 100644 .github/actions/genlayerlabs-nix-setup/action.yaml create mode 100644 docs/contributing/explanation/shared-submodule-cache.md create mode 100644 support/ci/unit_tests/conftest.py create mode 100644 support/ci/unit_tests/test_compose_message.py diff --git a/.genvm-tool.py b/.genvm-tool.py index 979bef33..5f43780f 100644 --- a/.genvm-tool.py +++ b/.genvm-tool.py @@ -123,6 +123,17 @@ def collect_pytest(ctx: genvm_tool.tests.stage.collection.Context): project_root_dir=p, ) + ci = ctx.shared.root_dir.joinpath('support', 'ci') + pytest.pytest( + ctx, + genvm_tool.tests.test.Description( + str(ci.relative_to(ctx.shared.root_dir)), + console_pool=True, + tags=frozenset({'unit'}), + ), + project_root_dir=ci, + ) + fuzz_files = list(p.glob('fuzz/src/*.py')) fuzz_files.sort() for fuzz_file in fuzz_files: diff --git a/.github/actions/genlayerlabs-nix-cache-push/action.yaml b/.github/actions/genlayerlabs-nix-cache-push/action.yaml deleted file mode 100644 index d5d64e79..00000000 --- a/.github/actions/genlayerlabs-nix-cache-push/action.yaml +++ /dev/null @@ -1,88 +0,0 @@ -name: GenLayer nix cache push -description: >- - Upload everything this job built locally to the attic binary cache, so the - next run substitutes it instead of rebuilding. -inputs: - token: - description: >- - push token for the cache. Empty disables the step entirely: a pull request - from a fork gets no secrets, and such a run must still pass. - required: false - default: "" - cache: - description: cache name on the server - required: false - default: genlayer - endpoint: - description: attic server the cache lives on - required: false - default: https://nix-cache.ygr.ai - include: - description: >- - Regex matched against the store path name with its hash stripped. Narrows - the locally-built set to our own derivations: a build also realizes - third-party intermediates that upstream caches already serve. - required: false - default: genlayer|genvm - exclude: - description: regex matched against the full store path; empty disables - required: false - default: "" -runs: - using: composite - steps: - # No `-x`: the token is passed to `attic login` as an argument. - - shell: bash -eo pipefail {0} - if: ${{ inputs.token != '' }} - env: - NIX_CACHE_PUSH_TOKEN: ${{ inputs.token }} - CACHE: ${{ inputs.cache }} - ENDPOINT: ${{ inputs.endpoint }} - INCLUDE: ${{ inputs.include }} - EXCLUDE: ${{ inputs.exclude }} - run: | - cd "$GITHUB_WORKSPACE" - - # A cache that refuses uploads must not turn a green build red, so every - # step of this script warns and gives up instead of failing the job. - attic_link="${RUNNER_TEMP:-/tmp}/attic-client" - # An out-link rather than `nix run`: it is a gcroot, and a second call - # in the same job skips both evaluation and substitution. - # `--inputs-from .` resolves `nixpkgs` through our flake lock, so the - # client comes from the pinned revision instead of the mutable registry. - if ! nix build --out-link "$attic_link" --inputs-from . nixpkgs#attic-client; then - echo "::warning::could not build the attic client; nothing pushed" - exit 0 - fi - - # `ultimate` is nix's own flag for "built on this machine", which is - # exactly the set worth uploading: everything else arrived from a - # substituter that already has it. It covers the intermediates the - # final artifact is assembled from, not just the artifact. - paths="${RUNNER_TEMP:-/tmp}/attic-paths" - nix path-info --all --json --json-format 1 \ - | jq -r --arg inc "$INCLUDE" --arg exc "$EXCLUDE" ' - to_entries[] - | select(.value.ultimate) - | select($inc == "" or (.key | sub("^/nix/store/[^-]+-"; "") | test($inc))) - | select(.key | test("(-|\\.)(source|socket|service|env|pam|drv)$") | not) - | select($exc == "" or (.key | test($exc) | not)) - | .key' > "$paths" || echo "::error::failed to enumerate locally-built paths" - - echo "::group::artifacts" - cat "$paths" - echo "::endgroup::" - - if [ ! -s "$paths" ]; then - echo "nothing was built locally; nothing to push" - exit 0 - fi - echo "pushing $(wc -l < "$paths") path(s) to $CACHE" - - if ! "$attic_link/bin/attic" login ci "$ENDPOINT" "$NIX_CACHE_PUSH_TOKEN" --set-default; then - echo "::warning::attic login failed; nothing pushed" - exit 0 - fi - - xargs -a "$paths" "$attic_link/bin/attic" push --jobs 4 "$CACHE" \ - || echo "::warning::attic push failed" diff --git a/.github/actions/genlayerlabs-nix-setup/action.yaml b/.github/actions/genlayerlabs-nix-setup/action.yaml deleted file mode 100644 index 099df2fc..00000000 --- a/.github/actions/genlayerlabs-nix-setup/action.yaml +++ /dev/null @@ -1,79 +0,0 @@ -name: GenLayer nix setup -description: Install nix, configured against the caches we substitute from. -inputs: - github_token: - description: token nix authenticates GitHub flake inputs with; they are rate-limited without one - required: false - enable_genlayerlabs_cache: - description: >- - substitute from our own cache. Turn off to measure a cold build, or to - keep a job from consuming paths a broken push may have poisoned. - required: false - default: "true" - extra_substituters: - description: >- - additional substituters, whitespace-separated. Each needs its key in - `extra_trusted_public_keys`, or nix refuses its paths and silently rebuilds. - required: false - default: "" - extra_trusted_public_keys: - description: additional trusted public keys, whitespace-separated - required: false - default: "" - extra_nix_config: - description: >- - lines appended verbatim to nix.conf, for settings this action has no input - for. Later lines win, so this can also override one it does. - required: false - default: "" -runs: - using: composite - steps: - - id: config - shell: bash -eo pipefail {0} - env: - GENLAYERLABS_CACHE: ${{ inputs.enable_genlayerlabs_cache }} - EXTRA_SUBSTITUTERS: ${{ inputs.extra_substituters }} - EXTRA_KEYS: ${{ inputs.extra_trusted_public_keys }} - EXTRA_CONFIG: ${{ inputs.extra_nix_config }} - run: | - substituters="https://cache.nixos.org/ https://nix-community.cachix.org" - keys="cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" - keys="$keys nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=" - - # The key stays trusted either way: disabling the cache stops us pulling - # from it, it does not make paths already in this store untrustworthy. - keys="$keys genlayer:hMvP8BkI2v8E3BtzMB9HQyIEZz/ahNdBZo/JEQ5hpVA=" - if [ "$GENLAYERLABS_CACHE" != "false" ]; then - substituters="$substituters https://nix-cache.ygr.ai/genlayer" - fi - - # nix.conf is line-based, so a multi-line input has to collapse to one. - append() { - local extra - extra="$(printf '%s' "$2" | tr -s '[:space:]' ' ' | sed 's/^ *//; s/ *$//')" - if [ -n "$extra" ]; then printf '%s %s' "$1" "$extra"; else printf '%s' "$1"; fi - } - substituters="$(append "$substituters" "$EXTRA_SUBSTITUTERS")" - keys="$(append "$keys" "$EXTRA_KEYS")" - - { - echo 'nix_config<<EOF_NIX_CONFIG' - echo "experimental-features = nix-command flakes recursive-nix" - echo "substituters = $substituters" - echo "trusted-public-keys = $keys" - echo "download-attempts = 10" - echo "connect-timeout = 15" - echo "stalled-download-timeout = 60" - echo "fallback = true" - if [ -n "$EXTRA_CONFIG" ]; then printf '%s\n' "$EXTRA_CONFIG"; fi - echo 'EOF_NIX_CONFIG' - } >> "$GITHUB_OUTPUT" - - # Pinned by commit: a tag can be moved, and this action runs with the - # workflow's token. - - uses: cachix/install-nix-action@a809471b5c7c913aa67bec8f459a11a0decc3fce # v31.6.2 - with: - extra_nix_config: ${{ steps.config.outputs.nix_config }} - github_access_token: ${{ inputs.github_token }} - install_url: https://releases.nixos.org/nix/nix-2.34.7/install diff --git a/.github/actions/get-src/action.yaml b/.github/actions/get-src/action.yaml index c52c9ef4..bbecea3a 100644 --- a/.github/actions/get-src/action.yaml +++ b/.github/actions/get-src/action.yaml @@ -12,6 +12,14 @@ inputs: github_token: description: GitHub token for nix setup required: false + nix_cache_pull_token: + description: >- + pull token for the GenLayer nix cache, which is private. A composite + action cannot read `secrets`, so the caller has to thread it through. + Empty leaves whatever credentials the runner already has, and none on a + clean runner β€” every nix build then starts from source. + required: false + default: "" pr: description: >- PR number to diff against, overriding `github.event.pull_request.number`. @@ -48,6 +56,9 @@ runs: python3 support/scripts/ci-changes.py --github-output - name: setup nix if: ${{ inputs.with_nix == 'true' }} - uses: ./.github/actions/genlayerlabs-nix-setup + # Pinned at every call site of these actions, `# <branch>` naming where the + # sha came from: a job re-run later has to run the action it first ran. + uses: genlayerlabs/github-actions/nix-setup@39b0a0d5e9bb27a1612d2e98b0f8509d31745157 # main with: github_token: ${{ inputs.github_token }} + cache_pull_token: ${{ inputs.nix_cache_pull_token }} diff --git a/.github/workflows/branch_merge_into_dev.yaml b/.github/workflows/branch_merge_into_dev.yaml index 59b37597..6d77ce27 100644 --- a/.github/workflows/branch_merge_into_dev.yaml +++ b/.github/workflows/branch_merge_into_dev.yaml @@ -10,7 +10,8 @@ name: branch / merge PR into dev # Gates (all required): # 1. base branch is a v<X>-dev branch # 2. a maintainer (push access) approved the head commit; pushing new commits -# revokes it, since the approval is matched against that exact sha +# revokes it, since the approval is matched against that exact sha. The +# `rtm` (ready-to-merge) label authorizes the merge in its place # 3. full GenVM CI (queue.yaml) concluded success on the head commit # 4. the cross-repo E2E check concluded success on the head commit # 5. the PR is 0 commits behind base (head already contains base tip) diff --git a/.github/workflows/branch_pr_checklist.yaml b/.github/workflows/branch_pr_checklist.yaml index 9096660a..9454209b 100644 --- a/.github/workflows/branch_pr_checklist.yaml +++ b/.github/workflows/branch_pr_checklist.yaml @@ -64,9 +64,23 @@ jobs: - [ ] Provision executor PRs <!-- action:provision --> - [ ] Merge into dev <!-- action:merge --> - <sub>Full GenVM CI runs only when **`run-full-tests`** is set β€” "Force run full tests" is a sticky toggle for it. **Merge** requires: an approving review from a maintainer on the head commit (pushing revokes it), green full tests, green E2E, and the branch 0 commits behind.</sub> + <details><summary><b>Merge</b></summary> - <sub>A **repo admin** can comment `/genvm-force-merge` to land the PR without the review, full-CI and E2E gates β€” for when those signals are unobtainable, not when they are red. The base, title and 0-behind checks still apply, and the skip is recorded on the PR.</sub> + Requires, on the exact head commit: + + - an approving review from a maintainer (any push revokes it), or the **`rtm`** label + - linear history β€” 0 commits behind base + - green full GenVM CI and green cross-repo E2E + + Full CI runs only when **`run-full-tests`** or **`rtm`** is set; "Force run full tests" is a sticky toggle for the former. + + Every repo lands ONE squashed commit, subject `<PR title> (#N)`. + </details> + + <details><summary><b>Commands</b></summary> + + - `/genvm-force-merge` β€” **repo admin** only: land without the review, full-CI and E2E gates, for when those signals are unobtainable, not when they are red. Base, title and 0-behind still apply, and the skip is recorded on the PR. + </details> EOF )" diff --git a/.github/workflows/docs_nightly.yaml b/.github/workflows/docs_nightly.yaml index 76505c55..e2a39282 100644 --- a/.github/workflows/docs_nightly.yaml +++ b/.github/workflows/docs_nightly.yaml @@ -62,6 +62,7 @@ jobs: with: with_nix: "true" github_token: ${{ secrets.GITHUB_TOKEN }} + nix_cache_pull_token: ${{ secrets.NIX_CACHE_PULL_TOKEN }} - name: Build docs run: ./support/ci/run.sh pipeline docs diff --git a/.github/workflows/incl_initial.yaml b/.github/workflows/incl_initial.yaml index 6fbc75d9..7b063f06 100644 --- a/.github/workflows/incl_initial.yaml +++ b/.github/workflows/incl_initial.yaml @@ -35,6 +35,7 @@ jobs: with: with_nix: "true" github_token: ${{ secrets.GITHUB_TOKEN }} + nix_cache_pull_token: ${{ secrets.NIX_CACHE_PULL_TOKEN }} # Each repo (manager + every executor submodule) supplies its own # pre-commit binary + generated config from its flake (git-hooks.nix), # with tools pinned per repo β€” so no separate toolchain setup is needed. @@ -47,7 +48,7 @@ jobs: # first realizes them after a `git-hooks.nix` bump. - name: Push to nix cache if: ${{ !cancelled() }} - uses: ./.github/actions/genlayerlabs-nix-cache-push + uses: genlayerlabs/github-actions/nix-cache-push@39b0a0d5e9bb27a1612d2e98b0f8509d31745157 # main with: token: ${{ secrets.NIX_CACHE_PUSH_TOKEN }} diff --git a/.github/workflows/incl_release_build_test_cell_build.yaml b/.github/workflows/incl_release_build_test_cell_build.yaml index 8f9212de..c39ed002 100644 --- a/.github/workflows/incl_release_build_test_cell_build.yaml +++ b/.github/workflows/incl_release_build_test_cell_build.yaml @@ -61,6 +61,7 @@ jobs: third_party: --all with_nix: "true" github_token: ${{ secrets.GITHUB_TOKEN }} + nix_cache_pull_token: ${{ secrets.NIX_CACHE_PULL_TOKEN }} - name: "Job: build" uses: ./.github/actions/matrix-cell-step @@ -72,9 +73,10 @@ jobs: # closure, and that is exactly the work the retry should not repeat. - name: Push to nix cache if: ${{ !cancelled() }} - uses: ./.github/actions/genlayerlabs-nix-cache-push + uses: genlayerlabs/github-actions/nix-cache-push@39b0a0d5e9bb27a1612d2e98b0f8509d31745157 # main with: token: ${{ secrets.NIX_CACHE_PUSH_TOKEN }} + include: genvm|genlayer|(^lief) - name: Upload artifacts uses: actions/upload-artifact@v4 diff --git a/.github/workflows/incl_release_build_test_cell_test.yaml b/.github/workflows/incl_release_build_test_cell_test.yaml index 43205f93..f500c572 100644 --- a/.github/workflows/incl_release_build_test_cell_test.yaml +++ b/.github/workflows/incl_release_build_test_cell_test.yaml @@ -60,6 +60,7 @@ jobs: third_party: --all with_nix: ${{ inputs.with_nix && 'true' || 'false' }} github_token: ${{ secrets.GITHUB_TOKEN }} + nix_cache_pull_token: ${{ secrets.NIX_CACHE_PULL_TOKEN }} - name: Setup Python if: ${{ inputs.setup_python }} @@ -112,7 +113,7 @@ jobs: # nix at all. - name: Push to nix cache if: ${{ !cancelled() && inputs.with_nix }} - uses: ./.github/actions/genlayerlabs-nix-cache-push + uses: genlayerlabs/github-actions/nix-cache-push@39b0a0d5e9bb27a1612d2e98b0f8509d31745157 # main with: token: ${{ secrets.NIX_CACHE_PUSH_TOKEN }} diff --git a/.github/workflows/queue.yaml b/.github/workflows/queue.yaml index ad5df699..b65ea9a3 100644 --- a/.github/workflows/queue.yaml +++ b/.github/workflows/queue.yaml @@ -10,8 +10,9 @@ name: GenVM full # executor landability precondition) ALWAYS run, in parallel; # - `gate-check` aggregates them and plans the test matrix (plan-queue-matrix); # the heavy `matrix-tests` cells run only when a run-full-tests marker is set -# β€” the `run-full-tests` label ("Force run full tests" sets it). Without a -# marker the plan is empty, so matrix-tests is skipped; +# β€” the `run-full-tests` label ("Force run full tests" sets it) or the `rtm` +# label, which marks the PR ready to merge and so needs the full run anyway. +# Without a marker the plan is empty, so matrix-tests is skipped; # - without a marker `validate-end` fails, so the CI check stays red # until a full run has happened. # @@ -33,7 +34,10 @@ on: workflow_dispatch: inputs: pr: - description: PR number this manual run validates (used only for the run name) + description: >- + PR number this manual run validates. Without it the PR gates below + have nothing to read and are skipped, leaving a green run that only + the heavy tests earned β€” pass it to authorize a merge. type: string required: false @@ -134,7 +138,7 @@ jobs: env: # Heavy cells run only with a marker; without one the plan is empty and # matrix-tests is skipped. - RUN_FULL_TESTS: ${{ github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run-full-tests') }} + RUN_FULL_TESTS: ${{ github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run-full-tests') || contains(github.event.pull_request.labels.*.name, 'rtm') }} run: ./support/ci/run.sh pipeline plan-queue-matrix matrix-tests: @@ -175,7 +179,7 @@ jobs: - gate-check - matrix-tests env: - HAS_MARKER: ${{ github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run-full-tests') }} + HAS_MARKER: ${{ github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run-full-tests') || contains(github.event.pull_request.labels.*.name, 'rtm') }} steps: - name: check run: | @@ -187,7 +191,7 @@ jobs: # No run-full-tests marker -> heavy tests were skipped; keep the # gate red so the PR is not mergeable until they run. if [ "$HAS_MARKER" != "true" ]; then - echo "::error::full tests have not run β€” check 'Force run full tests' on the action panel or add the 'run-full-tests' label" + echo "::error::full tests have not run β€” check 'Force run full tests' on the action panel or add the 'run-full-tests' or 'rtm' label" exit 1 fi results="${{ needs.matrix-tests.result }}" diff --git a/.github/workflows/queue_test_cell.yaml b/.github/workflows/queue_test_cell.yaml index aaf24dd4..9b67ad83 100644 --- a/.github/workflows/queue_test_cell.yaml +++ b/.github/workflows/queue_test_cell.yaml @@ -72,6 +72,7 @@ jobs: third_party: --all with_nix: "true" github_token: ${{ secrets.GITHUB_TOKEN }} + nix_cache_pull_token: ${{ secrets.NIX_CACHE_PULL_TOKEN }} - name: Configure fuzzing host if: ${{ inputs.fuzz_host }} @@ -114,7 +115,7 @@ jobs: - name: Push to nix cache if: ${{ !cancelled() }} - uses: ./.github/actions/genlayerlabs-nix-cache-push + uses: genlayerlabs/github-actions/nix-cache-push@39b0a0d5e9bb27a1612d2e98b0f8509d31745157 # main with: token: ${{ secrets.NIX_CACHE_PUSH_TOKEN }} diff --git a/CLAUDE.md b/CLAUDE.md index ad279be7..b62f2919 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -49,4 +49,5 @@ a task has a how-to, a design question has an explanation - [docs-layout.md](docs/contributing/explanation/docs-layout.md) β€” the 4 kinds of page and where each belongs - [executor-lines.md](docs/contributing/explanation/executor-lines.md) β€” why several executor lines ship side by side, and what it costs - [merge-model.md](docs/contributing/explanation/merge-model.md) β€” why merging is a maintainer panel rather than a merge queue +- [shared-submodule-cache.md](docs/contributing/explanation/shared-submodule-cache.md) β€” why submodules are worktrees of one cache repo, not clones - [vendored-trees.md](docs/contributing/explanation/vendored-trees.md) β€” why third-party sources are patch series, not forks diff --git a/docs/contributing/explanation/README.md b/docs/contributing/explanation/README.md index be68f9a2..918220f7 100644 --- a/docs/contributing/explanation/README.md +++ b/docs/contributing/explanation/README.md @@ -6,4 +6,5 @@ lives in the spec, and these pages only motivate it and link out - [docs-layout.md](docs-layout.md) β€” the 4 kinds of page and where each belongs - [executor-lines.md](executor-lines.md) β€” why several executor lines ship side by side, and what it costs - [merge-model.md](merge-model.md) β€” why merging is a maintainer panel rather than a merge queue +- [shared-submodule-cache.md](shared-submodule-cache.md) β€” why submodules are worktrees of one cache repo, not clones - [vendored-trees.md](vendored-trees.md) β€” why third-party sources are patch series, not forks diff --git a/docs/contributing/explanation/shared-submodule-cache.md b/docs/contributing/explanation/shared-submodule-cache.md new file mode 100644 index 00000000..74c56a9a --- /dev/null +++ b/docs/contributing/explanation/shared-submodule-cache.md @@ -0,0 +1,37 @@ +# Why Submodules Are Worktrees Of One Cache + +The submodule paths are not `git submodule` checkouts. One bare cache repo per +remote lives under `<manager .git>/genvm-submodule-cache/`, and every submodule +path is a [git worktree](https://git-scm.com/docs/git-worktree) of it, detached +on the gitlink, all created by `support/scripts/get-all-git.py` +([setup.md](../howto/setup.md)) + +The two executor lines are branches of the *same* repository +([executor-lines.md](executor-lines.md)), so a plain `git submodule update` +clones it twice, and once more for every git worktree of the manager. Sharing +the objects with `--reference` instead only trades that for a fragility: the +borrowing checkouts hold no reference the cache can see, so pruning or deleting +the cache corrupts them silently. A worktree's HEAD *is* a ref in that +repository β€” the objects it needs cannot be collected out from under it, and +the shared store is the only copy that exists + +A second git worktree of the manager makes that worse rather than better: its +submodule gitdirs are private to it (`.git/worktrees/<id>/modules/<path>`), so +nothing at all is shared with the checkout it was branched from. Under the cache +every manager checkout draws its executor trees from the same objects + +Converting an existing checkout is therefore non-destructive: only its `.git` +file is replaced, every ref it had is mirrored into the cache under +`refs/genvm/converted/<manager>/<path>/` β€” branches, tags, notes and the stash β€” +and the files, including the ignored third-party trees, are never touched + +## What It Costs + +`git submodule update` is no longer the way to materialize a checkout β€” it +would clone over the worktree β€” so a gitlink bump means re-running the script, +and the gitdir a converted checkout used (`.git/modules/<path>`, or +`.git/worktrees/<id>/modules/<path>` in a linked worktree) is left behind: its +refs were copied out, its reflogs were not, so it is yours to keep or delete. +Both executor lines also share one ref store, which is why an +executor feature branch is namespaced by line (`pr/<line>/…`) β€” that convention +predates this and is what makes the sharing safe diff --git a/docs/contributing/howto/committing/submodules.md b/docs/contributing/howto/committing/submodules.md index 77b1c7cd..fb364d77 100644 --- a/docs/contributing/howto/committing/submodules.md +++ b/docs/contributing/howto/committing/submodules.md @@ -15,6 +15,20 @@ Active lines and the manager version live in `.genvm-monorepo-root` `commit-hooks` pipeline model the manager and the active lines only. Branches: manager `feat/<name>` on `v<X>-dev`, each line `pr/<line>/feat/<name>` +## Materializing A Checkout + +`python3 support/scripts/get-all-git.py` β€” never `git submodule update`, which +would clone over the worktree each path actually is +([shared-submodule-cache.md](../../explanation/shared-submodule-cache.md)). Run +it after any gitlink bump; it fetches what is missing and parks each checkout on +the gitlink, exactly as `git submodule update` did. `origin` inside an executor +is the executor repo, so `git fetch` / `git push` work as usual + +An older checkout is converted on the spot, keeping every file and uncommitted +edit, and its branch when that branch is already at the gitlink. It refuses when +something is staged there, and when the path holds a standalone clone rather +than a submodule checkout β€” move that aside and re-run + ## The Critical Nix Flag Every manager flake ref needs `?submodules=1`, as `.envrc` does. Flake packages diff --git a/docs/contributing/howto/pr.md b/docs/contributing/howto/pr.md index 9a7037bf..971e5549 100644 --- a/docs/contributing/howto/pr.md +++ b/docs/contributing/howto/pr.md @@ -31,11 +31,13 @@ reading the diff `queue.yaml` always runs the pre-commit hooks. On a `pull_request` event it also requires 0 commits behind base and the executor precondition β€” every repo rebased, every pinned executor commit pushed β€” which is skipped for fork PRs. -The heavy matrix runs only with the `run-full-tests` marker; without it the -check stays red on purpose +The heavy matrix runs only with the `run-full-tests` or `rtm` marker; without +one the check stays red on purpose Merging re-verifies against the **exact head sha**: maintainer approval (any -push revokes it), green full CI, green cross-repo E2E, 0 behind. Merges are +push revokes it) or the `rtm` label (which survives pushes, so it authorizes +whatever lands next), green full CI, green cross-repo E2E, 0 behind. An +unwithdrawn *changes requested* blocks either way. Merges are serialised repository-wide, so a tick may wait for another PR to land first Each repo gains one commit per PR. Executor lines are always squashed onto @@ -50,5 +52,5 @@ manager PR that is a single commit and rewrites no gitlink keeps its sha | `full tests have not run` | tick *Force run full tests* | | `... commit(s) behind base` | rebase, `push --force-with-lease` | | `pinned commit ... is not on the executor repo` | push it, or tick *Provision executor PRs* | -| Approval revoked | you pushed after the review, re-request it | +| Approval revoked | you pushed after the review, re-request it or set `rtm` | | Merge run cancelled before it started | another merge was already queued, tick *Merge into dev* again | diff --git a/docs/contributing/howto/setup.md b/docs/contributing/howto/setup.md index 65cf4f34..8be85bdb 100644 --- a/docs/contributing/howto/setup.md +++ b/docs/contributing/howto/setup.md @@ -7,8 +7,10 @@ python3 support/scripts/get-all-git.py ``` - The defaults fetch every unique submodule remote once into a local cache, - initialize the submodules from it, and materialize all third-party trees + The defaults fetch every unique submodule remote once into a local cache + repo, check each submodule out of it as a git worktree, and materialize all + third-party trees. Re-run it after a gitlink bump; never + `git submodule update` ([submodules.md](committing/submodules.md)) 2. Enter the dev shell β€” normally direnv does this, `.envrc` is `use flake '.?submodules=1#full'`: diff --git a/docs/contributing/howto/testing/python.md b/docs/contributing/howto/testing/python.md index 74cb6b51..e2a6f9ab 100644 --- a/docs/contributing/howto/testing/python.md +++ b/docs/contributing/howto/testing/python.md @@ -4,9 +4,9 @@ genvm-tool test run --filter-tag python ``` -The suite is `genlayer-py-std`, which lives in the primary executor line. The -interpreter comes from a pinned standalone flake, so pytest can be run directly, -without the dev shell: +Two suites: `genlayer-py-std` (in the primary executor line) and `support/ci` +(`unit_tests/`, the stdlib-only CI tools). The interpreter comes from a pinned +standalone flake, so pytest can be run directly, without the dev shell: ```bash cd executors/v0.3.x/runners/genlayer-py-std @@ -16,3 +16,6 @@ PYTHONPATH="$PWD/src:$PWD/src-emb" "$env_dir/bin/pytest" tests/ Coverage is enforced (`--cov-fail-under=75`), and the numpy-dependent tests need that nix environment + +For `support/ci` the same env works with no PYTHONPATH: `cd support/ci && +"$env_dir/bin/pytest"` (its `conftest.py` puts the tools on the path) diff --git a/support/ci/pipelines/tests.py b/support/ci/pipelines/tests.py index 3b3a22a6..10654dcb 100644 --- a/support/ci/pipelines/tests.py +++ b/support/ci/pipelines/tests.py @@ -224,11 +224,11 @@ def group(gh_step: str, name: str, stage: Stage, body: GithubStep) -> None: ), ) -TEST_PYTHON = DefaultStepInfo( +TEST_OTHER = DefaultStepInfo( tool_step=Stage( data=ToolTest( tests_name_filter_for_or=None, - tests_tags_filter_for_or='python', + tests_tags_filter_for_or='!rust & !integration', ), ) ) @@ -297,7 +297,7 @@ def _cell( QUEUE_CELLS = [ - _cell('python', TEST_PYTHON), + _cell('other', TEST_OTHER), _cell('rust-executors', TEST_RUST_EXECUTORS, fuzz_host=True, disk_reclaim=True), _cell('rust-manager', TEST_RUST_MANAGER, fuzz_host=True, disk_reclaim=True), _cell( @@ -316,8 +316,8 @@ class PlanQueueMatrix(ci_lib.Pipeline): Emit queue.yaml's test matrix as a GitHub `matrix` output. The heavy cells run only when a marker is set (RUN_FULL_TESTS, from the - run-full-tests label or a manual dispatch); otherwise the matrix is empty and - matrix-tests is skipped. + run-full-tests / rtm labels or a manual dispatch); otherwise the matrix is + empty and matrix-tests is skipped. """ def name(self) -> str: diff --git a/support/ci/tools/genvm_merge_into_dev.py b/support/ci/tools/genvm_merge_into_dev.py index 5218b836..e5926a14 100755 --- a/support/ci/tools/genvm_merge_into_dev.py +++ b/support/ci/tools/genvm_merge_into_dev.py @@ -10,7 +10,8 @@ Gates (all required, all on the head commit): 1. base branch is a v<X>-dev branch -2. a maintainer (push access) approved the head commit β€” pushing revokes it +2. a maintainer (push access) approved the head commit β€” pushing revokes it β€” or + the PR carries the `rtm` (ready-to-merge) label 3. full GenVM CI (queue.yaml) concluded success 4. the cross-repo E2E check (exact name, from the exact app) concluded success 5. the PR is 0 commits behind base @@ -57,7 +58,10 @@ update to replay. A squashed message is the PR title plus one `* <subject>` bullet per collapsed -commit; the PR description never lands. Squashing also collapses several authors +commit, each followed by that commit's own body; the PR description never lands. +A collapsed commit that was itself a squash contributes its bullets to the same +flat list, so squashing a squash lands what squashing the originals would have. +Squashing also collapses several authors into one, so `Co-authored-by:` trailers are added for every other human author in the range (never tool/AI attribution β€” the commit-message hook rejects that). @@ -103,6 +107,8 @@ from tools.versions import active_lines +RTM_LABEL = 'rtm' + # Scratch ref that carries a merge's objects to both remotes before any branch # moves; outside refs/heads so nothing watches it and no PR is opened for it. LANDING_REF_PREFIX = 'refs/genvm-ci/landing' @@ -266,14 +272,17 @@ def can_maintain(login): return json.loads(out).get('permission') in ('admin', 'maintain', 'write') -def check_approval(head_sha): +def check_approval(head_sha, labels): """ - Require a maintainer's approving review of the exact commit being merged. + Require a maintainer's approving review of the exact commit being merged, or + the `rtm` (ready-to-merge) label as the equivalent manual authorization. - This replaces the `rtm` label. A label is held by whoever can edit labels and - survives every subsequent push, so it could authorize code nobody looked at; an - approval is tied to the commit it was left on, so pushing new commits silently - revokes it here without needing branch protection to dismiss anything. + An approval is tied to the commit it was left on, so pushing new commits + silently revokes it here without needing branch protection to dismiss + anything. The label is the weaker of the two β€” it is held by whoever can edit + labels and survives every subsequent push β€” so it authorizes only what its + holder is willing to vouch for. It stands in for a missing approval, not for + an unwithdrawn objection. Only the newest decision per reviewer counts, so a reviewer who approves and then requests changes on the same commit does not still authorize it. Reviews are @@ -323,11 +332,16 @@ def check_approval(head_sha): f'approve, re-review, or dismiss before this can merge.' ) + if any(label['name'] == RTM_LABEL for label in labels): + print(f'authorized by the `{RTM_LABEL}` label') + return + approvers = sorted(login for login, state in decision.items() if state == 'APPROVED') if not approvers: block( - f'no approving review on `{head_sha[:12]}`. A maintainer must approve this ' - f'exact commit β€” approvals of earlier commits do not carry over.' + f'no approving review on `{head_sha[:12]}`, and no `{RTM_LABEL}` label. A ' + f'maintainer must approve this exact commit β€” approvals of earlier commits ' + f'do not carry over β€” or set `{RTM_LABEL}`.' ) maintainers = [login for login in approvers if can_maintain(login)] @@ -450,8 +464,8 @@ def check_gates(pr): print(f'forced by {actor}: skipping the review, CI and E2E gates') return base, head_sha - # 2. a maintainer approved THIS commit - check_approval(head_sha) + # 2. a maintainer approved THIS commit, or the PR is labelled ready-to-merge + check_approval(head_sha, pr['labels']) # 3. full GenVM CI (queue.yaml) green on the head commit. The same head may # carry skipped runs (label events that didn't run full tests) alongside the @@ -701,30 +715,86 @@ def pr_reference(qualified=True): return f'({repo()}#{pr_number()})' if qualified else f'(#{pr_number()})' -def commit_subjects(repo, rev_range): +def commit_entries(repo, rev_range): """ - Subjects of the commits in `rev_range`, oldest first, merges excluded. + `(subject, body)` per commit in `rev_range`, oldest first, merges excluded. + + `-z` terminates the records, a body being able to contain anything. The two + halves need no separator: a subject is one line, so the first newline is it. """ - args = ('log', '--reverse', '--no-merges', '--format=%s', rev_range) + args = ('log', '--reverse', '--no-merges', '-z', '--format=%s%n%b', rev_range) out = (git(*args) if repo is None else egit(repo, *args)).stdout - return [line for line in out.split('\n') if line.strip()] + entries = [] + for record in out.split('\x00'): + if not record.strip(): + continue + subject, _, body = record.partition('\n') + entries.append((subject.strip(), body.strip('\n'))) + return entries + + +def flatten_body(body): + """ + A commit body as message lines, bullets kept at the top level. + + A body line starting with `*` IS a bullet -- that is the invariant the format + rests on -- so an already-squashed commit's bullets continue the outer list + instead of nesting under it. Prose keeps its own paragraph breaks, and gains + the blank line on either side that separates it from the bullets around it. + """ + lines = [] + prose = [] + + def flush(): + while prose and not prose[0].strip(): + prose.pop(0) + while prose and not prose[-1].strip(): + prose.pop() + if prose: + lines.extend(['', *prose, '']) + prose.clear() + + for line in body.split('\n'): + # Indentation stripped, as the commit-message hook does. + if line.strip().startswith('*'): + flush() + lines.append(line.strip()) + else: + prose.append(line) + flush() + return lines def compose_message(pr, submodule, rev_range, author, *, qualified=True): """ The squash message for `rev_range`: the PR title, one `* <subject>` bullet - per squashed commit, then `Co-authored-by:` for every other author. - - The bullets are the whole body. They are already conventional commits (the - commit-message hook saw them) and are what `make_release_notes` promotes into - the changelog, whereas a PR description is written for reviewers and only rots - on a dev branch. A range whose one commit already says the title gets no - bullets β€” the umbrella would just repeat it. + per squashed commit, each followed by that commit's own body, then + `Co-authored-by:` for every other author. + + The bullets are already conventional commits (the commit-message hook saw + them) and are what `make_release_notes` promotes into the changelog, whereas a + PR description is written for reviewers and only rots on a dev branch. Bodies + carry the `why`, which nothing else preserves once the originals are gone. A + range whose one commit already says the title gets no bullets, only its body. + + Squashing a squash flattens the inner bullets into the outer list instead of + nesting them, so every real entry stays at the level both readers of this + format look at. The inner umbrella subject rides along as one extra bullet, + and the changelog gains that one redundant line. """ - subjects = commit_subjects(submodule, rev_range) + entries = commit_entries(submodule, rev_range) + if [subject for subject, _ in entries] == [pr['title']]: + lines = flatten_body(entries[0][1]) + else: + lines = [] + for subject, body in entries: + lines.append(f'* {subject}') + lines.extend(flatten_body(body)) + message = f'{pr["title"]} {pr_reference(qualified)}\n' - if subjects != [pr['title']]: - message += '\n' + ''.join(f'* {subject}\n' for subject in subjects) + body = '\n'.join(lines).strip('\n') + if body: + message += f'\n{body}\n' trailers = coauthor_trailers(submodule, rev_range, author) if trailers: message += '\n' + '\n'.join(trailers) + '\n' @@ -1092,6 +1162,7 @@ def merge_into_dev(): 'headRefName', 'headRefOid', 'commits', + 'labels', 'state', 'isDraft', 'title', diff --git a/support/ci/tools/make_release_notes.py b/support/ci/tools/make_release_notes.py index 602d2d4d..d9689c2d 100755 --- a/support/ci/tools/make_release_notes.py +++ b/support/ci/tools/make_release_notes.py @@ -94,11 +94,13 @@ def parse_git_log(rev_range: str, *, dir: Path | None = None) -> list[CommitEntr lines = [l.strip() for l in body.strip().splitlines() if l.strip()] if not lines: continue - # body lines starting with * may carry their own conventional prefix + # Bullets only: a squashed message carries bodies too, and prose that + # reads like a subject is not a changelog entry. promoted = [ CommitEntry(cleaned, email) for line in lines[1:] - if COMMIT_RE.match(cleaned := re.sub(r'^\*\s*', '', line)) + if line.startswith('*') + and COMMIT_RE.match(cleaned := re.sub(r'^\*\s*', '', line)) ] if promoted: entries.extend(promoted) diff --git a/support/ci/unit_tests/conftest.py b/support/ci/unit_tests/conftest.py new file mode 100644 index 00000000..e11b495b --- /dev/null +++ b/support/ci/unit_tests/conftest.py @@ -0,0 +1,6 @@ +import sys +from pathlib import Path + +# `support/ci` is the import root the tools run under (run.sh puts it on +# PYTHONPATH); pytest is started from the repo root, so put it there ourselves. +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) diff --git a/support/ci/unit_tests/test_compose_message.py b/support/ci/unit_tests/test_compose_message.py new file mode 100644 index 00000000..b9750ca6 --- /dev/null +++ b/support/ci/unit_tests/test_compose_message.py @@ -0,0 +1,291 @@ +import os +import subprocess +import sys +from pathlib import Path +from textwrap import dedent + +import ci_lib +import gh_common +import pytest +from tools import genvm_merge_into_dev as merge +from tools import make_release_notes as notes + +CHECK_MESSAGE = ci_lib.ROOT_DIR / 'support' / 'scripts' / 'check-commit-message.py' + +AUTHOR = ('Landing Author', 'landing@example.com') +OTHER = ('Other Author', 'other@example.com') +PR = {'title': 'chore(ci): stitch the commits'} + + +def _env(author) -> dict: + name, email = author + return os.environ | { + 'GIT_AUTHOR_NAME': name, + 'GIT_AUTHOR_EMAIL': email, + 'GIT_COMMITTER_NAME': name, + 'GIT_COMMITTER_EMAIL': email, + # The tests assert on exact message text, so no user config may reach git. + 'GIT_CONFIG_GLOBAL': '/dev/null', + 'GIT_CONFIG_SYSTEM': '/dev/null', + } + + +def _git(repo: Path, *args: str, input: str | None = None, author=AUTHOR) -> str: + return subprocess.run( + ['git', '-C', str(repo), *args], + input=input, + capture_output=True, + text=True, + check=True, + env=_env(author), + ).stdout + + +def _add(repo: Path, message: str, *, author=AUTHOR) -> None: + """ + One commit carrying `message` verbatim -- `--cleanup=verbatim` so git keeps + the blank lines the format under test is made of. + """ + repo.joinpath(f'{len(list(repo.glob("*.txt")))}.txt').write_text(message) + _git(repo, 'add', '.', author=author) + _git(repo, 'commit', '--cleanup=verbatim', '-F', '-', input=message, author=author) + + +def _compose(since: str = 'HEAD~2') -> str: + return merge.compose_message(PR, None, f'{since}..HEAD', AUTHOR, qualified=False) + + +@pytest.fixture +def repo(tmp_path, monkeypatch): + path = tmp_path / 'repo' + path.mkdir() + _git(path, 'init', '-q', '-b', 'main') + monkeypatch.chdir(path) + monkeypatch.setattr( + gh_common, + '_CTX', + gh_common.Ctx( + manager_repo='genlayerlabs/genvm-manager', + executor_repo='genlayerlabs/genvm-executor', + _pr_number='7', + _head_ref='feat/x', + ), + ) + return path + + +def test_bodies_are_kept_after_their_bullet(repo): + _add(repo, 'chore: base\n') + _add(repo, 'feat(a): one\n') + _add(repo, 'fix(b): two\n\nThe parser lied about EOF.\nSecond line.\n') + + assert _compose() == dedent("""\ + chore(ci): stitch the commits (#7) + + * feat(a): one + * fix(b): two + + The parser lied about EOF. + Second line. + """) + + +def test_bullets_stay_adjacent_when_bodies_are_absent(repo): + _add(repo, 'chore: base\n') + _add(repo, 'feat(a): one\n') + _add(repo, 'fix(b): two\n') + + assert _compose() == dedent("""\ + chore(ci): stitch the commits (#7) + + * feat(a): one + * fix(b): two + """) + + +def test_a_body_is_blank_line_delimited_on_both_sides(repo): + _add(repo, 'chore: base\n') + _add(repo, 'feat(a): one\n\nWhy one.\n') + _add(repo, 'fix(b): two\n') + + assert _compose() == dedent("""\ + chore(ci): stitch the commits (#7) + + * feat(a): one + + Why one. + + * fix(b): two + """) + + +def test_squashing_a_squash_flattens_its_bullets(repo): + """ + Squashing is associative: the inner umbrella subject stays as a redundant + bullet, and everything under it keeps the level a first-hand squash gave it. + """ + _add(repo, 'chore: base\n') + _add(repo, 'chore(ci): landed earlier (#3)\n\n* feat(a): b\n* feat(a): c\n') + _add(repo, 'feat(d): d\n\n* feat(d): e\n* feat(d): f\n') + + assert _compose() == dedent("""\ + chore(ci): stitch the commits (#7) + + * chore(ci): landed earlier (#3) + * feat(a): b + * feat(a): c + * feat(d): d + * feat(d): e + * feat(d): f + """) + + +def test_prose_keeps_its_blank_lines_among_the_bullets(repo): + _add(repo, 'chore: base\n') + _add(repo, 'feat(a): one\n\n* feat(a): nested\nwhy nested.\n') + _add(repo, 'fix(b): two\n') + + assert _compose() == dedent("""\ + chore(ci): stitch the commits (#7) + + * feat(a): one + * feat(a): nested + + why nested. + + * fix(b): two + """) + + +def test_a_body_keeps_its_own_paragraph_breaks(repo): + _add(repo, 'chore: base\n') + _add(repo, 'feat(a): one\n\nwhat it does.\n\nwhy it does it.\n') + _add(repo, 'fix(b): two\n') + + assert _compose() == dedent("""\ + chore(ci): stitch the commits (#7) + + * feat(a): one + + what it does. + + why it does it. + + * fix(b): two + """) + + +def test_a_lone_commit_titled_like_the_pr_keeps_its_body(repo): + _add(repo, 'chore: base\n') + _add(repo, f'{PR["title"]}\n\nThe only rationale there is.\n') + + assert _compose('HEAD~1') == dedent("""\ + chore(ci): stitch the commits (#7) + + The only rationale there is. + """) + + +def test_coauthors_follow_the_last_body(repo): + _add(repo, 'chore: base\n') + _add(repo, 'feat(a): one\n\nWhy one.\n', author=OTHER) + + assert _compose('HEAD~1') == dedent("""\ + chore(ci): stitch the commits (#7) + + * feat(a): one + + Why one. + + Co-authored-by: Other Author <other@example.com> + """) + + +def test_the_composed_message_passes_the_commit_hook(repo): + """ + Nothing re-validates a generated message, so the hook is checked here instead. + """ + _add(repo, 'chore: base\n') + _add(repo, 'feat(a): one ✨\n\n* fix(a): nested ♻️\nWhy one, at length.\n') + _add(repo, 'fix(b): two πŸ›\n') + + result = subprocess.run( + [sys.executable, str(CHECK_MESSAGE), '--message-text', _compose()], + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stdout + + +def test_release_notes_promote_bullets_only(repo): + _add(repo, 'chore: base\n') + _add( + repo, + dedent("""\ + chore(ci): umbrella (#7) + + * feat(a): one + + fix: prose that merely reads like a subject + """), + ) + + entries = notes.parse_git_log('HEAD~1..HEAD', dir=repo) + + assert [e.message for e in entries] == ['feat(a): one'] + + +def test_a_malformed_bullet_is_an_error_not_body(repo): + """ + The format rests on `*` meaning bullet, so a `*` line that is not a subject + cannot be quietly kept as prose -- it would flatten into a broken entry. + """ + message = dedent("""\ + chore(ci): umbrella (#7) + + * feat(a): one + * just a note + """) + + result = subprocess.run( + [sys.executable, str(CHECK_MESSAGE), '--message-text', message], + capture_output=True, + text=True, + ) + + assert result.returncode == 1 + assert 'just a note' in result.stdout + + +def test_a_control_character_is_rejected(repo): + """ + `commit_entries` reads a range as lines, so a message may only be made of + those; a control character elsewhere in it is invisible where it is written. + """ + result = subprocess.run( + [ + sys.executable, + str(CHECK_MESSAGE), + '--message-text', + 'fix(a): before\x01after\n\nA body long enough to pass the length rule.\n', + ], + capture_output=True, + text=True, + ) + + assert result.returncode == 1 + assert 'Control character' in result.stdout + + +def test_an_emoji_sequence_is_not_a_control_character(repo): + _add(repo, 'chore: base\n') + _add(repo, 'feat(a): one πŸ‘¨β€πŸ’»\n') + + result = subprocess.run( + [sys.executable, str(CHECK_MESSAGE), '--message-text', _compose('HEAD~1')], + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stdout diff --git a/support/scripts/check-commit-message.py b/support/scripts/check-commit-message.py index e7e70db0..e82dc297 100755 --- a/support/scripts/check-commit-message.py +++ b/support/scripts/check-commit-message.py @@ -108,22 +108,25 @@ def _first_match(patterns: list[re.Pattern], text: str) -> str | None: class CommitPart: title: str body: str = '' + bullet: bool = False def parse_commit(message: str) -> list[CommitPart]: """Split a commit message into parts to validate independently. - A squash/PR-merge commit (GitHub style) has bullet lines in the body that - are themselves conventional commit subjects. Each becomes its own CommitPart; - the umbrella title becomes the first part with any non-bullet body text. - A regular commit returns a single CommitPart. + A squash carries a bullet per collapsed commit, each its own subject and + body; the umbrella takes what precedes the first bullet. + + A `*` line is a bullet, full stop -- that is what lets squashing a squash + flatten rather than nest -- so one that is not a conventional subject is a + malformed bullet, not body text. """ lines = message.splitlines() title = lines[0] if lines else '' body_lines = lines[1:] if len(lines) > 1 else [] - squash_parts: list[CommitPart] = [] - non_bullet: list[str] = [] + parts = [CommitPart(title=title)] + bodies: list[list[str]] = [[]] for line in body_lines: stripped = line.strip() @@ -132,14 +135,14 @@ def parse_commit(message: str) -> list[CommitPart]: m = SQUASH_BULLET_RE.match(stripped) if m: bullet = stripped[m.end() :] - clean, _ = _split_trailing_emojis(PR_REF_RE.sub('', bullet)) - if COMMIT_FORMAT_RE.match(clean): - squash_parts.append(CommitPart(title=bullet)) - continue - non_bullet.append(line) + parts.append(CommitPart(title=bullet, bullet=True)) + bodies.append([]) + else: + bodies[-1].append(line) - umbrella = CommitPart(title=title, body='\n'.join(non_bullet).strip()) - return [umbrella, *squash_parts] if squash_parts else [umbrella] + for part, body in zip(parts, bodies): + part.body = '\n'.join(body).strip() + return parts def _check_subject(subject: str, errors: list[str], warnings: list[str]) -> None: @@ -175,6 +178,11 @@ def validate_part(part: CommitPart) -> tuple[list[str], list[str]]: warnings: list[str] = [] _check_subject(part.title, errors, warnings) + if errors and part.bullet: + errors.append( + ' A body line starting with `*` is a squash bullet, so it must be a ' + 'conventional commit subject.' + ) if hit := _first_match(AI_PATTERNS, part.title + '\n' + part.body): errors.append(f"AI attribution detected: '{hit}'.") @@ -202,9 +210,26 @@ def validate_part(part: CommitPart) -> tuple[list[str], list[str]]: return errors, warnings +def _check_control_chars(message: str, errors: list[str]) -> None: + """Reject control characters: a message is text, newlines and tabs. + + Anything else is invisible where the message is written and meaningful to + something downstream that reads it. + """ + for index, char in enumerate(message): + # `Cc` alone: emoji sequences are joined with `Cf`. + if char in '\n\t' or unicodedata.category(char) != 'Cc': + continue + errors.append( + f'Control character {char!r} at offset {index}; a commit message is ' + f'text, newlines and tabs.' + ) + + def validate(message: str) -> tuple[list[str], list[str]]: all_errors: list[str] = [] all_warnings: list[str] = [] + _check_control_chars(message, all_errors) for part in parse_commit(message): errors, warnings = validate_part(part) all_errors.extend(errors) diff --git a/support/scripts/get-all-git.py b/support/scripts/get-all-git.py index c43a7b7d..c2293673 100755 --- a/support/scripts/get-all-git.py +++ b/support/scripts/get-all-git.py @@ -8,12 +8,20 @@ import posixpath import re import shlex +import shutil import subprocess from collections.abc import Iterator from dataclasses import dataclass from pathlib import Path from urllib.parse import urlparse, urlunparse +# Namespace of the cache-side refs that keep the pinned commits reachable. +PIN_REF_PREFIX = 'refs/genvm/pins/' +# Where the refs of a checkout converted to a worktree are mirrored. Namespaced +# by manager checkout as well as by path: several manager worktrees share one +# cache, and each converts the same submodule paths. +CONVERTED_REF_PREFIX = 'refs/genvm/converted/' + @dataclass(frozen=True) class RepoGitInfo: @@ -100,13 +108,22 @@ def _read_gitmodules_entries(root: Path) -> dict[str, dict[str, str]]: def _gitlink_hash(root: Path, path: str) -> str: - ls_tree = git_output(['git', 'ls-tree', 'HEAD', path], root) - if not ls_tree: - raise RuntimeError(f'submodule path is not present in HEAD: {path}') - fields = ls_tree.split() + """The commit ``path`` is pinned to, read from the index like git does. + + Staged and committed agree except right after a gitlink bump, where the + index is the one the next commit will carry. + """ + entry = git_output(['git', 'ls-files', '--stage', '--', path], root) + if not entry: + raise RuntimeError(f'submodule path is not present in the index: {path}') + # One line per conflict side when unmerged; any pick would be arbitrary. + lines = entry.splitlines() + if len(lines) > 1: + raise RuntimeError(f'submodule is unmerged in the index: {path}') + fields = lines[0].split() if len(fields) < 3 or fields[0] != '160000': - raise RuntimeError(f'path is not a submodule gitlink in HEAD: {path}') - return fields[2] + raise RuntimeError(f'path is not a submodule gitlink: {path}') + return fields[1] def get_submodule_git_info(root: Path) -> list[SubmoduleGitInfo]: @@ -171,9 +188,9 @@ def cache_lock(root: Path) -> Iterator[None]: Serialize the shared submodule cache against another copy of this script. CI runs it from several jobs against the same checkout, and both the bare - cache and the worktrees that borrow it with ``--reference`` are mutated in - place: an interleaved init/fetch loses to git's own lockfiles, or leaves a - cache another job is already reading from half-updated. + cache and the worktrees checked out of it are mutated in place: an + interleaved init/fetch loses to git's own lockfiles, or leaves a cache + another job is already reading from half-updated. """ lock = git_common_dir(root) / 'genvm-submodule-cache.lock' with open(lock, 'w') as handle: @@ -185,7 +202,23 @@ def cache_lock(root: Path) -> Iterator[None]: yield -def ensure_cache(root: Path, url: str, hashes: list[str], env: dict[str, str]) -> Path: +def has_commit(repo: Path, commit: str) -> bool: + return ( + subprocess.run( + ['git', '-C', str(repo), 'cat-file', '-e', f'{commit}^{{commit}}'], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ).returncode + == 0 + ) + + +def open_cache(root: Path, url: str, env: dict[str, str]) -> Path: + """Create (if needed) and configure the bare cache repo for ``url``. + + Every submodule sharing this URL is checked out of this one repository, so + its objects are fetched once and its refs are what keeps them alive. + """ cache = cache_path_for_url(root, url) cache.parent.mkdir(parents=True, exist_ok=True) if not (cache / 'HEAD').exists(): @@ -203,28 +236,311 @@ def ensure_cache(root: Path, url: str, hashes: list[str], env: dict[str, str]) - run(['git', '-C', str(cache), 'remote', 'set-url', 'origin', url], root, env) else: run(['git', '-C', str(cache), 'remote', 'add', 'origin', url], root, env) - + # A bare repo has no refspec by default, so a plain `git fetch` from an + # executor checkout would update nothing. Give it the non-bare one. run( - ['git', '-C', str(cache), 'fetch', '--no-tags', 'origin', *hashes], + [ + 'git', + '-C', + str(cache), + 'config', + # Plain `config` fails once the key holds a second refspec. + '--replace-all', + 'remote.origin.fetch', + '+refs/heads/*:refs/remotes/origin/*', + ], root, env, ) + # Off by default in a bare repo, which would leave a commit made in a + # worktree with no trace at all once the checkout is moved onto a new + # gitlink. A non-bare submodule clone logged it, so keep that. + run(['git', '-C', str(cache), 'config', 'core.logAllRefUpdates', 'true'], root, env) + # Drop administrative entries of checkouts that are gone (a deleted manager + # worktree), so re-adding the same path does not collide with a stale one. + run(['git', '-C', str(cache), 'worktree', 'prune'], root, env) return cache -def set_remote(repo: Path, name: str, url: str, env: dict[str, str]) -> None: - remote_exists = ( +def fetch_pins( + root: Path, cache: Path, submodules: list[SubmoduleGitInfo], env: dict[str, str] +) -> None: + """Make sure every pinned commit is in ``cache``, and keep it reachable. + + An unreferenced commit is only kept alive by the worktree HEADs pointing at + it, and those checkouts do not exist yet β€” hence a ref per pin. + """ + missing = sorted( + {s.gitlink_hash for s in submodules if not has_commit(cache, s.gitlink_hash)} + ) + if missing: + run(['git', '-C', str(cache), 'fetch', '--no-tags', 'origin', *missing], root, env) + + for submodule in submodules: + run( + [ + 'git', + '-C', + str(cache), + 'update-ref', + f'{PIN_REF_PREFIX}{submodule.path}', + submodule.gitlink_hash, + ], + root, + env, + ) + + +def repo_gitdir(path: Path) -> Path | None: + """Absolute git dir of the checkout rooted at ``path``, else None.""" + if not (path / '.git').exists(): + return None + result = subprocess.run( + ['git', '-C', str(path), 'rev-parse', '--path-format=absolute', '--git-dir'], + capture_output=True, + text=True, + ) + if result.returncode != 0: + return None + return Path(result.stdout.strip()).resolve() + + +def current_branch(path: Path) -> str: + """The branch checked out at ``path``, or '' when HEAD is detached.""" + result = subprocess.run( + ['git', '-C', str(path), 'symbolic-ref', '--quiet', '--short', 'HEAD'], + capture_output=True, + text=True, + ) + return result.stdout.strip() if result.returncode == 0 else '' + + +def ref_exists(repo: Path, ref: str) -> bool: + return ( subprocess.run( - ['git', '-C', str(repo), 'remote', 'get-url', name], + ['git', '-C', str(repo), 'show-ref', '--verify', '--quiet', ref], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ).returncode == 0 ) - if remote_exists: - run(['git', '-C', str(repo), 'remote', 'set-url', name, url], repo, env) + + +def has_staged_changes(path: Path) -> bool: + return ( + subprocess.run( + ['git', '-C', str(path), 'diff', '--cached', '--quiet'], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ).returncode + != 0 + ) + + +def clear_staging(staging: Path, name: str) -> None: + """Remove ``staging``, provided it is a leftover of an interrupted conversion. + + Which holds one `--no-checkout` worktree, so at most a `.git` file. Anything + else under that name is somebody's data. + """ + expected = {staging / name, staging / name / '.git'} + unexpected = sorted(p for p in staging.rglob('*') if p not in expected) + if unexpected: + raise RuntimeError( + f'{staging} is not a leftover conversion staging directory ' + f'({unexpected[0]} is not part of one); move it aside and re-run' + ) + shutil.rmtree(staging) + + +def adopt_as_worktree( + root: Path, cache: Path, dest: Path, head: str, env: dict[str, str] +) -> None: + """Give the populated directory ``dest`` the `.git` of a worktree at ``head``. + + `git worktree add` cannot be pointed at a directory that has files in it, so + the worktree is created empty beside it and only its `.git` file moves. No + file under ``dest`` is read, written or removed. + """ + # Same basename as the checkout, so the administrative entry the cache ends + # up with is named after the submodule rather than after this staging path. + staging = dest.parent / f'.genvm-convert-{dest.name}' + if staging.exists(): + clear_staging(staging, dest.name) + # Removing the staging tree of an interrupted conversion strands its entry, + # and `worktree add` refuses to reuse a path that is still registered. + run(['git', '-C', str(cache), 'worktree', 'prune'], root, env) + run( + [ + 'git', + '-C', + str(cache), + 'worktree', + 'add', + '--detach', + '--no-checkout', + str(staging / dest.name), + head, + ], + root, + env, + ) + # Fill the index `--no-checkout` left empty before the swap, so the state + # after it is complete. The index is built from HEAD alone; that the staging + # tree has no files in it does not matter, and none of the real ones move. + run( + ['git', '-C', str(staging / dest.name), 'reset', '--mixed', '--quiet', 'HEAD'], + root, + env, + ) + # One atomic rename over the old gitlink file: an interrupted conversion + # leaves a checkout that still belongs to one of the two repos, never one + # with no `.git` at all. + os.replace(staging / dest.name / '.git', dest / '.git') + # Before the staging path goes, or the next conversion's `worktree prune` + # drops an entry naming it and strands `dest` on a deleted gitdir. + run(['git', '-C', str(cache), 'worktree', 'repair', str(dest)], root, env) + shutil.rmtree(staging) + + +def convert_from_dangling( + root: Path, cache: Path, submodule: SubmoduleGitInfo, env: dict[str, str] +) -> None: + """Re-attach a checkout whose `.git` names a gitdir that is gone. + + The files are all there is to go on, so the checkout is re-attached at the + gitlink. Files that the old checkout had at some other commit stay as they + are and show up as ordinary modifications. + """ + adopt_as_worktree(root, cache, root / submodule.path, submodule.gitlink_hash, env) + + +def convert_to_worktree( + root: Path, + cache: Path, + submodule: SubmoduleGitInfo, + gitdir: Path | None, + env: dict[str, str], +) -> None: + """Re-point an existing checkout at ``cache``, keeping every file in place. + + Deleting and re-adding the directory would take the ignored content with it + β€” the materialized third-party trees above all β€” so only the `.git` file is + replaced: `git worktree add` cannot be pointed at a populated directory, but + an empty worktree can be created next to it and its `.git` moved over. The + abandoned gitdir may hold branches that exist nowhere else, so its refs are + mirrored into the cache first. ``gitdir`` is None when that file names one + that no longer exists, which leaves nothing to salvage but the files. + """ + dest = root / submodule.path + if (dest / '.git').is_dir(): + raise RuntimeError( + f'{submodule.path} is a standalone clone, not a submodule checkout; ' + f'move it aside and re-run' + ) + + print(f'+ converting {submodule.path} to a worktree of {cache}', flush=True) + if gitdir is None: + # Nothing is left to salvage but the files: whatever refs that gitdir + # held are wherever it went. + convert_from_dangling(root, cache, submodule, env) + return + + # The old index is abandoned with the gitdir, and `reset` rebuilds it from + # HEAD: unstaged and untracked work survives that, staged work would not. + if has_staged_changes(dest): + raise RuntimeError( + f'{submodule.path} has staged changes and cannot be converted to a ' + f'worktree; commit or unstage them and re-run' + ) + + branch = current_branch(dest) + head = git_output(['git', 'rev-parse', 'HEAD'], dest) + digest = hashlib.sha256(str(root).encode()).hexdigest()[:16] + namespace = f'{CONVERTED_REF_PREFIX}{digest}/{submodule.path}/' + # Everything under `refs/`, not just the branches: a stash, a local tag or a + # note can be the only thing holding on to a commit. + run( + [ + 'git', + '-C', + str(cache), + 'fetch', + '--no-tags', + str(gitdir), + f'+refs/*:{namespace}*', + f'+HEAD:{namespace}HEAD', + ], + root, + env, + ) + + adopt_as_worktree(root, cache, dest, head, env) + + # The worktree starts detached; put the branch back so a checkout converted + # mid-feature comes out on the branch it was on. The cache is shared, so a + # name already taken there belongs to another checkout and stays untouched. + if not branch: + return + if ref_exists(cache, f'refs/heads/{branch}'): + print( + f'+ {submodule.path} was on {branch}, which the cache already has; ' + f'leaving it detached, its refs are under {namespace}', + flush=True, + ) else: - run(['git', '-C', str(repo), 'remote', 'add', name, url], repo, env) + run(['git', '-C', str(dest), 'checkout', '-b', branch], root, env) + + +def convert_if_needed( + root: Path, cache: Path, submodule: SubmoduleGitInfo, env: dict[str, str] +) -> None: + """Convert the checkout at ``submodule.path`` unless it is already ours.""" + dest = root / submodule.path + if not (dest / '.git').exists(): + return + gitdir = repo_gitdir(dest) + if gitdir is None or cache.resolve() not in gitdir.parents: + convert_to_worktree(root, cache, submodule, gitdir, env) + + +def checkout_worktree( + root: Path, cache: Path, submodule: SubmoduleGitInfo, env: dict[str, str] +) -> None: + """Make ``submodule.path`` a worktree of ``cache`` parked on the gitlink.""" + dest = root / submodule.path + convert_if_needed(root, cache, submodule, env) + gitdir = repo_gitdir(dest) + + if gitdir is None: + if dest.exists() and any(dest.iterdir()): + raise RuntimeError(f'{submodule.path} is not empty and is not a git checkout') + run( + [ + 'git', + '-C', + str(cache), + 'worktree', + 'add', + '--detach', + str(dest), + submodule.gitlink_hash, + ], + root, + env, + ) + return + + # Idempotent, and the only thing that fixes the entry after the manager + # checkout is moved on disk or a conversion is interrupted before this step. + run(['git', '-C', str(cache), 'worktree', 'repair', str(dest)], root, env) + + if git_output(['git', 'rev-parse', 'HEAD'], dest) != submodule.gitlink_hash: + run( + ['git', '-C', str(dest), 'checkout', '--detach', submodule.gitlink_hash], + root, + env, + ) def update_submodules_with_cache(root: Path, env: dict[str, str]) -> None: @@ -238,38 +554,19 @@ def _update_submodules_with_cache(root: Path, env: dict[str, str]) -> None: for submodule in submodules: by_url.setdefault(submodule.url, []).append(submodule) - cache_by_url = { - url: ensure_cache( - root, - url, - sorted({submodule.gitlink_hash for submodule in url_submodules}), - env, - ) - for url, url_submodules in by_url.items() - } + cache_by_url = {url: open_cache(root, url, env) for url in by_url} + # Convert before fetching: a checkout being converted hands its refs to the + # cache, and one of them may be the pinned commit itself β€” an unpushed + # gitlink bump, which no fetch from `origin` could resolve. for submodule in submodules: - cache = cache_by_url[submodule.url] - run( - [ - 'git', - 'submodule', - 'update', - '--init', - '--recursive', - '--reference', - str(cache), - '--depth', - '1', - '--no-fetch', - '--', - submodule.path, - ], - root, - env, - ) - set_remote(root / submodule.path, 'origin', submodule.url, env) - set_remote(root / submodule.path, 'cache', str(cache), env) + convert_if_needed(root, cache_by_url[submodule.url], submodule, env) + + for url, url_submodules in by_url.items(): + fetch_pins(root, cache_by_url[url], url_submodules, env) + + for submodule in submodules: + checkout_worktree(root, cache_by_url[submodule.url], submodule, env) def main() -> None: diff --git a/support/tools/genvm-tool/genvm_tool/tests/__init__.py b/support/tools/genvm-tool/genvm_tool/tests/__init__.py index 12545693..3c372951 100644 --- a/support/tools/genvm-tool/genvm_tool/tests/__init__.py +++ b/support/tools/genvm-tool/genvm_tool/tests/__init__.py @@ -50,6 +50,8 @@ class SharedContext: logger: Formatter printer: Sink watchdog: Watchdog = field(default_factory=Watchdog.start) + metrics: dict[str, Any] = field(default_factory=dict) + _metrics_lock: threading.Lock = field(default_factory=threading.Lock) # When running a test case, this points to that case's per-test artifact # directory. ``None`` outside of a test case. @@ -66,6 +68,11 @@ class SharedContext: _interrupted: threading.Event = field(default_factory=threading.Event) _config: dict[str, Any] | None = None + def bump_metric[T](self, name: str, initial: T, delta: T) -> None: + """Increment a named metric by ``delta``, locked: collectors are threads.""" + with self._metrics_lock: + self.metrics[name] = self.metrics.setdefault(name, initial) + delta + @property def config(self) -> dict[str, Any]: """Runner config (``artifacts_dir`` / ``extra_python_paths``). diff --git a/support/tools/genvm-tool/genvm_tool/tests/cli.py b/support/tools/genvm-tool/genvm_tool/tests/cli.py index da95999e..4d83c6a5 100755 --- a/support/tools/genvm-tool/genvm_tool/tests/cli.py +++ b/support/tools/genvm-tool/genvm_tool/tests/cli.py @@ -440,4 +440,5 @@ async def foo(): else: func(shared_context, conf_env) finally: + shared_context.logger.debug('metrics', **shared_context.metrics) shared_context.watchdog.stop() diff --git a/support/tools/genvm-tool/genvm_tool/tests/stage/pipeline.py b/support/tools/genvm-tool/genvm_tool/tests/stage/pipeline.py index 7266315c..d58d1f63 100644 --- a/support/tools/genvm-tool/genvm_tool/tests/stage/pipeline.py +++ b/support/tools/genvm-tool/genvm_tool/tests/stage/pipeline.py @@ -30,6 +30,7 @@ async def run(ctx: SharedContext, arg: A) -> R: res = await stage.run(ctx, arg) end_t = time.monotonic() delta_t = end_t - start_t + ctx.bump_metric(f'stage_time.{stage.name}', 0.0, delta_t) ctx.logger.trace('stage end', name=stage.name, elapsed=delta_t) if delta_t > 1.0 and not stage.is_slow: ctx.logger.warning('stage is slow', name=stage.name, elapsed=delta_t) diff --git a/tests/runner/genvm_tool_plugins/integration.py b/tests/runner/genvm_tool_plugins/integration.py index 910e5b7a..b2c24198 100644 --- a/tests/runner/genvm_tool_plugins/integration.py +++ b/tests/runner/genvm_tool_plugins/integration.py @@ -6,6 +6,7 @@ """ import base64 +import contextlib import difflib import gzip import hashlib @@ -16,7 +17,10 @@ import pickle import re import shutil +import sqlite3 import sys +import threading +import time import typing from dataclasses import dataclass from pathlib import Path @@ -34,6 +38,9 @@ from gvm_extra.mock_host import MockStorage as MockStorage from origin.calldata import Address +if typing.TYPE_CHECKING: + import _jsonnet + # Get the local context local_ctx = genvm_tool.tests.stage.configuration.current_context() @@ -1011,6 +1018,215 @@ def integration_test( ) +class RWLock: + """ + Many concurrent readers or a single writer, writers taking priority. + + A reader that arrives while a writer waits queues behind it, so a steady + stream of lookups cannot starve a store. + """ + + def __init__(self) -> None: + self._cond = threading.Condition() + self._readers = 0 + self._writer = False + self._writers_waiting = 0 + + @contextlib.contextmanager + def read(self) -> typing.Iterator[None]: + with self._cond: + self._cond.wait_for(lambda: not self._writer and not self._writers_waiting) + self._readers += 1 + try: + yield + finally: + with self._cond: + self._readers -= 1 + if self._readers == 0: + self._cond.notify_all() + + @contextlib.contextmanager + def write(self) -> typing.Iterator[None]: + with self._cond: + self._writers_waiting += 1 + try: + self._cond.wait_for(lambda: not self._writer and not self._readers) + finally: + self._writers_waiting -= 1 + self._writer = True + try: + yield + finally: + with self._cond: + self._writer = False + self._cond.notify_all() + + +def _jsonnet_lib_dirs() -> list[str]: + """The search path a jsonnet vm starts with, in the order it is seeded.""" + return [ + f'/usr/share/jsonnet-{_jsonnet.version}/', + f'/usr/local/share/jsonnet-{_jsonnet.version}/', + ] + + +def _digest_file(path: Path) -> str: + """Digest of ``path``'s contents, or '' when it is gone.""" + try: + return hashlib.sha3_256(path.read_bytes()).hexdigest() + except OSError: + return '' + + +def _resolve_import(base: str, rel: str, jpathdir: list[str]) -> Path: + """Where jsonnet would find ``rel``, imported from a file in ``base``. + + Unresolved: a nested import resolves against this path. + """ + if os.path.isabs(rel): + candidates = [Path(rel)] + else: + candidates = [Path(base) / rel] + candidates.extend(Path(d) / rel for d in reversed(jpathdir)) + candidates.extend(Path(d) / rel for d in reversed(_jsonnet_lib_dirs())) + for candidate in candidates: + if candidate.is_file(): + return candidate + raise RuntimeError(f'no such file to import: {rel} (from {base})') + + +class JsonnetCache: + """ + Memoizes jsonnet evaluation in a sqlite file among the test artifacts. + + Keyed by the case file's path and contents, carrying every import it read so + that editing one of those misses too. Entries older than a day are dropped. + """ + + MAX_AGE = 24 * 60 * 60 + COLUMNS = {'digest', 'stored_at', 'deps', 'result'} + + def __init__(self) -> None: + # A native callback rewritten here, or a jsonnet upgrade, changes results + # with no input file changing. + self._evaluator_digest = hashlib.sha3_256( + _jsonnet.version.encode('utf-8') + Path(__file__).read_bytes() + ).digest() + + artifacts_dir = local_ctx.shared.artifacts_dir + artifacts_dir.mkdir(exist_ok=True, parents=True) + self._path = artifacts_dir / 'jsonnet-cache.sqlite' + self._lock = RWLock() + # One connection per thread: a shared one would serialize the readers + # that the lock lets run together. + self._local = threading.local() + + with self._lock.write(): + db = self._connection() + db.execute('PRAGMA journal_mode=WAL') + columns = { + row[1] for row in db.execute('PRAGMA table_info(evaluated)').fetchall() + } + if columns and columns != self.COLUMNS: + db.execute('DROP TABLE evaluated') + db.execute( + 'CREATE TABLE IF NOT EXISTS evaluated (' + 'digest BLOB PRIMARY KEY, stored_at REAL NOT NULL, ' + 'deps TEXT NOT NULL, result TEXT NOT NULL' + ') WITHOUT ROWID' + ) + db.execute( + 'DELETE FROM evaluated WHERE stored_at < ?', (time.time() - self.MAX_AGE,) + ) + db.commit() + + def _connection(self) -> sqlite3.Connection: + db = getattr(self._local, 'db', None) + if db is None: + db = sqlite3.connect(self._path) + # Per connection, unlike the journal mode. Losing the tail of a cache + # to a power cut costs a re-evaluation. + db.execute('PRAGMA synchronous=NORMAL') + self._local.db = db + return db + + @staticmethod + def _deps_hold(deps: list[list[str]], jpathdir: list[str]) -> bool: + """Whether every import still resolves where it did, to the same bytes. + + Re-resolved, not just re-digested: a new file of that name beside the + case steals an import that fell through to the search path. + """ + for base, rel, found, digest in deps: + try: + if str(_resolve_import(base, rel, jpathdir)) != found: + return False + except RuntimeError: + return False + if _digest_file(Path(found)) != digest: + return False + return True + + def eval(self, path: Path, *args, jpathdir: list[str], **kwargs) -> str: + # Keyed on the path too: two identical cases in different directories + # import different files under the same relative names. + full_digest = hashlib.sha3_256( + self._evaluator_digest + + str(path.resolve()).encode('utf-8') + + b'\0' + + path.read_bytes() + ).digest() + + eval_start = time.monotonic() + + with self._lock.read(): + row = ( + self._connection() + .execute('SELECT deps, result FROM evaluated WHERE digest = ?', (full_digest,)) + .fetchone() + ) + cache_check_end = time.monotonic() + local_ctx.shared.bump_metric( + 'jsonnet_cache_check_time', 0.0, cache_check_end - eval_start + ) + + if row is not None and self._deps_hold(json.loads(row[0]), jpathdir): + return row[1] + + deps: list[list[str]] = [] + + def import_callback(base: str, rel: str) -> tuple[str, bytes]: + found = _resolve_import(base, rel, jpathdir) + content = found.read_bytes() + deps.append([base, rel, str(found), hashlib.sha3_256(content).hexdigest()]) + return str(found), content + + # Replaces the importer wholesale, `jpathdir` included. + result = _jsonnet.evaluate_file( + str(path), + *args, + import_callback=import_callback, + **kwargs, + ) + + eval_end = time.monotonic() + local_ctx.shared.bump_metric('jsonnet_eval_time', 0.0, eval_end - cache_check_end) + + with self._lock.write(): + db = self._connection() + db.execute( + 'INSERT OR REPLACE INTO evaluated (digest, stored_at, deps, result) ' + 'VALUES (?, ?, ?, ?)', + (full_digest, time.time(), json.dumps(deps), result), + ) + db.commit() + + write_end = time.monotonic() + local_ctx.shared.bump_metric('jsonnet_cache_write_time', 0.0, write_end - eval_end) + + return result + + def integration_test_single_executor( ctx: genvm_tool.tests.stage.collection.Context, *, @@ -1030,6 +1246,10 @@ def integration_test_single_executor( Steps depend on /prepare, and child steps depend on their parent step. """ + import _jsonnet + + globals()['_jsonnet'] = _jsonnet # type: ignore + import genvm_tool.common EXEC_SUBDIR = os.environ.get( @@ -1053,21 +1273,26 @@ def integration_test_single_executor( 'save-hashes', not executor_integration_conf.get('ignore-hash', False) ) + glob_start = time.monotonic() jsonnet_files = list( x for x in CASES_DIR.glob('**/*.jsonnet') if not x.name.startswith('_') ) + glob_end = time.monotonic() + # Bumped, not assigned: this runs once per executor line. + local_ctx.shared.bump_metric('jsonnet_glob_time', 0.0, glob_end - glob_start) jsonnet_files.sort() - # pre-import - from concurrent.futures import ThreadPoolExecutor, as_completed + jsonnet_cache = JsonnetCache() + with ThreadPoolExecutor(thread_name_prefix='integration-test-collector') as executor: futures = { executor.submit( _single_integration_test, ctx, jsonnet_file, + jsonnet_cache, cases_dir=CASES_DIR, reroute_to=reroute_to, save_hashes=save_hashes, @@ -1090,6 +1315,7 @@ def _gvm32_of_str(text: str) -> str: def _single_integration_test( ctx: genvm_tool.tests.stage.collection.Context, jsonnet_file: Path, + jsonnet_cache: JsonnetCache, *, cases_dir: Path, reroute_to: str, @@ -1099,8 +1325,6 @@ def _single_integration_test( webdriver_service: genvm_tool.tests.stage.collection.Service, ci: bool, ) -> None: - import _jsonnet - rel_path = jsonnet_file.relative_to(cases_dir) # Determine stability tag from path @@ -1141,8 +1365,8 @@ def _single_integration_test( # Note the binding only marshals *primitives* β€” passing an array or object # to a native function is a jsonnet runtime error, so anything structured # has to cross as a JSON string. - jsonnet_result = _jsonnet.evaluate_file( - str(jsonnet_file), + jsonnet_result = jsonnet_cache.eval( + jsonnet_file, jpathdir=[str(TEMPLATES_DIR.parent)], native_callbacks={ # std.native('gvm32')('...') -> gvm32(sha3_256(utf8(s))), the id form @@ -1150,6 +1374,7 @@ def _single_integration_test( 'gvm32': (('text',), _gvm32_of_str), }, ) + jsonnet_parsed = json.loads(jsonnet_result) extra_tags = jsonnet_parsed.get('tags', []) if extra_tags: diff --git a/tests/system/parse_version/test.py b/tests/system/parse_version/test.py index fa162c9a..9cdbd943 100644 --- a/tests/system/parse_version/test.py +++ b/tests/system/parse_version/test.py @@ -33,7 +33,9 @@ def _add_case( artifacts_dir: Path, ) -> None: name = f'tests/system/parse_version/{wat_file.stem}' - desc = genvm_tool.tests.test.Description(name).with_tags(['parse_version']) + desc = genvm_tool.tests.test.Description(name).with_tags( + ['integration', 'parse_version'] + ) test_env = _default_env() diff --git a/tests/system/permits/test.py b/tests/system/permits/test.py index db38354e..97685f15 100644 --- a/tests/system/permits/test.py +++ b/tests/system/permits/test.py @@ -275,7 +275,7 @@ def collect( desc = genvm_tool.tests.test.Description( name='tests/system/permits', needed_services=frozenset({manager_service}), - tags=frozenset({'stable', 'permits'}), + tags=frozenset({'integration', 'stable', 'permits'}), console_pool=True, ) ctx.add_case( From 12a0b4d0f0e3a4c02cf3257177be0d728f09d540 Mon Sep 17 00:00:00 2001 From: kp2pml30 <kp2pml30@gmail.com> Date: Thu, 6 Aug 2026 16:38:24 +0900 Subject: [PATCH 08/45] feat: rework manager api for cross-major calls (#9) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(manager): websocket run protocol and cross-major contract calls * feat(manager): run cross-major contract calls as nested runs ✨ * feat(manager): make cross-contract call resolution opt-in ✨ * feat(manager): refuse nested envelopes asserting unserviceable permissions πŸ”’οΈ * fix(manager): refuse a nested reply carrying any reported effect πŸ”’οΈπŸ› * fix(manager): stop the call route from being consensus-visible πŸ›πŸ”’οΈ * refactor(manager): serve executions over an event-driven websocket ♻️ * fix(manager): harden the run socket protocol πŸ”’οΈπŸ› * fix(manager): drop runtime env mutation and open the lua stdlib πŸ”’οΈ * chore(codegen): generate host-fns and socket ids from one definition ♻️ * refactor(harness): compare the executor execution hash directly ♻️ * test(harness): route a call across a major boundary from jsonnet βœ… * fix(manager): report consumed_result as bytes, not an integer array πŸ› --- .genvm-tool.py | 3 + crates/modules-interfaces/Cargo.lock | 1 + crates/modules-interfaces/Cargo.toml | 1 + .../codegen/data/host-fns.json | 48 + .../codegen/data/manager-api.json | 37 + crates/modules-interfaces/src/domain.rs | 207 +- crates/modules-interfaces/src/host_fns.rs | 158 ++ crates/modules-interfaces/src/lib.rs | 4 + crates/modules-interfaces/src/manager_api.rs | 139 ++ crates/modules-interfaces/src/nested.rs | 286 +++ docs/adr/014. manager host socket protocol.md | 337 +++ docs/adr/015. cross-major contract calls.md | 363 ++++ .../explanation/executor-lines.md | 10 +- docs/contributing/howto/building/docs.md | 2 +- .../howto/extending/add-host-function.md | 6 +- docs/contributing/howto/genvm-tool.md | 5 +- .../contributing/howto/testing/integration.md | 17 + .../src/impl-spec/appendix/host-loop.rst | 80 +- docs/website/src/impl-spec/appendix/index.rst | 1 + .../src/impl-spec/appendix/manager-api.rst | 10 +- .../src/impl-spec/appendix/manager-api.yaml | 366 +++- .../appendix/manager-socket-consts.rst | 130 ++ .../src/impl-spec/appendix/manager-socket.rst | 319 +++ .../03-wasi_genlayer_sdk/02-gl_call.rst | 8 + docs/website/src/spec/03-vm/01-startup.rst | 14 +- .../src/spec/03-vm/02-meta-properties.rst | 5 + docs/website/src/spec/03-vm/05-result.rst | 6 + .../04-upgradability.rst | 9 +- docs/website/src/spec/appendix/constants.rst | 2 + executors/v0.2.x | 2 +- executors/v0.3.x | 2 +- implementation/Cargo.lock | 396 ++-- implementation/Cargo.toml | 14 +- implementation/src/manager/handlers.rs | 204 +- implementation/src/manager/handlers_test.rs | 56 + implementation/src/manager/mod.rs | 544 ++--- implementation/src/manager/run.rs | 1879 ++++++++++++++--- implementation/src/manager/run_test.rs | 832 ++++++++ implementation/src/manager/socket.rs | 775 +++++++ implementation/src/manager/socket_test.rs | 239 +++ implementation/src/manager/versioning.rs | 145 +- implementation/src/manager/versioning_test.rs | 66 + implementation/src/scripting/mod.rs | 3 +- install/config/genvm-manager.yaml | 2 + .../genvm-tool/genvm_tool/cmd_configure.py | 28 +- tests/runner/genvm_tool_plugins/genvm.py | 2 + .../runner/genvm_tool_plugins/integration.py | 135 +- tests/runner/genvm_tool_plugins/ninja.py | 5 - tests/runner/gvm_extra/mock_host.py | 187 +- tests/runner/origin/base_host.py | 1442 +++++++++---- tests/runner/origin/host_fns.py | 3 +- tests/runner/origin/manager_api.py | 31 + .../system/cross-major-observability/test.py | 674 ++++++ tests/system/cross-major/test.py | 1459 +++++++++++++ tests/system/manager-socket/test.py | 1034 +++++++++ tests/system/permits/test.py | 232 +- 56 files changed, 11318 insertions(+), 1647 deletions(-) create mode 100644 crates/modules-interfaces/codegen/data/host-fns.json create mode 100644 crates/modules-interfaces/codegen/data/manager-api.json create mode 100644 crates/modules-interfaces/src/host_fns.rs create mode 100644 crates/modules-interfaces/src/manager_api.rs create mode 100644 crates/modules-interfaces/src/nested.rs create mode 100644 docs/adr/014. manager host socket protocol.md create mode 100644 docs/adr/015. cross-major contract calls.md create mode 100644 docs/website/src/impl-spec/appendix/manager-socket-consts.rst create mode 100644 docs/website/src/impl-spec/appendix/manager-socket.rst create mode 100644 implementation/src/manager/handlers_test.rs create mode 100644 implementation/src/manager/run_test.rs create mode 100644 implementation/src/manager/socket.rs create mode 100644 implementation/src/manager/socket_test.rs create mode 100644 implementation/src/manager/versioning_test.rs create mode 100644 tests/runner/origin/manager_api.py create mode 100644 tests/system/cross-major-observability/test.py create mode 100644 tests/system/cross-major/test.py create mode 100644 tests/system/manager-socket/test.py diff --git a/.genvm-tool.py b/.genvm-tool.py index 5f43780f..35b4bf13 100644 --- a/.genvm-tool.py +++ b/.genvm-tool.py @@ -252,6 +252,9 @@ def collect_integration(ctx: genvm_tool.tests.stage.collection.Context): ) ctx.collect_dir('tests/system/permits', manager_service=manager_service) + ctx.collect_dir('tests/system/manager-socket') + ctx.collect_dir('tests/system/cross-major') + ctx.collect_dir('tests/system/cross-major-observability') ctx.add_collector(collect_integration) diff --git a/crates/modules-interfaces/Cargo.lock b/crates/modules-interfaces/Cargo.lock index 66beea9d..433aeb5d 100644 --- a/crates/modules-interfaces/Cargo.lock +++ b/crates/modules-interfaces/Cargo.lock @@ -293,6 +293,7 @@ dependencies = [ "serde_bytes", "serde_derive", "serde_json", + "sha3", "tokio", ] diff --git a/crates/modules-interfaces/Cargo.toml b/crates/modules-interfaces/Cargo.toml index bbc07fdd..63a37027 100644 --- a/crates/modules-interfaces/Cargo.toml +++ b/crates/modules-interfaces/Cargo.toml @@ -21,6 +21,7 @@ serde = { version = "1.0.219", features = ["rc", "derive"] } serde_bytes = "0.11.17" serde_derive = "1.0.219" serde_json = "1.0.140" +sha3 = "0.10" tokio = { version = "1.44.1", features = [ "rt", "rt-multi-thread", diff --git a/crates/modules-interfaces/codegen/data/host-fns.json b/crates/modules-interfaces/codegen/data/host-fns.json new file mode 100644 index 00000000..245044ac --- /dev/null +++ b/crates/modules-interfaces/codegen/data/host-fns.json @@ -0,0 +1,48 @@ +[ + { + "type": "enum", + "repr": "u8", + "name": "methods", + "values": { + "storage_read": 0, + "consume_fuel": 1, + "eth_call": 2, + "get_balance": 3, + "remaining_fuel_as_gen": 4, + "notify_nondet_disagreement": 5, + "consume_result": 6, + "resolve_callcontract_executor": 7, + "run_nested": 8 + } + }, + { + "type": "enum", + "repr": "u8", + "name": "errors", + "values": { + "ok": 0, + "evm_reverted": 1, + "forbidden": 2 + } + }, + { + "type": "str_trie", + "name": "vm_error_detail", + "values": [ + "internal", + "external" + ] + }, + { + "type": "const", + "name": "current_major", + "repr": "u8", + "value": 0 + }, + { + "type": "const", + "name": "current_major_str", + "repr": "str", + "value": "v0.0.0" + } +] diff --git a/crates/modules-interfaces/codegen/data/manager-api.json b/crates/modules-interfaces/codegen/data/manager-api.json new file mode 100644 index 00000000..a9bf09d7 --- /dev/null +++ b/crates/modules-interfaces/codegen/data/manager-api.json @@ -0,0 +1,37 @@ +[ + { + "type": "enum", + "repr": "u16", + "name": "methods", + "values": { + "error": 0, + "hello": 1, + "event": 2, + "run": 3, + "attach": 4, + "cancel": 5, + "ack": 6, + "get_artifact": 7 + } + }, + { + "type": "enum", + "repr": "u8", + "name": "errors", + "values": { + "internal": 0, + "malformed_frame": 1, + "unknown_method": 2, + "unknown_id": 3, + "boot_id_mismatch": 4, + "bad_request_id": 5, + "not_finished": 6 + } + }, + { + "type": "const", + "name": "current_major", + "repr": "u32", + "value": 0 + } +] diff --git a/crates/modules-interfaces/src/domain.rs b/crates/modules-interfaces/src/domain.rs index bba8b895..aec09051 100644 --- a/crates/modules-interfaces/src/domain.rs +++ b/crates/modules-interfaces/src/domain.rs @@ -7,6 +7,14 @@ fn default_record_actions() -> Vec<String> { Vec::new() } +fn default_host_hello_data() -> Vec<Bytes> { + Vec::new() +} + +fn default_none<T>() -> Option<T> { + None +} + #[derive( Debug, Clone, @@ -23,6 +31,10 @@ pub struct ExecutionData { pub leader_nondet_results: Option<Vec<Bytes>>, /// Maps each host method (by index) to a host id. When empty, all methods use host 0. pub method_hosts: Vec<u8>, + /// Raw bytes written to each host connection before the first method byte. + #[serde(default)] + #[calldata(default = default_host_hello_data)] + pub host_hello_data: Vec<Bytes>, pub bucket_totals: Vec<num_bigint::BigInt>, /// Host-provided `node` fee constants (moved off `host_data`). pub gas_data: std::collections::BTreeMap<String, String>, @@ -34,10 +46,49 @@ pub struct ExecutionData { #[serde(default)] #[calldata(default = default_record_actions)] pub record_actions: Vec<String>, + /// Recursion budget imported by a nested execution, or forced by a debug + /// override. Absent means the execution starts from the executor's own + /// limit. + #[serde(default)] + #[calldata(default = default_none)] + pub remaining_recursion: Option<u32>, + /// Present exactly when this execution is nested inside another one. + #[serde(default)] + #[calldata(default = default_none)] + pub nested: Option<NestedExecutionData>, +} + +/// The execution state a nested run imports from the caller it was delegated +/// by. +/// +/// These fields only mean anything together, so they travel together: a run is +/// nested or it is not, and there is no way to spell a half-nested one. Notably +/// *not* here is `remaining_recursion`, which a chain root can also carry. +#[derive( + Debug, + Clone, + serde::Serialize, + serde::Deserialize, + genlayer_calldata::Encode, + genlayer_calldata::Decode, +)] +pub struct NestedExecutionData { + /// Explicit memory budget for the nested executor process. + pub memory_limit: u32, + /// View-call stack imported from the caller. + pub stack: Vec<genlayer_calldata::Address>, + /// Permissions the caller derived for the callee. + pub permissions: crate::NestedPermissions, + /// Storage view the caller resolved for the callee. + pub state_mode: crate::NestedStorageType, + /// Entry runner the caller resolved for the callee. + pub topmost_runner_id: crate::NestedRunnerId, + /// Deterministic fuel left to the chain this run belongs to. + pub remaining_det_fuel: primitive_types::U256, } #[allow(clippy::enum_variant_names)] -#[derive(Debug, Clone, PartialEq, Eq, genlayer_calldata::Encode)] +#[derive(Debug, Clone, PartialEq, Eq, genlayer_calldata::Encode, genlayer_calldata::Decode)] #[calldata(tag = "type")] pub enum ExecutionEmission { EthSend { @@ -125,7 +176,15 @@ impl<W: genlayer_calldata::Writer> genlayer_calldata::codec::Encode<W> for Stora } } -#[derive(Debug, Clone, serde::Serialize, PartialEq, Eq, genlayer_calldata::Encode)] +#[derive( + Debug, + Clone, + serde::Serialize, + PartialEq, + Eq, + genlayer_calldata::Encode, + genlayer_calldata::Decode, +)] pub struct Frame { pub module_name: String, pub func: u32, @@ -146,6 +205,13 @@ impl<W: genlayer_calldata::Writer> genlayer_calldata::codec::Encode<W> for Backt } } +/// Decode mirror of [`ModuleFingerprint`]; the encoded form uses byte strings +/// and there is no `Decode` for fixed-size arrays. +#[derive(genlayer_calldata::Decode)] +struct ModuleFingerprintDe { + memories: Vec<Bytes>, +} + #[derive(Debug, Clone, serde::Serialize, PartialEq, Eq)] pub struct ModuleFingerprint { pub memories: Vec<[u8; 32]>, @@ -165,6 +231,26 @@ impl<W: genlayer_calldata::Writer> genlayer_calldata::codec::Encode<W> for Modul } } +impl genlayer_calldata::codec::Decode for ModuleFingerprint { + fn decode<D: genlayer_calldata::codec::Deserializer>( + deserializer: D, + ) -> Result<Self, genlayer_calldata::codec::DecodeError> { + let raw = ModuleFingerprintDe::decode(deserializer)?; + let memories = raw + .memories + .into_iter() + .map(|memory| { + <[u8; 32]>::try_from(memory.as_ref()).map_err(|_| { + genlayer_calldata::codec::DecodeError::Unexpected( + "module fingerprint memory hash must be 32 bytes", + ) + }) + }) + .collect::<Result<Vec<_>, _>>()?; + Ok(Self { memories }) + } +} + #[derive(Debug, Clone, PartialEq, Eq, Default)] pub struct WasmStoreHashes(pub std::collections::BTreeMap<String, ModuleFingerprint>); @@ -181,7 +267,38 @@ impl<W: genlayer_calldata::Writer> genlayer_calldata::codec::Encode<W> for WasmS } } -#[derive(Debug, Clone, Copy, Default, genlayer_calldata::Encode)] +impl genlayer_calldata::codec::Decode for WasmStoreHashes { + fn decode<D: genlayer_calldata::codec::Deserializer>( + deserializer: D, + ) -> Result<Self, genlayer_calldata::codec::DecodeError> { + std::collections::BTreeMap::<String, ModuleFingerprint>::decode(deserializer).map(Self) + } +} + +impl genlayer_calldata::codec::Decode for Backtrace { + fn decode<D: genlayer_calldata::codec::Deserializer>( + deserializer: D, + ) -> Result<Self, genlayer_calldata::codec::DecodeError> { + Vec::<Frame>::decode(deserializer).map(|frames| Self { frames }) + } +} + +impl genlayer_calldata::codec::Decode for StorageDelta { + fn decode<D: genlayer_calldata::codec::Deserializer>( + deserializer: D, + ) -> Result<Self, genlayer_calldata::codec::DecodeError> { + let parts = Vec::<Bytes>::decode(deserializer)?; + let [key, value] = <[Bytes; 2]>::try_from(parts).map_err(|_| { + genlayer_calldata::codec::DecodeError::Unexpected("storage delta must be a pair") + })?; + let key = <[u8; 36]>::try_from(key.as_ref()).map_err(|_| { + genlayer_calldata::codec::DecodeError::Unexpected("storage delta key must be 36 bytes") + })?; + Ok(Self::new(key, value.to_vec())) + } +} + +#[derive(Debug, Clone, Copy, Default, genlayer_calldata::Encode, genlayer_calldata::Decode)] pub struct BucketsConsumed { pub storage: primitive_types::U256, pub message_receipt: primitive_types::U256, @@ -190,9 +307,91 @@ pub struct BucketsConsumed { pub event: primitive_types::U256, } -#[derive(Debug, Clone, genlayer_calldata::Encode)] +/// Fingerprint of a run's outcome that a caller folds into its own sub-VM +/// hashes. +/// +/// It commits to the outcome only, never to how the run was hosted, so an +/// in-process child and one executed by another executor over the nested +/// protocol are indistinguishable to consensus. Every line compiles this one +/// definition; changing it changes every line at once, which is the point. +pub fn small_hash<D, H>( + kind: ResultCode, + data: &D, + subvm_hashes: &[u8], + wasm_store_hashes: &H, +) -> [u8; 32] +where + for<'a> D: + genlayer_calldata::codec::Encode<&'a mut HashWriter, Error = std::convert::Infallible>, + for<'a> H: + genlayer_calldata::codec::Encode<&'a mut HashWriter, Error = std::convert::Infallible>, +{ + use sha3::Digest as _; + + fn encode<D, H>( + enc: &mut genlayer_calldata::Encoder<&mut HashWriter>, + kind: ResultCode, + data: &D, + subvm_hashes: &[u8], + wasm_store_hashes: &H, + ) -> Result<(), std::convert::Infallible> + where + for<'a> D: + genlayer_calldata::codec::Encode<&'a mut HashWriter, Error = std::convert::Infallible>, + for<'a> H: + genlayer_calldata::codec::Encode<&'a mut HashWriter, Error = std::convert::Infallible>, + { + enc.start_map(4)?; + + enc.push_map_k("kind")?; + enc.push_str(match kind { + ResultCode::Return => "Return", + ResultCode::UserError => "UserError", + // An internal error is never handed to a caller as a result, so it + // shares the `VMError` label rather than inventing a fourth one. + ResultCode::VmError | ResultCode::InternalError => "VMError", + })?; + + enc.push_map_k("result")?; + genlayer_calldata::codec::Encode::encode(data, enc)?; + + enc.push_map_k("subvm_hashes")?; + enc.push_bytes(subvm_hashes)?; + + enc.push_map_k("wasm_store_hashes")?; + genlayer_calldata::codec::Encode::encode(wasm_store_hashes, enc)?; + + Ok(()) + } + + let mut hasher = HashWriter(sha3::Sha3_256::new()); + let mut enc = genlayer_calldata::Encoder::new(&mut hasher); + match encode(&mut enc, kind, data, subvm_hashes, wasm_store_hashes) { + Ok(()) => {} + Err(e) => match e {}, + } + hasher.0.finalize().into() +} + +/// Sink that feeds an encoded value straight into the [`small_hash`] digest, +/// so the hash never materializes the encoding. +pub struct HashWriter(sha3::Sha3_256); + +impl genlayer_calldata::Writer for &mut HashWriter { + type Error = std::convert::Infallible; + + fn write_all(&mut self, data: &[u8]) -> Result<(), Self::Error> { + sha3::Digest::update(&mut self.0, data); + Ok(()) + } +} + +#[derive(Debug, Clone, genlayer_calldata::Encode, genlayer_calldata::Decode)] pub struct ReportedResult { pub execution_hash: bytes::Bytes, + /// See [`small_hash`]: the route-invariant part of the outcome, which is + /// what a caller in another executor folds. + pub small_hash: bytes::Bytes, pub kind: ResultCode, pub data: genlayer_calldata::unparsed::Maybe<genlayer_calldata::Value>, diff --git a/crates/modules-interfaces/src/host_fns.rs b/crates/modules-interfaces/src/host_fns.rs new file mode 100644 index 00000000..d3cfcbe1 --- /dev/null +++ b/crates/modules-interfaces/src/host_fns.rs @@ -0,0 +1,158 @@ +// This file is auto-generated. Do not edit! + +#![allow(dead_code, clippy::redundant_static_lifetimes)] + +use serde::{Deserialize, Serialize}; + +use std::borrow::Cow; + +#[derive( + Debug, + PartialEq, + Clone, + Copy, + Serialize, + Deserialize, + ::genlayer_calldata::Encode, + ::genlayer_calldata::Decode, +)] +#[repr(u8)] +pub enum Methods { + StorageRead = 0, + ConsumeFuel = 1, + EthCall = 2, + GetBalance = 3, + RemainingFuelAsGen = 4, + NotifyNondetDisagreement = 5, + ConsumeResult = 6, + ResolveCallcontractExecutor = 7, + RunNested = 8, +} + +impl Methods { + pub const SIZE: usize = 9; + pub fn value(self) -> u8 { + match self { + Methods::StorageRead => 0, + Methods::ConsumeFuel => 1, + Methods::EthCall => 2, + Methods::GetBalance => 3, + Methods::RemainingFuelAsGen => 4, + Methods::NotifyNondetDisagreement => 5, + Methods::ConsumeResult => 6, + Methods::ResolveCallcontractExecutor => 7, + Methods::RunNested => 8, + } + } + pub fn str_snake_case(self) -> &'static str { + match self { + Methods::StorageRead => "storage_read", + Methods::ConsumeFuel => "consume_fuel", + Methods::EthCall => "eth_call", + Methods::GetBalance => "get_balance", + Methods::RemainingFuelAsGen => "remaining_fuel_as_gen", + Methods::NotifyNondetDisagreement => "notify_nondet_disagreement", + Methods::ConsumeResult => "consume_result", + Methods::ResolveCallcontractExecutor => "resolve_callcontract_executor", + Methods::RunNested => "run_nested", + } + } +} + +impl TryFrom<u8> for Methods { + type Error = (); + + fn try_from(value: u8) -> Result<Self, ()> { + match value { + 0 => Ok(Methods::StorageRead), + 1 => Ok(Methods::ConsumeFuel), + 2 => Ok(Methods::EthCall), + 3 => Ok(Methods::GetBalance), + 4 => Ok(Methods::RemainingFuelAsGen), + 5 => Ok(Methods::NotifyNondetDisagreement), + 6 => Ok(Methods::ConsumeResult), + 7 => Ok(Methods::ResolveCallcontractExecutor), + 8 => Ok(Methods::RunNested), + _ => Err(()), + } + } +} +#[derive( + Debug, + PartialEq, + Clone, + Copy, + Serialize, + Deserialize, + ::genlayer_calldata::Encode, + ::genlayer_calldata::Decode, +)] +#[repr(u8)] +pub enum Errors { + Ok = 0, + EvmReverted = 1, + Forbidden = 2, +} + +impl Errors { + pub const SIZE: usize = 3; + pub fn value(self) -> u8 { + match self { + Errors::Ok => 0, + Errors::EvmReverted => 1, + Errors::Forbidden => 2, + } + } + pub fn str_snake_case(self) -> &'static str { + match self { + Errors::Ok => "ok", + Errors::EvmReverted => "evm_reverted", + Errors::Forbidden => "forbidden", + } + } +} + +impl TryFrom<u8> for Errors { + type Error = (); + + fn try_from(value: u8) -> Result<Self, ()> { + match value { + 0 => Ok(Errors::Ok), + 1 => Ok(Errors::EvmReverted), + 2 => Ok(Errors::Forbidden), + _ => Err(()), + } + } +} +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct VmErrorDetail(pub Cow<'static, str>); + +impl From<VmErrorDetail> for String { + fn from(val: VmErrorDetail) -> String { + val.0.into() + } +} +#[rustfmt::skip] +impl VmErrorDetail { + pub const fn internal() -> Self { Self(Cow::Borrowed("internal")) } + pub const fn external() -> Self { Self(Cow::Borrowed("external")) } +} + +#[rustfmt::skip] +impl VmErrorDetail { + /// Whether `s` is a well-formed `vm_error_detail` path. + pub fn is_valid_(s: &str) -> bool { + if matches!(s, + "internal" | + "external" + ) { + return true; + } + false + } +} + +pub const CURRENT_MAJOR: u8 = 0; +pub const CURRENT_MAJOR_STR: &'static str = "v0.0.0"; + +// EOF diff --git a/crates/modules-interfaces/src/lib.rs b/crates/modules-interfaces/src/lib.rs index 5dc956ef..9217ad97 100644 --- a/crates/modules-interfaces/src/lib.rs +++ b/crates/modules-interfaces/src/lib.rs @@ -17,8 +17,12 @@ pub trait Web { pub mod abi_stub; pub mod domain; +pub mod host_fns; +pub mod manager_api; +pub mod nested; pub use abi_stub::*; pub use domain::*; +pub use nested::*; #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(untagged)] diff --git a/crates/modules-interfaces/src/manager_api.rs b/crates/modules-interfaces/src/manager_api.rs new file mode 100644 index 00000000..8b46009a --- /dev/null +++ b/crates/modules-interfaces/src/manager_api.rs @@ -0,0 +1,139 @@ +// This file is auto-generated. Do not edit! + +#![allow(dead_code, clippy::redundant_static_lifetimes)] + +use serde::{Deserialize, Serialize}; + +#[derive( + Debug, + PartialEq, + Clone, + Copy, + Serialize, + Deserialize, + ::genlayer_calldata::Encode, + ::genlayer_calldata::Decode, +)] +#[repr(u16)] +pub enum Methods { + Error = 0, + Hello = 1, + Event = 2, + Run = 3, + Attach = 4, + Cancel = 5, + Ack = 6, + GetArtifact = 7, +} + +impl Methods { + pub const SIZE: usize = 8; + pub fn value(self) -> u16 { + match self { + Methods::Error => 0, + Methods::Hello => 1, + Methods::Event => 2, + Methods::Run => 3, + Methods::Attach => 4, + Methods::Cancel => 5, + Methods::Ack => 6, + Methods::GetArtifact => 7, + } + } + pub fn str_snake_case(self) -> &'static str { + match self { + Methods::Error => "error", + Methods::Hello => "hello", + Methods::Event => "event", + Methods::Run => "run", + Methods::Attach => "attach", + Methods::Cancel => "cancel", + Methods::Ack => "ack", + Methods::GetArtifact => "get_artifact", + } + } +} + +impl TryFrom<u16> for Methods { + type Error = (); + + fn try_from(value: u16) -> Result<Self, ()> { + match value { + 0 => Ok(Methods::Error), + 1 => Ok(Methods::Hello), + 2 => Ok(Methods::Event), + 3 => Ok(Methods::Run), + 4 => Ok(Methods::Attach), + 5 => Ok(Methods::Cancel), + 6 => Ok(Methods::Ack), + 7 => Ok(Methods::GetArtifact), + _ => Err(()), + } + } +} +#[derive( + Debug, + PartialEq, + Clone, + Copy, + Serialize, + Deserialize, + ::genlayer_calldata::Encode, + ::genlayer_calldata::Decode, +)] +#[repr(u8)] +pub enum Errors { + Internal = 0, + MalformedFrame = 1, + UnknownMethod = 2, + UnknownId = 3, + BootIdMismatch = 4, + BadRequestId = 5, + NotFinished = 6, +} + +impl Errors { + pub const SIZE: usize = 7; + pub fn value(self) -> u8 { + match self { + Errors::Internal => 0, + Errors::MalformedFrame => 1, + Errors::UnknownMethod => 2, + Errors::UnknownId => 3, + Errors::BootIdMismatch => 4, + Errors::BadRequestId => 5, + Errors::NotFinished => 6, + } + } + pub fn str_snake_case(self) -> &'static str { + match self { + Errors::Internal => "internal", + Errors::MalformedFrame => "malformed_frame", + Errors::UnknownMethod => "unknown_method", + Errors::UnknownId => "unknown_id", + Errors::BootIdMismatch => "boot_id_mismatch", + Errors::BadRequestId => "bad_request_id", + Errors::NotFinished => "not_finished", + } + } +} + +impl TryFrom<u8> for Errors { + type Error = (); + + fn try_from(value: u8) -> Result<Self, ()> { + match value { + 0 => Ok(Errors::Internal), + 1 => Ok(Errors::MalformedFrame), + 2 => Ok(Errors::UnknownMethod), + 3 => Ok(Errors::UnknownId), + 4 => Ok(Errors::BootIdMismatch), + 5 => Ok(Errors::BadRequestId), + 6 => Ok(Errors::NotFinished), + _ => Err(()), + } + } +} +pub const CURRENT_MAJOR: u32 = 0; + +// EOF diff --git a/crates/modules-interfaces/src/nested.rs b/crates/modules-interfaces/src/nested.rs new file mode 100644 index 00000000..57021cf8 --- /dev/null +++ b/crates/modules-interfaces/src/nested.rs @@ -0,0 +1,286 @@ +use bytes::Bytes; +use genlayer_calldata::codec::{Decode, DecodeError, Deserializer, Encode}; + +use crate::{MessageData, ResultCode}; + +/// Prefix marking a version string as a regular expression rather than a +/// directory name. Universal: it holds wherever a version is named, from a +/// nested run's routing to a request's debug reroute. +pub const VERSION_REGEX_PREFIX: &str = "re:"; + +/// What a version string names, once [`VERSION_REGEX_PREFIX`] is accounted for. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum VersionMatch<'a> { + /// An executor directory, used as it stands. + Exact(&'a str), + /// A pattern over manifest version keys, resolved by the manifest's rules. + Regex(&'a str), +} + +/// Reads a version string by the universal rule. +pub fn parse_version_match(version: &str) -> VersionMatch<'_> { + match version.strip_prefix(VERSION_REGEX_PREFIX) { + Some(pattern) => VersionMatch::Regex(pattern), + None => VersionMatch::Exact(version), + } +} + +/// Which executor line a run must use: a major, left to the manifest's rules, +/// or a version naming one directly by [`parse_version_match`]. +/// +/// As a nested run's routing this is normally minted by the host, which executor +/// lines carry without reading. An executor mints one itself only for a callee +/// whose major it does not serve and the host declined to place, because the +/// mapping from a major to a line belongs to the manager rather than to any +/// line. +#[derive( + Debug, + Clone, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + genlayer_calldata::Encode, + genlayer_calldata::Decode, +)] +#[calldata(tag = "kind")] +#[serde(tag = "kind")] +pub enum ExecutorSelector { + #[calldata(rename = "major")] + #[serde(rename = "major")] + MajorOverride { major: u32 }, + #[calldata(rename = "version")] + #[serde(rename = "version")] + VersionOverride { version: String }, +} + +/// Storage view carried across an executor boundary. +/// +/// Executor lines map variants by meaning to their local `StorageType`; they +/// must not cast local discriminants to or from this type. +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + genlayer_calldata::Encode, + genlayer_calldata::Decode, +)] +#[serde(rename_all = "snake_case")] +pub enum NestedStorageType { + Default, + LatestFinal, + LatestNonFinal, +} + +/// Permission bits carried across an executor boundary. +/// +/// Each executor maps its local permission fields by meaning. Bits unknown to +/// this shared representation are cleared while decoding and therefore deny +/// rather than grant a permission. `READ_STORAGE` maps the legacy line's +/// separate read flag and is ignored by lines where reads are not gated; +/// permissions with no local equivalent are likewise left denied. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize)] +#[serde(transparent)] +pub struct NestedPermissions(u32); + +impl NestedPermissions { + pub const DETERMINISTIC: Self = Self(1 << 0); + pub const READ_STORAGE: Self = Self(1 << 1); + pub const WRITE_STORAGE: Self = Self(1 << 2); + pub const SEND_MESSAGES: Self = Self(1 << 3); + pub const CALL_OTHERS: Self = Self(1 << 4); + pub const SPAWN_NONDET: Self = Self(1 << 5); + pub const REGISTER_RUNNERS: Self = Self(1 << 6); + pub const USE_BALANCE_FOR_MESSAGE_FEES: Self = Self(1 << 7); + + const KNOWN_BITS: u32 = Self::DETERMINISTIC.0 + | Self::READ_STORAGE.0 + | Self::WRITE_STORAGE.0 + | Self::SEND_MESSAGES.0 + | Self::CALL_OTHERS.0 + | Self::SPAWN_NONDET.0 + | Self::REGISTER_RUNNERS.0 + | Self::USE_BALANCE_FOR_MESSAGE_FEES.0; + + pub const fn from_bits(bits: u32) -> Self { + Self(bits & Self::KNOWN_BITS) + } + + pub const fn bits(self) -> u32 { + self.0 + } + + pub const fn contains(self, permission: Self) -> bool { + self.0 & permission.0 == permission.0 + } +} + +impl std::ops::BitOr for NestedPermissions { + type Output = Self; + + fn bitor(self, rhs: Self) -> Self::Output { + Self(self.0 | rhs.0) + } +} + +impl std::ops::BitOrAssign for NestedPermissions { + fn bitor_assign(&mut self, rhs: Self) { + self.0 |= rhs.0; + } +} + +impl<'de> serde::Deserialize<'de> for NestedPermissions { + fn deserialize<D>(deserializer: D) -> Result<Self, D::Error> + where + D: serde::Deserializer<'de>, + { + u32::deserialize(deserializer).map(Self::from_bits) + } +} + +impl<W: genlayer_calldata::Writer> Encode<W> for NestedPermissions { + type Error = W::Error; + + fn encode(&self, encoder: &mut genlayer_calldata::Encoder<W>) -> Result<(), Self::Error> { + self.0.encode(encoder) + } +} + +impl Decode for NestedPermissions { + fn decode<D: Deserializer>(deserializer: D) -> Result<Self, DecodeError> { + u32::decode(deserializer).map(Self::from_bits) + } +} + +/// Textual runner reference carried across an executor boundary. +/// +/// A `CallContract` child uses `contract`, which the receiving executor +/// resolves against the callee code slot using its own storage layout. Local +/// enum encodings and caller-derived code slots are never placed on this +/// boundary. +#[derive( + Debug, + Clone, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + genlayer_calldata::Encode, + genlayer_calldata::Decode, +)] +#[serde(transparent)] +pub struct NestedRunnerId(pub String); + +/// Complete input needed to run a `CallContract` child in another executor. +/// +/// Carries no wire version: every executor line compiles this very definition, +/// so a change is a compile error at each construction site rather than +/// something a peer could observe at runtime. +#[derive( + Debug, + Clone, + serde::Serialize, + serde::Deserialize, + genlayer_calldata::Encode, + genlayer_calldata::Decode, +)] +pub struct NestedRunEnvelope { + /// Host-minted routing data, opaque to both executor lines. + pub routing_payload: Bytes, + pub calldata: Bytes, + /// The caller sets `value` to zero and `is_init` to false. + pub message: MessageData, + /// View-call stack with the caller's contract address appended. + pub stack: Vec<genlayer_calldata::Address>, + pub permissions: NestedPermissions, + pub state_mode: NestedStorageType, + pub topmost_runner_id: NestedRunnerId, + /// Recursion budget left for the whole chain, minted by its root. Carried + /// as a remainder rather than a spent count so the bound does not depend + /// on each line's own `VM_RECURSION`. + pub remaining_recursion: u32, + pub remaining_det_fuel: primitive_types::U256, + pub memory_limit: u32, +} + +/// ABI-neutral result of a nested contract call. +#[derive(Debug, Clone, genlayer_calldata::Encode, genlayer_calldata::Decode)] +pub struct NestedRunResult { + pub kind: ResultCode, + pub data: genlayer_calldata::unparsed::Maybe<genlayer_calldata::Value>, +} + +/// Result returned to the caller executor after a nested run. +#[derive(Debug, Clone, genlayer_calldata::Encode, genlayer_calldata::Decode)] +pub struct NestedRunReply { + pub result: NestedRunResult, + /// The callee's [`crate::small_hash`], forwarded unchanged, and the only + /// hash the caller folds: an execution hash also commits to fee accounting + /// and storage effects, which would make the same call hash differently + /// depending on whether the host routed it in-process or across majors. + pub small_hash: Bytes, + /// True only when the reported result proves that no effects were produced. + pub effect_free: bool, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unknown_permission_bits_are_denied() { + let encoded = + genlayer_calldata::encode_obj(&(NestedPermissions::CALL_OTHERS.bits() | (1 << 31))); + let decoded: NestedPermissions = genlayer_calldata::decode_obj(&encoded).unwrap(); + + assert_eq!(decoded.bits(), NestedPermissions::CALL_OTHERS.bits()); + } + + #[test] + fn small_hash_separates_outcome_kinds() { + use crate::domain::{small_hash, ResultCode, WasmStoreHashes}; + + let data = genlayer_calldata::Value::Str("x".to_owned()); + let hash = |kind| small_hash(kind, &data, &[0u8; 32], &WasmStoreHashes::default()); + + assert_ne!(hash(ResultCode::Return), hash(ResultCode::UserError)); + assert_ne!(hash(ResultCode::Return), hash(ResultCode::VmError)); + // An internal error is never a caller-visible outcome, so it shares the + // `VMError` label rather than adding a kind a peer line might not know. + assert_eq!(hash(ResultCode::VmError), hash(ResultCode::InternalError)); + } + + #[test] + fn routing_payload_round_trips() { + for selector in [ + ExecutorSelector::MajorOverride { major: 3 }, + ExecutorSelector::VersionOverride { + version: "v0.2.17".to_owned(), + }, + ExecutorSelector::VersionOverride { + version: "re:^v0\\.2\\..*$".to_owned(), + }, + ] { + let encoded = genlayer_calldata::encode_obj(&selector); + let decoded: ExecutorSelector = genlayer_calldata::decode_obj(&encoded).unwrap(); + + assert_eq!(decoded, selector); + } + } + + #[test] + fn a_version_string_carries_its_own_rule() { + assert_eq!( + parse_version_match("v0.2.17"), + VersionMatch::Exact("v0.2.17") + ); + assert_eq!( + parse_version_match("re:^v0\\.2\\..*$"), + VersionMatch::Regex("^v0\\.2\\..*$") + ); + } +} diff --git a/docs/adr/014. manager host socket protocol.md b/docs/adr/014. manager host socket protocol.md new file mode 100644 index 00000000..c7036c2c --- /dev/null +++ b/docs/adr/014. manager host socket protocol.md @@ -0,0 +1,337 @@ +# ADR-014: Manager Host Socket Protocol + +## Status + +- **Status**: Accepted +- **Date**: 2026-07-27 +- **Amended**: 2026-07-27 β€” transport is a WebSocket on the existing HTTP + listener, not a second listener. See [Amendment 1](#amendment-1-websocket-transport). + +## Context + +The host (node) drives GenVM executions through the manager's HTTP API: `POST +/genvm/run`, poll `GET /genvm/{id}`, `DELETE /genvm/{id}`. That shape has +accumulated real defects: + +- **No push.** The manager cannot tell the host anything. When an executor + fails before dialling the host's socket (e.g. clap rejects its arguments), + the host waits on `accept()` forever β€” the observed node hang. The + workaround is a `sleep(1)`-then-GET probe (`tests/runner/origin/base_host.py:661`) + plus a client-side timeout task that fires `DELETE`. +- **Destructive read.** `GET /genvm/{id}` removes the execution on first read + (`implementation/src/manager/run.rs:226`), so a second reader gets `null` + and retention is an implicit 60 s GC sweep (`run.rs:314-324`). Multiple + clients per execution are structurally impossible. +- **Poll-driven lifecycle.** `check_proc` (`run.rs:732`) discovers exits via + `try_wait` from the GC loop and status handlers, and holds the process mutex + across an `acquire_many` on the output semaphore (`run.rs:757`) β€” a latent + ordering hazard. +- The host has no way to hand the executor per-connection preamble bytes; the + executor's first write to host 0 is always the first method byte. +- `warp` forces a second hyper (0.14 next to reqwest's 1.x) and dominates + manager compile time. + +The executor side of host communication is [ADR-003](<003. host communication.md>); +this ADR changes how the *host talks to the manager*, and two narrow things on +the executor↔host wire: a hello preamble and the removal of `notify_finished`. + +## Decision + +### Transport and framing + +A long-lived WebSocket at `GET /ws` on the manager's existing HTTP listener +(see [Amendment 1](#amendment-1-websocket-transport) for why this is not a +separate listener). Each binary message carries exactly one protocol message: + +``` +u16 method_id (BE) | u64 request_id (BE) | calldata(payload) +``` + +WebSocket already delimits messages, so there is no length prefix of our own. +Integers are big-endian, matching the manager/module framing +(`implementation/src/common/mod.rs:193-229`) and *not* the executor host +protocol's u32 LE (`executors/v0.3.x/executor/src/host/mod.rs:64-88`); the two +wires stay distinct. + +Routing lives **outside** the calldata payload because calldata sorts map keys +bytewise and rejects unsorted keys on decode +(`executors/v0.3.x/executor/crates/calldata/src/bin.rs`, spec +`04-contract-interface/01-calldata.rst`) β€” a `"method"` key cannot be forced to +the front of the encoded map, so envelope-in-calldata would require decoding +the full payload just to dispatch. With the fixed header, dispatch, request +correlation and error replies need zero decoding and are immune to key order. +The payload itself stays an externally-tagged enum (single-key map), so the +variant name also appears first inside the blob β€” redundant, but good for logs. + +`request_id != 0` marks a request expecting exactly one reply frame with the +same `method_id` and `request_id` echoed. `request_id == 0` marks a +notification (no reply); only the manager sends notifications. Errors are +replied with the dedicated `error` method id, `request_id` echoed, payload +`{code, message}`. Malformed payloads and unknown method ids get an error +frame and the connection survives. A message above the configured cap +(`max_message_bytes`) is refused by the WebSocket layer itself, which drops the +connection; the manager does not buffer it. + +Method ids and error codes are codegen data β€” +`crates/modules-interfaces/codegen/data/manager-api.json`, a sibling of the +relocated host-function data (see Implementation Notes) β€” generated for Rust, +Python (`tests/runner/origin/`) and the impl-spec. Payload structs are +hand-written; only the id/const enums are generated. Full wire reference: +impl-spec appendix `manager-socket`. + +### Run identity and reconnect + +- `run` allocates and returns a manager-assigned `GenVMId` **immediately**; + spawning proceeds asynchronously and outcomes arrive as events. +- `next_genvm_id` starts at 1 and resets on process start (`run.rs:277`), so a + bare id is ambiguous across manager restarts. The manager generates a random + **`boot_id`** per process start and announces it in a `hello` notification + on every connection; the durable run key is `(boot_id, genvm_id)`. `attach` + carries both and fails with `boot_id_mismatch` after a restart instead of + binding to an unrelated reused id. +- The client may pass a **`host_genvm_id`** correlation token in `run`: it is + echoed in events and doubles as an idempotency key β€” repeating a `run` with + the same token before the retention TTL returns the already-allocated id + rather than spawning twice (covers a reconnect racing an in-flight `run`). + +### Push lifecycle, pull artifacts + +The manager pushes lifecycle notifications to every subscribed connection: +`started`, `failed_to_start` (spawn/validation failure β€” this is what removes +the accept-hang class), and a single terminal `finished` event carrying: cause +(exited / cancelled / deadline / shutdown), exit code, the `consume_result` +bytes, metrics, timestamps, executor version, and the **sizes** of the +diagnostic blobs. The blobs themselves β€” stdout/stderr tails and `genvm_log` β€” +are **pulled** via `get_artifact {genvm_id, field, offset, max_len}` in +bounded chunks. + +Bulk artifact replies travel on the **same connection** through a +low-priority writer queue (control frames are high-priority), so one run's +huge stdout cannot starve another run's `finished` event. A second +artifacts-only socket was rejected: it would need identity and run binding +re-solved just to fix the same starvation problem a two-queue writer fixes in +a few lines. + +### Attach, ack, retention + +- `attach {boot_id, genvm_id}` returns a snapshot of the current state and + subscribes the connection to further events. Subscriber fan-out gives + "many clients observe one execution" for free. +- Reads are non-destructive. The destructive `GET` semantics are replaced by + an explicit `ack {genvm_id}` that releases the retained result, plus a + retention TTL from config (`execution_retention`, default `5m`, replacing + the implicit 60 s) for clients that never come back. +- **Disconnect does not kill the run.** This matches today (HTTP was + stateless). Termination happens only via explicit `cancel`, the strict + deadline (now enforced manager-side, pushed as a terminal event β€” the + client-side timeout task and `DELETE` disappear), or manager shutdown. + `cancel` of a run still queued on permits aborts it without consuming a + permit and still emits exactly one terminal event. + +### Durations + +Every duration crossing this wire or sitting in the manager configuration is a +string with a mandatory unit: `"30s"`, `"10.5m"`, `"1h"`, `"250ms"`, resolved +at millisecond granularity. A bare number is rejected. + +The existing `max_execution_minutes` is a plain integer whose scale lives only +in its name β€” callers that want seconds cannot express them, and a caller that +misreads the unit is off by 60x with no error. The socket `run` therefore takes +`deadline` as a duration string, overriding `max_execution_minutes` when +present; `max_execution_minutes` stays on the deprecated HTTP body only. + +### `host_hello_data` + +`ExecutionData` (`crates/modules-interfaces/src/domain.rs:18`) gains +`host_hello_data: Vec<Bytes>`, indexed by **host connection index** β€” the same +index space `method_hosts` values point into. On connecting host *i* +(`executors/v0.3.x/executor/src/host/mod.rs:41-61`), the executor writes entry +*i* **verbatim, unframed, before the first method byte**. A missing, short, or +empty entry means no write β€” an empty vector is byte-identical to today's +wire. No length prefix: the host supplied the bytes, so it knows the length; +this is the only shape where "0 bytes = exactly like now" falls out. If a host +wants structure, it embeds its own framing. + +The manager does not write hello data itself β€” it does not hold host 0's +socket, only the executor does. The manager forwards the vector from the `run` +request into `ExecutionData` and rejects a non-empty entry for a host +connection it owns (today index 1, its `consume_result` socketpair). + +Both executor lines are patched β€” `ExecutionData` is a single shared +definition (each line's `modules-interfaces` is a stub manifest over +`crates/modules-interfaces/`), so v0.2.x support status is irrelevant here: +one field edit covers both, and each line adds the write-on-connect. + +### `notify_finished` removed; `consume_result` stays + +`notify_finished` routed to host 0 and served two jobs: the host loop's +termination signal (`base_host.py:331`), and β€” via its 1-byte ACK β€” a sync +point before the executor's `process::exit(0)` letting the host distinguish +clean finish from crash. The pushed terminal event now carries exit status and +result, so both jobs are gone: the host loop ends on EOF (on a stream socket +all written data precedes FIN, so this is well-ordered β€” provided the executor +flushes its buffered host writers before exiting, which the ACK round-trip +used to force implicitly). The method is deleted from the shared `host-fns` +enum and from both executor lines. + +`consume_result` is untouched: it routes to host 1 and is how the manager +obtains a trusted copy of the result without round-tripping it through the +node. Orthogonal to this change. + +### HTTP surface + +- **Kept**: the admin endpoints β€” `/status` (also the test harness health + check, `tests/runner/genvm_tool_plugins/genvm.py:90`), `/module/*`, + `/permits`, `/log/level`, `/manifest/reload`, `/llm/check`, + `/vm-error/describe`, `/contract/detect-version`. Low-frequency, + curl-debuggable, no reason to move. +- **Temporary adapter**: `POST /genvm/run`, `GET|DELETE /genvm/{id}` become + thin shims over the same lifecycle core the socket uses (`run::Ctx` grows a + real API: start / attach / subscribe / cancel / ack / get artifact). The old + destructive `GET` maps to read-then-ack. There is exactly one lifecycle + implementation; the shims live one release train and are deleted once the + node has migrated to the socket. +- `GET /genvm/{id}/artifact` stays available over HTTP as a curl/debug + convenience only. +- **warp is replaced by axum 0.8**: hyper unifies on 1.x (shared with + reqwest), `axum::serve` accepts a `tokio::net::UnixListener` natively so the + hand-rolled `UnixAcceptor` (`implementation/src/manager/mod.rs:369-385`) is + deleted. The socket protocol rides the same router as `/ws`. + +### Stated assumptions and accepted limitations + +- **No authentication.** The manager runs in a trusted architecture β€” unix + socket or loopback, host and manager co-located and mutually trusting. This + is an assumption stated here, not a mechanism; nothing in the protocol + reserves space for auth. +- **Multi-attach is not multi-host.** Many clients can attach to a run on the + *manager* channel, but host 0 β€” the socket the executor dials and speaks the + [ADR-003](<003. host communication.md>) protocol on β€” is still owned by + whoever supplied its address in `run`. `host_hello_data` is the enabling + token for future shared-host schemes (the connecting side can present it to + a broker), together with the reworked Python host library and, later, node + changes. Explicitly out of scope now. + +## Consequences + +**Positive.** A dead-at-startup executor becomes a pushed `failed_to_start` +instead of a host-side hang; the probe, retry, and client-timeout scaffolding +in the host library is deleted. Results survive multiple readers and manager +restarts are detectable (`boot_id`). Exactly-once terminal events replace +poll-races; the per-run supervisor task (`child.wait().await`) removes +`check_proc`'s poll state machine and the mutex-held `acquire_many` hazard. +One hyper, faster manager builds. Deadlines are enforced in one place, with an +event, instead of in every client. + +**Negative.** A new wire protocol to maintain, spec and codegen included. The +host library becomes stateful (persistent connection, run table) β€” a real +rewrite of `tests/runner/origin/base_host.py` and its consumers. Two run entry +paths (socket + HTTP shims) exist for one release train. Executor lines and +manager must move together on `notify_finished`/hello (same-train lockstep, +enforced by the shared `host-fns` data). The executor must now explicitly +flush host sockets before exit, since the `notify_finished` ACK no longer +forces it. + +**Neutral.** The executor↔host method protocol is otherwise unchanged; an +empty `host_hello_data` is byte-identical to today, so existing goldens hold. +The admin HTTP surface is unchanged. Modules (`llm`/`web`) keep their existing +framing and `GenVMHello` prologue β€” `handle_stream`/`MessageHandler` is +strict request-reply and is deliberately *not* reused for this (it cannot +push); only `read_message`/`write_message` and the listener loops are shared. + +## Alternatives Considered + +- ~~**WebSocket**~~ β€” originally rejected as buying only browser clients and + proxy traversal, neither of which a co-located host needs. Reversed by + [Amendment 1](#amendment-1-websocket-transport): that reasoning priced the + client and ignored the cost of a second listener and a second exposed port. +- **HTTP long-poll / SSE on the existing server** β€” still no clientβ†’manager + multiplexing on one connection, keeps per-request overhead, and ties the + execution protocol to the HTTP framework. +- **Envelope inside calldata** β€” decode-before-dispatch, and calldata's + canonical key order forbids putting the method first. See framing rationale. +- **Full result blobs in the terminal event** β€” multi-megabyte stdout tails + (`run.rs:612`) would head-of-line-block control traffic; sizes-plus-pull + keeps the control path small with the same information. +- **Second socket for artifacts** β€” re-solves identity and run binding to fix + starvation that a two-priority writer queue already fixes. +- **Kill runs on disconnect** β€” punishes transient client failures, diverges + from today's semantics, and ownership can be added later per-run + (`kill_on_last_detach`) if the node wants it. +- **Keep `notify_finished` alongside the terminal event** β€” two finish + signals with two orderings; one of them necessarily lies in crash cases. +- **Delete the HTTP run path in lockstep** β€” forces the node and every test + to migrate atomically with the manager release; the shim is ~50 lines. +- **Auth mechanism now** β€” no threat model demands it in a trusted, loopback + deployment; speculative surface. + +## Amendment 1: WebSocket transport + +*2026-07-27, after the plain-socket server was implemented and green.* + +The original decision put the protocol on its own listener with its own +`socket_address` config key, and rejected WebSocket in Alternatives. That +rejection weighed only what WebSocket buys the *client* β€” browser reach, proxy +traversal β€” and never priced what a second listener costs the *manager*. In +the implemented code that cost was visible: + +- two accept loops, the second one re-deriving unix-vs-tcp dispatch, stale + socket removal and parent-directory creation that `--socket` already does; +- a second address the deployment and the test harness must configure, + publish and agree on, on top of `--host/--port`/`--socket` β€” and, when the + manager is containerised, a second published port or a second bind-mounted + socket in every compose file and orchestrator manifest that reaches it; +- a hand-rolled length prefix, an oversize-discard loop and a + shorter-than-header check, all of which are solved problems one layer down. + +`GET /ws` on the existing listener removes all three. It **inherits the +listener's transport**: `--socket` gives the protocol over a unix socket and +`--port` gives it over TCP, from one address, with no second config key and no +second bind. `socket_address` is deleted; `max_frame_bytes` becomes +`max_message_bytes` and is handed to the WebSocket layer as its message cap. + +Client cost is not the argument it looked like either. The node is Go and the +test host is Python; both have a WebSocket client in hand (`gorilla/websocket`, +`aiohttp`) and neither had a length-prefixed-framing client in hand. The +protocol's own clients get *simpler*, not harder. + +The "frame layer over a frame layer" objection stands, and is the price: a +WebSocket header (2-14 bytes) per message where a `u32` would do. That buys +message delimitation, the size cap, ping/pong keepalive and close codes as +library behaviour rather than as our code β€” a trade the original text never +put on the scale. + +Nothing else changes. Method ids, payload shapes, `boot_id`, attach/ack +semantics, retention, the two-priority writer and the artifact chunking are +untouched; only the bytes around a message differ, plus the removal of the +now-unreachable `oversized_frame` error code β€” the transport refuses the +message before we have buffered anything, which is the outcome that mattered. +Note the refusal is a dropped connection, not a 1009 close: the cap is enforced +in the reader and surfaces as a read error, so no close handshake happens. A +client sees an abnormal closure. That is a downgrade in diagnosability against +the old explicit error frame, and the price of not buffering. + +## Implementation Notes + +- Wire reference: impl-spec appendix `manager-socket` (methods, ids, payload + shapes, error codes, hello). `host-loop` documents the hello-data write and + the EOF-terminated loop. +- Shared protocol data moves to the umbrella crate: + `crates/modules-interfaces/codegen/data/` (single host-function JSON copy β€” the + per-line copies had already drifted on the `errors` enum) with + `manager-api.json` as a sibling; Rust is generated once into + `crates/modules-interfaces/src/`, each line's `common` re-exports it, and + `tests/runner/origin/host_fns.py` regenerates from the shared file instead + of the primary line's copy (`support/tools/genvm-tool/genvm_tool/cmd_configure.py:291-302`). +- The manager keeps `implementation/src/common/mod.rs` framing helpers and + listeners; the socket server is a new connection loop (reader task + writer + task with control/bulk queues), not a `MessageHandler`. +- Rollout is staged so every step lands green on the existing HTTP surface: + axum swap first, then the event-driven core behind unchanged HTTP, then the + socket, then executors (hello + `notify_finished`), then the host library. + The Python host loop must learn EOF termination *before* an executor that no + longer sends `notify_finished` can land. +- `POST /env` (and its `std::env::set_var`) is removed in the same effort but + is an independent decision: deleting the only writer makes the remaining + `getenv` readers (reqwest proxy vars, chrono TZ, Lua's `loadlib`, ...) + sound without a lock that libc would not honor anyway. diff --git a/docs/adr/015. cross-major contract calls.md b/docs/adr/015. cross-major contract calls.md new file mode 100644 index 00000000..eff91c2e --- /dev/null +++ b/docs/adr/015. cross-major contract calls.md @@ -0,0 +1,363 @@ +# ADR-015: Cross-Major Contract Calls + +## Status + +- **Status**: Accepted +- **Date**: 2026-07-30 + +## Context + +The versioning specification already committed GenVM to supporting messages and +calls across majors by delegating to an older executor +([`03-versioning.rst`, lines 31–36](../website/src/spec/01-core-architecture/03-versioning.rst)). +This decision defines that delegation boundary. + +Today the answer is: it fails. The contract's `major` byte is read from the root +slot and compared against the running executor's `CURRENT_MAJOR` at three +points, each raising `major_mismatch`: + +- top-level load (`executor/src/lib.rs`, via + `Storage::check_major_and_resolve_code_slot`), +- `gl_call`'s `CallContract` (`wasi/genlayer_sdk/run.rs`, same helper), +- chain-runner resolution (`rt/supervisor/actions.rs`), where the *runner* is + itself a deployed contract. + +The check is right β€” without it a newer executor would mis-read an older +contract's calldata and storage. But it is a dead end, and the cost is +structural: the moment a major is bumped, every existing contract becomes +uncallable from every new one. A chain cannot roll a major forward that way. + +Both directions matter. A new contract calling an old library is the obvious +case; an old contract calling a newly deployed one is the case that decides +whether the old line stays useful after the bump. + +The two lines this bridges today are not peers. v0.3 is the current line. v0.2 +is legacy β€” it only ever ran in studio and early development, never on a +production chain, and it is kept alive solely so that work written against it +can migrate. That asymmetry is why decisions here are allowed to resolve +against v0.2 rather than in it: where the two lines cannot agree, the loss is +taken on the side that exists for migration convenience, and no v0.2 concern +justifies constraining the current line. + +### Terminology + +**Major** here is the public-ABI major a contract is deployed against β€” the +value `major_mismatch` names. It is *not* the semver major of an executor +version: every line released so far is semver major `0`, so `v0.2.17` and +`v0.3.0-rc7` are indistinguishable by that number. A major therefore cannot name +a line, which is why routing carries a selector and not an integer. + +A **delegated call** is one the manager runs as a separate executor process. +Crossing a major is what motivates delegation and is what this ADR is named for, +but the host decides the route, so a delegated call may also have both ends on +one line. The boundary rules below hold for any delegated call. + +## Decision + +`major_mismatch` stops being terminal on the `CallContract` path. Top-level and +runner loads retain their existing checks. Every `CallContract`, not only an +observed mismatch, asks the host which executor should run the callee. + +### Two steps, two host connections + +The resolution and the execution are separate calls, to different peers, +because they answer to different authorities. + +1. **`resolve_callcontract_executor`**, on host connection 0 β€” the host that + initiated the transaction β€” when the run request sets + `hook_cross_contract_calls`. Otherwise it goes to the manager on host 1, + which answers null; see the implementation notes. + The executor reports the address, storage state, and an advisory major. The + host owns version detection and its answer is authoritative. The advisory + major carries no information: v0.2 storage has no major field at all, and the + byte v0.3 reads from a root records the semver major. A host must decide from + the address, not from the advisory value. + +2. **Run**, on host connection 1 β€” the manager socketpair. The request carries a + routing selector plus the preserved envelope below. The manager starts the + named binary; the callee dials the same host connections, writes its hello + data, and executes as an ordinary run. Its result returns as the reply. + + The caller does not interpret the selector. It is normally minted by the host + and read by the manager, so the routing vocabulary can change without an + executor release on either line. + +Splitting the steps keeps the node's host loop responsible for exactly the +question it alone can answer, and keeps process lifecycle with the manager, +which already owns it. + +### What a null resolution means + +Null means the host declines to place the callee; it does not mean the call +stays in-process, and it does not suppress the major check. + +The caller keeps its local check and applies it to the advisory major. If the +major is one this line serves, the call runs in-process exactly as before. If it +is not, the line has no way to run the callee and no answer from the host β€” so +rather than reject a major it does not own, it mints a `major` selector itself +and delegates through step 2. The manager then resolves that major; when no +installed line provides it, the newest line runs and raises the canonical +`invalid_contract major_mismatch` itself. + +So the floor is preserved β€” a mismatch the host declines to route still ends in +`major_mismatch` β€” but it is produced by the line the manager selected rather +than by the caller. A host that implements nothing keeps working; it pays one +process spawn on the failing path. + +### The executor selector + +Routing data is an `ExecutorSelector`, tagged on `kind`: + +- `major` β€” resolved by the manifest's rules, falling back to the newest line + when no installed line provides it, as above. +- `version` β€” names a line outright: an executor directory used as it stands, or + a `re:`-prefixed regular expression over manifest version keys, the newest + match winning. No match fails the run rather than silently running another + line. + +Both forms exist because of the collapse described under Terminology: only a +version can mean one particular line while every line is semver major `0`. A +version is taken on trust and is not checked against the manifest β€” the host is +not an adversary, and a pin it stores is a statement about a contract it already +owns. The same two forms, spelled the same way, drive the debug +`unsafe_overrides.reroute_to`, so there is one rule for naming a line rather than +two. + +### The preserved envelope + +An in-process sub-VM inherits its parent by construction: every meta-property +starts as a copy, and `CallContract` deviates from that copy exactly as +[Meta-Property Derivation](../website/src/spec/03-vm/01-startup.rst) specifies. +Across a process boundary nothing is inherited, so the derived values have to be +computed by the caller and carried on the wire. The table is complete against +`ExecutionData` in `crates/modules-interfaces/src/domain.rs`. + +| `ExecutionData` field | Nested value | +| --- | --- | +| `calldata` | Derived `CallContract` calldata. The child is always `Main`; v0.2 rewrites its legacy `method` key to the current empty key at the boundary. | +| `message` | Derived child message: target contract, inherited sender/origin/signer, chain and datetime, `value = 0`, `is_init = false`. v0.2's contract-facing message has no signer field, so its executor carries the signer beside the message and forwards it unchanged; contracts on that line still cannot observe it. | +| `code` | `None`; the child reads the target code through host 0 storage. | +| `host_data`, `gas_data` | Copied by the manager from the outer request, not carried in the envelope. | +| `method_hosts` | Constructed by the manager: host 0 by default, `consume_result` and `run_nested` on host 1. | +| `host_hello_data` | Copied from the outer request; entry 1 is cleared for the new manager socketpair. | +| Effectful and nondeterministic allocations | Cleared: `leader_nondet_results` and `record_actions` are empty because a deterministic child produces neither; `bucket_totals`, `message_fee_allocation` and `initial_time_units_allocation` are empty or zero because the child imports its remaining deterministic budget instead of being granted a fresh one. | +| `remaining_recursion` | Parent budget minus one. Carried as a remainder, not a spent count, so the bound stays the one the chain's root minted even when the lines disagree on `VM_RECURSION` -- but the receiving executor clamps it to its own limit, since a remainder is not an authority to exceed one. Also set, for a chain root, by the `initial_recursion` debug override. | +| `nested.memory_limit` | The derived limiter's remaining memory, made an explicit process-spawn limit. | +| `nested.stack` | Caller stack with the caller contract appended. | +| `nested.permissions` | `CallContract`-derived permissions. Unknown envelope bits decode as denied. | +| `nested.state_mode` | Resolved child storage view. | +| `nested.topmost_runner_id` | `contract`; the receiving executor resolves the target's code slot in its own layout. | +| `nested.remaining_det_fuel` | Remaining deterministic budget. | + +`nested` is one optional group rather than six independent options, so a run is +nested or it is not and there is no way to spell a half-nested one. + +`granted_custom` is deliberately absent: runtime-registered runner pins are +process-local, and a delegated call with a non-empty loaded custom set is invalid +rather than pretending the grants travelled (see below). Data-fee state is absent +because the child must be effect-free; module addresses are absent because a +deterministic child cannot reach nondeterministic modules. The manager +additionally assigns a fresh `GenVMId`, disables debug mode, and starts the child +synchronously. The routing selector is consumed at the boundary and does not +become `ExecutionData`. + +Spelling the envelope out as spec-derived rather than as its own list is the +point: the boundary must not become a second place where derivation is defined, +free to drift from the one in the spec. + +### Delegated-call boundary restrictions + +A delegated callee never carries `spawn_nondet`, `write_storage`, +`send_messages` or `use_balance_for_message_fees`, whatever its caller holds. +This is normative β€” it is stated in the spec's permission model, not left to each +line's derivation. The manager gives a nested run `no_modules: true` and no +effect-bearing connections, so granting any of the four would hand out authority +nothing downstream can honour. The manager rejects an envelope asserting one +rather than translating it: every line already clears them, so a well-behaved +caller never trips it, and the manager is the only party that can hold this line +against an executor line it did not derive. + +A delegated call is also refused outright when the caller has any custom runner +loaded β€” the caller sees the ordinary `inval` errno from `gl_call_contract`, a +contract-visible failure it can handle. Custom pins are process-local archives +and the envelope carries none, so the alternative is a `custom:` id that resolves +in-process and fails when delegated. Transferring the archives would mean +re-charging the child's RAM for them and defining a canonical transfer order; +nothing needs it today. + +Both restrictions make delegation admissible in strictly fewer cases than the +in-process route, and v0.2 adds one more: its in-process derivation keeps +`can_spawn_nondet` for a `CallContract`, which is sound there because the callee +shares the caller's module connections. So a delegated v0.2 -> v0.2 call is +refused `RunNondet` where the in-process route would have served it. The +consequence is accepted rather than fixed: all honest nodes share one routing +policy within a round, so this is not an intra-round split, and v0.2 will not be +used for same-chain sync. v0.3 derives the two routes identically. + +### Nested runs share the caller's permit + +A nested run does not acquire a manager permit. The caller already holds one and +is blocked; charging the chain again is both wrong β€” one logical execution β€” +and deadlocking: at saturation every permit is held by a caller waiting for a +callee that can never be admitted. The permit is held for the whole chain and +released when the outermost run terminates. + +Cancellation, the deadline, and shutdown propagate from the outermost run +downward. A nested run has no independent lifetime. + +### Determinism constraints + +Two properties are load-bearing and neither is automatic: + +- **The resolution must be a pure function of `(address, state, major)`.** The + host answers it, so the constraint lands on the host: two validators that + resolve the same triple differently execute the same call on different + binaries and disagree. Availability and load are not admissible inputs. +- **For a call admissible on both routes, the folded fingerprint must be + identical across lines and across routes.** The callee reports a `small_hash` + next to its execution hash, and that is what the caller folds into + `det_subvm_hashes`; the manager forwards it unchanged. The two are different + objects on purpose: the execution hash also commits to the backtrace, storage + changes and fee accounting, so folding it would make the same call hash + differently depending on the route. `small_hash` commits to the outcome only + (kind, result, sub-VM hashes, wasm store hashes), is defined once in + `crates/modules-interfaces`, and is the same value an in-process caller folds. + The reply must also prove the call was effect-free. + +The route is therefore invisible to a call that succeeds either way β€” but not +unobservable in general, since the restrictions above make some calls fail only +when delegated. Routing must be deterministic for that reason too, not merely +for hash stability. + +The call and return envelope is independent of the contract ABI it carries. +`major` gates calldata layout; the frame that transports calldata across the +boundary cannot itself be gated by it, or the two ends could never agree on how +to read the frame that tells them which layout to use. + +It carries no version of its own either, and neither does the `ReportedResult` +the manager decodes from a run. Both live in `crates/modules-interfaces`, whose +`src` is the single copy every executor line compiles: each line's +`crates/modules-interfaces/Cargo.toml` is a shim whose `[lib] path` points back +at it. Changing a field is therefore a compile error at every construction site +in the same build, and a runtime version could never disagree with itself. The +shared crate *is* the compatibility policy. This holds only while executors ship +from the same checkout as the manager that spawns them; if a line ever ships +independently, the frames need a version again. + +### The envelope is not subject to a configurable bound + +A contract chooses the calldata it forwards, so any tunable ceiling on the +envelope is contract-reachable and therefore consensus-visible: two honest nodes +configured differently disagree on whether the same call succeeds. The nested +read is consequently not bounded by the manager's `max_message_bytes`, which +bounds inbound client frames on `/ws` and nothing else β€” exactly like +`consume_result` on the same stream. + +The protocol maximum is the `u32` length prefix the frame carries, and the +practical bound is the memory limit the child runs under, a per-line constant. +Both are fixed properties of the build rather than deployment configuration, +which is what consensus requires. A regression case runs the manager with +`max_message_bytes` below the envelope it then sends successfully. + +## Consequences + +- A major bump stops orphaning deployed contracts, in both call directions. +- The resolution hook gives the host a say in routing it did not have; the same + mechanism can later serve migrations that are not major bumps. +- **The frozen line stops being frozen.** v0.2.x has to gain the hook to call + forward, which edits a binary whose value is that it does not change. Runs + already recorded against it were produced by a build that raised + `major_mismatch` where this one delegates. The behaviour only differs on + inputs that used to fail outright, which is the narrowest possible widening, + but it is not zero and the line's hash changes. +- A call chain is now a chain of processes. Spawn cost per delegated hop is real, + and it is now also paid on the failing path where a host declines to route an + unservable major. Routing loops are allowed and terminate when the imported + recursion budget runs out; ordinary resource limits may be reached first. +- The envelope is a new compatibility surface next to the ABI major it exists to + bridge. It cannot silently drift, since every line compiles the one definition, + but a change to it now touches every line at once. +- Nested runs are invisible to permit accounting, so `/status` permit counts + describe outermost executions, not processes. + +## Failure taxonomy + +Infrastructure faults are internal errors; contract-produced outcomes stay +contract-visible. The dividing line is whether a contract can reach the +condition. + +Internal: a host that cannot answer `resolve_callcontract_executor` (the host is +the authority on which line owns an address); a nested callee reporting an +internal error, which propagates unchanged; a callee reporting any effect, which +the manager refuses outright since a `CallContract` child holds none of the +permissions behind storage writes, messages, events, nondet results or any +consumption bucket. + +A nested executor dying on a signal is deliberately *not* translated into a +contract-visible error β€” the executor has no idea why the process died. The +reason travels as the run's terminal cause instead, so a caller whose chain ran +out of time observes `timeout` rather than an internal error. A system test +covers that inference. + +Ruled, because a contract controls the input: **a user error whose payload is +not a string, returned into v0.2.** That line's user errors are strings, so a +callee raising structured data produces something the caller's ABI cannot carry. +It stays the callee's ordinary failure and degrades to the vm error +`invalid_contract major_mismatch` β€” a contract must not be able to manufacture an +internal error out of its own failure. That code is reused because v0.2 never +emits it otherwise and it already names the condition. A vm error is a string in +the execution hash, so it must be a fixed literal; the payload is dropped and +recorded only in an error log. A caller consequently learns that the callee +failed and that the boundary could not carry the reason, but not the reason +itself. + +A stable contract-visible taxonomy for the rest is deferred to +[GVM-325](https://linear.app/genlayer-labs/issue/GVM-325). + +## Alternatives Considered + +### Keep failing, require callers to pin a compatible callee + +Costs nothing to build and is what happens today. Rejected because it makes a +major bump a chain-wide coordinated migration: every contract that wants to be +callable from the new major must be redeployed, and contracts that cannot be +redeployed are stranded. + +### Translate the older major's ABI in the newer executor + +One binary, no nested processes, no envelope. Rejected because it puts every +retired ABI into every future executor permanently, and the translation is +exactly the mis-reading the `major` check exists to prevent β€” written by hand, +per major pair, and wrong silently. + +### Let the manager detect which line owns an address + +The manager does resolve a selector to a line β€” that is the selected design. What +was rejected is the manager deriving *ownership* from the address and storage on +its own, without a selector from the host or the caller. Rejected because the +manager does not know the deployment context the hello data identifies, and +because routing would then depend on manager-local state, which is the +determinism hazard named above. + +## Open questions + +- The stable failure taxonomy (GVM-325). +- Whether top-level and runner-load mismatches should ever use the resolver. + They keep their existing checks today. +- Compatibility if an executor line ever ships from a different checkout than + the manager, which is what the unversioned shared frames rely on. + +## Implementation Notes + +- Both methods are host-protocol methods and live in + `crates/modules-interfaces/codegen/data/host-fns.json`; the manager socketpair + speaks the host protocol, not the manager socket protocol. Regenerating + updates Rust, Python and RST β€” see + [genvm-tool.md](../contributing/howto/genvm-tool.md). +- **Resolution is opt-in.** A run request carries `hook_cross_contract_calls`, + default false. False routes `resolve_callcontract_executor` to host 1, where + the manager answers null; true routes it to host 0 so the node decides. The + default is "manager answers null" because a node that does not route across + majors should pay neither the round-trip nor the cost of implementing a host + method it has no answer for. A nested run inherits the parent's value, since a + child that may itself cross a boundary needs the same routing as its parent. diff --git a/docs/contributing/explanation/executor-lines.md b/docs/contributing/explanation/executor-lines.md index 98e6fd03..23377f98 100644 --- a/docs/contributing/explanation/executor-lines.md +++ b/docs/contributing/explanation/executor-lines.md @@ -26,5 +26,11 @@ This is the price of the guarantee, not complexity to be simplified away ## Lines Are Not Peers `active-versions` is ordered: the first entry is the current line, the rest are -kept so existing work keeps running. `support-only-versions` goes further β€” still -shipped, no longer built from full sources +kept so existing work keeps running. Where 2 lines cannot agree, the loss is +taken on the older side ([ADR-015](<../../adr/015. cross-major contract calls.md>)). +`support-only-versions` goes further β€” still shipped, no longer built from full +sources + +A major bump would be worthless if it cut new contracts off from existing ones, +so a call crossing majors is delegated to the callee's executor rather than +rejected. Where that boundary falls is specified, not explained here diff --git a/docs/contributing/howto/building/docs.md b/docs/contributing/howto/building/docs.md index b90d54fc..d7a48f63 100644 --- a/docs/contributing/howto/building/docs.md +++ b/docs/contributing/howto/building/docs.md @@ -30,7 +30,7 @@ Generated, never edited by hand: | Page | Generator | |---|---| -| `spec/appendix/constants.rst` | codegen ([genvm-tool.md](../genvm-tool.md)) | +| `spec/appendix/constants.rst`, `impl-spec/appendix/manager-socket-consts.rst` | codegen ([genvm-tool.md](../genvm-tool.md)) | | `available-runners.rst`, `runners-versions.json`, SDK `changelog.rst`, in the primary line | `docs/website/generate.py` | | `overview/executor-lines_generated.rst` | `docs/website/generate.py` | diff --git a/docs/contributing/howto/extending/add-host-function.md b/docs/contributing/howto/extending/add-host-function.md index f123f9ef..4a9fa43c 100644 --- a/docs/contributing/howto/extending/add-host-function.md +++ b/docs/contributing/howto/extending/add-host-function.md @@ -1,9 +1,9 @@ # Adding a Host Function -1. `executors/<line>.x/executor/codegen/data/host-fns.json` β€” add the method id, +1. `crates/modules-interfaces/codegen/data/host-fns.json` β€” add the method id, then `ninja -C build codegen` ([genvm-tool.md](../genvm-tool.md)). That - regenerates the line's `executor/crates/common/src/host_fns.rs` and the - manager's `tests/runner/origin/host_fns.py` + regenerates `crates/modules-interfaces/src/host_fns.rs` and + `tests/runner/origin/host_fns.py` 2. `executors/<line>.x/executor/src/host/mod.rs` β€” wire the executor-side client 3. `tests/runner/origin/base_host.py` β€” handle the new case in the read loop and add the method to the `IHost` protocol. `origin/` is mirrored into the node, diff --git a/docs/contributing/howto/genvm-tool.md b/docs/contributing/howto/genvm-tool.md index e7b3742b..3fbd80d6 100644 --- a/docs/contributing/howto/genvm-tool.md +++ b/docs/contributing/howto/genvm-tool.md @@ -18,11 +18,12 @@ has no wrapper ## Codegen `ninja -C build codegen` regenerates from data JSON; never edit an output by -hand. Host protocol data is per line, public ABI data comes from the primary one +hand. Host protocol data is shared, public ABI data comes from the primary line | Data | Generated | |---|---| -| `executors/<line>.x/executor/codegen/data/host-fns.json` | that line's `executor/crates/common/src/host_fns.rs`, `tests/runner/origin/host_fns.py` | +| `crates/modules-interfaces/codegen/data/host-fns.json` | `…/src/host_fns.rs`, `tests/runner/origin/host_fns.py` | +| `crates/modules-interfaces/codegen/data/manager-api.json` | `…/src/manager_api.rs`, `tests/runner/origin/manager_api.py`, `docs/website/src/impl-spec/appendix/manager-socket-consts.rst` | | `executors/<primary>.x/executor/codegen/data/public-abi.json` | `tests/runner/origin/public_abi.py`, `docs/website/src/spec/appendix/constants.rst` | ## Suite Layout diff --git a/docs/contributing/howto/testing/integration.md b/docs/contributing/howto/testing/integration.md index 97e36183..7593f763 100644 --- a/docs/contributing/howto/testing/integration.md +++ b/docs/contributing/howto/testing/integration.md @@ -9,6 +9,23 @@ yields a `<name>/prepare` case plus one per step (`/0l`, `/0v`, `/0s`, …), and a sibling `.skip` file marks a case skipped. Every line in `active-versions` runs its own cases against its own built executor, via `build/info.json` +A step may declare `executor_routes: {'<address>': <route>}` to send a +`CallContract` on that address to another executor line instead of running it +in-process β€” the mock host answers `resolve_callcontract_executor` with that +route, and the manager spawns the nested run. A route is a major (an integer, +resolved by the manifest's rules) or a version string naming the line outright, +`re:`-prefixed to match manifest keys rather than name a directory. Prefer a +version: every released line is semver major `0`, so a major cannot pick +between them and resolves to the newest one. Omit the whole key and every call +stays in-process, which is what all other cases do. A routed callee must pin +its runner by hash: the nested executor runs with debug mode disabled, where +`:test` does not resolve. See `misc/routed_call` on v0.3.x + +A step's declared public-ABI `major` is read from the contract's root slot, so +the manager routes it the way a real host would. A step may set `major` +explicitly when its subject is what the executor does with a major the manager +would otherwise refuse to route β€” see `runner/major_mismatch` + ## Tags | Tag | Meaning | diff --git a/docs/website/src/impl-spec/appendix/host-loop.rst b/docs/website/src/impl-spec/appendix/host-loop.rst index 12ecf0fc..d79258aa 100644 --- a/docs/website/src/impl-spec/appendix/host-loop.rst +++ b/docs/website/src/impl-spec/appendix/host-loop.rst @@ -34,14 +34,49 @@ The executor accepts multiple ``--host`` arguments. Each host connection independently runs the protocol described below. In the manager deployment, host 0 is the node's host loop and host 1 is a -socketpair to the manager. The manager routes ``consume_result`` to itself -(host 1) while all other methods go to host 0. +socketpair to the manager. The manager serves ``consume_result``, ``run_nested`` +and -- by default -- ``resolve_callcontract_executor`` on host 1. + +``resolve_callcontract_executor`` moves to host 0 only when the run request sets +``hook_cross_contract_calls`` (see :doc:`manager-socket`). Otherwise the manager +answers it with a null reply, which keeps every ``CallContract`` in-process. A +host that does not route calls across major boundaries therefore need not +implement that method at all. + +A non-null answer is a calldata map naming the line the callee runs on, tagged on +``kind``: ``{"kind": "major", "major": u32}`` resolves by the manifest's rules, +falling back to the newest line when none provides that major, while +``{"kind": "version", "version": str}`` names a line outright -- an executor +directory as it stands, or a ``re:``-prefixed regex over manifest version keys, +which fails the run if it matches nothing. Only a version can mean one +particular line, since every line released so far is semver major ``0``. +Executor lines carry the payload without reading it; only the manager decodes +it. + +Connection Hello Data +~~~~~~~~~~~~~~~~~~~~~ + +``ExecutionData`` carries ``host_hello_data``, an array of byte strings +indexed by host connection index (the same index space ``method_hosts`` +values point into). Immediately after connecting host *i*, before writing the +first method byte, the executor writes entry *i* to that connection +**verbatim** -- raw bytes, no length prefix or framing. A missing, short, or +empty entry means nothing is written; an empty (or absent) array is +byte-identical to a connection without hello data. The bytes are supplied by +the host in the run request (see :doc:`manager-socket`), so the host knows +their length; a host wanting structure embeds its own framing. Protocol Loop ~~~~~~~~~~~~~ The :term:`host` processes requests in a loop. Each host connection runs -independently, handling only the methods routed to it: +independently, handling only the methods routed to it. The loop ends when the +executor closes the connection (EOF on the method-id read): on a stream +socket all written data precedes the FIN, so every response the host owes has +been requested and every result byte has arrived by then. The executor +flushes all host connections before exiting. Completion status (exit code, +result, clean-finish vs crash) is reported by the manager's terminal event +(:doc:`manager-socket`), not by a host method: :: @@ -90,19 +125,48 @@ independently, handling only the methods routed to it: write_byte json/errors/ok write_bytes balance.to_le_bytes(32) # 256-bit integer + json/methods/resolve_callcontract_executor: + address := read_bytes(ACCOUNT_ADDR_SIZE) + state := read_byte as json/storage_type + advisory_major := read_byte + payload, err := host_resolve_callcontract_executor( + address, state, advisory_major) + if err != json/errors/ok: + write_byte err + else: + write_byte json/errors/ok + # Calldata optional bytes preserve null vs an empty payload. + write_byte_slice calldata_encode(payload) + + json/methods/run_nested: + envelope := calldata_decode(read_slice()) + # The envelope carries no version of its own -- every executor line + # compiles the one definition. The manager validates the routing + # payload, resolves the executor at the inherited timestamp, and mints + # a fresh execution id. It uses the outer deadline/cancellation and no + # permit. + child := manager_start_nested_sync( + envelope, + storage_host = host_0, + result_host = host_1) + reply := { + result: child.result, + # The child's sub-VM small hash, not its execution hash: the caller + # folds this, and it must not depend on the route the host chose. + small_hash: child.small_hash, + effect_free: child.has_no_effects + } + write_byte_slice calldata_encode(reply) + json/methods/remaining_fuel_as_gen: fuel, err := host_remaining_fuel_as_gen() if err != json/errors/ok: write_byte err else: write_byte json/errors/ok - write_bytes fuel.to_le_bytes(8) # 64-bit integer, must be safe integer (fits in double) + write_bytes fuel.to_le_bytes(32) # 256-bit unsigned, little-endian, always 32 bytes json/methods/notify_nondet_disagreement: call_no := read_u32_le host_notify_nondet_disagreement(call_no) # note: this method doesn't send any response - - json/methods/notify_finished: - # signals that all execution is complete - write_byte 0x00 diff --git a/docs/website/src/impl-spec/appendix/index.rst b/docs/website/src/impl-spec/appendix/index.rst index 54dede99..5569c364 100644 --- a/docs/website/src/impl-spec/appendix/index.rst +++ b/docs/website/src/impl-spec/appendix/index.rst @@ -6,3 +6,4 @@ Appendix host-loop manager-api + manager-socket diff --git a/docs/website/src/impl-spec/appendix/manager-api.rst b/docs/website/src/impl-spec/appendix/manager-api.rst index 4edba3b5..f061263a 100644 --- a/docs/website/src/impl-spec/appendix/manager-api.rst +++ b/docs/website/src/impl-spec/appendix/manager-api.rst @@ -1,7 +1,15 @@ Manager API =========== -The GenVM Manager is an HTTP server that provides an API for managing GenVM instances, modules, and related operations. +The GenVM Manager exposes an HTTP server for administration (status, modules, +permits, log level, manifest, LLM checks, error descriptions, contract version +detection) and a framed socket for driving executions +(:doc:`manager-socket`). + +The execution endpoints below -- ``POST /genvm/run``, ``GET /genvm/{id}``, +``DELETE /genvm/{id}`` -- are **deprecated**: they remain for one release +train as a thin adapter over the same execution core the socket uses, then +get removed. New host integrations MUST use the socket protocol. Two endpoints below are tightly coupled with the runner manifest (the per-line available-runners listing, generated from ``runners-versions.json``, published on diff --git a/docs/website/src/impl-spec/appendix/manager-api.yaml b/docs/website/src/impl-spec/appendix/manager-api.yaml index 4d4647f7..54a52ef2 100644 --- a/docs/website/src/impl-spec/appendix/manager-api.yaml +++ b/docs/website/src/impl-spec/appendix/manager-api.yaml @@ -9,6 +9,24 @@ servers: description: Default local development server paths: + /ws: + get: + summary: Upgrade to the manager socket protocol + description: | + WebSocket upgrade. Everything past the handshake -- framing, methods, + events, error codes -- is specified in the Manager Socket Protocol + appendix, not here; OpenAPI cannot describe a bidirectional message + stream. Listed so this file stays a complete route table. + + This is the supported way to drive executions. The `/genvm/*` paths + below are deprecated adapters over the same core. + operationId: managerSocket + responses: + '101': + description: Switching protocols; the connection becomes a manager socket + '426': + description: Upgrade required (the request was not a WebSocket handshake) + /status: get: summary: Get manager status @@ -162,9 +180,22 @@ paths: /genvm/run: post: - summary: Start a GenVM execution + summary: Start a GenVM execution (deprecated) + deprecated: true description: | - Start a new GenVM instance for contract execution. + Deprecated adapter over the socket protocol's `run` (see the Manager + Socket Protocol appendix); kept for one release train while hosts + migrate. Unlike the socket `run`, this call responds only after the + execution has started (or failed to start). + + Both entry points decode the same request type and share the same start + path, so every field of `GenvmRunRequest` behaves identically here -- + including the ones the socket protocol introduced (`deadline`, + `host_hello_data`, `hook_cross_contract_calls`) and the `host_genvm_id` + idempotency token, whose reservation happens below both. That is a + guarantee for as long as this adapter exists, not an accident of shared + types. What differs is only what this call *returns*: an id after the + execution has started, rather than immediately. The request body uses calldata binary encoding (not JSON). The schema below shows the logical structure that must be encoded using the calldata format. @@ -195,8 +226,20 @@ paths: /genvm/{genvm_id}: get: - summary: Get GenVM status - description: Get the status of a specific GenVM instance. + summary: Get GenVM status (deprecated) + deprecated: true + description: | + Deprecated adapter over the socket protocol's `attach` + `ack`; kept + for one release train while hosts migrate + + The optional `wait` parameter exists so hosts that cannot use the + socket protocol can stop busy-polling. It does not un-deprecate this + route; the socket protocol remains the supported interface + + The route preserves the historical destructive semantics: a returned + non-null status acknowledges the execution, so a second GET answers + null. Socket clients get non-destructive reads with an explicit `ack` + instead operationId: getGenvmStatus parameters: - name: genvm_id @@ -205,6 +248,18 @@ paths: schema: type: integer description: Unique identifier of the GenVM instance + - name: wait + in: query + required: false + schema: + type: string + example: 30s + description: | + Maximum time to wait for a terminal status, as a number followed + by `ms`, `s`, `m`, or `h`, for example `500ms`, `30s`, or `1m` + + Values above 120 seconds are clamped to 120 seconds. If the wait + times out, the response has a null status rather than an error responses: '200': description: GenVM status @@ -218,11 +273,21 @@ paths: status: type: object description: Current status of the GenVM instance + properties: + consumed_result: + type: string + format: byte + nullable: true '500': $ref: '#/components/responses/ErrorResponse' delete: - summary: Shutdown GenVM instance - description: Gracefully shutdown a GenVM instance. + summary: Shutdown GenVM instance (deprecated) + deprecated: true + description: | + Deprecated adapter over the socket protocol's `cancel`; kept for one + release train while hosts migrate. Socket clients should rely on the + request-carried deadline plus `cancel` instead of a client-side + timeout that fires DELETE. operationId: shutdownGenvm parameters: - name: genvm_id @@ -253,6 +318,64 @@ paths: genvm_id: type: integer + /genvm/{genvm_id}/artifact: + get: + summary: Get retained GenVM artifact + description: | + Debug convenience adapter over the socket protocol's `get_artifact`. + Returns a base64-encoded chunk from one retained artifact. Socket + clients should use the framed `get_artifact` method instead. + operationId: getGenvmArtifact + parameters: + - name: genvm_id + in: path + required: true + schema: + type: integer + description: Unique identifier of the GenVM instance + - name: field + in: query + required: true + schema: + type: string + enum: [stdout, stderr, genvm_log] + description: Artifact field to read + - name: offset + in: query + required: false + schema: + type: integer + format: uint64 + default: 0 + description: Byte offset within the artifact + - name: max_len + in: query + required: true + schema: + type: integer + format: uint32 + description: Maximum bytes to return before server-side chunk capping + responses: + '200': + description: Artifact chunk + content: + application/json: + schema: + type: object + properties: + genvm_id: + type: integer + field: + type: string + total_len: + type: integer + format: uint64 + data_base64: + type: string + format: byte + '500': + $ref: '#/components/responses/ErrorResponse' + /contract/detect-version: post: summary: Detect contract version @@ -343,46 +466,6 @@ paths: '500': $ref: '#/components/responses/ErrorResponse' - /env: - post: - summary: Set environment variable - description: Set an environment variable in the manager process. - operationId: setEnv - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - key - - value - properties: - key: - type: string - description: Environment variable name - value: - type: string - description: Environment variable value - example: - key: DEBUG_MODE - value: "true" - responses: - '200': - description: Environment variable set - content: - application/json: - schema: - type: object - properties: - result: - type: string - example: env_var_set - key: - type: string - '500': - $ref: '#/components/responses/ErrorResponse' - /permits: get: summary: Get permits @@ -570,6 +653,51 @@ components: description: 20-byte Ethereum-style address as hex string example: "0x1234567890abcdef1234567890abcdef12345678" + ExecutorSelector: + description: Which executor line runs a request. + oneOf: + - $ref: '#/components/schemas/ExecutorSelectorMajor' + - $ref: '#/components/schemas/ExecutorSelectorVersion' + discriminator: + propertyName: kind + mapping: + major: '#/components/schemas/ExecutorSelectorMajor' + version: '#/components/schemas/ExecutorSelectorVersion' + + ExecutorSelectorMajor: + type: object + description: | + Resolved by the manifest's rules, falling back to the newest line when + none provides this major. Cannot separate lines that share a semver + major, which every released line does. + required: + - kind + - major + properties: + kind: + type: string + enum: [major] + major: + type: integer + format: uint32 + + ExecutorSelectorVersion: + type: object + description: | + Names a line: an executor directory used as it stands, or a `re:`-prefixed + regular expression over manifest version keys, the newest match winning. + No match fails the run. + required: + - kind + - version + properties: + kind: + type: string + enum: [version] + version: + type: string + example: "v0.2.17" + MessageData: type: object description: Contract execution message data @@ -577,7 +705,9 @@ components: - contract_address - sender_address - origin_address + - signer_address - chain_id + - value - is_init properties: contract_address: @@ -586,12 +716,8 @@ components: $ref: '#/components/schemas/Address' origin_address: $ref: '#/components/schemas/Address' - stack: - type: array - items: - $ref: '#/components/schemas/Address' - default: [] - description: View methods call chain (empty for entrypoint) + signer_address: + $ref: '#/components/schemas/Address' chain_id: type: string description: Chain ID as big integer string @@ -599,7 +725,7 @@ components: value: type: string description: Transaction value as big integer string - default: "0" + example: "0" is_init: type: boolean description: Whether this is a contract initialization call @@ -614,19 +740,20 @@ components: GenVM execution request. This structure must be encoded using the calldata binary format before sending. required: - - major + - selector - message - is_sync - - storage_pages + - bucket_totals - host_data - timestamp - host - calldata + - code + - leader_nondet_results + - initial_time_units_allocation properties: - major: - type: integer - format: uint32 - description: Major version specification + selector: + $ref: '#/components/schemas/ExecutorSelector' message: $ref: '#/components/schemas/MessageData' is_sync: @@ -650,10 +777,14 @@ components: format: uint64 default: 20 description: Maximum execution time in minutes - storage_pages: - type: integer - format: uint64 - description: Number of storage pages allocated + bucket_totals: + type: array + items: + type: string + description: | + Per-bucket data-fee balances, as big integer strings, indexed by the + bucket numbers the executor's fee config references. No default: a + request that omits it is rejected. host_data: type: string description: Host-specific data as JSON string @@ -678,15 +809,110 @@ components: type: string format: binary nullable: true - description: Optional contract bytecode (for deployment) - reroute_to: + description: | + Contract bytecode for a deployment, or null for a call. Required to be + present; only its value is nullable. + leader_nondet_results: + type: array + nullable: true + items: + type: string + format: binary + description: | + The leader's non-deterministic results for a validator run, or null for + a leader run. Required to be present; only its value is nullable. + initial_time_units_allocation: + type: integer + format: uint32 + description: Initial time-unit budget for the execution. + permissions: + type: string + default: wscn + description: | + Permission meta-properties granted to the entry VM, as a subset of the + characters `w` (write storage), `s` (send messages), `c` (call other + contracts) and `n` (spawn nondet). **Defaults to all four**, so a host + that computes narrower permissions must send this field explicitly. + no_modules: + type: boolean + default: false + description: | + Run without the LLM and web modules even when the permissions would + otherwise require them. + gas_data: + type: object + additionalProperties: + type: string + default: {} + description: Host-provided `node` fee constants. + message_fee_allocation: + type: array + items: + type: object + default: [] + description: | + Message-fee allocation tree. See `MessageAllocationNode` in + `crates/modules-interfaces/src/domain/fees/mod.rs` for the node shape. + record_actions: + type: array + items: + type: string + enum: [runner_load, vm_spawn] + default: [] + description: Auditable supervisor action kinds to return in the result. + host_genvm_id: + type: string + nullable: true + default: null + description: | + Client correlation token, echoed in this run's events. Also an + idempotency key on the socket protocol; see the `run` method in the + manager socket protocol page. + deadline: type: string - default: "" + nullable: true + default: null description: | - Debug-only override of the executor directory (a version dir under the - manager's executors path) to run, bypassing manifest resolution. Honored - only when `debug_mode` is `safe` or above; ignored in production - (`disabled`) so consensus always runs the manifest-resolved version. + Duration string that overrides `max_execution_minutes` as the strict + deadline when set. + host_hello_data: + type: array + items: + type: string + format: binary + default: [] + description: | + Bytes written verbatim to each host connection, indexed by connection + index, before the first method byte. The manager rejects a non-empty + entry for a connection it owns itself. + hook_cross_contract_calls: + type: boolean + default: false + description: | + Whether the host wants to answer `resolve_callcontract_executor`. When + false the manager answers it with a null reply and every `CallContract` + stays in-process. + unsafe_overrides: + type: object + description: | + Overrides that reach boundaries production traffic cannot. Each member + states the `debug_mode` it needs; with debugging disabled none apply. + properties: + reroute_to: + type: string + default: "" + description: | + Run this version instead of the one `selector` resolves to, in + the same two forms `selector`'s version variant accepts. Empty is + no override. Honored from `safe` up. + initial_recursion: + type: integer + format: uint32 + nullable: true + default: null + description: | + Seeds the chain's recursion budget, replacing the executor's own + `VM_RECURSION`. Honored from `unsafe` up. responses: ErrorResponse: diff --git a/docs/website/src/impl-spec/appendix/manager-socket-consts.rst b/docs/website/src/impl-spec/appendix/manager-socket-consts.rst new file mode 100644 index 00000000..08f57844 --- /dev/null +++ b/docs/website/src/impl-spec/appendix/manager-socket-consts.rst @@ -0,0 +1,130 @@ +Constants +========= + +.. _gvm-def-enum-methods: + +methods +------- + +Type: u16 + +.. _gvm-def-enum-value-methods-error: + +error +~~~~~ + +Value: ``0`` + +.. _gvm-def-enum-value-methods-hello: + +hello +~~~~~ + +Value: ``1`` + +.. _gvm-def-enum-value-methods-event: + +event +~~~~~ + +Value: ``2`` + +.. _gvm-def-enum-value-methods-run: + +run +~~~ + +Value: ``3`` + +.. _gvm-def-enum-value-methods-attach: + +attach +~~~~~~ + +Value: ``4`` + +.. _gvm-def-enum-value-methods-cancel: + +cancel +~~~~~~ + +Value: ``5`` + +.. _gvm-def-enum-value-methods-ack: + +ack +~~~ + +Value: ``6`` + +.. _gvm-def-enum-value-methods-get-artifact: + +get_artifact +~~~~~~~~~~~~ + +Value: ``7`` + +.. _gvm-def-enum-errors: + +errors +------ + +Type: u8 + +.. _gvm-def-enum-value-errors-internal: + +internal +~~~~~~~~ + +Value: ``0`` + +.. _gvm-def-enum-value-errors-malformed-frame: + +malformed_frame +~~~~~~~~~~~~~~~ + +Value: ``1`` + +.. _gvm-def-enum-value-errors-unknown-method: + +unknown_method +~~~~~~~~~~~~~~ + +Value: ``2`` + +.. _gvm-def-enum-value-errors-unknown-id: + +unknown_id +~~~~~~~~~~ + +Value: ``3`` + +.. _gvm-def-enum-value-errors-boot-id-mismatch: + +boot_id_mismatch +~~~~~~~~~~~~~~~~ + +Value: ``4`` + +.. _gvm-def-enum-value-errors-bad-request-id: + +bad_request_id +~~~~~~~~~~~~~~ + +Value: ``5`` + +.. _gvm-def-enum-value-errors-not-finished: + +not_finished +~~~~~~~~~~~~ + +Value: ``6`` + +.. _gvm-def-const-current-major: + +current_major +------------- + +Type: u32 + +Value: ``0`` diff --git a/docs/website/src/impl-spec/appendix/manager-socket.rst b/docs/website/src/impl-spec/appendix/manager-socket.rst new file mode 100644 index 00000000..12d49409 --- /dev/null +++ b/docs/website/src/impl-spec/appendix/manager-socket.rst @@ -0,0 +1,319 @@ +Manager Socket Protocol +======================= + +The manager exposes a long-lived WebSocket for driving executions. It replaces +the HTTP run flow (``POST /genvm/run`` + poll + ``DELETE``), which remains +available for one release train as a deprecated adapter over the same core +(see :doc:`manager-api`). The admin HTTP endpoints are unaffected. + +Clients connect by upgrading ``GET /ws`` on the manager's HTTP listener, so the +protocol is reachable wherever that listener is: over the unix socket when the +manager runs with ``--socket``, over TCP when it runs with ``--host``/``--port``. +There is no separate address to configure. + +No authentication: the manager assumes a trusted deployment (unix socket or +loopback); do not expose this listener beyond the machine boundary. + +.. toctree:: + :hidden: + :maxdepth: 1 + + manager-socket-consts + +Framing +------- + +Every protocol message is one **binary** WebSocket message; WebSocket already +delimits messages, so there is no length prefix. Header integers are +big-endian. Payloads are calldata-encoded (see +:doc:`/spec/04-contract-interface/01-calldata`). + +:: + + message := method_id:u16_be request_id:u64_be payload + payload := calldata(value) + +A message is therefore ``payload length + 10`` bytes. Note the byte order +differs from the executor host protocol, which is little-endian +(:doc:`host-loop`). + +- ``request_id != 0``: a request. Exactly one reply message follows, with the + same ``method_id`` and ``request_id`` echoed (or an ``error`` message with the + ``request_id`` echoed). +- ``request_id == 0``: a notification; no reply. Only the manager sends + notifications. A client message with ``request_id == 0`` is answered with an + ``error`` (``bad_request_id``, ``request_id == 0``). + +Request ids are chosen by the client and scoped to the connection; the manager +never initiates requests, so there is no id-space split. + +Unknown method ids, undecodable payloads, messages shorter than the 10-byte +header, and text messages each produce an ``error`` message without closing the +connection. + +A message larger than the manager's ``max_message_bytes`` is refused by the +WebSocket layer before the manager sees it. The cap is enforced while reading, +and reports as a read failure rather than a close handshake, so the connection +is **dropped without a close frame** -- a client observes an abnormal closure +(1006), not 1009, and no ``error`` payload is sent. Clients that page artifacts +within the documented chunk cap never approach the cap. + +Payloads are externally-tagged: a single-key map whose key names the variant. +The ``method_id`` header is authoritative for routing; the in-payload tag is +redundant by design (log readability). + +Method ids +---------- + +Generated from ``crates/modules-interfaces/codegen/data/manager-api.json`` +(Rust, Python and this page share one source). + +The generated constants are rendered in :doc:`manager-socket-consts`; see +:ref:`gvm-def-enum-methods` for the method id enum. Directions and request +kinds are described in each method section below. + +Connection hello +---------------- + +Immediately after accepting a connection the manager sends a ``hello`` +notification:: + + { "hello": { "boot_id": u64, "protocol_major": u32 } } + +``boot_id`` is random, generated once per manager process start. GenVM ids +restart from 1 with the process, so the durable identity of a run is the pair +``(boot_id, genvm_id)``. Clients MUST remember the ``boot_id`` and pass it to +``attach``; a mismatch after a manager restart is surfaced as +``boot_id_mismatch`` instead of silently binding to an unrelated, reused id. + +``run`` +------- + +Request payload: the same logical structure as the deprecated +``POST /genvm/run`` body (``GenvmRunRequest`` in :doc:`manager-api`), plus: + +- ``host_genvm_id`` (string, optional) -- client correlation token, echoed in + events for this run. Also an idempotency key: a ``run`` repeated with the + same token before the retention TTL expires returns the id already + allocated for it instead of starting a second execution. +- ``host_hello_data`` (array of bytes, optional, default ``[]``) -- indexed + by host connection index; the executor writes entry *i* verbatim to host + *i* on connect, before the first method byte (see :doc:`host-loop`). The + manager rejects a non-empty entry for a host connection it owns itself + (currently index 1, the ``consume_result`` socketpair). +- ``hook_cross_contract_calls`` (bool, optional, default ``false``) -- whether + this host wants to be asked where a ``CallContract`` runs. When false the + manager answers ``resolve_callcontract_executor`` itself with a null reply, + so every call stays in-process and the host need not implement that method. + When true the question is routed to host 0 and the host may send the caller + across a major boundary (see :doc:`host-loop`). A nested run inherits the + value from its parent. +- ``deadline`` (duration string, optional) -- when set, overrides + ``max_execution_minutes`` as the strict deadline. The manager enforces it + and pushes the terminal event; clients need no timeout timer of their own. + Either way the deadline is capped at 24 hours: a longer one is silently + shortened, not rejected. See `Duration strings`_. +- ``unsafe_overrides`` (map, optional) -- overrides that reach boundaries + production traffic cannot. Each member states the ``debug_mode`` it needs; + with debugging disabled none of them apply, so consensus traffic always runs + the manifest-resolved version with the executor's own limits. + + - ``reroute_to`` (string, optional) -- run this version instead of the one + ``selector`` resolves to. A plain string is an executor directory used as it + stands; a ``re:`` prefix makes the rest a regular expression matched against + manifest version keys, and the newest match wins. Honored from ``safe``. + - ``initial_recursion`` (u32, optional) -- seeds the chain's recursion + budget, replacing the executor's own ``VM_RECURSION``, so a boundary test + need not spend one executor process per unit of budget. Honored from + ``unsafe``. + +Response:: + + { "genvm_id": u64 } + +The id is allocated and returned immediately; validation, permit acquisition +and process spawn continue asynchronously and report through ``event`` +notifications. The requesting connection is subscribed to the run's events +automatically. + +``attach`` +---------- + +:: + + { "attach": { "boot_id": u64, "genvm_id": u64 } } + +Subscribes the connection to the run's events and returns a snapshot -- the +most recent lifecycle event for the run, in the same shape as an ``event`` +payload:: + + { "snapshot": <event payload> } + +Errors: ``boot_id_mismatch`` if ``boot_id`` is not the current process's; +``unknown_id`` if the run does not exist, was acked, or its retention TTL +expired. Disconnecting drops all of a connection's subscriptions; it does not +affect the run (see below). + +``cancel`` +---------- + +:: + + { "cancel": { "genvm_id": u64 } } + +Requests termination. Response is an empty map; the outcome arrives as the +run's single terminal event. Cancelling a run still queued on permits aborts +it before spawn (no permit is consumed) and still yields exactly one terminal +event with cause ``cancelled``. + +``ack`` +------- + +:: + + { "ack": { "genvm_id": u64 } } + +Releases the retained result and state for a finished run. Response is an +empty map. An ``ack`` before a terminal event or result is refused with +``not_finished`` and leaves the run fully usable. After ``ack`` (or after the +retention TTL expires), ``attach`` and ``get_artifact`` answer ``unknown_id``. +Reads before ``ack`` are non-destructive and repeatable from any number of +connections. + +``get_artifact`` +---------------- + +:: + + { "get_artifact": { "genvm_id": u64, "field": str, + "offset": u64, "max_len": u32 } } + +``field`` is one of ``stdout``, ``stderr``, ``genvm_log``. Response:: + + { "total_len": u64, "data": bytes } + +``data`` is at most ``min(max_len, chunk_cap)`` bytes starting at ``offset`` +(``chunk_cap`` is a server constant, 256 KiB); clients page until +``offset + len(data) == total_len``. ``genvm_log`` is served as JSON Lines +(one structured record per line). Artifact replies are sent on the same +connection through a low-priority writer queue, so bulk transfers cannot +starve lifecycle events. + +The terminal event carries each artifact's total size, so clients can skip +the calls entirely when the blobs are empty. + +``event`` notifications +----------------------- + +Externally-tagged; every event carries ``genvm_id`` and, when the run was +started with one, ``host_genvm_id``. Variants: + +``queued`` + The run is allocated but no executor process exists yet, because it is + waiting for a permit. Non-terminal. Mainly seen as the ``attach`` snapshot of + a run that has not spawned; a client that attaches this early always observes + the current lifecycle state and every terminal event, but intermediate states + may be coalesced, so a fast run can go straight from ``queued`` to a terminal + event. :: + + { "queued": { "genvm_id": u64, "host_genvm_id": str? } } + +``started`` + The executor process was spawned. :: + + { "started": { "genvm_id": u64, "host_genvm_id": str? } } + +``failed_to_start`` + Terminal. The run never reached a spawned executor: request validation, + version resolution, or the spawn itself failed. Permits are released. :: + + { "failed_to_start": { "genvm_id": u64, "host_genvm_id": str?, + "error": str } } + + An executor that *was* spawned and then died on its own -- rejecting its + arguments, say -- reports ``finished`` with that exit code, not this event. + The two are distinguished by whether a process was created, which the + manager knows exactly; "started, then died immediately" is not a state it + can observe without guessing, and a timer that guesses it would both tax + every healthy run and still race a slow failure. + + Either way the host receives a terminal event promptly, which is what + matters: this is the class of failure that used to leave it waiting on + ``accept()`` forever. + +``finished`` + Terminal, sent exactly once per run. :: + + { "finished": { + "genvm_id": u64, "host_genvm_id": str?, + "cause": str, # exited | cancelled | deadline | shutdown + "exit_code": i64?, # null when killed before exit code known + "consumed_result": bytes?, # what the executor sent via consume_result + "metrics": map?, + "finished_at": str, # RFC3339 + "version_major": u32, "version_minor": u32, + "artifact_sizes": { "stdout": u64, "stderr": u64, + "genvm_log": u64 } } } + +Lifecycle guarantees: + +- Exactly one terminal event (``failed_to_start`` or ``finished``) per run, + delivered to every connection subscribed at the time. +- **Disconnect does not kill a run.** Runs terminate only via ``cancel``, the + deadline, or manager shutdown. A client may disconnect, reconnect, and + ``attach`` by ``(boot_id, genvm_id)`` to recover the state and result. +- Results are retained until ``ack`` or the retention TTL (manager + configuration ``execution_retention``, a duration string, default ``5m``). + +Duration strings +---------------- + +Every duration on this wire and in the manager configuration is a string: +a decimal number followed immediately by a unit, with no space. + +============ ====================== +Unit suffix Meaning +============ ====================== +``ms`` milliseconds +``s`` seconds +``m`` minutes +``h`` hours +============ ====================== + +Examples: ``"30s"``, ``"10.5m"``, ``"1h"``, ``"250ms"``. The fractional part +is optional and is resolved at millisecond granularity; a value that is not a +number followed by one of the units above is rejected as ``malformed_frame`` +(or as a configuration error at startup). + +A bare number is **not** accepted -- the unit is mandatory, so that a value +can never be silently misread as the wrong scale. + +Error messages +-------------- + +``method_id = error``, ``request_id`` echoed from the failing request +(``0`` when no request id could be attributed):: + + { "code": u8, "message": str } + +See :ref:`gvm-def-enum-errors` for the generated error code enum. Meanings: +``internal`` is a handler failure with a diagnostic message; +``malformed_frame`` is failed calldata decode, a bad payload shape, a message +shorter than the header, or a non-binary message; ``unknown_method`` is a +method id not in the generated table; ``unknown_id`` means the run never +existed, was acked, or expired; ``boot_id_mismatch`` is ``attach`` across a +manager restart; ``bad_request_id`` is a client request with +``request_id == 0``; and ``not_finished`` is ``ack`` before a terminal event +or result. + +An oversized message has no error code: the connection is dropped, as described +under `Framing`_. + +Scope note +---------- + +Multiple connections attaching to one run observe it through the *manager* +channel only. Host connection 0 -- the socket the executor dials and speaks +the host protocol on (:doc:`host-loop`) -- is owned by whichever address the +``run`` request supplied; attaching does not share it. ``host_hello_data`` is +the building block for future shared-host schemes. diff --git a/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/02-gl_call.rst b/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/02-gl_call.rst index ba7b5592..8aafb847 100644 --- a/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/02-gl_call.rst +++ b/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/02-gl_call.rst @@ -78,6 +78,14 @@ Requirements Creates a :term:`sub-VM`. See :ref:`gvm-meta-property-derivation`. +Before every call, :term:`GenVM` asks the :term:`host` whether to delegate the +callee to another executor. A null answer preserves the local path, including +:ref:`gvm-def-str-trie-value-vm-error-invalid-contract-major-mismatch`. A +non-null answer executes the same derived :term:`sub-VM` in the selected +executor. The call result and its contribution to the execution hash are the +same kinds of observable output in either path. Routing cycles are permitted +and remain bounded by :ref:`gvm-def-consts-value-top-limits-vm-recursion`. + The calling convention is checked in the calling :term:`sub-VM`, before the callee is spawned. A violation is the caller's own malformed argument, so it is answered with ``Errno::Inval`` like the other argument checks and the caller can diff --git a/docs/website/src/spec/03-vm/01-startup.rst b/docs/website/src/spec/03-vm/01-startup.rst index 1f4cbc4a..97c50836 100644 --- a/docs/website/src/spec/03-vm/01-startup.rst +++ b/docs/website/src/spec/03-vm/01-startup.rst @@ -16,7 +16,9 @@ Before executing the root :term:`sub-VM`, :term:`GenVM`: #. Validates the entry payload against :ref:`gvm-def-contract-call-conv` (see below). -#. Reads the contract's locked slots and root-slot data. +#. Reads the contract's root-slot data, and its locked slots when the run may + write storage. Locked slots only constrain writes, so a run without that + permission does not read them. #. Resolves the contract code from deployment input or from the contract's configured code slot. #. Checks upgrade and ABI-major compatibility as specified in @@ -158,7 +160,9 @@ The child is read-only. writes (see :ref:`contract-execution-flow`). - :ref:`gvm_vm_field_topmost_runner_id` is the callee's contract runner. - :ref:`gvm_vm_field_granted_custom` is the caller's entire loaded - custom-runner set (see `Custom-Runner Grants`_). + custom-runner set (see `Custom-Runner Grants`_). A call the host delegates to + another executor cannot carry them, so it is rejected instead --- see + `Custom-Runner Grants`_. - The child's startup message is a copy of the caller's, except: - ``contract_address`` is the callee's address @@ -229,7 +233,11 @@ the caller's loaded custom-runner set the caller and is granted implicitly. - :ref:`gvm-def-gl-call-call-contract` children receive the caller's entire - custom set. + custom set. Grants are process-local, so this holds only for a child the + executor runs itself: when the host delegates the callee to another executor + (see :ref:`gvm-def-contract-version`) and the caller has any custom runner + loaded, the call fails with the ``error_inval`` code instead, rather than + running the callee with a set that depends on the route. Each grant is a load action in the child, charged against the child's RAM budget. Grants are applied before the entry-runner load, so a ``custom:`` diff --git a/docs/website/src/spec/03-vm/02-meta-properties.rst b/docs/website/src/spec/03-vm/02-meta-properties.rst index b1a72623..d66e3bf1 100644 --- a/docs/website/src/spec/03-vm/02-meta-properties.rst +++ b/docs/website/src/spec/03-vm/02-meta-properties.rst @@ -123,6 +123,11 @@ Requires :ref:`gvm-perm-deterministic`. Allows spawning :ref:`gvm-def-non-det-mode` :term:`sub-VM` instances via :ref:`gvm-def-gl-call-run-nondet`. +A :term:`sub-VM` that a host delegated to another executor (see +:ref:`gvm-def-contract-version`) never holds this permission, whatever its +caller holds. Such a :term:`sub-VM` has no connection to the non-deterministic +modules, so the permission cannot be served there. + .. _gvm-perm-register-runners: ``register_runners`` diff --git a/docs/website/src/spec/03-vm/05-result.rst b/docs/website/src/spec/03-vm/05-result.rst index e3147b0a..7cd20124 100644 --- a/docs/website/src/spec/03-vm/05-result.rst +++ b/docs/website/src/spec/03-vm/05-result.rst @@ -158,6 +158,12 @@ Non-deterministic sub-calls do not contribute. A run with no deterministic sub-c finalizes its accumulator to a fixed digest, so that error and edge results hash uniformly. +The small hash of a sub-VM the host delegated to another executor +(see :ref:`gvm-def-contract-version`) is computed the same way, from the values +that executor reports. Where the callee ran is not an input: a call that a host +routes to another executor MUST fold the same value it would have folded had the +callee run in-process. + Post-Execution Result Validation -------------------------------- diff --git a/docs/website/src/spec/04-contract-interface/04-upgradability.rst b/docs/website/src/spec/04-contract-interface/04-upgradability.rst index acd3a474..41c441aa 100644 --- a/docs/website/src/spec/04-contract-interface/04-upgradability.rst +++ b/docs/website/src/spec/04-contract-interface/04-upgradability.rst @@ -51,9 +51,12 @@ Contract Major Version ---------------------- The ``major`` field of :ref:`genvm-def-root-slot` is at -:ref:`gvm-def-consts-value-root-offsets-major` and stores the public-ABI major version that the -contract was built against. On every load :term:`GenVM` compares this byte to its own -``CURRENT_MAJOR`` constant and refuses to execute a contract whose major differs. +:ref:`gvm-def-consts-value-root-offsets-major` and stores the public-ABI major +version that the contract was built against. Top-level and runner loads compare +this byte to ``CURRENT_MAJOR`` and fail with +:ref:`gvm-def-str-trie-value-vm-error-invalid-contract-major-mismatch` when it +differs. :ref:`gvm-def-gl-call-call-contract` may instead delegate the callee to +another executor selected by the host. The value is **not** modifiable by the contract itself: it is written once at deploy time by the host from a value detected in the contract package (a ``genvm.version`` custom WASM diff --git a/docs/website/src/spec/appendix/constants.rst b/docs/website/src/spec/appendix/constants.rst index 3ce82216..aa114299 100644 --- a/docs/website/src/spec/appendix/constants.rst +++ b/docs/website/src/spec/appendix/constants.rst @@ -519,6 +519,8 @@ Param: i32 ``invalid_contract major_mismatch`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +This error remains terminal for top-level and runner loads. During :ref:`gvm-def-gl-call-call-contract`, the host may select another executor before the local major check. + .. _gvm-def-str-trie-value-vm-error-invalid-contract-wasm-validating: ``invalid_contract wasm validating`` diff --git a/executors/v0.2.x b/executors/v0.2.x index 1c8126d6..e3e1ffa8 160000 --- a/executors/v0.2.x +++ b/executors/v0.2.x @@ -1 +1 @@ -Subproject commit 1c8126d6efb6cab9fdce02b1745217fc571a343b +Subproject commit e3e1ffa8d2aa7560a1d0ca4a3f8788ea5e287b42 diff --git a/executors/v0.3.x b/executors/v0.3.x index 9d2db0dd..ef7dd2fb 160000 --- a/executors/v0.3.x +++ b/executors/v0.3.x @@ -1 +1 @@ -Subproject commit 9d2db0dd98506cca712c18af4cde7b5dd926bf0d +Subproject commit ef7dd2fb9bab428827972ca67e5e92a18be8d82b diff --git a/implementation/Cargo.lock b/implementation/Cargo.lock index 89914f17..37e629a6 100644 --- a/implementation/Cargo.lock +++ b/implementation/Cargo.lock @@ -212,7 +212,7 @@ dependencies = [ "num-traits", "pastey", "rayon", - "thiserror 2.0.18", + "thiserror", "v_frame", "y4m", ] @@ -241,10 +241,59 @@ dependencies = [ ] [[package]] -name = "base64" -version = "0.21.7" +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "base64", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sha1", + "sync_wrapper", + "tokio", + "tokio-tungstenite 0.29.0", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] [[package]] name = "base64" @@ -905,7 +954,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" dependencies = [ "futures-core", - "futures-sink", ] [[package]] @@ -1036,7 +1084,7 @@ dependencies = [ "serde_json", "serde_yaml", "symbol_table", - "thiserror 2.0.18", + "thiserror", "tokio", ] @@ -1047,7 +1095,8 @@ dependencies = [ "afl", "anyhow", "async-trait", - "base64 0.22.1", + "axum", + "base64", "bytes", "cc", "chrono", @@ -1059,7 +1108,6 @@ dependencies = [ "genvm_modules_interfaces", "hex", "hickory-resolver", - "hyper 0.14.32", "image", "libc", "mlua", @@ -1087,7 +1135,6 @@ dependencies = [ "unicode-normalization", "unicode-width", "url", - "warp", ] [[package]] @@ -1106,6 +1153,7 @@ dependencies = [ "serde_bytes", "serde_derive", "serde_json", + "sha3", "tokio", ] @@ -1159,25 +1207,6 @@ dependencies = [ "weezl", ] -[[package]] -name = "h2" -version = "0.3.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d" -dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http 0.2.12", - "indexmap 2.14.0", - "slab", - "tokio", - "tokio-util", - "tracing", -] - [[package]] name = "h2" version = "0.4.13" @@ -1189,7 +1218,7 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http 1.4.0", + "http", "indexmap 2.14.0", "slab", "tokio", @@ -1237,30 +1266,6 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" -[[package]] -name = "headers" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06683b93020a07e3dbcf5f8c0f6d40080d725bea7936fc01ad345c01b97dc270" -dependencies = [ - "base64 0.21.7", - "bytes", - "headers-core", - "http 0.2.12", - "httpdate", - "mime", - "sha1", -] - -[[package]] -name = "headers-core" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7f66481bfee273957b1f20485a4ff3362987f85b2c236580d81b4eb7a326429" -dependencies = [ - "http 0.2.12", -] - [[package]] name = "heck" version = "0.5.0" @@ -1291,7 +1296,7 @@ dependencies = [ "once_cell", "rand 0.9.4", "ring", - "thiserror 2.0.18", + "thiserror", "tinyvec", "tokio", "tracing", @@ -1314,7 +1319,7 @@ dependencies = [ "rand 0.9.4", "resolv-conf", "smallvec", - "thiserror 2.0.18", + "thiserror", "tokio", "tracing", ] @@ -1328,17 +1333,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "http" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - [[package]] name = "http" version = "1.4.0" @@ -1349,17 +1343,6 @@ dependencies = [ "itoa", ] -[[package]] -name = "http-body" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" -dependencies = [ - "bytes", - "http 0.2.12", - "pin-project-lite", -] - [[package]] name = "http-body" version = "1.0.1" @@ -1367,7 +1350,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.4.0", + "http", ] [[package]] @@ -1378,8 +1361,8 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http 1.4.0", - "http-body 1.0.1", + "http", + "http-body", "pin-project-lite", ] @@ -1395,30 +1378,6 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" -[[package]] -name = "hyper" -version = "0.14.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" -dependencies = [ - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "h2 0.3.27", - "http 0.2.12", - "http-body 0.4.6", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "socket2 0.5.10", - "tokio", - "tower-service", - "tracing", - "want", -] - [[package]] name = "hyper" version = "1.9.0" @@ -1429,10 +1388,11 @@ dependencies = [ "bytes", "futures-channel", "futures-core", - "h2 0.4.13", - "http 1.4.0", - "http-body 1.0.1", + "h2", + "http", + "http-body", "httparse", + "httpdate", "itoa", "pin-project-lite", "smallvec", @@ -1446,8 +1406,8 @@ version = "0.27.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ - "http 1.4.0", - "hyper 1.9.0", + "http", + "hyper", "hyper-util", "rustls", "tokio", @@ -1462,18 +1422,18 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64 0.22.1", + "base64", "bytes", "futures-channel", "futures-util", - "http 1.4.0", - "http-body 1.0.1", - "hyper 1.9.0", + "http", + "http-body", + "hyper", "ipnet", "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.3", + "socket2", "tokio", "tower-service", "tracing", @@ -1727,7 +1687,7 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d40460c0ce33d6ce4b0630ad68ff63d6661961c48b6dba35e5a4d81cfb48222" dependencies = [ - "socket2 0.6.3", + "socket2", "widestring", "windows-registry", "windows-result 0.4.1", @@ -1918,6 +1878,12 @@ dependencies = [ "which", ] +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + [[package]] name = "maybe-rayon" version = "0.1.1" @@ -1940,16 +1906,6 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" -[[package]] -name = "mime_guess" -version = "2.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" -dependencies = [ - "mime", - "unicase", -] - [[package]] name = "miniz_oxide" version = "0.8.9" @@ -2030,24 +1986,6 @@ dependencies = [ "pxfm", ] -[[package]] -name = "multer" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01acbdc23469fd8fe07ab135923371d5f5a422fbf9c522158677c8eb15bc51c2" -dependencies = [ - "bytes", - "encoding_rs", - "futures-util", - "http 0.2.12", - "httparse", - "log", - "memchr", - "mime", - "spin", - "version_check", -] - [[package]] name = "new_debug_unreachable" version = "1.0.6" @@ -2267,26 +2205,6 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" -[[package]] -name = "pin-project" -version = "1.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1749c7ed4bcaf4c3d0a3efc28538844fb29bcdd7d2b67b2be7e20ba861ff517" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "pin-project-lite" version = "0.2.17" @@ -2451,8 +2369,8 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.6.3", - "thiserror 2.0.18", + "socket2", + "thiserror", "tokio", "tracing", "web-time", @@ -2473,7 +2391,7 @@ dependencies = [ "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.18", + "thiserror", "tinyvec", "tracing", "web-time", @@ -2488,7 +2406,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.3", + "socket2", "tracing", "windows-sys 0.60.2", ] @@ -2609,7 +2527,7 @@ dependencies = [ "rand 0.9.4", "rand_chacha 0.9.0", "simd_helpers", - "thiserror 2.0.18", + "thiserror", "v_frame", "wasm-bindgen", ] @@ -2713,17 +2631,17 @@ version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ - "base64 0.22.1", + "base64", "bytes", "cookie", "cookie_store", "encoding_rs", "futures-core", - "h2 0.4.13", - "http 1.4.0", - "http-body 1.0.1", + "h2", + "http", + "http-body", "http-body-util", - "hyper 1.9.0", + "hyper", "hyper-rustls", "hyper-util", "js-sys", @@ -2903,12 +2821,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "scoped-tls" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" - [[package]] name = "scopeguard" version = "1.2.0" @@ -2995,6 +2907,17 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -3013,7 +2936,7 @@ version = "3.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd5414fad8e6907dbdd5bc441a50ae8d6e26151a03b1de04d89a5576de61d01f" dependencies = [ - "base64 0.22.1", + "base64", "chrono", "hex", "indexmap 1.9.3", @@ -3125,16 +3048,6 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" -[[package]] -name = "socket2" -version = "0.5.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - [[package]] name = "socket2" version = "0.6.3" @@ -3145,12 +3058,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "spin" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" - [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -3243,33 +3150,13 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" -[[package]] -name = "thiserror" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" -dependencies = [ - "thiserror-impl 1.0.69", -] - [[package]] name = "thiserror" version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" dependencies = [ - "thiserror-impl 2.0.18", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" -dependencies = [ - "proc-macro2", - "quote", - "syn", + "thiserror-impl", ] [[package]] @@ -3364,7 +3251,7 @@ dependencies = [ "mio", "pin-project-lite", "signal-hook-registry", - "socket2 0.6.3", + "socket2", "tokio-macros", "windows-sys 0.61.2", ] @@ -3392,26 +3279,26 @@ dependencies = [ [[package]] name = "tokio-tungstenite" -version = "0.21.0" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c83b561d025642014097b66e6c1bb422783339e0909e4429cde4749d1990bc38" +checksum = "7a9daff607c6d2bf6c16fd681ccb7eecc83e4e2cdc1ca067ffaadfca5de7f084" dependencies = [ "futures-util", "log", "tokio", - "tungstenite 0.21.0", + "tungstenite 0.26.2", ] [[package]] name = "tokio-tungstenite" -version = "0.26.2" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a9daff607c6d2bf6c16fd681ccb7eecc83e4e2cdc1ca067ffaadfca5de7f084" +checksum = "8f72a05e828585856dacd553fba484c242c46e391fb0e58917c942ee9202915c" dependencies = [ "futures-util", "log", "tokio", - "tungstenite 0.26.2", + "tungstenite 0.29.0", ] [[package]] @@ -3470,6 +3357,7 @@ dependencies = [ "tokio", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -3483,8 +3371,8 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "http 1.4.0", - "http-body 1.0.1", + "http", + "http-body", "http-body-util", "iri-string", "pin-project-lite", @@ -3541,38 +3429,35 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "tungstenite" -version = "0.21.0" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ef1a641ea34f399a848dea702823bbecfb4c486f911735368f1f137cb8257e1" +checksum = "4793cb5e56680ecbb1d843515b23b6de9a75eb04b66643e256a396d43be33c13" dependencies = [ - "byteorder", "bytes", "data-encoding", - "http 1.4.0", + "http", "httparse", "log", - "rand 0.8.6", + "rand 0.9.4", "sha1", - "thiserror 1.0.69", - "url", + "thiserror", "utf-8", ] [[package]] name = "tungstenite" -version = "0.26.2" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4793cb5e56680ecbb1d843515b23b6de9a75eb04b66643e256a396d43be33c13" +checksum = "6c01152af293afb9c7c2a57e4b559c5620b421f6d133261c60dd2d0cdb38e6b8" dependencies = [ "bytes", "data-encoding", - "http 1.4.0", + "http", "httparse", "log", "rand 0.9.4", "sha1", - "thiserror 2.0.18", - "utf-8", + "thiserror", ] [[package]] @@ -3599,12 +3484,6 @@ dependencies = [ "static_assertions", ] -[[package]] -name = "unicase" -version = "2.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" - [[package]] name = "unicode-ident" version = "1.0.24" @@ -3712,35 +3591,6 @@ dependencies = [ "try-lock", ] -[[package]] -name = "warp" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4378d202ff965b011c64817db11d5829506d3404edeadb61f190d111da3f231c" -dependencies = [ - "bytes", - "futures-channel", - "futures-util", - "headers", - "http 0.2.12", - "hyper 0.14.32", - "log", - "mime", - "mime_guess", - "multer", - "percent-encoding", - "pin-project", - "scoped-tls", - "serde", - "serde_json", - "serde_urlencoded", - "tokio", - "tokio-tungstenite 0.21.0", - "tokio-util", - "tower-service", - "tracing", -] - [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" diff --git a/implementation/Cargo.toml b/implementation/Cargo.toml index 504d7271..80a3b907 100644 --- a/implementation/Cargo.toml +++ b/implementation/Cargo.toml @@ -30,13 +30,13 @@ genvm_modules_interfaces = { path = "../executors/v0.3.x/executor/crates/modules anyhow = { version = "1.0.98", features = [] } async-trait = "0.1.88" +axum = { version = "0.8", features = ["ws"] } base64 = "0.22.1" bytes = "1.10.1" clap = { version = "4.5.35", features = ["derive"] } crossbeam = "0.8.4" futures-util = "0.3.31" hex = "0.4.3" -hyper = { version = "0.14", features = ["server", "http1", "tcp"] } image = { version = "0.25.8", features = ["png", "jpeg", "webp"] } libc = "0.2" num-bigint = { version = "0.4.6", features = ["serde"] } @@ -62,7 +62,6 @@ tokio-tungstenite = "0.26.2" unicode-normalization = "0.1.20" unicode-width = "0.2.1" url = { version = "2.5.4", features = ["serde"] } -warp = "0.3.7" # kept in sync with the version reqwest depends on, to avoid a duplicate hickory # in the tree hickory-resolver = { version = "0.25", features = ["tokio"] } @@ -79,7 +78,16 @@ version = "0.12.15" [dependencies.tokio] default-features = false -features = ["rt", "rt-multi-thread", "net", "time", "macros", "sync", "process"] +features = [ + "rt", + "rt-multi-thread", + "net", + "time", + "macros", + "sync", + "process", + "fs", +] version = "1.44.1" # we do not use anyhow feature because it does not work with boxed downcast_ref diff --git a/implementation/src/manager/handlers.rs b/implementation/src/manager/handlers.rs index 35f73402..79721ab0 100644 --- a/implementation/src/manager/handlers.rs +++ b/implementation/src/manager/handlers.rs @@ -1,4 +1,5 @@ use anyhow::Result; +use axum::Json; use genlayer_calldata as calldata; use genvm_common::*; use std::collections::{BTreeMap, HashMap}; @@ -13,8 +14,8 @@ use crate::{common, llm, scripting}; use super::AppContext; -pub async fn handle_status(ctx: sync::DArc<AppContext>) -> Result<impl warp::Reply> { - Ok(warp::reply::json(&serde_json::json!({ +pub async fn handle_status(ctx: sync::DArc<AppContext>) -> Result<Json<serde_json::Value>> { + Ok(Json(serde_json::json!({ "llm_module": ctx.mod_ctx.get_status(modules::Type::Llm).await, "web_module": ctx.mod_ctx.get_status(modules::Type::Web).await, "permits": { @@ -33,7 +34,7 @@ struct StopRequest { pub async fn handle_module_stop( ctx: sync::DArc<AppContext>, calldata: serde_json::Value, -) -> Result<impl warp::Reply, anyhow::Error> { +) -> Result<Json<serde_json::Value>, anyhow::Error> { let stop_request = serde_json::from_value::<StopRequest>(calldata.clone())?; let res = ctx.mod_ctx.stop(stop_request.module_type).await?; @@ -44,39 +45,35 @@ pub async fn handle_module_stop( "module_not_running" }; - Ok(warp::reply::json(&serde_json::json!({"result": res}))) + Ok(Json(serde_json::json!({"result": res}))) } pub async fn handle_module_start( ctx: sync::DArc<AppContext>, calldata: serde_json::Value, -) -> Result<impl warp::Reply> { +) -> Result<Json<serde_json::Value>> { let req = serde_json::from_value::<modules::StartRequest>(calldata)?; ctx.mod_ctx.start(req).await?; - Ok(warp::reply::json( - &serde_json::json!({"result": "module_started"}), - )) + Ok(Json(serde_json::json!({"result": "module_started"}))) } pub async fn handle_module_restart( ctx: sync::DArc<AppContext>, calldata: serde_json::Value, -) -> Result<impl warp::Reply> { +) -> Result<Json<serde_json::Value>> { let req = serde_json::from_value::<modules::StartRequest>(calldata)?; ctx.mod_ctx.restart(req).await?; - Ok(warp::reply::json( - &serde_json::json!({"result": "module_restarted"}), - )) + Ok(Json(serde_json::json!({"result": "module_restarted"}))) } pub async fn handle_genvm_run( ctx: sync::DArc<AppContext>, data: &[u8], -) -> Result<impl warp::Reply> { +) -> Result<Json<serde_json::Value>> { let mut res: super::run::Request = calldata::decode_obj(data)?; res.patch_legacy_method_key(); @@ -92,22 +89,36 @@ pub async fn handle_genvm_run( None }; - let (id, _) = super::run::start_genvm(ctx, res, Box::new(modules_lock)).await?; + let run_ctx = ctx.gep(|x| &x.run_ctx); + let id = run_ctx.start(ctx, res, Box::new(modules_lock)).await?; + let (snapshot, mut events) = run_ctx.attach(run_ctx.boot_id(), id)?; + + let start_event = match snapshot { + run::Snapshot::Event(event) => event, + run::Snapshot::Queued { .. } => loop { + events.changed().await?; + if let run::Snapshot::Event(event) = events.borrow_and_update().clone() { + break event; + } + }, + }; - Ok(warp::reply::json( - &serde_json::json!({"result": "started", "id": id}), - )) + if let run::Event::FailedToStart { error, .. } = start_event { + anyhow::bail!(error); + } + + Ok(Json(serde_json::json!({"result": "started", "id": id}))) } pub async fn handle_contract_detect_version( ctx: sync::DArc<AppContext>, contract_code: bytes::Bytes, deployment_timestamp: String, -) -> Result<impl warp::Reply> { +) -> Result<Json<serde_json::Value>> { let deployment_timestamp = chrono::DateTime::parse_from_rfc3339(&deployment_timestamp)?.with_timezone(&chrono::Utc); let major = versioning::detect_major_spec(&ctx, &contract_code, deployment_timestamp).await?; - Ok(warp::reply::json(&serde_json::json!({ + Ok(Json(serde_json::json!({ "specified_major": major, }))) } @@ -115,7 +126,7 @@ pub async fn handle_contract_detect_version( pub async fn handle_set_log_level( _ctx: sync::DArc<AppContext>, data: serde_json::Value, -) -> Result<impl warp::Reply> { +) -> Result<Json<serde_json::Value>> { let level = data .get("level") .and_then(|v| v.as_str()) @@ -127,49 +138,28 @@ pub async fn handle_set_log_level( }; logger.set_filter(level); - Ok(warp::reply::json( - &serde_json::json!({"result": "log_level_set", "level": level}), + Ok(Json( + serde_json::json!({"result": "log_level_set", "level": level}), )) } -pub async fn handle_manifest_reload(ctx: sync::DArc<AppContext>) -> Result<impl warp::Reply> { +pub async fn handle_manifest_reload( + ctx: sync::DArc<AppContext>, +) -> Result<Json<serde_json::Value>> { ctx.ver_ctx.reload_manifest().await?; - Ok(warp::reply::json( - &serde_json::json!({"result": "manifest_reloaded"}), - )) -} - -pub async fn handle_set_env( - _ctx: sync::DArc<AppContext>, - data: serde_json::Value, -) -> Result<impl warp::Reply> { - let key = data - .get("key") - .and_then(|v| v.as_str()) - .ok_or_else(|| anyhow::anyhow!("invalid env var key"))?; - - let value = data - .get("value") - .and_then(|v| v.as_str()) - .ok_or_else(|| anyhow::anyhow!("invalid env var value"))?; - - std::env::set_var(key, value); - - Ok(warp::reply::json( - &serde_json::json!({"result": "env_var_set", "key": key}), - )) + Ok(Json(serde_json::json!({"result": "manifest_reloaded"}))) } -pub async fn handle_get_permits(ctx: sync::DArc<AppContext>) -> Result<impl warp::Reply> { +pub async fn handle_get_permits(ctx: sync::DArc<AppContext>) -> Result<Json<serde_json::Value>> { let permits = ctx.run_ctx.get_max_permits().await; - Ok(warp::reply::json(&serde_json::json!({"permits": permits}))) + Ok(Json(serde_json::json!({"permits": permits}))) } pub async fn handle_set_permits( ctx: sync::DArc<AppContext>, data: serde_json::Value, -) -> Result<impl warp::Reply> { +) -> Result<Json<serde_json::Value>> { let permits = data .get("permits") .and_then(|v| v.as_u64()) @@ -178,8 +168,8 @@ pub async fn handle_set_permits( let new_permits = ctx.run_ctx.set_permits(permits).await; - Ok(warp::reply::json( - &serde_json::json!({"result": "permits_set", "permits": new_permits}), + Ok(Json( + serde_json::json!({"result": "permits_set", "permits": new_permits}), )) } @@ -190,33 +180,93 @@ pub async fn handle_genvm_shutdown( ctx: sync::DArc<AppContext>, genvm_id: run::GenVMId, _req: ShutdownRequest, -) -> Result<impl warp::Reply> { - let result = ctx.run_ctx.graceful_shutdown(genvm_id).await; +) -> Result<Json<serde_json::Value>> { + let result = async { + ctx.run_ctx.cancel(genvm_id).await?; + let _ = ctx.run_ctx.await_terminal(genvm_id).await?; + anyhow::Ok(()) + } + .await; match result { - Ok(()) => Ok(warp::reply::json(&serde_json::json!({ + Ok(()) => Ok(Json(serde_json::json!({ "result": "shutdown_completed", "genvm_id": genvm_id }))), - Err(e) => Ok(warp::reply::json(&serde_json::json!({ + Err(e) => Ok(Json(serde_json::json!({ "error": format!("{}", e), "genvm_id": genvm_id }))), } } +const STATUS_WAIT_CAP: std::time::Duration = std::time::Duration::from_secs(120); + +#[derive(Debug, serde::Deserialize)] +pub struct StatusRequest { + #[serde(default)] + pub wait: Option<String>, +} + +fn parse_status_wait(req: &StatusRequest) -> Result<Option<std::time::Duration>> { + req.wait + .as_deref() + .map(run::ManagerDuration::try_from) + .transpose() + .map(|wait| { + wait.map(run::ManagerDuration::to_std) + .map(|wait| wait.min(STATUS_WAIT_CAP)) + }) +} + pub async fn handle_genvm_status( ctx: sync::DArc<AppContext>, genvm_id: run::GenVMId, -) -> Result<impl warp::Reply> { - let status = ctx.run_ctx.fetch_genvm_status(genvm_id).await; + req: StatusRequest, +) -> Result<Json<serde_json::Value>> { + if let Some(wait) = parse_status_wait(&req)? { + if ctx.run_ctx.status(genvm_id).is_none() { + let _ = tokio::time::timeout(wait, ctx.run_ctx.await_terminal(genvm_id)).await; + } + } + + let status = ctx.run_ctx.status(genvm_id); + if status.is_some() { + ctx.run_ctx.ack(genvm_id); + } - Ok(warp::reply::json(&serde_json::json!({ + Ok(Json(serde_json::json!({ "genvm_id": genvm_id, "status": status }))) } +#[derive(Debug, serde::Deserialize)] +pub struct ArtifactRequest { + pub field: String, + #[serde(default)] + pub offset: u64, + pub max_len: u32, +} + +pub async fn handle_genvm_artifact( + ctx: sync::DArc<AppContext>, + genvm_id: run::GenVMId, + req: ArtifactRequest, +) -> Result<Json<serde_json::Value>> { + use base64::Engine; + + let artifact = ctx + .run_ctx + .get_artifact(genvm_id, &req.field, req.offset, req.max_len)?; + Ok(Json(serde_json::json!({ + "genvm_id": genvm_id, + "field": req.field, + "total_len": artifact.total_len, + "data_base64": base64::engine::general_purpose::STANDARD.encode(&artifact.data), + }))) +} + #[derive(serde::Deserialize)] pub struct LlmCheckRequest { pub configs: Vec<LlmProviderConfig>, @@ -243,7 +293,7 @@ pub struct LlmAvailabilityResult { pub async fn handle_llm_check( _ctx: sync::DArc<AppContext>, data: serde_json::Value, -) -> Result<impl warp::Reply> { +) -> Result<Json<Vec<LlmAvailabilityResult>>> { let request: LlmCheckRequest = serde_json::from_value(data)?; let mut results = Vec::new(); @@ -273,7 +323,7 @@ pub async fn handle_llm_check( } } - Ok(warp::reply::json(&results)) + Ok(Json(results)) } #[derive(Debug, serde::Deserialize)] @@ -315,18 +365,13 @@ fn describe_vm_error(error: &str) -> Option<&'static str> { pub async fn handle_describe_vm_error( _ctx: sync::DArc<AppContext>, request: DescribeVmErrorRequest, -) -> Result<impl warp::Reply> { +) -> Result<Json<serde_json::Value>> { let description = describe_vm_error(&request.error); - Ok(warp::reply::json( - &serde_json::json!({ "description": description }), - )) + Ok(Json(serde_json::json!({ "description": description }))) } -async fn check_llm_availability( - config_data: &LlmProviderConfig, - test_prompt: &llm::prompt::Internal, -) -> Result<String> { +fn build_check_backend(config_data: &LlmProviderConfig) -> Result<llm::config::BackendConfig> { let backend = serde_json::json!({ "host": config_data.host, "provider": config_data.provider, @@ -336,12 +381,10 @@ async fn check_llm_availability( "key": config_data.key }); + // the key arrives as `${ENV[NAME]}` and is resolved against the manager's own + // environment; without these vars the templater expands it to an empty string let mut vars = HashMap::new(); - for (mut name, value) in std::env::vars() { - name.insert_str(0, "ENV["); - name.push(']'); - vars.insert(name, value); - } + genvm_common::populate_default_config_vars(&mut vars)?; let backend = genvm_common::templater::patch_json( &vars, @@ -349,7 +392,14 @@ async fn check_llm_availability( &genvm_common::templater::DOLLAR_UNFOLDER_RE, )?; - let backend: llm::config::BackendConfig = serde_json::from_value(backend)?; + Ok(serde_json::from_value(backend)?) +} + +async fn check_llm_availability( + config_data: &LlmProviderConfig, + test_prompt: &llm::prompt::Internal, +) -> Result<String> { + let backend = build_check_backend(config_data)?; let provider = backend.to_provider(); let ctx = scripting::CtxPart { @@ -379,3 +429,7 @@ async fn check_llm_availability( Ok(response.result) } + +#[cfg(test)] +#[path = "handlers_test.rs"] +mod tests; diff --git a/implementation/src/manager/handlers_test.rs b/implementation/src/manager/handlers_test.rs new file mode 100644 index 00000000..d5f179d4 --- /dev/null +++ b/implementation/src/manager/handlers_test.rs @@ -0,0 +1,56 @@ +use super::*; + +#[test] +fn status_wait_can_be_absent() { + let req = StatusRequest { wait: None }; + + assert_eq!(parse_status_wait(&req).unwrap(), None); +} + +#[test] +fn status_wait_parses_duration() { + let req = StatusRequest { + wait: Some("30s".to_owned()), + }; + + assert_eq!( + parse_status_wait(&req).unwrap(), + Some(std::time::Duration::from_secs(30)) + ); +} + +#[test] +fn status_wait_is_clamped() { + let req = StatusRequest { + wait: Some("5m".to_owned()), + }; + + assert_eq!(parse_status_wait(&req).unwrap(), Some(STATUS_WAIT_CAP)); +} + +#[test] +fn status_wait_rejects_malformed_duration() { + let req = StatusRequest { + wait: Some("soon".to_owned()), + }; + + assert!(parse_status_wait(&req).is_err()); +} + +#[test] +fn availability_check_resolves_the_key_from_the_environment() { + // callers send `${ENV[NAME]}` rather than the secret itself; an unresolved + // one expands to an empty string instead of failing, so every provider + // would silently report itself unavailable + let path = std::env::var("PATH").unwrap(); + + let backend = build_check_backend(&LlmProviderConfig { + host: "http://127.0.0.1:0".to_owned(), + provider: llm::config::Provider::OpenaiCompatible, + model: "some-model".to_owned(), + key: "${ENV[PATH]}".to_owned(), + }) + .unwrap(); + + assert_eq!(backend.key, path); +} diff --git a/implementation/src/manager/mod.rs b/implementation/src/manager/mod.rs index 05a9a265..c8f262e5 100644 --- a/implementation/src/manager/mod.rs +++ b/implementation/src/manager/mod.rs @@ -2,7 +2,16 @@ use genvm_common::*; use std::{collections::HashMap, sync::Arc}; use anyhow::{Context, Result}; -use warp::Filter; +use axum::{ + body::Bytes, + extract::rejection::QueryRejection, + extract::ws::WebSocketUpgrade, + extract::{Path, Query, State}, + http::{HeaderMap, StatusCode}, + response::{IntoResponse, Response}, + routing::{delete, get, post}, + Json, Router, +}; use crate::common; @@ -11,13 +20,9 @@ pub mod execution_context; mod handlers; pub mod modules; mod run; +mod socket; mod versioning; -#[derive(Debug)] -struct AnyhowRejection(anyhow::Error); - -impl warp::reject::Reject for AnyhowRejection {} - #[derive(serde::Serialize, serde::Deserialize)] pub struct Config { #[serde(flatten)] @@ -25,6 +30,18 @@ pub struct Config { pub manifest_path: String, pub permits: Option<usize>, + #[serde(default = "default_execution_retention")] + pub execution_retention: run::ManagerDuration, + #[serde(default = "default_max_message_bytes")] + pub max_message_bytes: usize, +} + +fn default_execution_retention() -> run::ManagerDuration { + run::ManagerDuration::try_from("5m").expect("default execution_retention is valid") +} + +fn default_max_message_bytes() -> usize { + 64 * 1024 * 1024 } #[derive(serde::Serialize, serde::Deserialize, Clone)] @@ -103,29 +120,253 @@ pub struct CliArgs { pub manifest_path: Option<String>, } -fn unwrap_all_anyhow<R: warp::Reply + 'static>( - route: impl warp::Filter<Extract = (anyhow::Result<R>,), Error = warp::Rejection> - + Clone - + Send - + Sync - + 'static, -) -> impl warp::Filter<Error = warp::Rejection, Extract: warp::Reply> + Clone + Send + Sync + 'static -{ - route - .boxed() - .and_then(|x: anyhow::Result<R>| async move { - x.map_err(|e| warp::reject::custom(AnyhowRejection(e))) - }) - .recover(|err: warp::reject::Rejection| async move { - let Some(AnyhowRejection(inner)) = err.find::<AnyhowRejection>() else { - return Err(err); - }; - log_error!(err:ah = inner; "internal server error"); - Ok(warp::reply::with_status( - warp::reply::json(&serde_json::json!({"error": format!("{:#}", inner)})), - warp::http::StatusCode::INTERNAL_SERVER_ERROR, - )) - }) +fn error_response(status: StatusCode, error: impl Into<String>) -> Response { + (status, Json(serde_json::json!({"error": error.into()}))).into_response() +} + +fn internal_error(error: anyhow::Error) -> Response { + log_error!(err:ah = &error; "internal server error"); + error_response(StatusCode::INTERNAL_SERVER_ERROR, format!("{:#}", error)) +} + +fn unwrap_all_anyhow<R: IntoResponse>(result: anyhow::Result<R>) -> Response { + match result { + Ok(response) => response.into_response(), + Err(error) => internal_error(error), + } +} + +async fn not_found() -> Response { + error_response(StatusCode::NOT_FOUND, "Not Found") +} + +/// Parses a json body without requiring a `Content-Type`. warp accepted a body +/// with the header absent, and clients rely on that. +fn parse_json_body(body: Bytes) -> anyhow::Result<serde_json::Value> { + Ok(serde_json::from_slice(&body)?) +} + +fn parse_query<T>(query: std::result::Result<Query<T>, QueryRejection>) -> anyhow::Result<T> { + Ok(query?.0) +} + +fn parse_genvm_id(raw: &str) -> Option<run::GenVMId> { + if raw.is_empty() || !raw.bytes().all(|byte| byte.is_ascii_digit()) { + return None; + } + raw.parse::<u64>().ok().map(run::GenVMId) +} + +fn deployment_timestamp(headers: &HeaderMap) -> anyhow::Result<String> { + let value = headers + .get("Deployment-Timestamp") + .ok_or_else(|| anyhow::anyhow!("missing Deployment-Timestamp header"))?; + Ok(value.to_str()?.to_owned()) +} + +fn create_router(app_ctx: sync::DArc<AppContext>) -> Router { + // NOTE: when changing routes, also update docs/website/src/impl-spec/appendix/manager-api.yaml + Router::new() + .route( + "/ws", + get( + |ws: WebSocketUpgrade, State(ctx): State<sync::DArc<AppContext>>| async move { + ws.max_message_size(ctx.config.max_message_bytes) + .on_upgrade(move |socket| socket::handle_connection(socket, ctx)) + }, + ), + ) + .route( + "/status", + get(|State(ctx): State<sync::DArc<AppContext>>| async move { + unwrap_all_anyhow(handlers::handle_status(ctx).await) + }), + ) + .route( + "/module/start", + post( + |State(ctx): State<sync::DArc<AppContext>>, + body: Bytes| async move { + let body = match parse_json_body(body) { + Ok(body) => body, + Err(error) => return internal_error(error), + }; + unwrap_all_anyhow(handlers::handle_module_start(ctx, body).await) + }, + ), + ) + .route( + "/module/stop", + post( + |State(ctx): State<sync::DArc<AppContext>>, + body: Bytes| async move { + let body = match parse_json_body(body) { + Ok(body) => body, + Err(error) => return internal_error(error), + }; + unwrap_all_anyhow(handlers::handle_module_stop(ctx, body).await) + }, + ), + ) + .route( + "/module/restart", + post( + |State(ctx): State<sync::DArc<AppContext>>, + body: Bytes| async move { + let body = match parse_json_body(body) { + Ok(body) => body, + Err(error) => return internal_error(error), + }; + unwrap_all_anyhow(handlers::handle_module_restart(ctx, body).await) + }, + ), + ) + .route( + "/genvm/run", + post( + |State(ctx): State<sync::DArc<AppContext>>, body: Bytes| async move { + unwrap_all_anyhow(handlers::handle_genvm_run(ctx, &body).await) + }, + ), + ) + .route( + "/contract/detect-version", + post( + |State(ctx): State<sync::DArc<AppContext>>, + headers: HeaderMap, + contract_code: Bytes| async move { + let deployment_timestamp = match deployment_timestamp(&headers) { + Ok(deployment_timestamp) => deployment_timestamp, + Err(error) => return internal_error(error), + }; + unwrap_all_anyhow( + handlers::handle_contract_detect_version( + ctx, + contract_code, + deployment_timestamp, + ) + .await, + ) + }, + ), + ) + .route( + "/log/level", + post( + |State(ctx): State<sync::DArc<AppContext>>, + body: Bytes| async move { + let body = match parse_json_body(body) { + Ok(body) => body, + Err(error) => return internal_error(error), + }; + unwrap_all_anyhow(handlers::handle_set_log_level(ctx, body).await) + }, + ), + ) + .route( + "/manifest/reload", + post(|State(ctx): State<sync::DArc<AppContext>>| async move { + unwrap_all_anyhow(handlers::handle_manifest_reload(ctx).await) + }), + ) + .route( + "/permits", + get(|State(ctx): State<sync::DArc<AppContext>>| async move { + unwrap_all_anyhow(handlers::handle_get_permits(ctx).await) + }) + .post( + |State(ctx): State<sync::DArc<AppContext>>, + body: Bytes| async move { + let body = match parse_json_body(body) { + Ok(body) => body, + Err(error) => return internal_error(error), + }; + unwrap_all_anyhow(handlers::handle_set_permits(ctx, body).await) + }, + ), + ) + .route( + "/genvm/{genvm_id}", + delete( + |State(ctx): State<sync::DArc<AppContext>>, + Path(raw_id): Path<String>, + query: std::result::Result<Query<handlers::ShutdownRequest>, QueryRejection>| async move { + let Some(genvm_id) = parse_genvm_id(&raw_id) else { + return not_found().await; + }; + let query = match parse_query(query) { + Ok(query) => query, + Err(error) => return internal_error(error), + }; + unwrap_all_anyhow(handlers::handle_genvm_shutdown(ctx, genvm_id, query).await) + }, + ) + .get( + |State(ctx): State<sync::DArc<AppContext>>, + Path(raw_id): Path<String>, + query: std::result::Result<Query<handlers::StatusRequest>, QueryRejection>| async move { + let Some(genvm_id) = parse_genvm_id(&raw_id) else { + return not_found().await; + }; + let query = match parse_query(query) { + Ok(query) => query, + Err(error) => return internal_error(error), + }; + unwrap_all_anyhow( + handlers::handle_genvm_status(ctx, genvm_id, query).await, + ) + }, + ), + ) + .route( + "/genvm/{genvm_id}/artifact", + get( + |State(ctx): State<sync::DArc<AppContext>>, + Path(raw_id): Path<String>, + query: std::result::Result<Query<handlers::ArtifactRequest>, QueryRejection>| async move { + let Some(genvm_id) = parse_genvm_id(&raw_id) else { + return not_found().await; + }; + let query = match parse_query(query) { + Ok(query) => query, + Err(error) => return internal_error(error), + }; + unwrap_all_anyhow(handlers::handle_genvm_artifact(ctx, genvm_id, query).await) + }, + ), + ) + .route( + "/llm/check", + post( + |State(ctx): State<sync::DArc<AppContext>>, + body: Bytes| async move { + let body = match parse_json_body(body) { + Ok(body) => body, + Err(error) => return internal_error(error), + }; + unwrap_all_anyhow(handlers::handle_llm_check(ctx, body).await) + }, + ), + ) + .route( + "/vm-error/describe", + get( + |State(ctx): State<sync::DArc<AppContext>>, + query: std::result::Result<Query<handlers::DescribeVmErrorRequest>, QueryRejection>| async move { + let query = match parse_query(query) { + Ok(query) => query, + Err(error) => return internal_error(error), + }; + unwrap_all_anyhow(handlers::handle_describe_vm_error(ctx, query).await) + }, + ), + ) + .fallback(not_found) + .method_not_allowed_fallback(not_found) + // Contract code and calldata routinely exceed the default cap, and an + // exceeded cap answers with a plain-text 413 rather than our error shape. + .layer(axum::extract::DefaultBodyLimit::disable()) + .with_state(app_ctx) } async fn run_http_server( @@ -143,217 +384,18 @@ async fn run_http_server( run::start_service(app_ctx.gep(|x| &x.run_ctx), cancel.clone()).await?; - let ctx = app_ctx.clone(); - let status_route = unwrap_all_anyhow(warp::path("status").and(warp::get()).then(move || { - let ctx = ctx.clone(); - async move { handlers::handle_status(ctx).await } - })); - - let ctx = app_ctx.clone(); - let start_route = unwrap_all_anyhow( - warp::path!("module" / "start") - .and(warp::post()) - .and(warp::body::json()) - .then(move |calldata| { - let ctx = ctx.clone(); - async move { handlers::handle_module_start(ctx, calldata).await } - }), - ); - - let ctx = app_ctx.clone(); - let stop_route = unwrap_all_anyhow( - warp::path!("module" / "stop") - .and(warp::post()) - .and(warp::body::json()) - .then(move |calldata| { - let ctx = ctx.clone(); - async move { handlers::handle_module_stop(ctx, calldata).await } - }), - ); - - let ctx = app_ctx.clone(); - let restart_route = unwrap_all_anyhow( - warp::path!("module" / "restart") - .and(warp::post()) - .and(warp::body::json()) - .then(move |calldata| { - let ctx = ctx.clone(); - async move { handlers::handle_module_restart(ctx, calldata).await } - }), - ); - - let ctx = app_ctx.clone(); - let genvm_run_route = unwrap_all_anyhow( - warp::path!("genvm" / "run") - .and(warp::post()) - .and(warp::body::bytes()) - .then(move |data: bytes::Bytes| { - let ctx = ctx.clone(); - async move { handlers::handle_genvm_run(ctx, &data).await } - }), - ); - - let ctx = app_ctx.clone(); - let contract_detect_version_route = unwrap_all_anyhow( - warp::path!("contract" / "detect-version") - .and(warp::post()) - .and(warp::body::bytes()) - .and(warp::header::<String>("Deployment-Timestamp")) - .then(move |contract_code, deployment_timestamp| { - let ctx = ctx.clone(); - async move { - handlers::handle_contract_detect_version( - ctx, - contract_code, - deployment_timestamp, - ) - .await - } - }), - ); - - let ctx = app_ctx.clone(); - let set_log_level_route = unwrap_all_anyhow( - warp::path!("log" / "level") - .and(warp::post()) - .and(warp::body::json()) - .then(move |data| { - let ctx = ctx.clone(); - async move { handlers::handle_set_log_level(ctx, data).await } - }), - ); - - let ctx = app_ctx.clone(); - let manifest_reload_route = unwrap_all_anyhow( - warp::path!("manifest" / "reload") - .and(warp::post()) - .then(move || { - let ctx = ctx.clone(); - async move { handlers::handle_manifest_reload(ctx).await } - }), - ); - - let ctx = app_ctx.clone(); - let set_env_route = unwrap_all_anyhow( - warp::path("env") - .and(warp::post()) - .and(warp::body::json()) - .then(move |data| { - let ctx = ctx.clone(); - async move { handlers::handle_set_env(ctx, data).await } - }), - ); - - let ctx = app_ctx.clone(); - let get_permits_route = - unwrap_all_anyhow(warp::path("permits").and(warp::get()).then(move || { - let ctx = ctx.clone(); - async move { handlers::handle_get_permits(ctx).await } - })); - - let ctx = app_ctx.clone(); - let set_permits_route = unwrap_all_anyhow( - warp::path("permits") - .and(warp::post()) - .and(warp::body::json()) - .then(move |data| { - let ctx = ctx.clone(); - async move { handlers::handle_set_permits(ctx, data).await } - }), - ); - - let ctx = app_ctx.clone(); - let genvm_shutdown_route = unwrap_all_anyhow( - warp::path!("genvm" / u64) - .and(warp::delete()) - .and(warp::query::<handlers::ShutdownRequest>()) - .then(move |genvm_id, shutdown_req| { - let ctx = ctx.clone(); - async move { - handlers::handle_genvm_shutdown(ctx, run::GenVMId(genvm_id), shutdown_req).await - } - }), - ); - - let ctx = app_ctx.clone(); - let genvm_status_route = unwrap_all_anyhow(warp::path!("genvm" / u64).and(warp::get()).then( - move |genvm_id| { - let ctx = ctx.clone(); - async move { handlers::handle_genvm_status(ctx, run::GenVMId(genvm_id)).await } - }, - )); - - let ctx = app_ctx.clone(); - let llm_check_route = unwrap_all_anyhow( - warp::path!("llm" / "check") - .and(warp::post()) - .and(warp::body::json()) - .then(move |data| { - let ctx = ctx.clone(); - async move { handlers::handle_llm_check(ctx, data).await } - }), - ); - - let ctx = app_ctx.clone(); - let describe_vm_error_route = unwrap_all_anyhow( - warp::path!("vm-error" / "describe") - .and(warp::get()) - .and(warp::query::<handlers::DescribeVmErrorRequest>()) - .then(move |query| { - let ctx = ctx.clone(); - async move { handlers::handle_describe_vm_error(ctx, query).await } - }), - ); - - // NOTE: when changing routes, also update doc/website/src/impl-spec/appendix/manager-api.yaml - let routes = status_route - .or(start_route) - .or(stop_route) - .or(restart_route) - .or(genvm_run_route) - .or(contract_detect_version_route) - .or(set_log_level_route) - .or(manifest_reload_route) - .or(set_env_route) - .or(get_permits_route) - .or(set_permits_route) - .or(genvm_shutdown_route) - .or(genvm_status_route) - .or(llm_check_route) - .or(describe_vm_error_route); - - let routes = routes.recover(|err: warp::reject::Rejection| async move { - if err.is_not_found() { - Ok::<_, std::convert::Infallible>(warp::reply::with_status( - warp::reply::json(&serde_json::json!({"error": "Not Found"})), - warp::http::StatusCode::NOT_FOUND, - )) - } else { - let err_format = format!("{:?}", err); - Ok(warp::reply::with_status( - warp::reply::json(&serde_json::json!({"error": err_format})), - warp::http::StatusCode::INTERNAL_SERVER_ERROR, - )) - } - }); + let routes = create_router(app_ctx); let cancellation = cancel.clone(); if let Some(socket_path) = &args.socket { - // Unix socket mode - use hyper::server::accept::Accept; - use std::pin::Pin; - use std::task::{Context, Poll}; - let path = std::path::Path::new(socket_path); - // Clean up stale socket file if path.exists() { std::fs::remove_file(path) .with_context(|| format!("removing stale socket {}", path.display()))?; } - // Ensure parent directory exists if let Some(parent) = path.parent() { if !parent.as_os_str().is_empty() && !parent.exists() { std::fs::create_dir_all(parent)?; @@ -365,36 +407,9 @@ async fn run_http_server( log_info!(socket_path:? = path; "HTTP server started on Unix socket"); - // Custom Accept implementation for UnixListener - struct UnixAcceptor(tokio::net::UnixListener); - - impl Accept for UnixAcceptor { - type Conn = tokio::net::UnixStream; - type Error = std::io::Error; - - fn poll_accept( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - ) -> Poll<Option<Result<Self::Conn, Self::Error>>> { - match self.get_mut().0.poll_accept(cx) { - Poll::Ready(Ok((stream, _addr))) => Poll::Ready(Some(Ok(stream))), - Poll::Ready(Err(e)) => Poll::Ready(Some(Err(e))), - Poll::Pending => Poll::Pending, - } - } - } - - let warp_svc = warp::service(routes); - let make_svc = hyper::service::make_service_fn(move |_| { - let svc = warp_svc.clone(); - async move { Ok::<_, std::convert::Infallible>(svc) } - }); - - let server = hyper::Server::builder(UnixAcceptor(listener)) - .serve(make_svc) + let server = axum::serve(listener, routes) .with_graceful_shutdown(async move { cancellation.chan.closed().await }); - // Cleanup socket on shutdown let cleanup_path = path.to_owned(); let result = server.await; let _ = std::fs::remove_file(&cleanup_path); @@ -402,15 +417,16 @@ async fn run_http_server( log_info!(socket_path:? = cleanup_path; "HTTP server stopped"); result?; } else { - // TCP mode (existing behavior) - let serv = warp::serve(routes); - let (addr, fut) = serv.bind_with_graceful_shutdown( - (args.host.parse::<std::net::IpAddr>()?, args.port), - async move { cancellation.chan.closed().await }, - ); + let addr = std::net::SocketAddr::new(args.host.parse()?, args.port); + let listener = tokio::net::TcpListener::bind(addr) + .await + .with_context(|| format!("binding to tcp address {}", addr))?; + let addr = listener.local_addr()?; log_info!(address:? = addr; "HTTP server started"); - fut.await; + axum::serve(listener, routes) + .with_graceful_shutdown(async move { cancellation.chan.closed().await }) + .await?; log_info!(address:? = addr; "HTTP server stopped"); } diff --git a/implementation/src/manager/run.rs b/implementation/src/manager/run.rs index 6b3124b9..c67ed736 100644 --- a/implementation/src/manager/run.rs +++ b/implementation/src/manager/run.rs @@ -1,19 +1,207 @@ use std::{ + collections::HashMap, ops::DerefMut, os::fd::{AsRawFd, FromRawFd}, str::FromStr, - sync::Arc, + sync::{ + atomic::{AtomicBool, AtomicU16, AtomicUsize, Ordering}, + Arc, + }, }; +use anyhow::Context as _; use genlayer_calldata as calldata; +use genlayer_calldata::codec::{Decode, Encode}; use genvm_common::io::{set_fd_nonblocking, AsyncCustomFD, FdWrapper}; use genvm_common::*; +use serde::{Deserialize, Serialize}; use tokio::io::AsyncBufReadExt; pub use genvm_modules_interfaces::GenVMId; use crate::common::{LogSink, LogSinkElement, LogSinkInner, LoggerWithId, GENVM_BY_ID_LOGGER}; +const ARTIFACT_CHUNK_CAP: usize = 256 * 1024; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ManagerDuration(std::time::Duration); + +impl ManagerDuration { + pub fn to_std(self) -> std::time::Duration { + self.0 + } + + fn parse_decimal_millis(number: &str, unit_millis: u128) -> anyhow::Result<u128> { + if number.is_empty() { + anyhow::bail!("duration number is empty"); + } + + let mut parts = number.split('.'); + let whole = parts.next().unwrap(); + let frac = parts.next(); + if parts.next().is_some() { + anyhow::bail!("duration has more than one decimal point"); + } + if whole.is_empty() && frac.unwrap_or("").is_empty() { + anyhow::bail!("duration number is empty"); + } + if !whole.bytes().all(|b| b.is_ascii_digit()) { + anyhow::bail!("duration whole part is not a number"); + } + let whole_value = if whole.is_empty() { + 0 + } else { + whole.parse::<u128>()? + }; + + let frac_millis = if let Some(frac) = frac { + if frac.is_empty() || !frac.bytes().all(|b| b.is_ascii_digit()) { + anyhow::bail!("duration fractional part is not a number"); + } + let scale = 10_u128 + .checked_pow(frac.len() as u32) + .ok_or_else(|| anyhow::anyhow!("duration fractional part is too long"))?; + frac.parse::<u128>()? + .checked_mul(unit_millis) + .ok_or_else(|| anyhow::anyhow!("duration is too large"))? + / scale + } else { + 0 + }; + + whole_value + .checked_mul(unit_millis) + .and_then(|v| v.checked_add(frac_millis)) + .ok_or_else(|| anyhow::anyhow!("duration is too large")) + } +} + +impl TryFrom<&str> for ManagerDuration { + type Error = anyhow::Error; + + fn try_from(value: &str) -> Result<Self, Self::Error> { + let (number, unit_millis) = if let Some(number) = value.strip_suffix("ms") { + (number, 1) + } else if let Some(number) = value.strip_suffix('s') { + (number, 1_000) + } else if let Some(number) = value.strip_suffix('m') { + (number, 60_000) + } else if let Some(number) = value.strip_suffix('h') { + (number, 3_600_000) + } else { + anyhow::bail!("duration must end with ms, s, m, or h"); + }; + + let millis = Self::parse_decimal_millis(number, unit_millis)?; + let millis = u64::try_from(millis)?; + Ok(Self(std::time::Duration::from_millis(millis))) + } +} + +impl Serialize for ManagerDuration { + fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> { + serializer.serialize_str(&format!("{}ms", self.0.as_millis())) + } +} + +impl<'de> Deserialize<'de> for ManagerDuration { + fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> { + let s = String::deserialize(deserializer)?; + Self::try_from(s.as_str()).map_err(serde::de::Error::custom) + } +} + +impl<W: calldata::Writer> Encode<W> for ManagerDuration { + type Error = W::Error; + + fn encode(&self, enc: &mut calldata::Encoder<W>) -> Result<(), Self::Error> { + enc.push_str(&format!("{}ms", self.0.as_millis())) + } +} + +impl Decode for ManagerDuration { + fn decode<D: calldata::codec::Deserializer>( + deserializer: D, + ) -> Result<Self, calldata::codec::DecodeError> { + <String as Decode>::decode(deserializer).and_then(|s| { + Self::try_from(s.as_str()) + .map_err(|e| calldata::codec::DecodeError::UserError(e.into())) + }) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FinishCause { + Exited, + Cancelled, + Deadline, + Shutdown, +} + +impl FinishCause { + pub fn as_str(self) -> &'static str { + match self { + FinishCause::Exited => "exited", + FinishCause::Cancelled => "cancelled", + FinishCause::Deadline => "deadline", + FinishCause::Shutdown => "shutdown", + } + } +} + +#[derive(Debug, Clone)] +pub struct ArtifactSizes { + pub stdout: u64, + pub stderr: u64, + pub genvm_log: u64, +} + +#[derive(Debug, Clone)] +pub enum Event { + Started { + genvm_id: GenVMId, + host_genvm_id: Option<String>, + }, + FailedToStart { + genvm_id: GenVMId, + host_genvm_id: Option<String>, + error: String, + }, + Finished { + genvm_id: GenVMId, + host_genvm_id: Option<String>, + cause: FinishCause, + exit_code: Option<i64>, + consumed_result: Option<Vec<u8>>, + metrics: serde_json::Value, + finished_at: chrono::DateTime<chrono::Utc>, + version_major: u16, + version_minor: u16, + artifact_sizes: ArtifactSizes, + }, +} + +impl Event { + pub fn is_terminal(&self) -> bool { + matches!(self, Event::FailedToStart { .. } | Event::Finished { .. }) + } +} + +#[derive(Debug, Clone)] +pub enum Snapshot { + Queued { + genvm_id: GenVMId, + host_genvm_id: Option<String>, + }, + Event(Event), +} + +#[derive(Debug, Clone)] +pub struct Artifact { + pub total_len: u64, + pub data: bytes::Bytes, +} + /// Spawn relay that passes the stream to the module handler async fn spawn_module_relay( parent_fd: FdWrapper, @@ -46,6 +234,8 @@ pub struct SingleGenVMContextDone { pub consumed_result: Option<Vec<u8>>, pub version_major: u16, pub version_minor: u16, + pub cause: FinishCause, + pub exit_code: Option<i64>, } impl serde::Serialize for SingleGenVMContextDone { @@ -53,6 +243,7 @@ impl serde::Serialize for SingleGenVMContextDone { where S: serde::Serializer, { + use base64::Engine; use serde::ser::SerializeStruct; let mut state = serializer.serialize_struct("SingleGenVMContextDone", 8)?; @@ -61,7 +252,13 @@ impl serde::Serialize for SingleGenVMContextDone { state.serialize_field("stderr", &self.stderr)?; state.serialize_field("genvm_log", &self.genvm_log)?; state.serialize_field("metrics", &self.metrics)?; - state.serialize_field("consumed_result", &self.consumed_result)?; + state.serialize_field( + "consumed_result", + &self + .consumed_result + .as_ref() + .map(|v| base64::engine::general_purpose::STANDARD.encode(v)), + )?; state.serialize_field("version_major", &self.version_major)?; state.serialize_field("version_minor", &self.version_minor)?; state.end() @@ -70,10 +267,15 @@ impl serde::Serialize for SingleGenVMContextDone { struct SingleGenVMContext { id: GenVMId, - version: String, - version_major: u16, - version_minor: u16, + host_genvm_id: Option<String>, + version: std::sync::RwLock<String>, + version_major: AtomicU16, + version_minor: AtomicU16, result: tokio::sync::OnceCell<SingleGenVMContextDone>, + started_event: tokio::sync::OnceCell<Event>, + terminal_event: tokio::sync::OnceCell<Event>, + terminal_at: tokio::sync::OnceCell<chrono::DateTime<chrono::Utc>>, + events: tokio::sync::watch::Sender<Snapshot>, started_at: chrono::DateTime<chrono::Utc>, strict_deadline: chrono::DateTime<chrono::Utc>, @@ -83,9 +285,135 @@ struct SingleGenVMContext { log_sink: LogSink, consumed_result: tokio::sync::OnceCell<Vec<u8>>, - process_handle: tokio::sync::Mutex<tokio::process::Child>, + process_handle: tokio::sync::Mutex<Option<tokio::process::Child>>, + cancel_requested: AtomicBool, + cancel_notify: tokio::sync::Notify, + finish_cause: std::sync::Mutex<Option<FinishCause>>, all_permits: crossbeam::atomic::AtomicCell<Option<Box<dyn std::any::Any + Send + Sync>>>, - _execution_context: Option<sync::DArc<super::execution_context::ExecutionContext>>, + nested_runs: AtomicUsize, + nested_runs_done: tokio::sync::Notify, + execution_context: + tokio::sync::OnceCell<sync::DArc<super::execution_context::ExecutionContext>>, +} + +impl SingleGenVMContext { + fn set_version(&self, version: String, major: u16, minor: u16) { + if let Ok(mut current) = self.version.write() { + *current = version; + } + self.version_major.store(major, Ordering::SeqCst); + self.version_minor.store(minor, Ordering::SeqCst); + } + + fn version(&self) -> String { + self.version + .read() + .map(|v| v.clone()) + .unwrap_or_else(|_| String::new()) + } + + fn request_finish(&self, cause: FinishCause) { + let should_notify = !self.cancel_requested.swap(true, Ordering::SeqCst); + if let Ok(mut stored_cause) = self.finish_cause.lock() { + if stored_cause.is_none() { + *stored_cause = Some(cause); + } + } + if should_notify { + self.cancel_notify.notify_waiters(); + } + } + + fn finish_cause(&self) -> Option<FinishCause> { + self.finish_cause.lock().ok().and_then(|cause| *cause) + } + + async fn wait_cancelled(&self) { + loop { + let notified = self.cancel_notify.notified(); + if self.cancel_requested.load(Ordering::SeqCst) { + return; + } + notified.await; + } + } + + fn nested_run_started(&self) { + self.nested_runs.fetch_add(1, Ordering::SeqCst); + } + + fn nested_run_finished(&self) { + if self.nested_runs.fetch_sub(1, Ordering::SeqCst) == 1 { + self.nested_runs_done.notify_waiters(); + } + } + + async fn wait_for_nested_runs(&self) { + loop { + let notified = self.nested_runs_done.notified(); + if self.nested_runs.load(Ordering::SeqCst) == 0 { + return; + } + notified.await; + } + } + + fn genvm_log_json_lines(genvm_log: &[serde_json::Map<String, serde_json::Value>]) -> Vec<u8> { + let mut out = Vec::new(); + for item in genvm_log { + match serde_json::to_vec(item) { + Ok(mut line) => { + out.append(&mut line); + out.push(b'\n'); + } + Err(e) => { + log_error!(error:err = e; "failed to serialize genvm log artifact line"); + } + } + } + out + } + + fn artifact_sizes_for(result: &SingleGenVMContextDone) -> ArtifactSizes { + ArtifactSizes { + stdout: result.stdout.len() as u64, + stderr: result.stderr.len() as u64, + genvm_log: Self::genvm_log_json_lines(&result.genvm_log).len() as u64, + } + } + + fn finished_event(&self, result: &SingleGenVMContextDone) -> Event { + Event::Finished { + genvm_id: self.id, + host_genvm_id: self.host_genvm_id.clone(), + cause: result.cause, + exit_code: result.exit_code, + consumed_result: result.consumed_result.clone(), + metrics: result.metrics.clone(), + finished_at: result.finished_at, + version_major: result.version_major, + version_minor: result.version_minor, + artifact_sizes: Self::artifact_sizes_for(result), + } + } + + fn publish_terminal(&self, event: Event) -> bool { + let _ = self.terminal_at.set(chrono::Utc::now()); + if self.terminal_event.set(event.clone()).is_err() { + return false; + } + self.events.send_replace(Snapshot::Event(event)); + true + } + + fn publish_started(&self) { + let event = Event::Started { + genvm_id: self.id, + host_genvm_id: self.host_genvm_id.clone(), + }; + let _ = self.started_event.set(event.clone()); + self.events.send_replace(Snapshot::Event(event)); + } } struct PermitsData { @@ -94,8 +422,18 @@ struct PermitsData { throttled: Option<tokio::sync::OwnedSemaphorePermit>, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AckOutcome { + Acked, + NotFinished, + Unknown, +} + pub struct Ctx { known_executions: papaya::HashMap<GenVMId, sync::DArc<SingleGenVMContext>>, + host_genvm_ids: std::sync::Mutex<HashMap<String, GenVMId>>, + boot_id: u64, + execution_retention: ManagerDuration, next_genvm_id: std::sync::atomic::AtomicU64, pub permits: Arc<tokio::sync::Semaphore>, max_permits: tokio::sync::Mutex<PermitsData>, @@ -104,6 +442,10 @@ pub struct Ctx { } impl Ctx { + pub fn boot_id(&self) -> u64 { + self.boot_id + } + pub fn executors_path(&self) -> &std::path::Path { &self.executors_path } @@ -129,7 +471,7 @@ impl Ctx { genvm_id.to_string(), serde_json::json!({ "result": result, - "version": exec_ctx.version, + "version": exec_ctx.version(), "started_at": exec_ctx.started_at.to_rfc3339(), "strict_deadline": exec_ctx.strict_deadline.to_rfc3339() }), @@ -177,7 +519,7 @@ impl Ctx { permits_lock.max } - pub async fn graceful_shutdown(&self, genvm_id: GenVMId) -> anyhow::Result<()> { + async fn request_finish(&self, genvm_id: GenVMId, cause: FinishCause) -> anyhow::Result<()> { let Some(exec_ctx) = self.known_executions.pin().get(&genvm_id).cloned() else { anyhow::bail!("GenVM with id {} not found", genvm_id); }; @@ -186,47 +528,137 @@ impl Ctx { return Ok(()); } - let mut child = exec_ctx.process_handle.lock().await; + exec_ctx.request_finish(cause); - if let Ok(Some(_)) = child.try_wait() { - log_trace_into!(&LoggerWithId, genvm_id:id = genvm_id.0; "GenVM process already exited"); - return Ok(()); + let mut child = exec_ctx.process_handle.lock().await; + if let Some(child) = child.as_mut() { + let _ = child.start_kill(); } - let _ = child.start_kill(); - child.wait().await?; + Ok(()) + } + pub async fn graceful_shutdown(&self, genvm_id: GenVMId) -> anyhow::Result<()> { + self.cancel(genvm_id).await?; + let _ = self.await_terminal(genvm_id).await; Ok(()) } - pub async fn get_genvm_status( + pub fn attach( &self, + boot_id: u64, genvm_id: GenVMId, - ) -> Option<sync::DArc<SingleGenVMContextDone>> { + ) -> anyhow::Result<(Snapshot, tokio::sync::watch::Receiver<Snapshot>)> { + if boot_id != self.boot_id { + anyhow::bail!("boot_id_mismatch"); + } + let Some(exec_ctx) = self.known_executions.pin().get(&genvm_id).cloned() else { + anyhow::bail!("unknown_id"); + }; + let mut rx = exec_ctx.events.subscribe(); + let snapshot = rx.borrow_and_update().clone(); + Ok((snapshot, rx)) + } + + pub async fn cancel(&self, genvm_id: GenVMId) -> anyhow::Result<()> { + self.request_finish(genvm_id, FinishCause::Cancelled).await + } + + pub fn ack(&self, genvm_id: GenVMId) -> AckOutcome { + let pinned = self.known_executions.pin(); + let Some(exec_ctx) = pinned.get(&genvm_id) else { + return AckOutcome::Unknown; + }; + if exec_ctx.result.get().is_none() && exec_ctx.terminal_event.get().is_none() { + return AckOutcome::NotFinished; + } + let Some(exec_ctx) = pinned.remove(&genvm_id) else { + return AckOutcome::Unknown; + }; + if let Some(host_genvm_id) = &exec_ctx.host_genvm_id { + self.release_token(host_genvm_id, genvm_id); + } + AckOutcome::Acked + } + + fn lock_host_ids(&self) -> std::sync::MutexGuard<'_, HashMap<String, GenVMId>> { + match self.host_genvm_ids.lock() { + Ok(host_ids) => host_ids, + Err(poisoned) => poisoned.into_inner(), + } + } + + /// Drops a token only while it still maps to `genvm_id`. Between removing an + /// execution and taking the lock, another start can reuse the token for a + /// fresh run, and that mapping must survive. + fn release_token(&self, host_genvm_id: &str, genvm_id: GenVMId) { + let mut host_ids = self.lock_host_ids(); + if host_ids.get(host_genvm_id) == Some(&genvm_id) { + host_ids.remove(host_genvm_id); + } + } + + pub fn status(&self, genvm_id: GenVMId) -> Option<sync::DArc<SingleGenVMContextDone>> { let Some(exec_ctx) = self.known_executions.pin().get(&genvm_id).cloned() else { log_trace_into!(&LoggerWithId, genvm_id:id = genvm_id.0; "genvm status requested for unknown id"); return None; }; - let proc_check = self.check_proc(exec_ctx.clone()).await; - log_debug_into!(&LoggerWithId, genvm_id:id = genvm_id.0, proc_exited = proc_check; "genvm status checked"); - exec_ctx .into_get_sub(|data| data.result.get()) .lift_option() .map(sync::DArcStruct::into_arc) } - pub async fn fetch_genvm_status( + pub fn get_artifact( &self, genvm_id: GenVMId, - ) -> Option<sync::DArc<SingleGenVMContextDone>> { - let res = self.get_genvm_status(genvm_id).await; - if res.is_some() { - self.known_executions.pin().remove(&genvm_id); + field: &str, + offset: u64, + max_len: u32, + ) -> anyhow::Result<Artifact> { + let Some(result) = self.status(genvm_id) else { + anyhow::bail!("unknown_id"); + }; + + let data = match field { + "stdout" => result.stdout.as_bytes().to_vec(), + "stderr" => result.stderr.as_bytes().to_vec(), + "genvm_log" => SingleGenVMContext::genvm_log_json_lines(&result.genvm_log), + _ => anyhow::bail!("unknown artifact field {}", field), + }; + + let total_len = data.len() as u64; + let offset = usize::try_from(offset) + .unwrap_or(usize::MAX) + .min(data.len()); + let len = usize::try_from(max_len) + .unwrap_or(usize::MAX) + .min(ARTIFACT_CHUNK_CAP) + .min(data.len().saturating_sub(offset)); + + Ok(Artifact { + total_len, + data: bytes::Bytes::copy_from_slice(&data[offset..offset + len]), + }) + } + + pub async fn await_terminal(&self, genvm_id: GenVMId) -> anyhow::Result<Event> { + let (snapshot, mut rx) = self.attach(self.boot_id, genvm_id)?; + if let Snapshot::Event(event) = snapshot { + if event.is_terminal() { + return Ok(event); + } } - res + loop { + rx.changed().await?; + if let Snapshot::Event(event) = rx.borrow_and_update().clone() { + if event.is_terminal() { + return Ok(event); + } + } + } } #[cfg(target_os = "macos")] @@ -274,6 +706,9 @@ impl Ctx { Ok(Self { known_executions: Default::default(), + host_genvm_ids: std::sync::Mutex::new(HashMap::new()), + boot_id: rand::random::<u64>(), + execution_retention: config.execution_retention, next_genvm_id: std::sync::atomic::AtomicU64::new(1), permits: Arc::new(tokio::sync::Semaphore::new(permits)), max_permits: tokio::sync::Mutex::new(PermitsData { @@ -287,6 +722,19 @@ impl Ctx { } } +/// How often to sweep for expired deadlines and retained results. +/// +/// Derived from the retention, because a fixed sweep decides how far past its +/// TTL a result can linger: a one minute sweep against a retention configured +/// in seconds keeps results for a minute regardless of the setting. The floor +/// keeps a tiny retention from spinning the loop. +fn gc_interval(ctx: &Ctx) -> std::time::Duration { + const FLOOR: std::time::Duration = std::time::Duration::from_millis(250); + const CEILING: std::time::Duration = std::time::Duration::from_secs(60); + + (ctx.execution_retention.to_std() / 4).clamp(FLOOR, CEILING) +} + async fn gc_step(ctx: &sync::DArc<Ctx>) { let now = chrono::Utc::now(); @@ -303,25 +751,37 @@ async fn gc_step(ctx: &sync::DArc<Ctx>) { continue; }; - if val.strict_deadline < now { + if val.strict_deadline < now && val.terminal_event.get().is_none() { log_warn_into!(&LoggerWithId, genvm_id:id = key.0; "genvm execution exceeded strict deadline, terminating"); - let _ = ctx.graceful_shutdown(key).await; + let _ = ctx.request_finish(key, FinishCause::Deadline).await; } - let _ = ctx.check_proc(val.clone()).await; } // Remove old finished executions + let retention = chrono::Duration::from_std(ctx.execution_retention.to_std()) + .unwrap_or_else(|_| chrono::Duration::hours(24)); + let mut expired_host_ids = Vec::new(); ctx.known_executions.pin().retain(|k, v| { - let Some(result) = v.result.get() else { + let terminal_at = if let Some(result) = v.result.get() { + result.finished_at + } else if let Some(terminal_at) = v.terminal_at.get() { + *terminal_at + } else { return true; }; - let passed = now.signed_duration_since(result.finished_at); - if passed > chrono::Duration::seconds(60) { + let passed = now.signed_duration_since(terminal_at); + if passed > retention { log_warn_into!(&LoggerWithId, genvm_id:id = k.0; "removing zombie genvm execution context"); + if let Some(host_genvm_id) = &v.host_genvm_id { + expired_host_ids.push((host_genvm_id.clone(), *k)); + } return false; } true }); + for (host_genvm_id, genvm_id) in expired_host_ids { + ctx.release_token(&host_genvm_id, genvm_id); + } } pub async fn start_service( @@ -333,9 +793,18 @@ pub async fn start_service( tokio::select! { _ = cancel.chan.closed() => { log_info!("shutting down run service waiter"); + let keys = ctx + .known_executions + .pin() + .iter() + .map(|kv| *kv.0) + .collect::<Vec<_>>(); + for key in keys { + let _ = ctx.request_finish(key, FinishCause::Shutdown).await; + } break; } - _ = tokio::time::sleep(std::time::Duration::from_secs(60)) => { + _ = tokio::time::sleep(gc_interval(&ctx)) => { gc_step(&ctx).await; } } @@ -385,12 +854,54 @@ fn default_reroute_to() -> String { String::new() } +fn default_unsafe_overrides() -> UnsafeOverrides { + UnsafeOverrides::default() +} + +fn default_initial_recursion() -> Option<u32> { + None +} + +/// Overrides that exist to reach boundaries production traffic cannot. Each +/// field states the debug level it needs; none of them are honored with +/// debugging disabled, so consensus traffic is unaffected. +#[derive( + Clone, + Default, + serde::Serialize, + serde::Deserialize, + genlayer_calldata::Decode, + genlayer_calldata::Encode, +)] +pub struct UnsafeOverrides { + /// Run this version instead of the one the request's major resolves to: an + /// executor directory as it stands, or a `re:`-prefixed pattern over + /// manifest keys. Empty is no override. Honored from `debug_mode >= Safe`. + #[serde(default = "default_reroute_to")] + #[calldata(default = default_reroute_to)] + pub reroute_to: String, + /// Initial recursion budget for the chain, replacing the executor's own + /// `VM_RECURSION`. Exhausting the real limit costs one executor process per + /// unit of budget, so a boundary test seeds a small one instead. Honored + /// from `debug_mode >= Unsafe`. + #[serde(default)] + #[calldata(default = default_initial_recursion)] + pub initial_recursion: Option<u32>, +} + #[serde_as] #[derive( - serde::Serialize, serde::Deserialize, genlayer_calldata::Decode, genlayer_calldata::Encode, + Clone, + serde::Serialize, + serde::Deserialize, + genlayer_calldata::Decode, + genlayer_calldata::Encode, )] pub struct Request { - pub major: u32, + /// Which executor line runs this: for a top-level run the public-ABI major + /// the host read off the contract's root slot, for a nested one the routing + /// its caller was given. + pub selector: genvm_modules_interfaces::ExecutorSelector, pub message: genvm_modules_interfaces::MessageData, pub is_sync: bool, /// Executor debug level. Controls captured-output bounding, tracing, runner @@ -419,13 +930,10 @@ pub struct Request { #[serde(default)] #[calldata(default = default_no_modules)] pub no_modules: bool, - /// Debug-only override: force the executor directory (a version dir under the - /// executors path) instead of resolving it from the manifest. Honored only - /// when `debug_mode >= Safe`; ignored in production (`disabled`) so consensus - /// always runs the manifest-resolved version. - #[serde(default = "default_reroute_to")] - #[calldata(default = default_reroute_to)] - pub reroute_to: String, + /// Debug-only overrides; see [`UnsafeOverrides`]. + #[serde(default)] + #[calldata(default = default_unsafe_overrides)] + pub unsafe_overrides: UnsafeOverrides, pub leader_nondet_results: Option<Vec<bytes::Bytes>>, /// Host-provided `node` fee constants (moved off `host_data`). #[serde(default)] @@ -441,6 +949,26 @@ pub struct Request { #[serde(default)] #[calldata(default = default_record_actions)] pub record_actions: Vec<String>, + #[serde(default)] + #[calldata(default = default_host_genvm_id)] + pub host_genvm_id: Option<String>, + #[serde(default)] + #[calldata(default = default_deadline)] + pub deadline: Option<ManagerDuration>, + #[serde(default)] + #[calldata(default = default_host_hello_data)] + pub host_hello_data: Vec<bytes::Bytes>, + /// Whether the host wants to be asked where a `CallContract` should run. + /// When false the manager answers `resolve_callcontract_executor` itself + /// with a null reply, so every call stays in-process and the host never + /// has to implement that method. + #[serde(default)] + #[calldata(default = default_hook_cross_contract_calls)] + pub hook_cross_contract_calls: bool, +} + +fn default_hook_cross_contract_calls() -> bool { + false } fn default_gas_data() -> std::collections::BTreeMap<String, String> { @@ -455,6 +983,18 @@ fn default_record_actions() -> Vec<String> { Vec::new() } +fn default_host_genvm_id() -> Option<String> { + None +} + +fn default_deadline() -> Option<ManagerDuration> { + None +} + +fn default_host_hello_data() -> Vec<bytes::Bytes> { + Vec::new() +} + fn default_permissions() -> String { "wscn".to_owned() } @@ -544,7 +1084,7 @@ impl LogAppender for LogAppenderToLog { } } -struct LogAppenderToValue(LogSink, GenVMId); +struct LogAppenderToValue(LogSink); impl LogAppender for LogAppenderToValue { #[inline(always)] @@ -671,124 +1211,453 @@ async fn pipe_read<P: tokio::io::AsyncReadExt + Unpin>( std::mem::drop(permit); } -async fn read_manager_host_stream( - parent_fd: FdWrapper, - consumed_result: sync::DArc<tokio::sync::OnceCell<Vec<u8>>>, - genvm_id: GenVMId, -) { - use tokio::io::{AsyncReadExt, AsyncWriteExt}; +#[derive(Default)] +struct ManagerHostStreamState { + closed: AtomicBool, + closed_notify: tokio::sync::Notify, +} - let mut file = match parent_fd.into_async_fd() { - Ok(f) => f, - Err(e) => { - log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, error:err = e; "failed to create async fd for manager host stream"); - return; +impl ManagerHostStreamState { + fn close(&self) { + if !self.closed.swap(true, Ordering::SeqCst) { + self.closed_notify.notify_waiters(); } - }; + } - loop { - let mut method_buf = [0u8; 1]; - match file.read_exact(&mut method_buf).await { - Ok(_) => {} - Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => { - log_debug_into!(&LoggerWithId, genvm_id:id = genvm_id.0; "manager host stream closed"); - return; - } - Err(e) => { - log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, error:err = e; "failed to read method from manager host stream"); + async fn wait_closed(&self) { + loop { + let notified = self.closed_notify.notified(); + if self.closed.load(Ordering::SeqCst) { return; } + notified.await; } + } +} - if method_buf[0] == host_fns::Methods::ConsumeResult as u8 { - // Read length-prefixed data - let mut len_buf = [0u8; 4]; - if let Err(e) = file.read_exact(&mut len_buf).await { - log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, error:err = e; "failed to read result length"); - return; - } - let len = u32::from_le_bytes(len_buf) as usize; - let mut data = vec![0u8; len]; - if let Err(e) = file.read_exact(&mut data).await { - log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, error:err = e; "failed to read result data"); - return; - } - log_debug_into!(&LoggerWithId, genvm_id:id = genvm_id.0, len = len; "manager received consume_result"); - let _ = consumed_result.set(data); - // Send ACK - if let Err(e) = file.write_all(&[0u8]).await { - log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, error:err = e; "failed to send ACK for consume_result"); - return; - } - let _ = file.flush().await; - } else { - log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, method = method_buf[0]; "unexpected method on manager host stream"); - return; - } +struct NestedRunRegistration(sync::DArc<SingleGenVMContext>); + +impl NestedRunRegistration { + fn new(exec_ctx: sync::DArc<SingleGenVMContext>) -> Self { + exec_ctx.nested_run_started(); + Self(exec_ctx) } } -impl Ctx { - async fn check_proc(&self, exec: sync::DArc<SingleGenVMContext>) -> bool { - if exec.result.initialized() { - return true; - } +impl Drop for NestedRunRegistration { + fn drop(&mut self) { + self.0.nested_run_finished(); + } +} - tokio::task::spawn(async move { - let mut proc = exec.process_handle.lock().await; +async fn read_length_prefixed<R: tokio::io::AsyncRead + Unpin>( + reader: &mut R, + max_len: usize, +) -> anyhow::Result<Vec<u8>> { + use tokio::io::AsyncReadExt; - match proc.try_wait() { - Ok(Some(status)) => { - log_debug!(id = exec.id, status = status; "genvm exited"); + let len = reader.read_u32_le().await? as usize; + anyhow::ensure!( + len <= max_len, + "manager host frame is too large: {len} > {max_len}" + ); + let mut data = vec![0; len]; + reader.read_exact(&mut data).await?; + Ok(data) +} - let _ = sync::DropGuard::new(|| { - GENVM_BY_ID_LOGGER.pin().remove(&exec.id); - }); +async fn write_length_prefixed<W: tokio::io::AsyncWrite + Unpin>( + writer: &mut W, + data: &[u8], +) -> anyhow::Result<()> { + use tokio::io::AsyncWriteExt; - let metrics = exec - ._execution_context - .as_ref() - .map(|ctx| ctx.collect_metrics()) - .unwrap_or(serde_json::Value::Null); + let len = u32::try_from(data.len()).map_err(|_| anyhow::anyhow!("reply is too large"))?; + writer.write_u32_le(len).await?; + writer.write_all(data).await?; + writer.flush().await?; + Ok(()) +} - log_debug!(id = exec.id, metrics:serde = metrics; "metrics collected"); +/// A nested run gets `no_modules: true`, no message-fee allocation and a state +/// the manager did not derive, so these permissions are unserviceable across the +/// boundary by construction. Refusing an envelope that asserts one is the only +/// enforcement available against a line the manager did not derive: granting it +/// would hand out authority nothing downstream can honour. +fn check_nested_permissions( + permissions: genvm_modules_interfaces::NestedPermissions, +) -> anyhow::Result<()> { + use genvm_modules_interfaces::NestedPermissions as P; + + for (bit, name) in [ + (P::SPAWN_NONDET, "spawn_nondet"), + (P::WRITE_STORAGE, "write_storage"), + (P::SEND_MESSAGES, "send_messages"), + ( + P::USE_BALANCE_FOR_MESSAGE_FEES, + "use_balance_for_message_fees", + ), + ] { + anyhow::ensure!( + !permissions.contains(bit), + "nested run envelope asserts `{name}`, which a call crossing a major boundary never carries" + ); + } - exec.all_permits.store(None); // drop all resources it owns - let _ = exec.stdout_stderr_sem.acquire_many(2).await; // wait for stderr/stdout to be fully read - log_debug!(id = exec.id, status = status; "stdout/stderr sem acquired"); - let stdout = exec.stdout.get().map(|x| x.as_str()).unwrap_or(""); - let stderr = exec.stderr.get().map(|x| x.as_str()).unwrap_or(""); + Ok(()) +} - let genvm_log = { - let mut as_vec = Vec::new(); - while let Some(data) = exec.log_sink.pop() { - as_vec.push(data.into_json()); - } - as_vec - }; +fn nested_internal_error() -> genvm_modules_interfaces::NestedRunReply { + genvm_modules_interfaces::NestedRunReply { + result: genvm_modules_interfaces::NestedRunResult { + kind: genvm_modules_interfaces::ResultCode::InternalError, + data: calldata::Value::Str("cross-major nested run failed".to_owned()).into(), + }, + small_hash: bytes::Bytes::new(), + effect_free: false, + } +} + +fn nested_reply_from_consumed_result( + data: &[u8], +) -> anyhow::Result<genvm_modules_interfaces::NestedRunReply> { + let (&kind, encoded) = data + .split_first() + .ok_or_else(|| anyhow::anyhow!("nested executor returned an empty result"))?; + let kind = match kind { + 0 => genvm_modules_interfaces::ResultCode::Return, + 1 => genvm_modules_interfaces::ResultCode::UserError, + 2 => genvm_modules_interfaces::ResultCode::VmError, + 3 => genvm_modules_interfaces::ResultCode::InternalError, + value => anyhow::bail!("nested executor returned unknown result code {value}"), + }; + let reported: genvm_modules_interfaces::ReportedResult = calldata::decode_obj(encoded)?; + // The code is stated twice: once as the framing byte, once inside the + // reported map that the execution hash commits to. A disagreement means the + // callee is not the implementation we think it is. + anyhow::ensure!( + kind == reported.kind, + "nested executor result code {kind:?} disagrees with the reported {:?}", + reported.kind + ); + if kind != genvm_modules_interfaces::ResultCode::InternalError { + anyhow::ensure!( + reported.small_hash.len() == 32, + "nested executor returned an invalid small hash length" + ); + } + // Every effect a nested run could report is gated behind a permission the + // boundary derivation clears, so a non-empty one means the callee did + // something its permissions forbade. Refuse the reply instead of passing it + // up flagged: the manager is the only party that can enforce this against a + // line it did not derive. + if let Some(effect) = nested_effect(&reported) { + anyhow::bail!("nested executor reported `{effect}`, which its permissions forbid"); + } - if let Err(e) = exec.result.set(SingleGenVMContextDone { - finished_at: chrono::Utc::now(), - stdout: stdout.to_owned(), - stderr: stderr.to_owned(), - genvm_log, - metrics, - consumed_result: exec.consumed_result.get().cloned(), - version_major: exec.version_major, - version_minor: exec.version_minor, - }) { - log_warn!(error:err = e; "error setting genvm result; it can happen rarely due to concurrency"); + Ok(genvm_modules_interfaces::NestedRunReply { + result: genvm_modules_interfaces::NestedRunResult { + kind, + data: reported.data, + }, + small_hash: reported.small_hash, + effect_free: true, + }) +} + +/// Names the first field of `reported` that proves the callee produced an effect +/// or consumed a shared budget. +/// +/// The destructuring is the point: adding a field to `ReportedResult` stops +/// compiling until it is classified here, so a new effect cannot be dropped +/// silently the way the ignored ones below once were. +fn nested_effect(reported: &genvm_modules_interfaces::ReportedResult) -> Option<&'static str> { + let genvm_modules_interfaces::ReportedResult { + // The outcome and the hash accounting, not effects. The caller folds + // `small_hash` into its own `det_subvm_hashes`; the execution hash stays + // here for the node's own result comparison. + execution_hash: _, + small_hash: _, + kind: _, + data: _, + backtrace: _, + wasm_store_hashes: _, + // A remaining budget is a report, not a consumption. + data_fees_remaining: _, + storage_changes, + emissions, + nondet_disagreement, + nondet_results, + data_fees_consumed: + genvm_modules_interfaces::BucketsConsumed { + storage, + message_receipt, + nondet_output, + message_fee, + event, + }, + llm_consumption, + } = reported; + + if !storage_changes.is_empty() { + return Some("storage_changes"); + } + if !emissions.is_empty() { + return Some("emissions"); + } + if nondet_disagreement.is_some() { + return Some("nondet_disagreement"); + } + if !nondet_results.is_empty() { + return Some("nondet_results"); + } + if !llm_consumption.is_zero() { + return Some("llm_consumption"); + } + + for (bucket, name) in [ + (storage, "data_fees_consumed.storage"), + (message_receipt, "data_fees_consumed.message_receipt"), + (nondet_output, "data_fees_consumed.nondet_output"), + (message_fee, "data_fees_consumed.message_fee"), + (event, "data_fees_consumed.event"), + ] { + if !bucket.is_zero() { + return Some(name); + } + } + + None +} + +fn read_manager_host_stream( + parent_fd: FdWrapper, + full_ctx: sync::DArc<crate::manager::AppContext>, + exec_ctx: sync::DArc<SingleGenVMContext>, + parent_req: Arc<Request>, + consumed_result: sync::DArc<tokio::sync::OnceCell<Vec<u8>>>, + genvm_id: GenVMId, + stream_state: Arc<ManagerHostStreamState>, +) -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send>> { + Box::pin(async move { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + let file = match parent_fd.into_async_fd() { + Ok(f) => f, + Err(e) => { + log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, error:err = e; "failed to create async fd for manager host stream"); + stream_state.close(); + return; + } + }; + let (mut reader, writer) = tokio::io::split(file); + let writer = Arc::new(tokio::sync::Mutex::new(writer)); + let stream_state_on_drop = stream_state.clone(); + let _close_guard = sync::DropGuard::new(move || stream_state_on_drop.close()); + + loop { + let mut method_buf = [0u8; 1]; + match reader.read_exact(&mut method_buf).await { + Ok(_) => {} + Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => { + log_debug_into!(&LoggerWithId, genvm_id:id = genvm_id.0; "manager host stream closed"); + return; + } + Err(e) => { + log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, error:err = e; "failed to read method from manager host stream"); + return; } - true } - Ok(None) => false, - Err(e) => { - log_error!(error:err = e; "error checking process status"); - true + + match host_fns::Methods::try_from(method_buf[0]) { + Ok(host_fns::Methods::ConsumeResult) => { + let data = match read_length_prefixed(&mut reader, u32::MAX as usize).await { + Ok(data) => data, + Err(e) => { + log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, error:ah = &e; "failed to read consumed result"); + return; + } + }; + log_debug_into!(&LoggerWithId, genvm_id:id = genvm_id.0, len = data.len(); "manager received consume_result"); + let _ = consumed_result.set(data); + + let mut writer = writer.lock().await; + if let Err(e) = writer.write_all(&[0]).await { + log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, error:err = e; "failed to send ACK for consume_result"); + return; + } + if let Err(e) = writer.flush().await { + log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, error:err = e; "failed to flush ACK for consume_result"); + return; + } + } + Ok(host_fns::Methods::RunNested) => { + // Unbounded, like `consume_result` above: the peer is a child + // process this manager spawned, and the envelope's real bound + // is the memory limit that child runs under. + let data = match read_length_prefixed(&mut reader, u32::MAX as usize).await { + Ok(data) => data, + Err(e) => { + log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, error:ah = &e; "failed to read run_nested request"); + return; + } + }; + // Served inline, not spawned: replies carry no request id, so + // the only thing that pairs one with its request is arrival + // order on this stream. The caller is blocked inside its own + // call while this runs, so there is nothing to overlap with. + let _registration = NestedRunRegistration::new(exec_ctx.clone()); + let reply = match calldata::decode_obj::< + genvm_modules_interfaces::NestedRunEnvelope, + >(&data) + { + Ok(envelope) => { + let run_ctx = full_ctx.gep(|ctx| &ctx.run_ctx); + match run_ctx + .start_nested( + full_ctx.clone(), + exec_ctx.clone(), + parent_req.clone(), + envelope, + stream_state.clone(), + ) + .await + { + Ok(reply) => reply, + Err(e) => { + log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, error:ah = &e; "nested run failed"); + nested_internal_error() + } + } + } + Err(e) => { + log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, error:err = e; "failed to decode run_nested envelope"); + nested_internal_error() + } + }; + let encoded = calldata::encode_obj(&reply); + let mut writer = writer.lock().await; + if let Err(e) = write_length_prefixed(&mut *writer, &encoded).await { + log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, error:ah = &e; "failed to send run_nested reply"); + } + } + Ok(host_fns::Methods::ResolveCallcontractExecutor) => { + // The do-nothing route: a request that did not opt into + // `hook_cross_contract_calls` gets this arm instead of the + // node's host, and a null reply means "stay in-process". + let mut request = [0u8; calldata::ADDRESS_SIZE + 2]; + if let Err(e) = reader.read_exact(&mut request).await { + log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, error:err = e; "failed to read resolve_callcontract_executor request"); + return; + } + let encoded = calldata::encode_obj(&calldata::Value::Null); + let mut writer = writer.lock().await; + if let Err(e) = writer.write_all(&[host_fns::Errors::Ok as u8]).await { + log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, error:err = e; "failed to send resolve_callcontract_executor status"); + return; + } + if let Err(e) = write_length_prefixed(&mut *writer, &encoded).await { + log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, error:ah = &e; "failed to send resolve_callcontract_executor reply"); + return; + } + } + Ok(method) => { + log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, method = method as u8; "unexpected method on manager host stream"); + return; + } + Err(()) => { + log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, method = method_buf[0]; "unknown method on manager host stream"); + return; + } } } - }).await.unwrap_or(false) + }) +} + +async fn acquire_run_permits( + permits: Arc<tokio::sync::Semaphore>, + count: u32, +) -> anyhow::Result<tokio::sync::OwnedSemaphorePermit> { + if count == 1 { + Ok(permits.acquire_owned().await?) + } else { + Ok(permits.acquire_many_owned(count).await?) + } +} + +async fn wait_for_output(exec: &SingleGenVMContext) { + let _ = exec.stdout_stderr_sem.acquire_many(2).await; + log_debug!(id = exec.id; "stdout/stderr sem acquired"); +} + +fn drain_log_sink(log_sink: &LogSink) -> Vec<serde_json::Map<String, serde_json::Value>> { + let mut as_vec = Vec::new(); + while let Some(data) = log_sink.pop() { + as_vec.push(data.into_json()); } + as_vec +} + +async fn finish_execution( + exec: &SingleGenVMContext, + status: Option<std::process::ExitStatus>, + default_cause: FinishCause, +) -> bool { + if exec.result.initialized() || exec.terminal_event.get().is_some() { + return false; + } + + // Bound to a name on purpose: `let _ =` would unregister the sink right + // here, losing every line this function still logs before it drains it. + let _by_id_logger_guard = sync::DropGuard::new(|| { + GENVM_BY_ID_LOGGER.pin().remove(&exec.id); + }); + + let metrics = exec + .execution_context + .get() + .map(|ctx| ctx.collect_metrics()) + .unwrap_or(serde_json::Value::Null); + + log_debug!(id = exec.id, metrics:serde = metrics; "metrics collected"); + + exec.all_permits.store(None); + wait_for_output(exec).await; + + let stdout = exec.stdout.get().map(|x| x.as_str()).unwrap_or(""); + let stderr = exec.stderr.get().map(|x| x.as_str()).unwrap_or(""); + let genvm_log = drain_log_sink(&exec.log_sink); + let cause = exec.finish_cause().unwrap_or(default_cause); + let exit_code = status.and_then(|status| status.code()).map(i64::from); + + let done = SingleGenVMContextDone { + finished_at: chrono::Utc::now(), + stdout: stdout.to_owned(), + stderr: stderr.to_owned(), + genvm_log, + metrics, + consumed_result: exec.consumed_result.get().cloned(), + version_major: exec.version_major.load(Ordering::SeqCst), + version_minor: exec.version_minor.load(Ordering::SeqCst), + cause, + exit_code, + }; + let event = exec.finished_event(&done); + if let Err(e) = exec.result.set(done) { + log_warn!(error:err = e; "error setting genvm result; it can happen rarely due to concurrency"); + return false; + } + exec.publish_terminal(event) +} + +fn fail_to_start(exec: &SingleGenVMContext, error: anyhow::Error) { + exec.all_permits.store(None); + let event = Event::FailedToStart { + genvm_id: exec.id, + host_genvm_id: exec.host_genvm_id.clone(), + error: format!("{:#}", error), + }; + let _ = exec.publish_terminal(event); } /// Creates a pipe for GenVM logging. Returns `(read_fd, write_fd)` with @@ -912,40 +1781,440 @@ fn spawn_stdin_writer( }) } -/// Spawns a task that awaits the stdin writer and kills the process on failure. -fn spawn_stdin_monitor( - stdin_task: tokio::task::JoinHandle<std::io::Result<()>>, - exec_ctx: sync::DArc<SingleGenVMContext>, - genvm_id: GenVMId, -) { - tokio::task::spawn(async move { - let stdin_failed = match stdin_task.await { - Ok(Ok(())) => { - log_debug_into!(&LoggerWithId, genvm_id:id = genvm_id.0; "execution data written"); - false - } - Ok(Err(e)) => { - log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, error:err = e; "failed to write execution data to child stdin, killing process"); - true +fn strict_deadline_from_request(req: &Request) -> chrono::DateTime<chrono::Utc> { + let max = std::time::Duration::from_secs(24 * 60 * 60); + let duration = req + .deadline + .map(ManagerDuration::to_std) + .unwrap_or_else(|| { + std::time::Duration::from_secs(req.max_execution_minutes.min(24 * 60) * 60) + }) + .min(max); + chrono::Utc::now() + + chrono::Duration::from_std(duration).unwrap_or_else(|_| chrono::Duration::hours(24)) +} + +fn execution_data_from_request(req: &Request) -> genvm_modules_interfaces::ExecutionData { + let mut method_hosts = vec![0; host_fns::Methods::SIZE]; + method_hosts[host_fns::Methods::ConsumeResult as usize] = 1; + method_hosts[host_fns::Methods::RunNested as usize] = 1; + if !req.hook_cross_contract_calls { + method_hosts[host_fns::Methods::ResolveCallcontractExecutor as usize] = 1; + } + + genvm_modules_interfaces::ExecutionData { + calldata: req.calldata.clone(), + message: req.message.clone(), + host_data: req.host_data.clone(), + code: req.code.clone(), + leader_nondet_results: req.leader_nondet_results.clone(), + host_hello_data: req.host_hello_data.clone(), + method_hosts, + bucket_totals: req.bucket_totals.clone(), + gas_data: req.gas_data.clone(), + message_fee_allocation: req.message_fee_allocation.clone(), + initial_time_units_allocation: req.initial_time_units_allocation, + record_actions: req.record_actions.clone(), + remaining_recursion: if req.debug_mode >= genvm_common::DebugMode::Unsafe { + req.unsafe_overrides.initial_recursion + } else { + None + }, + nested: None, + } +} + +enum RunResources { + TopLevel { + permits: tokio::sync::OwnedSemaphorePermit, + modules_lock: Box<dyn std::any::Any + Send + Sync>, + }, + Nested { + caller_stream: Arc<ManagerHostStreamState>, + }, +} + +impl RunResources { + fn is_top_level(&self) -> bool { + matches!(self, Self::TopLevel { .. }) + } + + fn caller_stream(&self) -> Option<Arc<ManagerHostStreamState>> { + match self { + Self::TopLevel { .. } => None, + Self::Nested { caller_stream, .. } => Some(caller_stream.clone()), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ProcessStop { + Cancelled, + Deadline, + CallerClosed, +} + +enum Reservation { + Existing(GenVMId), + Reserved(sync::DArc<SingleGenVMContext>), +} + +async fn wait_for_process_stop( + exec_ctx: &SingleGenVMContext, + caller_stream: Option<&ManagerHostStreamState>, + deadline_duration: std::time::Duration, +) -> ProcessStop { + tokio::select! { + _ = exec_ctx.wait_cancelled() => ProcessStop::Cancelled, + _ = tokio::time::sleep(deadline_duration) => ProcessStop::Deadline, + _ = async { + match caller_stream { + Some(stream) => stream.wait_closed().await, + None => std::future::pending().await, } - Err(e) => { - log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, error:err = e; "stdin write task panicked, killing process"); - true + } => ProcessStop::CallerClosed, + } +} + +impl Ctx { + fn reserve_execution( + &self, + host_genvm_id: Option<&str>, + build: impl FnOnce(GenVMId) -> sync::DArc<SingleGenVMContext>, + ) -> Reservation { + // The guard is held across the lookup, the id allocation and both + // inserts, so two starts sharing a token cannot both reserve. Keeping + // this section free of await points is what makes that safe: a papaya + // pin is an epoch guard rather than a lock, so taking one under the + // mutex cannot block on `ack` or `gc_step` taking them the other way. + let mut host_ids = self.lock_host_ids(); + + if let Some(host_genvm_id) = host_genvm_id { + if let Some(genvm_id) = host_ids.get(host_genvm_id).copied() { + if self.known_executions.pin().get(&genvm_id).is_some() { + return Reservation::Existing(genvm_id); + } + host_ids.remove(host_genvm_id); } + } + + let genvm_id = GenVMId( + self.next_genvm_id + .fetch_add(1, std::sync::atomic::Ordering::SeqCst), + ); + let exec_ctx = build(genvm_id); + self.known_executions + .pin() + .insert(genvm_id, exec_ctx.clone()); + if let Some(host_genvm_id) = host_genvm_id { + host_ids.insert(host_genvm_id.to_owned(), genvm_id); + } + + Reservation::Reserved(exec_ctx) + } + + pub async fn start( + &self, + full_ctx: sync::DArc<crate::manager::AppContext>, + req: Request, + modules_lock: Box<dyn std::any::Any + Send + Sync>, + ) -> anyhow::Result<GenVMId> { + let reservation = self.reserve_execution(req.host_genvm_id.as_deref(), |genvm_id| { + let events = tokio::sync::watch::Sender::new(Snapshot::Queued { + genvm_id, + host_genvm_id: req.host_genvm_id.clone(), + }); + sync::DArc::new(SingleGenVMContext { + result: tokio::sync::OnceCell::new(), + started_event: tokio::sync::OnceCell::new(), + terminal_event: tokio::sync::OnceCell::new(), + terminal_at: tokio::sync::OnceCell::new(), + events, + version: std::sync::RwLock::new(String::new()), + version_major: AtomicU16::new(0), + version_minor: AtomicU16::new(0), + id: genvm_id, + host_genvm_id: req.host_genvm_id.clone(), + process_handle: tokio::sync::Mutex::new(None), + started_at: chrono::Utc::now(), + strict_deadline: strict_deadline_from_request(&req), + + stdout_stderr_sem: Arc::new(tokio::sync::Semaphore::new(2)), + stdout: tokio::sync::OnceCell::new(), + stderr: tokio::sync::OnceCell::new(), + log_sink: Arc::new(LogSinkInner::new( + req.debug_mode.capture() == genvm_common::debug_mode::Capture::Unbounded, + )), + consumed_result: tokio::sync::OnceCell::new(), + + cancel_requested: AtomicBool::new(false), + cancel_notify: tokio::sync::Notify::new(), + finish_cause: std::sync::Mutex::new(None), + all_permits: crossbeam::atomic::AtomicCell::new(None), + nested_runs: AtomicUsize::new(0), + nested_runs_done: tokio::sync::Notify::new(), + execution_context: tokio::sync::OnceCell::new(), + }) + }); + + let exec_ctx = match reservation { + Reservation::Existing(genvm_id) => return Ok(genvm_id), + Reservation::Reserved(exec_ctx) => exec_ctx, }; - if stdin_failed { - let mut proc = exec_ctx.process_handle.lock().await; - let _ = proc.start_kill(); + let genvm_id = exec_ctx.id; + + tokio::spawn(supervise_genvm(full_ctx, exec_ctx, req, modules_lock)); + + Ok(genvm_id) + } + + async fn start_nested( + &self, + full_ctx: sync::DArc<crate::manager::AppContext>, + exec_ctx: sync::DArc<SingleGenVMContext>, + parent_req: Arc<Request>, + envelope: genvm_modules_interfaces::NestedRunEnvelope, + caller_stream: Arc<ManagerHostStreamState>, + ) -> anyhow::Result<genvm_modules_interfaces::NestedRunReply> { + anyhow::ensure!( + !envelope.message.is_init, + "nested CallContract message cannot be an init" + ); + anyhow::ensure!( + envelope.message.value == num_bigint::BigInt::from(0), + "nested CallContract message value must be zero" + ); + if parent_req.host.trim_start().starts_with("fd://") { + log_error_into!(&LoggerWithId, genvm_id:id = exec_ctx.id.0, host = parent_req.host; "cannot start nested executor because host 0 is not addressable"); + anyhow::bail!("host 0 uses fd:// and cannot be re-dialed by a nested executor"); } - }); + + let routing: genvm_modules_interfaces::ExecutorSelector = + calldata::decode_obj(&envelope.routing_payload)?; + + let genvm_id = GenVMId( + self.next_genvm_id + .fetch_add(1, std::sync::atomic::Ordering::SeqCst), + ); + let mut host_hello_data = parent_req.host_hello_data.clone(); + if host_hello_data.len() < 2 { + host_hello_data.resize(2, bytes::Bytes::new()); + } + host_hello_data[1] = bytes::Bytes::new(); + + check_nested_permissions(envelope.permissions)?; + let permissions = if envelope + .permissions + .contains(genvm_modules_interfaces::NestedPermissions::CALL_OTHERS) + { + "c".to_owned() + } else { + String::new() + }; + + let req = Request { + selector: routing, + message: envelope.message.clone(), + is_sync: true, + debug_mode: genvm_common::DebugMode::Disabled, + max_execution_minutes: parent_req.max_execution_minutes, + bucket_totals: Vec::new(), + host_data: parent_req.host_data.clone(), + timestamp: envelope.message.datetime, + host: parent_req.host.clone(), + extra_args: Vec::new(), + calldata: envelope.calldata.clone(), + code: None, + permissions, + no_modules: true, + unsafe_overrides: Default::default(), + leader_nondet_results: None, + gas_data: parent_req.gas_data.clone(), + message_fee_allocation: Vec::new(), + initial_time_units_allocation: 0, + record_actions: Vec::new(), + host_genvm_id: None, + deadline: None, + host_hello_data, + // A child that may itself call across a boundary needs the same + // routing as its parent. + hook_cross_contract_calls: parent_req.hook_cross_contract_calls, + }; + let mut execution_data = execution_data_from_request(&req); + execution_data.code = None; + execution_data.leader_nondet_results = None; + execution_data.record_actions.clear(); + execution_data.message_fee_allocation.clear(); + execution_data.remaining_recursion = Some(envelope.remaining_recursion); + execution_data.nested = Some(genvm_modules_interfaces::NestedExecutionData { + memory_limit: envelope.memory_limit, + stack: envelope.stack, + permissions: envelope.permissions, + state_mode: envelope.state_mode, + topmost_runner_id: envelope.topmost_runner_id, + remaining_det_fuel: envelope.remaining_det_fuel, + }); + + let consumed_result = sync::DArc::new(tokio::sync::OnceCell::new()); + let status = run_genvm_process( + full_ctx, + exec_ctx, + req, + execution_data, + genvm_id, + consumed_result.clone(), + RunResources::Nested { caller_stream }, + ) + .await?; + anyhow::ensure!( + status.success(), + "nested executor exited unsuccessfully: {status}" + ); + let result = consumed_result + .get() + .ok_or_else(|| anyhow::anyhow!("nested executor exited without consume_result"))?; + nested_reply_from_consumed_result(result) + } } -pub async fn start_genvm( +async fn supervise_genvm( full_ctx: sync::DArc<crate::manager::AppContext>, + exec_ctx: sync::DArc<SingleGenVMContext>, req: Request, modules_lock: Box<dyn std::any::Any + Send + Sync>, -) -> anyhow::Result<(GenVMId, tokio::sync::oneshot::Receiver<()>)> { - // Get module handlers early, before consuming full_ctx +) { + let ctx = full_ctx.gep(|x| &x.run_ctx); + let permit_count = if req.needs_modules() { 2 } else { 1 }; + + let permit_future = acquire_run_permits(ctx.permits.clone(), permit_count); + tokio::pin!(permit_future); + let permits = tokio::select! { + _ = exec_ctx.wait_cancelled() => { + let cause = exec_ctx.finish_cause().unwrap_or(FinishCause::Cancelled); + let _ = finish_execution(&exec_ctx, None, cause).await; + return; + } + permits = &mut permit_future => match permits { + Ok(permits) => permits, + Err(e) => { + fail_to_start(&exec_ctx, e); + return; + } + }, + }; + + if exec_ctx.cancel_requested.load(Ordering::SeqCst) { + let cause = exec_ctx.finish_cause().unwrap_or(FinishCause::Cancelled); + drop(permits); + let _ = finish_execution(&exec_ctx, None, cause).await; + return; + } + + if let Err(e) = + supervise_genvm_inner(full_ctx, exec_ctx.clone(), req, modules_lock, permits).await + { + fail_to_start(&exec_ctx, e); + } +} + +async fn supervise_genvm_inner( + full_ctx: sync::DArc<crate::manager::AppContext>, + exec_ctx: sync::DArc<SingleGenVMContext>, + req: Request, + modules_lock: Box<dyn std::any::Any + Send + Sync>, + permits: tokio::sync::OwnedSemaphorePermit, +) -> anyhow::Result<()> { + let execution_data = execution_data_from_request(&req); + let result = run_genvm_process( + full_ctx, + exec_ctx.clone(), + req, + execution_data, + exec_ctx.id, + exec_ctx.gep(|ctx| &ctx.consumed_result), + RunResources::TopLevel { + permits, + modules_lock, + }, + ) + .await; + exec_ctx.wait_for_nested_runs().await; + let status = result?; + + let default_cause = if exec_ctx.cancel_requested.load(Ordering::SeqCst) { + exec_ctx.finish_cause().unwrap_or(FinishCause::Cancelled) + } else { + FinishCause::Exited + }; + let _ = finish_execution(&exec_ctx, Some(status), default_cause).await; + Ok(()) +} + +/// The executor line a selector names. +/// +/// A major no line provides is contract input, not a node failure, so it falls +/// back to the newest line and lets that line's own check answer with +/// `invalid_contract major_mismatch`. A version is a direct statement by a +/// trusted party and gets no such benefit of the doubt: no match fails the run. +async fn resolve_selector( + ver_ctx: &crate::manager::versioning::Ctx, + selector: &genvm_modules_interfaces::ExecutorSelector, + timestamp: chrono::DateTime<chrono::Utc>, + genvm_id: GenVMId, +) -> anyhow::Result<crate::manager::versioning::ResolvedVersion> { + use genvm_modules_interfaces::{ExecutorSelector, VersionMatch}; + + let major = match selector { + ExecutorSelector::VersionOverride { version } => { + return match genvm_modules_interfaces::parse_version_match(version) { + VersionMatch::Exact(version) => { + Ok(crate::manager::versioning::exact_version(version)) + } + VersionMatch::Regex(pattern) => { + let pattern = regex::Regex::new(pattern).with_context(|| { + format!("version selector `{pattern}` is not a valid regex") + })?; + + ver_ctx + .get_matching_version(&pattern, timestamp) + .await + .ok_or_else(|| { + anyhow::anyhow!("no line matches version selector `{pattern}`") + }) + } + }; + } + ExecutorSelector::MajorOverride { major } => *major, + }; + + if let Some(version) = ver_ctx.get_version(major, timestamp).await { + return Ok(version); + } + + log_warn!( + major = major, + genvm_id:id = genvm_id.0; + "no line provides the requested major, falling back to the newest one" + ); + + ver_ctx.get_newest_version(timestamp).await.ok_or_else(|| { + anyhow::anyhow!( + "no compatible version found for major {major} and no line is available at all" + ) + }) +} + +async fn run_genvm_process( + full_ctx: sync::DArc<crate::manager::AppContext>, + exec_ctx: sync::DArc<SingleGenVMContext>, + req: Request, + execution_data: genvm_modules_interfaces::ExecutionData, + genvm_id: GenVMId, + consumed_result: sync::DArc<tokio::sync::OnceCell<Vec<u8>>>, + resources: RunResources, +) -> anyhow::Result<std::process::ExitStatus> { + let is_top_level = resources.is_top_level(); + let caller_stream = resources.caller_stream(); let module_handlers = if req.needs_modules() { let (llm, web) = full_ctx .mod_ctx @@ -957,14 +2226,6 @@ pub async fn start_genvm( None }; - let genvm_id = GenVMId( - full_ctx - .run_ctx - .next_genvm_id - .fetch_add(1, std::sync::atomic::Ordering::SeqCst), - ); - - // Create execution context if modules are needed let execution_context = if req.needs_modules() { let host_data: genvm_modules_interfaces::HostData = serde_json::from_str(&req.host_data)?; let role = if req.leader_nondet_results.is_none() { @@ -979,30 +2240,33 @@ pub async fn start_genvm( gas_data: req.gas_data.clone(), initial_time_units_allocation: req.initial_time_units_allocation, }); - Some(full_ctx.mod_ctx.create_execution_context(hello).await?) + let ctx = full_ctx.mod_ctx.create_execution_context(hello).await?; + let _ = exec_ctx.execution_context.set(ctx.clone()); + Some(ctx) } else { None }; - let version = full_ctx - .ver_ctx - .get_version(req.major, req.timestamp) - .await - .ok_or_else(|| anyhow::anyhow!("no compatible version found for major {}", req.major))?; - - let ctx = full_ctx.into_gep(|x| &x.run_ctx); - - let permits = if req.needs_modules() { - ctx.permits.clone().acquire_many_owned(2).await? - } else { - ctx.permits.clone().acquire_owned().await? + // A debug reroute replaces the request's own selector, but only from `safe` + // up -- production (`disabled`) always resolves what the request states, so + // consensus can't be steered to a different binary. + let selector = match &req.unsafe_overrides.reroute_to { + reroute if !reroute.is_empty() && req.debug_mode >= genvm_common::DebugMode::Safe => { + genvm_modules_interfaces::ExecutorSelector::VersionOverride { + version: reroute.clone(), + } + } + _ => req.selector.clone(), }; + let version = resolve_selector(&full_ctx.ver_ctx, &selector, req.timestamp, genvm_id).await?; + + let ctx = full_ctx.clone().into_gep(|x| &x.run_ctx); // Capture controls how logs and stdout/stderr are kept: disabled (forwarded // to the manager log only), bounded, or unbounded. use genvm_common::debug_mode::Capture; let capture = req.debug_mode.capture(); - let log_sink: LogSink = Arc::new(LogSinkInner::new(capture == Capture::Unbounded)); + let log_sink = exec_ctx.log_sink.clone(); // Only route per-id logs into the result sink when we actually capture; under // `disabled` the sink stays unregistered so manager-internal logs (and the // forwarded executor logs) go to the manager log, leaving `genvm_log` empty. @@ -1013,15 +2277,14 @@ pub async fn start_genvm( GENVM_BY_ID_LOGGER.pin().remove(&genvm_id); }); - // Resolve command path. A request may reroute to an explicit executor dir, - // but only when running with a debug mode of `safe` or above -- production - // (`disabled`) always uses the manifest-resolved version so consensus can't - // be steered to a different binary. let mut command_path = ctx.executors_path.clone(); - let version_str_owned = version.orig_key.clone(); - let mut version_str: &str = &version_str_owned; - if !req.reroute_to.is_empty() && req.debug_mode >= genvm_common::DebugMode::Safe { - version_str = &req.reroute_to; + let version_str: &str = &version.orig_key; + if is_top_level { + exec_ctx.set_version( + version_str.to_owned(), + version.version.major as u16, + version.version.minor as u16, + ); } command_path.push(version_str); command_path.push("bin"); @@ -1033,7 +2296,8 @@ pub async fn start_genvm( let (read_fd, write_fd) = create_log_pipe()?; let mut proc = build_genvm_command(command_path, &req, genvm_id, &write_fd); - // Setup manager host socketpair (host id=1 for consume_result and notify_finished) + // Setup manager host socketpair (host id=1 for consume_result, run_nested + // and -- unless the request hooks them -- resolve_callcontract_executor) let (manager_parent, manager_child) = FdWrapper::socketpair()?; manager_parent.set_cloexec(true)?; manager_child.set_cloexec(false)?; @@ -1052,25 +2316,15 @@ pub async fn start_genvm( None }; - let mut method_hosts: Vec<u8> = vec![0; host_fns::Methods::SIZE]; - method_hosts[host_fns::Methods::ConsumeResult as usize] = 1; - - let execution_data = genvm_modules_interfaces::ExecutionData { - calldata: req.calldata.clone(), - message: req.message.clone(), - host_data: req.host_data.clone(), - code: req.code.clone(), - leader_nondet_results: req.leader_nondet_results.clone(), - method_hosts, - bucket_totals: req.bucket_totals.clone(), - gas_data: req.gas_data.clone(), - message_fee_allocation: req.message_fee_allocation.clone(), - initial_time_units_allocation: req.initial_time_units_allocation, - record_actions: req.record_actions.clone(), - }; - let execution_data_bytes = calldata::encode_obj(&execution_data); + if req + .host_hello_data + .get(1) + .is_some_and(|data| !data.is_empty()) + { + anyhow::bail!("host_hello_data for manager-owned host index 1 is not allowed"); + } - let execution_data_bytes = bytes::Bytes::from(execution_data_bytes); + let execution_data_bytes = bytes::Bytes::from(calldata::encode_obj(&execution_data)); // Spawn log reader: capture into the sink, or (when capture is disabled) // forward to the manager log instead of buffering into the result. @@ -1079,7 +2333,6 @@ pub async fn start_genvm( } else { let logger = Arc::new(tokio::sync::Mutex::new(LogAppenderToValue( log_sink.clone(), - genvm_id, ))); let l = logger.clone().lock_owned().await; tokio::spawn(read_log_pipe(read_fd, l)); @@ -1091,55 +2344,97 @@ pub async fn start_genvm( std::mem::drop(module_child_fds); std::mem::drop(manager_child); - // Spawn stdin writer let stdin_task = child .stdin .take() .map(|stdin| spawn_stdin_writer(stdin, execution_data_bytes)); - // Create exec context and stdout/stderr permits - let stdout_stderr_sem = Arc::new(tokio::sync::Semaphore::new(2)); - let stdout_perm = stdout_stderr_sem.clone().acquire_owned().await?; - let stderr_perm = stdout_stderr_sem.clone().acquire_owned().await?; - let stdout = child.stdout.take(); let stderr = child.stderr.take(); + let stdout_perm = if stdout.is_some() { + Some(exec_ctx.stdout_stderr_sem.clone().acquire_owned().await?) + } else { + None + }; + let stderr_perm = if stderr.is_some() { + Some(exec_ctx.stdout_stderr_sem.clone().acquire_owned().await?) + } else { + None + }; - let (tx, rx) = tokio::sync::oneshot::channel::<()>(); - let all_resources = (permits, tx, modules_lock); - - let exec_ctx = sync::DArc::new(SingleGenVMContext { - result: tokio::sync::OnceCell::new(), - version: version_str.to_owned(), - version_major: version.version.major as u16, - version_minor: version.version.minor as u16, - id: genvm_id, - process_handle: tokio::sync::Mutex::new(child), - started_at: chrono::Utc::now(), - // Cap at 24h so a huge value can't overflow the i64 cast / Duration. - strict_deadline: chrono::Utc::now() - + chrono::Duration::minutes(req.max_execution_minutes.min(24 * 60) as i64), - - stdout_stderr_sem, - stdout: tokio::sync::OnceCell::new(), - stderr: tokio::sync::OnceCell::new(), - log_sink, - consumed_result: tokio::sync::OnceCell::new(), - - all_permits: crossbeam::atomic::AtomicCell::new(Some(Box::new(all_resources))), - _execution_context: execution_context, - }); - - // Spawn manager host protocol handler + let manager_stream_state = Arc::new(ManagerHostStreamState::default()); + let manager_stream_state_on_drop = manager_stream_state.clone(); + let _manager_stream_guard = sync::DropGuard::new(move || manager_stream_state_on_drop.close()); tokio::spawn(read_manager_host_stream( manager_parent, - exec_ctx.gep(|x| &x.consumed_result), + full_ctx.clone(), + exec_ctx.clone(), + Arc::new(req.clone()), + consumed_result, genvm_id, + manager_stream_state.clone(), )); - // Spawn stdin monitor and pipe readers - if let Some(stdin_task) = stdin_task { - spawn_stdin_monitor(stdin_task, exec_ctx.clone(), genvm_id); + if let Some(mut stdin_task) = stdin_task { + let deadline_duration = exec_ctx + .strict_deadline + .signed_duration_since(chrono::Utc::now()) + .to_std() + .unwrap_or_default(); + let stop = wait_for_process_stop(&exec_ctx, caller_stream.as_deref(), deadline_duration); + tokio::pin!(stop); + tokio::select! { + result = &mut stdin_task => match result { + Ok(Ok(())) => { + log_debug_into!(&LoggerWithId, genvm_id:id = genvm_id.0; "execution data written"); + } + Ok(Err(e)) => { + let _ = child.start_kill(); + let _ = child.wait().await; + anyhow::bail!("failed to write execution data to child stdin: {e}"); + } + Err(e) => { + let _ = child.start_kill(); + let _ = child.wait().await; + anyhow::bail!("stdin write task panicked: {e}"); + } + }, + status = child.wait() => { + stdin_task.abort(); + return match status { + Ok(status) => Ok(status), + Err(e) => { + let _ = child.start_kill(); + let _ = child.wait().await; + Err(e.into()) + } + }; + } + reason = &mut stop => { + if reason == ProcessStop::Deadline { + exec_ctx.request_finish(FinishCause::Deadline); + } + stdin_task.abort(); + let _ = child.start_kill(); + return child.wait().await.map_err(Into::into); + } + } + } + + let mut child = Some(child); + if is_top_level { + let mut stored_child = exec_ctx.process_handle.lock().await; + *stored_child = child.take(); + } + + if let RunResources::TopLevel { + permits, + modules_lock, + } = resources + { + exec_ctx + .all_permits + .store(Some(Box::new((permits, modules_lock)))); } let out_limit = (capture == Capture::Bounded).then_some(OUTPUT_TAIL_LIMIT); @@ -1147,7 +2442,7 @@ pub async fn start_genvm( tokio::spawn(pipe_read( stdout, exec_ctx.gep(|x| &x.stdout), - stdout_perm, + stdout_perm.expect("stdout permit must exist when stdout is piped"), out_limit, )); } @@ -1155,46 +2450,62 @@ pub async fn start_genvm( tokio::spawn(pipe_read( stderr, exec_ctx.gep(|x| &x.stderr), - stderr_perm, + stderr_perm.expect("stderr permit must exist when stderr is piped"), out_limit, )); } - ctx.known_executions - .pin() - .insert(genvm_id, exec_ctx.clone()); log_sink_guard.forget(); - - Ok((genvm_id, rx)) -} - -#[cfg(test)] -mod tests { - use super::keep_tail; - - #[test] - fn under_or_at_limit_is_unchanged() { - let mut s = String::from("hello"); - keep_tail(&mut s, 10); - assert_eq!(s, "hello"); - keep_tail(&mut s, 5); - assert_eq!(s, "hello"); + if is_top_level { + exec_ctx.publish_started(); } - #[test] - fn keeps_the_tail() { - let mut s = String::from("0123456789"); - keep_tail(&mut s, 4); - assert_eq!(s, "6789"); - } + let deadline_duration = exec_ctx + .strict_deadline + .signed_duration_since(chrono::Utc::now()) + .to_std() + .unwrap_or_default(); + let mut child = if is_top_level { + let mut stored_child = exec_ctx.process_handle.lock().await; + stored_child + .take() + .ok_or_else(|| anyhow::anyhow!("process handle missing"))? + } else { + child + .take() + .ok_or_else(|| anyhow::anyhow!("nested process handle missing"))? + }; - #[test] - fn never_cuts_mid_char() { - // "aΓ©b" = a(1) Γ©(2) b(1) = 4 bytes; cutting at byte 2 is mid-'Γ©', so the - // cut advances to the next boundary, dropping the partial char. - let mut s = String::from("aΓ©b"); - keep_tail(&mut s, 2); - assert_eq!(s, "b"); - assert!(std::str::from_utf8(s.as_bytes()).is_ok()); - } + let mut kill_sent = false; + let stop = wait_for_process_stop(&exec_ctx, caller_stream.as_deref(), deadline_duration); + tokio::pin!(stop); + let status = loop { + if kill_sent { + break child.wait().await?; + } + tokio::select! { + status = child.wait() => match status { + Ok(status) => break status, + Err(e) => { + let _ = child.start_kill(); + let _ = child.wait().await; + return Err(e.into()); + } + }, + reason = &mut stop => { + if reason == ProcessStop::Deadline { + exec_ctx.request_finish(FinishCause::Deadline); + } + let _ = child.start_kill(); + kill_sent = true; + } + } + }; + manager_stream_state.close(); + log_debug_into!(&LoggerWithId, genvm_id:id = genvm_id.0, status = status; "genvm exited"); + Ok(status) } + +#[cfg(test)] +#[path = "run_test.rs"] +mod tests; diff --git a/implementation/src/manager/run_test.rs b/implementation/src/manager/run_test.rs new file mode 100644 index 00000000..9744ad02 --- /dev/null +++ b/implementation/src/manager/run_test.rs @@ -0,0 +1,832 @@ +use super::*; + +fn test_ctx(retention: &str, permits: usize) -> Ctx { + Ctx { + known_executions: Default::default(), + host_genvm_ids: std::sync::Mutex::new(HashMap::new()), + boot_id: 7, + execution_retention: ManagerDuration::try_from(retention).unwrap(), + next_genvm_id: std::sync::atomic::AtomicU64::new(1), + permits: Arc::new(tokio::sync::Semaphore::new(permits)), + max_permits: tokio::sync::Mutex::new(PermitsData { + max: permits, + num_throttled: 0, + throttled: None, + }), + executors_path: std::path::PathBuf::new(), + } +} + +fn fake_execution( + genvm_id: GenVMId, + finished_at: Option<chrono::DateTime<chrono::Utc>>, +) -> sync::DArc<SingleGenVMContext> { + fake_execution_with_host_id(genvm_id, finished_at, None) +} + +fn fake_execution_with_host_id( + genvm_id: GenVMId, + finished_at: Option<chrono::DateTime<chrono::Utc>>, + host_genvm_id: Option<String>, +) -> sync::DArc<SingleGenVMContext> { + let events = tokio::sync::watch::Sender::new(Snapshot::Queued { + genvm_id, + host_genvm_id: host_genvm_id.clone(), + }); + let exec = sync::DArc::new(SingleGenVMContext { + id: genvm_id, + host_genvm_id, + version: std::sync::RwLock::new("v0.test".to_owned()), + version_major: AtomicU16::new(0), + version_minor: AtomicU16::new(0), + result: tokio::sync::OnceCell::new(), + started_event: tokio::sync::OnceCell::new(), + terminal_event: tokio::sync::OnceCell::new(), + terminal_at: tokio::sync::OnceCell::new(), + events, + started_at: chrono::Utc::now(), + strict_deadline: chrono::Utc::now() + chrono::Duration::hours(1), + stdout_stderr_sem: Arc::new(tokio::sync::Semaphore::new(2)), + stdout: tokio::sync::OnceCell::new(), + stderr: tokio::sync::OnceCell::new(), + log_sink: Arc::new(LogSinkInner::new(false)), + consumed_result: tokio::sync::OnceCell::new(), + process_handle: tokio::sync::Mutex::new(None), + cancel_requested: AtomicBool::new(false), + cancel_notify: tokio::sync::Notify::new(), + finish_cause: std::sync::Mutex::new(None), + all_permits: crossbeam::atomic::AtomicCell::new(None), + nested_runs: AtomicUsize::new(0), + nested_runs_done: tokio::sync::Notify::new(), + execution_context: tokio::sync::OnceCell::new(), + }); + + if let Some(finished_at) = finished_at { + exec.result + .set(SingleGenVMContextDone { + finished_at, + stdout: "out".to_owned(), + stderr: "err".to_owned(), + genvm_log: Vec::new(), + metrics: serde_json::Value::Null, + consumed_result: Some(vec![1, 2, 3]), + version_major: 0, + version_minor: 0, + cause: FinishCause::Exited, + exit_code: Some(0), + }) + .unwrap(); + } + + exec +} + +#[test] +fn duration_strings_parse_with_units() { + assert_eq!( + ManagerDuration::try_from("30s").unwrap().to_std(), + std::time::Duration::from_secs(30) + ); + assert_eq!( + ManagerDuration::try_from("10.5m").unwrap().to_std(), + std::time::Duration::from_secs(630) + ); + assert_eq!( + ManagerDuration::try_from("250ms").unwrap().to_std(), + std::time::Duration::from_millis(250) + ); + assert_eq!( + ManagerDuration::try_from("1h").unwrap().to_std(), + std::time::Duration::from_secs(3600) + ); +} + +#[test] +fn duration_strings_reject_missing_or_unknown_units() { + assert!(ManagerDuration::try_from("30").is_err()); + assert!(ManagerDuration::try_from("30q").is_err()); +} + +#[test] +fn reserve_execution_is_atomic_for_shared_host_id() { + const TASKS: usize = 32; + + let ctx = test_ctx("5m", 2); + let host_genvm_id = "host-run-1"; + let barrier = std::sync::Barrier::new(TASKS); + let builds = AtomicUsize::new(0); + let reservations = std::thread::scope(|s| { + let mut handles = Vec::with_capacity(TASKS); + for _ in 0..TASKS { + let ctx = &ctx; + let barrier = &barrier; + let builds = &builds; + handles.push(s.spawn(move || { + barrier.wait(); + ctx.reserve_execution(Some(host_genvm_id), |genvm_id| { + builds.fetch_add(1, Ordering::SeqCst); + fake_execution_with_host_id(genvm_id, None, Some(host_genvm_id.to_owned())) + }) + })); + } + handles + .into_iter() + .map(|handle| handle.join().unwrap()) + .collect::<Vec<_>>() + }); + + let mut existing = 0; + let mut reserved = 0; + let genvm_ids = reservations + .into_iter() + .map(|reservation| match reservation { + Reservation::Existing(genvm_id) => { + existing += 1; + genvm_id + } + Reservation::Reserved(exec_ctx) => { + reserved += 1; + exec_ctx.id + } + }) + .collect::<Vec<_>>(); + + assert_eq!(builds.load(Ordering::SeqCst), 1); + assert_eq!((reserved, existing), (1, TASKS - 1)); + assert!(genvm_ids.iter().all(|genvm_id| *genvm_id == genvm_ids[0])); +} + +#[test] +fn ack_removes_retained_result_but_status_is_not_destructive() { + let ctx = test_ctx("5m", 2); + let genvm_id = GenVMId(1); + let host_genvm_id = "host-run-1"; + ctx.known_executions.pin().insert( + genvm_id, + fake_execution_with_host_id( + genvm_id, + Some(chrono::Utc::now()), + Some(host_genvm_id.to_owned()), + ), + ); + ctx.host_genvm_ids + .lock() + .unwrap() + .insert(host_genvm_id.to_owned(), genvm_id); + + assert!(ctx.status(genvm_id).is_some()); + assert!(ctx.status(genvm_id).is_some()); + assert_eq!(ctx.ack(genvm_id), AckOutcome::Acked); + assert!(ctx.status(genvm_id).is_none()); + assert!(!ctx + .host_genvm_ids + .lock() + .unwrap() + .contains_key(host_genvm_id)); +} + +#[test] +fn ack_before_terminal_is_refused_and_preserves_run() { + let ctx = test_ctx("5m", 2); + let genvm_id = GenVMId(1); + let host_genvm_id = "host-run-1"; + ctx.known_executions.pin().insert( + genvm_id, + fake_execution_with_host_id(genvm_id, None, Some(host_genvm_id.to_owned())), + ); + ctx.host_genvm_ids + .lock() + .unwrap() + .insert(host_genvm_id.to_owned(), genvm_id); + + let acked = ctx.ack(genvm_id); + let attach_succeeds = ctx.attach(ctx.boot_id(), genvm_id).is_ok(); + let status_contains_run = ctx.status_executions().get(genvm_id.to_string()).is_some(); + let token_mapping = ctx + .host_genvm_ids + .lock() + .unwrap() + .get(host_genvm_id) + .copied(); + + assert_eq!( + (acked, attach_succeeds, status_contains_run, token_mapping), + (AckOutcome::NotFinished, true, true, Some(genvm_id)) + ); +} + +#[test] +fn ack_does_not_orphan_reused_token() { + let ctx = test_ctx("5m", 2); + let host_genvm_id = "host-run-1"; + let old_id = GenVMId(1); + let new_id = GenVMId(2); + ctx.known_executions.pin().insert( + old_id, + fake_execution_with_host_id( + old_id, + Some(chrono::Utc::now()), + Some(host_genvm_id.to_owned()), + ), + ); + ctx.host_genvm_ids + .lock() + .unwrap() + .insert(host_genvm_id.to_owned(), old_id); + ctx.known_executions.pin().insert( + new_id, + fake_execution_with_host_id(new_id, None, Some(host_genvm_id.to_owned())), + ); + ctx.host_genvm_ids + .lock() + .unwrap() + .insert(host_genvm_id.to_owned(), new_id); + + assert_eq!(ctx.ack(old_id), AckOutcome::Acked); + + let token_mapping = ctx + .host_genvm_ids + .lock() + .unwrap() + .get(host_genvm_id) + .copied(); + assert!(ctx.attach(ctx.boot_id(), new_id).is_ok()); + assert_eq!(token_mapping, Some(new_id)); +} + +#[test] +fn token_is_reusable_once_its_execution_is_gone() { + let ctx = test_ctx("5m", 2); + let host_genvm_id = "host-run-1"; + let first_id = match ctx.reserve_execution(Some(host_genvm_id), |genvm_id| { + fake_execution_with_host_id(genvm_id, None, Some(host_genvm_id.to_owned())) + }) { + Reservation::Reserved(exec_ctx) => exec_ctx.id, + Reservation::Existing(_) => panic!("first reservation already existed"), + }; + ctx.known_executions.pin().remove(&first_id); + + let second_id = match ctx.reserve_execution(Some(host_genvm_id), |genvm_id| { + fake_execution_with_host_id(genvm_id, None, Some(host_genvm_id.to_owned())) + }) { + Reservation::Reserved(exec_ctx) => exec_ctx.id, + Reservation::Existing(_) => panic!("stale token returned an existing execution"), + }; + + assert_ne!(second_id, first_id); +} + +#[tokio::test] +async fn ttl_expiry_removes_finished_execution() { + let ctx = sync::DArc::new(test_ctx("1ms", 2)); + let genvm_id = GenVMId(1); + ctx.known_executions.pin().insert( + genvm_id, + fake_execution( + genvm_id, + Some(chrono::Utc::now() - chrono::Duration::seconds(2)), + ), + ); + + gc_step(&ctx).await; + assert!(ctx.status(genvm_id).is_none()); +} + +#[tokio::test] +async fn gc_does_not_orphan_reused_token() { + // A finished run past retention and a fresh run that reused its idempotency + // token: GC must drop the old execution without touching the token mapping + // the new run installed. + let ctx = sync::DArc::new(test_ctx("1ms", 2)); + let host_genvm_id = "host-run-1"; + let old_id = GenVMId(1); + let new_id = GenVMId(2); + ctx.known_executions.pin().insert( + old_id, + fake_execution_with_host_id( + old_id, + Some(chrono::Utc::now() - chrono::Duration::seconds(2)), + Some(host_genvm_id.to_owned()), + ), + ); + ctx.known_executions.pin().insert( + new_id, + fake_execution_with_host_id(new_id, None, Some(host_genvm_id.to_owned())), + ); + ctx.host_genvm_ids + .lock() + .unwrap() + .insert(host_genvm_id.to_owned(), new_id); + + gc_step(&ctx).await; + + let token_mapping = ctx + .host_genvm_ids + .lock() + .unwrap() + .get(host_genvm_id) + .copied(); + assert!(ctx.status(old_id).is_none()); + assert!(ctx.attach(ctx.boot_id(), new_id).is_ok()); + assert_eq!(token_mapping, Some(new_id)); +} + +#[test] +fn attach_observes_concurrent_terminal() { + // Races a terminal publish against `attach`. Before the watch channel the + // window was between taking the snapshot and subscribing: a terminal + // landing there was in neither. The snapshot now comes from the receiver + // itself, so the sweep passes by construction rather than by luck. + const ROUNDS: u64 = 10_000; + let ctx = test_ctx("5m", 2); + let execs: Vec<_> = (1..=ROUNDS) + .map(|i| { + let genvm_id = GenVMId(i); + let exec = fake_execution(genvm_id, None); + ctx.known_executions.pin().insert(genvm_id, exec.clone()); + exec + }) + .collect(); + + // A rendezvous channel rather than a barrier, and no panics inside the + // scope: a failure breaks out of the loop, the sender drops when the + // closure ends, the publisher unblocks, and the scope joins cleanly. + // Panicking inside would leave the publisher parked in `recv()` forever. + let (start_tx, start_rx) = std::sync::mpsc::sync_channel::<()>(0); + let execs_ref = &execs; + let mut failure = None; + std::thread::scope(|s| { + s.spawn(move || { + for (round, exec) in execs_ref.iter().enumerate() { + let event = Event::FailedToStart { + genvm_id: exec.id, + host_genvm_id: None, + error: "raced".to_owned(), + }; + if start_rx.recv().is_err() { + return; + } + for _ in 0..(round % 16) { + std::hint::spin_loop(); + } + exec.publish_terminal(event); + } + }); + + let start_tx = start_tx; + 'rounds: for exec in &execs { + start_tx.send(()).unwrap(); + let (snapshot, mut rx) = ctx.attach(ctx.boot_id(), exec.id).unwrap(); + if matches!(&snapshot, Snapshot::Event(event) if event.is_terminal()) { + continue; + } + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(1); + loop { + match rx.has_changed() { + Ok(true) => { + if matches!( + rx.borrow_and_update().clone(), + Snapshot::Event(event) if event.is_terminal() + ) { + break; + } + } + Ok(false) => { + if std::time::Instant::now() >= deadline { + failure = Some(format!( + "genvm {}: terminal event in neither snapshot nor subscription", + exec.id.0 + )); + break 'rounds; + } + std::thread::yield_now(); + } + Err(e) => { + failure = Some(format!("genvm {}: subscription broke: {e}", exec.id.0)); + break 'rounds; + } + } + } + } + }); + + if let Some(failure) = failure { + panic!("{failure}"); + } +} + +#[tokio::test] +async fn cancel_while_queued_consumes_no_permit() { + let permits = Arc::new(tokio::sync::Semaphore::new(0)); + let exec = fake_execution(GenVMId(1), None); + let waiter = tokio::spawn({ + let permits = permits.clone(); + let exec = exec.clone(); + async move { + let permit_future = acquire_run_permits(permits, 1); + tokio::pin!(permit_future); + tokio::select! { + _ = exec.wait_cancelled() => false, + permit = &mut permit_future => { + drop(permit); + true + } + } + } + }); + + tokio::task::yield_now().await; + exec.request_finish(FinishCause::Cancelled); + assert!(!waiter.await.unwrap()); + assert_eq!(permits.available_permits(), 0); + permits.add_permits(1); + assert_eq!(permits.available_permits(), 1); +} + +#[tokio::test] +async fn nested_run_does_not_consume_parent_permit() { + let ctx = test_ctx("5m", 1); + let exec = fake_execution(GenVMId(1), None); + let permit = ctx.permits.clone().acquire_owned().await.unwrap(); + exec.all_permits.store(Some(Box::new(permit))); + + let registration = NestedRunRegistration::new(exec.clone()); + assert_eq!(ctx.permits.available_permits(), 0); + drop(registration); + exec.wait_for_nested_runs().await; + assert_eq!(ctx.permits.available_permits(), 0); + + exec.all_permits.store(None); + assert_eq!(ctx.permits.available_permits(), 1); +} + +#[tokio::test] +async fn nested_run_observes_cancel_and_deadline() { + let cancelled = fake_execution(GenVMId(1), None); + cancelled.request_finish(FinishCause::Cancelled); + assert_eq!( + wait_for_process_stop(&cancelled, None, std::time::Duration::from_secs(60)).await, + ProcessStop::Cancelled + ); + + let deadline = fake_execution(GenVMId(2), None); + assert_eq!( + wait_for_process_stop(&deadline, None, std::time::Duration::from_millis(1)).await, + ProcessStop::Deadline + ); +} + +#[tokio::test] +async fn parent_stream_close_stops_and_reaps_nested_run() { + let exec = fake_execution(GenVMId(1), None); + let stream = ManagerHostStreamState::default(); + let registration = NestedRunRegistration::new(exec.clone()); + + stream.close(); + assert_eq!( + wait_for_process_stop(&exec, Some(&stream), std::time::Duration::from_secs(60)).await, + ProcessStop::CallerClosed + ); + + drop(registration); + tokio::time::timeout( + std::time::Duration::from_secs(1), + exec.wait_for_nested_runs(), + ) + .await + .unwrap(); +} + +#[tokio::test] +async fn nested_reply_uses_little_endian_length_prefix() { + use tokio::io::AsyncReadExt; + + let reply = nested_internal_error(); + let body = calldata::encode_obj(&reply); + let (mut writer, mut reader) = tokio::io::duplex(1024); + let expected = body.clone(); + let write = tokio::spawn(async move { + write_length_prefixed(&mut writer, &body).await.unwrap(); + }); + + let mut prefix = [0; 4]; + reader.read_exact(&mut prefix).await.unwrap(); + assert_eq!(prefix, (expected.len() as u32).to_le_bytes()); + let mut actual = vec![0; expected.len()]; + reader.read_exact(&mut actual).await.unwrap(); + assert_eq!(actual, expected); + write.await.unwrap(); + + let decoded: genvm_modules_interfaces::NestedRunReply = calldata::decode_obj(&actual).unwrap(); + assert_eq!( + decoded.result.kind, + genvm_modules_interfaces::ResultCode::InternalError + ); + assert!(!decoded.effect_free); +} + +/// The result a well-behaved nested callee reports: an outcome and nothing +/// else. Built from the real `ReportedResult`, so a field added to it breaks +/// this at compile time rather than at runtime. +fn clean_reported() -> genvm_modules_interfaces::ReportedResult { + genvm_modules_interfaces::ReportedResult { + execution_hash: bytes::Bytes::from(vec![1; 32]), + small_hash: bytes::Bytes::from(vec![2; 32]), + kind: genvm_modules_interfaces::ResultCode::Return, + data: calldata::Value::Null.into(), + backtrace: None, + wasm_store_hashes: Default::default(), + storage_changes: Vec::new(), + emissions: Vec::new(), + nondet_disagreement: None, + nondet_results: Vec::new(), + data_fees_remaining: Vec::new(), + data_fees_consumed: Default::default(), + llm_consumption: primitive_types::U256::zero(), + } +} + +/// Builds the frame an executor sends. +fn encoded_nested_result(reported: &genvm_modules_interfaces::ReportedResult) -> Vec<u8> { + let mut encoded = vec![reported.kind as u8]; + encoded.extend(calldata::encode_obj(reported)); + encoded +} + +fn some_storage_delta() -> genvm_modules_interfaces::StorageDelta { + genvm_modules_interfaces::StorageDelta::new([0; 36], vec![1]) +} + +fn some_emission() -> genvm_modules_interfaces::ExecutionEmission { + genvm_modules_interfaces::ExecutionEmission::EmitEvent { + topics: Vec::new(), + blob: calldata::Map::new().into(), + storage_fee: primitive_types::U256::zero(), + } +} + +#[test] +fn nested_reply_accepts_an_effect_free_report() { + let reply = nested_reply_from_consumed_result(&encoded_nested_result(&clean_reported())) + .expect("a report with no effects is accepted"); + assert!(reply.effect_free); +} + +#[test] +fn nested_reply_refuses_every_reported_effect() { + // One case per field `nested_effect` classifies as an effect. Not reachable + // end to end -- the boundary derivation clears the permission behind each of + // them -- so this is the only place the refusal is exercised. + let mutate: [(&str, fn(&mut genvm_modules_interfaces::ReportedResult)); 10] = [ + ("storage_changes", |r| { + r.storage_changes.push(some_storage_delta()) + }), + ("emissions", |r| r.emissions.push(some_emission())), + ("nondet_disagreement", |r| r.nondet_disagreement = Some(0)), + ("nondet_results", |r| { + r.nondet_results.push(bytes::Bytes::from_static(&[1])) + }), + ("llm_consumption", |r| { + r.llm_consumption = primitive_types::U256::one() + }), + ("storage", |r| { + r.data_fees_consumed.storage = primitive_types::U256::one() + }), + ("message_receipt", |r| { + r.data_fees_consumed.message_receipt = primitive_types::U256::one() + }), + ("nondet_output", |r| { + r.data_fees_consumed.nondet_output = primitive_types::U256::one() + }), + ("message_fee", |r| { + r.data_fees_consumed.message_fee = primitive_types::U256::one() + }), + ("event", |r| { + r.data_fees_consumed.event = primitive_types::U256::one() + }), + ]; + + for (name, mutate) in mutate { + let mut reported = clean_reported(); + mutate(&mut reported); + assert!( + nested_reply_from_consumed_result(&encoded_nested_result(&reported)).is_err(), + "a report carrying `{name}` must be refused" + ); + } +} + +#[test] +fn nested_permissions_a_boundary_can_serve_are_accepted() { + use genvm_modules_interfaces::NestedPermissions as P; + + for permissions in [ + P::default(), + P::DETERMINISTIC, + P::READ_STORAGE, + P::CALL_OTHERS, + P::REGISTER_RUNNERS, + P::DETERMINISTIC | P::READ_STORAGE | P::CALL_OTHERS, + ] { + assert!( + check_nested_permissions(permissions).is_ok(), + "{permissions:?}" + ); + } +} + +#[test] +fn nested_permissions_the_boundary_cannot_serve_are_refused() { + use genvm_modules_interfaces::NestedPermissions as P; + + for permissions in [ + P::SPAWN_NONDET, + P::WRITE_STORAGE, + P::SEND_MESSAGES, + P::USE_BALANCE_FOR_MESSAGE_FEES, + P::CALL_OTHERS | P::SPAWN_NONDET, + ] { + assert!( + check_nested_permissions(permissions).is_err(), + "{permissions:?}" + ); + } +} + +#[tokio::test] +async fn terminal_event_is_published_once() { + let exec = fake_execution(GenVMId(1), None); + let mut rx = exec.events.subscribe(); + + assert!(finish_execution(&exec, None, FinishCause::Exited).await); + assert!(!finish_execution(&exec, None, FinishCause::Cancelled).await); + + rx.changed().await.unwrap(); + assert!(matches!( + rx.borrow_and_update().clone(), + Snapshot::Event(Event::Finished { .. }) + )); + assert!(matches!(rx.has_changed(), Ok(false))); +} + +#[test] +fn under_or_at_limit_is_unchanged() { + let mut s = String::from("hello"); + keep_tail(&mut s, 10); + assert_eq!(s, "hello"); + keep_tail(&mut s, 5); + assert_eq!(s, "hello"); +} + +#[test] +fn keeps_the_tail() { + let mut s = String::from("0123456789"); + keep_tail(&mut s, 4); + assert_eq!(s, "6789"); +} + +#[test] +fn never_cuts_mid_char() { + // "aΓ©b" = a(1) Γ©(2) b(1) = 4 bytes; cutting at byte 2 is mid-'Γ©', so the + // cut advances to the next boundary, dropping the partial char. + let mut s = String::from("aΓ©b"); + keep_tail(&mut s, 2); + assert_eq!(s, "b"); + assert!(std::str::from_utf8(s.as_bytes()).is_ok()); +} + +/// Field names of [`Request`] as the wire sees them. +/// +/// The destructure is load-bearing: adding a field to `Request` stops this +/// compiling until the name is listed, which is what keeps +/// [`request_schema_matches_the_rust_struct`] from silently going stale. +fn request_field_names() -> Vec<&'static str> { + // A value is needed only to destructure against; nothing reads it. + #[allow(unused_variables)] + let name_of = |req: Request| { + let Request { + selector, + message, + is_sync, + debug_mode, + max_execution_minutes, + bucket_totals, + host_data, + timestamp, + host, + extra_args, + calldata, + code, + permissions, + no_modules, + unsafe_overrides, + leader_nondet_results, + gas_data, + message_fee_allocation, + initial_time_units_allocation, + record_actions, + host_genvm_id, + deadline, + host_hello_data, + hook_cross_contract_calls, + } = req; + }; + + vec![ + "selector", + "message", + "is_sync", + "debug_mode", + "max_execution_minutes", + "bucket_totals", + "host_data", + "timestamp", + "host", + "extra_args", + "calldata", + "code", + "permissions", + "no_modules", + "unsafe_overrides", + "leader_nondet_results", + "gas_data", + "message_fee_allocation", + "initial_time_units_allocation", + "record_actions", + "host_genvm_id", + "deadline", + "host_hello_data", + "hook_cross_contract_calls", + ] +} + +fn documented_properties(schema: &str) -> Vec<String> { + let doc: serde_yaml::Value = serde_yaml::from_str( + &std::fs::read_to_string(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../docs/website/src/impl-spec/appendix/manager-api.yaml" + )) + .expect("reading the documented manager API"), + ) + .expect("parsing the documented manager API"); + + let properties = doc["components"]["schemas"][schema]["properties"] + .as_mapping() + .unwrap_or_else(|| panic!("{schema} has no documented properties")); + + let mut names: Vec<String> = properties + .keys() + .map(|k| k.as_str().expect("property name is a string").to_owned()) + .collect(); + names.sort(); + names +} + +/// A host implementing against the schema alone must see every field the +/// manager accepts, and none it does not. Drift here is not cosmetic: an +/// undocumented `permissions` silently grants the `wscn` default, and a +/// documented-but-absent field makes a host send something that is ignored. +#[test] +fn request_schema_matches_the_rust_struct() { + let mut accepted = request_field_names() + .into_iter() + .map(str::to_owned) + .collect::<Vec<_>>(); + accepted.sort(); + + assert_eq!(documented_properties("GenvmRunRequest"), accepted); +} + +/// Same contract for the message the host builds by hand. +#[test] +fn message_schema_matches_the_rust_struct() { + #[allow(unused_variables)] + let name_of = |message: genvm_modules_interfaces::MessageData| { + let genvm_modules_interfaces::MessageData { + contract_address, + sender_address, + origin_address, + signer_address, + chain_id, + value, + is_init, + datetime, + } = message; + }; + + let mut accepted = [ + "contract_address", + "sender_address", + "origin_address", + "signer_address", + "chain_id", + "value", + "is_init", + "datetime", + ] + .map(str::to_owned) + .to_vec(); + accepted.sort(); + + assert_eq!(documented_properties("MessageData"), accepted); +} diff --git a/implementation/src/manager/socket.rs b/implementation/src/manager/socket.rs new file mode 100644 index 00000000..e0dd4dab --- /dev/null +++ b/implementation/src/manager/socket.rs @@ -0,0 +1,775 @@ +use std::collections::HashMap; + +use anyhow::{Context, Result}; +use axum::extract::ws::{Message, WebSocket}; +use futures_util::{Sink, SinkExt, Stream, StreamExt}; +use genlayer_calldata as calldata; +use genlayer_calldata::codec::{Decode as DecodeTrait, Encode as EncodeTrait}; +use genlayer_calldata::{Decode, Encode}; +use genvm_common::*; +use genvm_modules_interfaces::manager_api::{self, Errors, Methods}; + +use super::{run, AppContext}; + +const HEADER_LEN: usize = 10; +const ARTIFACT_CHUNK_CAP: u32 = 256 * 1024; +const CONTROL_QUEUE_CAP: usize = 256; +const ARTIFACT_QUEUE_CAP: usize = 8; + +#[derive(Debug, Clone)] +struct Frame { + method: Methods, + request_id: u64, + payload: Vec<u8>, +} + +impl Frame { + fn new( + method: Methods, + request_id: u64, + payload: &impl EncodeTrait<Vec<u8>, Error = std::convert::Infallible>, + ) -> Self { + Self { + method, + request_id, + payload: calldata::encode_obj(payload), + } + } + + fn body_len(&self) -> usize { + HEADER_LEN + self.payload.len() + } +} + +#[derive(Clone)] +struct Writer { + control: tokio::sync::mpsc::Sender<Frame>, + artifact: tokio::sync::mpsc::Sender<Frame>, +} + +impl Writer { + /// Non-blocking, for the paths that run inside the read loop's error + /// handling and must not park. Everything that can await uses the async + /// twin instead, so a stalled peer applies backpressure rather than + /// costing frames. + fn send_control(&self, frame: Frame) -> Result<()> { + self.control + .try_send(frame) + .context("connection writer queue is full") + } + + async fn send_control_async(&self, frame: Frame) -> Result<()> { + self.control + .send(frame) + .await + .context("connection writer queue is closed") + } + + async fn send_artifact_async(&self, frame: Frame) -> Result<()> { + self.artifact + .send(frame) + .await + .context("connection writer queue is closed") + } + + fn error(&self, request_id: u64, code: Errors, message: impl Into<String>) -> Result<()> { + self.send_control(error_frame(request_id, code, message)) + } +} + +async fn writer_loop<S>( + mut sink: S, + mut control_rx: tokio::sync::mpsc::Receiver<Frame>, + mut artifact_rx: tokio::sync::mpsc::Receiver<Frame>, +) -> Result<()> +where + S: Sink<Message> + Unpin, + S::Error: std::error::Error + Send + Sync + 'static, +{ + loop { + while let Ok(frame) = control_rx.try_recv() { + write_frame(&mut sink, frame).await?; + } + + tokio::select! { + biased; + frame = control_rx.recv() => { + let Some(frame) = frame else { + break; + }; + write_frame(&mut sink, frame).await?; + } + frame = artifact_rx.recv() => { + let Some(frame) = frame else { + if control_rx.is_closed() { + break; + } + continue; + }; + write_frame(&mut sink, frame).await?; + } + } + } + Ok(()) +} + +async fn write_frame<S>(sink: &mut S, frame: Frame) -> Result<()> +where + S: Sink<Message> + Unpin, + S::Error: std::error::Error + Send + Sync + 'static, +{ + let mut message = Vec::with_capacity(frame.body_len()); + message.extend_from_slice(&frame.method.value().to_be_bytes()); + message.extend_from_slice(&frame.request_id.to_be_bytes()); + message.extend_from_slice(&frame.payload); + sink.send(Message::Binary(message.into())) + .await + .context("writing websocket message")?; + Ok(()) +} + +#[derive(Decode)] +enum RunPayload { + #[calldata(rename = "run")] + Run(run::Request), +} + +#[derive(Decode)] +enum AttachPayload { + #[calldata(rename = "attach")] + Attach(AttachRequest), +} + +#[derive(Decode)] +enum CancelPayload { + #[calldata(rename = "cancel")] + Cancel(IdRequest), +} + +#[derive(Decode)] +enum AckPayload { + #[calldata(rename = "ack")] + Ack(IdRequest), +} + +#[derive(Decode)] +enum GetArtifactPayload { + #[calldata(rename = "get_artifact")] + GetArtifact(GetArtifactRequest), +} + +#[derive(Decode)] +struct AttachRequest { + boot_id: u64, + genvm_id: run::GenVMId, +} + +#[derive(Decode)] +struct IdRequest { + genvm_id: run::GenVMId, +} + +#[derive(Decode)] +struct GetArtifactRequest { + genvm_id: run::GenVMId, + field: String, + offset: u64, + max_len: u32, +} + +#[derive(Encode)] +enum HelloPayload { + #[calldata(rename = "hello")] + Hello(Hello), +} + +#[derive(Encode)] +struct Hello { + boot_id: u64, + protocol_major: u32, +} + +#[derive(Encode)] +struct RunResponse { + genvm_id: run::GenVMId, +} + +#[derive(Encode)] +struct EmptyResponse {} + +#[derive(Encode)] +struct ErrorPayload { + code: u8, + message: String, +} + +#[derive(Encode)] +struct ArtifactResponse { + total_len: u64, + data: bytes::Bytes, +} + +#[derive(Encode)] +struct AttachResponse { + snapshot: EventPayload, +} + +#[derive(Encode)] +enum EventPayload { + #[calldata(rename = "queued")] + Queued { + genvm_id: run::GenVMId, + host_genvm_id: Option<String>, + }, + #[calldata(rename = "started")] + Started { + genvm_id: run::GenVMId, + host_genvm_id: Option<String>, + }, + #[calldata(rename = "failed_to_start")] + FailedToStart { + genvm_id: run::GenVMId, + host_genvm_id: Option<String>, + error: String, + }, + #[calldata(rename = "finished")] + Finished { + genvm_id: run::GenVMId, + host_genvm_id: Option<String>, + cause: String, + exit_code: Option<i64>, + consumed_result: Option<bytes::Bytes>, + #[calldata(serialize_with = encode_json_value)] + metrics: serde_json::Value, + finished_at: String, + version_major: u32, + version_minor: u32, + artifact_sizes: ArtifactSizesPayload, + }, +} + +#[derive(Encode)] +struct ArtifactSizesPayload { + stdout: u64, + stderr: u64, + genvm_log: u64, +} + +fn encode_json_value<W: calldata::Writer>( + value: &serde_json::Value, + enc: &mut calldata::Encoder<W>, +) -> std::result::Result<(), W::Error> { + match value { + serde_json::Value::Null => enc.push_null(), + serde_json::Value::Bool(value) => enc.push_bool(*value), + serde_json::Value::Number(value) => { + if let Some(value) = value.as_i64() { + enc.push_i64(value) + } else if let Some(value) = value.as_u64() { + enc.push_u64(value) + } else { + enc.push_null() + } + } + serde_json::Value::String(value) => enc.push_str(value), + serde_json::Value::Array(values) => { + enc.start_array(values.len() as u64)?; + for value in values { + encode_json_value(value, enc)?; + } + Ok(()) + } + serde_json::Value::Object(map) => { + let mut entries = map.iter().collect::<Vec<_>>(); + entries.sort_by_key(|(key, _)| key.as_str()); + enc.start_map(entries.len() as u64)?; + for (key, value) in entries { + enc.push_map_k(key)?; + encode_json_value(value, enc)?; + } + Ok(()) + } + } +} + +fn error_frame(request_id: u64, code: Errors, message: impl Into<String>) -> Frame { + Frame::new( + Methods::Error, + request_id, + &ErrorPayload { + code: code.value(), + message: message.into(), + }, + ) +} + +fn hello_frame(boot_id: u64) -> Frame { + Frame::new( + Methods::Hello, + 0, + &HelloPayload::Hello(Hello { + boot_id, + protocol_major: manager_api::CURRENT_MAJOR, + }), + ) +} + +fn event_frame(event: run::Event) -> Frame { + Frame::new(Methods::Event, 0, &event_payload(event)) +} + +fn event_payload(event: run::Event) -> EventPayload { + match event { + run::Event::Started { + genvm_id, + host_genvm_id, + } => EventPayload::Started { + genvm_id, + host_genvm_id, + }, + run::Event::FailedToStart { + genvm_id, + host_genvm_id, + error, + } => EventPayload::FailedToStart { + genvm_id, + host_genvm_id, + error, + }, + run::Event::Finished { + genvm_id, + host_genvm_id, + cause, + exit_code, + consumed_result, + metrics, + finished_at, + version_major, + version_minor, + artifact_sizes, + } => EventPayload::Finished { + genvm_id, + host_genvm_id, + cause: cause.as_str().to_owned(), + exit_code, + consumed_result: consumed_result.map(bytes::Bytes::from), + metrics, + finished_at: finished_at.to_rfc3339_opts(chrono::SecondsFormat::AutoSi, true), + version_major: u32::from(version_major), + version_minor: u32::from(version_minor), + artifact_sizes: ArtifactSizesPayload { + stdout: artifact_sizes.stdout, + stderr: artifact_sizes.stderr, + genvm_log: artifact_sizes.genvm_log, + }, + }, + } +} + +fn snapshot_payload(snapshot: run::Snapshot) -> EventPayload { + match snapshot { + run::Snapshot::Queued { + genvm_id, + host_genvm_id, + } => EventPayload::Queued { + genvm_id, + host_genvm_id, + }, + run::Snapshot::Event(event) => event_payload(event), + } +} + +async fn read_frame<R>(reader: &mut R, writer: &Writer) -> Result<Option<(Methods, u64, Vec<u8>)>> +where + R: Stream<Item = std::result::Result<Message, axum::Error>> + Unpin, +{ + loop { + let Some(message) = reader.next().await else { + return Ok(None); + }; + let body = match message.context("reading websocket message")? { + Message::Binary(body) => body, + Message::Text(_) => { + writer.error(0, Errors::MalformedFrame, "text messages are not valid")?; + continue; + } + Message::Close(_) => return Ok(None), + // axum answers pings itself. + Message::Ping(_) | Message::Pong(_) => continue, + }; + + if body.len() < HEADER_LEN { + writer.error(0, Errors::MalformedFrame, "message is shorter than header")?; + continue; + } + + let method_id = u16::from_be_bytes(body[..2].try_into().expect("fixed method length")); + let request_id = + u64::from_be_bytes(body[2..HEADER_LEN].try_into().expect("fixed id length")); + let method = match Methods::try_from(method_id) { + Ok(method) => method, + Err(()) => { + writer.error( + request_id, + Errors::UnknownMethod, + format!("unknown method id {method_id}"), + )?; + continue; + } + }; + + return Ok(Some((method, request_id, body[HEADER_LEN..].to_vec()))); + } +} + +fn decode_payload<T: DecodeTrait>( + payload: &[u8], +) -> std::result::Result<T, calldata::codec::DecodeError> { + calldata::decode_obj(payload) +} + +struct Connection<R> { + ctx: sync::DArc<AppContext>, + reader: R, + writer: Writer, + subscriptions: HashMap<run::GenVMId, tokio::task::JoinHandle<()>>, +} + +impl<R> Connection<R> +where + R: Stream<Item = std::result::Result<Message, axum::Error>> + Unpin, +{ + async fn run(mut self) -> Result<()> { + loop { + let Some((method, request_id, payload)) = + read_frame(&mut self.reader, &self.writer).await? + else { + break; + }; + self.dispatch(method, request_id, &payload).await?; + } + Ok(()) + } + + async fn dispatch(&mut self, method: Methods, request_id: u64, payload: &[u8]) -> Result<()> { + if request_id == 0 { + self.writer.error( + 0, + Errors::BadRequestId, + "client requests must use request_id != 0", + )?; + return Ok(()); + } + + match method { + Methods::Run => self.handle_run(request_id, payload).await, + Methods::Attach => self.handle_attach(request_id, payload).await, + Methods::Cancel => self.handle_cancel(request_id, payload).await, + Methods::Ack => self.handle_ack(request_id, payload).await, + Methods::GetArtifact => self.handle_get_artifact(request_id, payload).await, + Methods::Error | Methods::Hello | Methods::Event => { + self.writer.error( + request_id, + Errors::UnknownMethod, + format!("method {} is not a client request", method.value()), + )?; + Ok(()) + } + } + } + + async fn handle_run(&mut self, request_id: u64, payload: &[u8]) -> Result<()> { + let RunPayload::Run(mut req) = match decode_payload::<RunPayload>(payload) { + Ok(req) => req, + Err(e) => { + self.writer + .error(request_id, Errors::MalformedFrame, e.to_string())?; + return Ok(()); + } + }; + req.patch_legacy_method_key(); + if req + .host_hello_data + .get(1) + .is_some_and(|data| !data.is_empty()) + { + self.writer.error( + request_id, + Errors::MalformedFrame, + "host_hello_data for manager-owned host index 1 is not allowed", + )?; + return Ok(()); + } + + let modules_lock = if req.needs_modules() { + match super::modules::Ctx::get_module_locks(self.ctx.gep(|x| &x.mod_ctx)).await { + Some(lock) => Some(lock), + None => { + self.writer.error( + request_id, + Errors::Internal, + "modules are required but not running", + )?; + return Ok(()); + } + } + } else { + None + }; + let run_ctx = self.ctx.gep(|x| &x.run_ctx); + let genvm_id = match run_ctx + .start(self.ctx.clone(), req, Box::new(modules_lock)) + .await + { + Ok(id) => id, + Err(e) => { + self.writer + .error(request_id, Errors::Internal, format!("{:#}", e))?; + return Ok(()); + } + }; + let (snapshot, rx) = match run_ctx.attach(run_ctx.boot_id(), genvm_id) { + Ok(result) => result, + Err(e) => { + self.writer + .error(request_id, Errors::UnknownId, format!("{:#}", e))?; + return Ok(()); + } + }; + self.writer.send_control(Frame::new( + Methods::Run, + request_id, + &RunResponse { genvm_id }, + ))?; + if let run::Snapshot::Event(event) = snapshot { + self.writer.send_control(event_frame(event))?; + } + // Subscribing last keeps the frames ordered: the receiver was taken + // above, so nothing is missed, but a terminal landing meanwhile cannot + // produce an event for a genvm_id the client has not been told yet. + if let Err(e) = self.subscribe(genvm_id, Ok(rx)) { + self.writer + .error(request_id, Errors::UnknownId, format!("{:#}", e))?; + return Ok(()); + } + Ok(()) + } + + async fn handle_attach(&mut self, request_id: u64, payload: &[u8]) -> Result<()> { + let AttachPayload::Attach(req) = match decode_payload::<AttachPayload>(payload) { + Ok(req) => req, + Err(e) => { + self.writer + .error(request_id, Errors::MalformedFrame, e.to_string())?; + return Ok(()); + } + }; + + let run_ctx = self.ctx.gep(|x| &x.run_ctx); + let (snapshot, rx) = match run_ctx.attach(req.boot_id, req.genvm_id) { + Ok(result) => result, + Err(e) if e.to_string() == "boot_id_mismatch" => { + self.writer + .error(request_id, Errors::BootIdMismatch, "boot_id_mismatch")?; + return Ok(()); + } + Err(e) => { + self.writer + .error(request_id, Errors::UnknownId, format!("{:#}", e))?; + return Ok(()); + } + }; + self.subscribe(req.genvm_id, Ok(rx))?; + self.writer.send_control(Frame::new( + Methods::Attach, + request_id, + &AttachResponse { + snapshot: snapshot_payload(snapshot), + }, + ))?; + Ok(()) + } + + async fn handle_cancel(&mut self, request_id: u64, payload: &[u8]) -> Result<()> { + let CancelPayload::Cancel(req) = match decode_payload::<CancelPayload>(payload) { + Ok(req) => req, + Err(e) => { + self.writer + .error(request_id, Errors::MalformedFrame, e.to_string())?; + return Ok(()); + } + }; + match self.ctx.run_ctx.cancel(req.genvm_id).await { + Ok(()) => self.writer.send_control(Frame::new( + Methods::Cancel, + request_id, + &EmptyResponse {}, + ))?, + Err(e) => self + .writer + .error(request_id, Errors::UnknownId, format!("{:#}", e))?, + } + Ok(()) + } + + async fn handle_ack(&mut self, request_id: u64, payload: &[u8]) -> Result<()> { + let AckPayload::Ack(req) = match decode_payload::<AckPayload>(payload) { + Ok(req) => req, + Err(e) => { + self.writer + .error(request_id, Errors::MalformedFrame, e.to_string())?; + return Ok(()); + } + }; + match self.ctx.run_ctx.ack(req.genvm_id) { + run::AckOutcome::Acked => { + if let Some(handle) = self.subscriptions.remove(&req.genvm_id) { + handle.abort(); + } + self.writer.send_control(Frame::new( + Methods::Ack, + request_id, + &EmptyResponse {}, + ))?; + } + run::AckOutcome::NotFinished => { + self.writer + .error(request_id, Errors::NotFinished, "run has not finished")? + } + run::AckOutcome::Unknown => { + self.writer + .error(request_id, Errors::UnknownId, "unknown_id")? + } + } + Ok(()) + } + + async fn handle_get_artifact(&mut self, request_id: u64, payload: &[u8]) -> Result<()> { + let GetArtifactPayload::GetArtifact(mut req) = + match decode_payload::<GetArtifactPayload>(payload) { + Ok(req) => req, + Err(e) => { + self.writer + .error(request_id, Errors::MalformedFrame, e.to_string())?; + return Ok(()); + } + }; + req.max_len = req.max_len.min(ARTIFACT_CHUNK_CAP); + match self + .ctx + .run_ctx + .get_artifact(req.genvm_id, &req.field, req.offset, req.max_len) + { + Ok(artifact) => { + self.writer + .send_artifact_async(Frame::new( + Methods::GetArtifact, + request_id, + &ArtifactResponse { + total_len: artifact.total_len, + data: artifact.data, + }, + )) + .await? + } + Err(e) if e.to_string() == "unknown_id" => { + self.writer + .error(request_id, Errors::UnknownId, "unknown_id")? + } + Err(e) => self + .writer + .error(request_id, Errors::MalformedFrame, format!("{:#}", e))?, + } + Ok(()) + } + + fn subscribe( + &mut self, + genvm_id: run::GenVMId, + rx: Result<tokio::sync::watch::Receiver<run::Snapshot>>, + ) -> Result<()> { + if self.subscriptions.contains_key(&genvm_id) { + return Ok(()); + } + let rx = rx?; + let handle = tokio::spawn(forward_events(self.writer.clone(), rx)); + self.subscriptions.insert(genvm_id, handle); + Ok(()) + } +} + +async fn forward_events(writer: Writer, mut rx: tokio::sync::watch::Receiver<run::Snapshot>) { + loop { + if rx.changed().await.is_err() { + return; + } + let run::Snapshot::Event(event) = rx.borrow_and_update().clone() else { + continue; + }; + let is_terminal = event.is_terminal(); + if writer.send_control_async(event_frame(event)).await.is_err() || is_terminal { + return; + } + } +} + +impl<R> Drop for Connection<R> { + fn drop(&mut self) { + // Each subscriber task holds a `Writer` clone, so without this they + // outlive the connection and keep its writer task alive until every run + // they watch terminates. + for handle in self.subscriptions.values() { + handle.abort(); + } + } +} + +pub async fn handle_connection(socket: WebSocket, ctx: sync::DArc<AppContext>) { + let (writer_stream, reader) = socket.split(); + let (control_tx, control_rx) = tokio::sync::mpsc::channel(CONTROL_QUEUE_CAP); + let (artifact_tx, artifact_rx) = tokio::sync::mpsc::channel(ARTIFACT_QUEUE_CAP); + let writer = Writer { + control: control_tx, + artifact: artifact_tx, + }; + + let writer_task = tokio::spawn(writer_loop(writer_stream, control_rx, artifact_rx)); + if writer + .send_control(hello_frame(ctx.run_ctx.boot_id())) + .is_err() + { + writer_task.abort(); + return; + } + + let cancel = ctx.cancel.clone(); + let connection = Connection { + ctx, + reader, + writer, + subscriptions: HashMap::new(), + }; + // A client is entitled to hold this connection open indefinitely, so the + // shutdown signal has to reach the read loop directly: axum's graceful + // shutdown waits for in-flight connections, and would wait on this one + // forever. + tokio::select! { + result = connection.run() => { + if let Err(e) = result { + log_debug!(error:ah = e; "manager websocket connection closed"); + } + } + _ = cancel.chan.closed() => {} + } + + match writer_task.await { + Ok(Ok(())) => {} + Ok(Err(e)) => log_debug!(error:ah = e; "manager websocket writer stopped"), + Err(e) if e.is_cancelled() => {} + Err(e) => log_warn!(error:err = e; "manager websocket writer task failed"), + } +} + +#[cfg(test)] +#[path = "socket_test.rs"] +mod tests; diff --git a/implementation/src/manager/socket_test.rs b/implementation/src/manager/socket_test.rs new file mode 100644 index 00000000..452f16c8 --- /dev/null +++ b/implementation/src/manager/socket_test.rs @@ -0,0 +1,239 @@ +use super::*; +use std::sync::Arc; + +fn test_frame(method: Methods, request_id: u64, text: &str) -> Frame { + Frame { + method, + request_id, + payload: text.as_bytes().to_vec(), + } +} + +fn decode_written_message(message: &Message) -> (u16, u64, Vec<u8>) { + let Message::Binary(body) = message else { + panic!("expected binary websocket message"); + }; + let method = u16::from_be_bytes(body[..2].try_into().unwrap()); + let request_id = u64::from_be_bytes(body[2..HEADER_LEN].try_into().unwrap()); + (method, request_id, body[HEADER_LEN..].to_vec()) +} + +fn test_writer( + cap: usize, +) -> ( + Writer, + tokio::sync::mpsc::Receiver<Frame>, + tokio::sync::mpsc::Receiver<Frame>, +) { + let (control, control_rx) = tokio::sync::mpsc::channel(cap); + let (artifact, artifact_rx) = tokio::sync::mpsc::channel(cap); + (Writer { control, artifact }, control_rx, artifact_rx) +} + +fn started_event(id: u64) -> run::Event { + run::Event::Started { + genvm_id: run::GenVMId(id), + host_genvm_id: None, + } +} + +fn terminal_event(id: u64) -> run::Event { + run::Event::FailedToStart { + genvm_id: run::GenVMId(id), + host_genvm_id: None, + error: "failed".to_owned(), + } +} + +#[test] +fn finished_event_encodes_consumed_result_as_bytes() { + let expected = vec![0, 1, 2, 255]; + let payload = EventPayload::Finished { + genvm_id: run::GenVMId(1), + host_genvm_id: None, + cause: "exited".to_owned(), + exit_code: Some(0), + consumed_result: Some(expected.clone().into()), + metrics: serde_json::Value::Null, + finished_at: "2026-08-06T00:00:00Z".to_owned(), + version_major: 0, + version_minor: 3, + artifact_sizes: ArtifactSizesPayload { + stdout: 0, + stderr: 0, + genvm_log: 0, + }, + }; + + let encoded = calldata::encode_obj(&payload); + let calldata::Value::Map(event) = calldata::decode_obj(&encoded).unwrap() else { + panic!("expected event map"); + }; + let Some(calldata::Value::Map(finished)) = event.get("finished") else { + panic!("expected finished event"); + }; + assert_eq!( + finished.get("consumed_result"), + Some(&calldata::Value::Bytes(expected)) + ); +} + +#[tokio::test] +async fn writer_drains_control_before_artifacts() { + let written = Arc::new(tokio::sync::Mutex::new(Vec::new())); + let sink = Box::pin(futures_util::sink::unfold( + written.clone(), + |written, message| async move { + written.lock().await.push(message); + Ok::<_, std::convert::Infallible>(written) + }, + )); + let (control_tx, control_rx) = tokio::sync::mpsc::channel(8); + let (artifact_tx, artifact_rx) = tokio::sync::mpsc::channel(8); + + artifact_tx + .send(test_frame(Methods::GetArtifact, 1, "artifact-1")) + .await + .unwrap(); + artifact_tx + .send(test_frame(Methods::GetArtifact, 2, "artifact-2")) + .await + .unwrap(); + control_tx + .send(test_frame(Methods::Event, 0, "event")) + .await + .unwrap(); + let writer = tokio::spawn(writer_loop(sink, control_rx, artifact_rx)); + + let first = tokio::time::timeout(std::time::Duration::from_secs(1), async { + loop { + if let Some(message) = written.lock().await.first().cloned() { + return message; + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + + let (_, request_id, payload) = decode_written_message(&first); + assert_eq!(request_id, 0); + assert_eq!(payload, b"event"); + + drop(control_tx); + drop(artifact_tx); + writer.await.unwrap().unwrap(); +} + +#[tokio::test] +async fn full_control_queue_does_not_kill_the_subscription() { + let (writer, mut control_rx, _artifact_rx) = test_writer(1); + let (events, events_rx) = tokio::sync::watch::channel(run::Snapshot::Queued { + genvm_id: run::GenVMId(1), + host_genvm_id: None, + }); + + // Occupy the capacity-1 control queue, so the forwarder's send cannot land + // until the peer reads. A stalled client's connection looks exactly like + // this from the manager's side. + writer + .send_control(test_frame(Methods::Event, 99, "occupier")) + .unwrap(); + let forwarder = tokio::spawn(forward_events(writer.clone(), events_rx)); + + events.send_replace(run::Snapshot::Event(started_event(1))); + // Let the forwarder reach its send and find the queue full. + for _ in 0..4 { + tokio::task::yield_now().await; + } + + let occupier = control_rx.recv().await.expect("occupier frame"); + assert_eq!(occupier.request_id, 99); + + // The queue has room again, so the event must still reach the writer. A + // subscription that silently died while the connection lives on is the bug. + let event = tokio::time::timeout(std::time::Duration::from_secs(1), control_rx.recv()) + .await + .expect("event never delivered: subscription died silently") + .expect("control channel closed instead of delivering the event"); + assert_eq!(event.method, Methods::Event); + + // And the subscription must still be live afterwards, not merely have + // survived long enough to flush one frame. + events.send_replace(run::Snapshot::Event(started_event(2))); + let next = tokio::time::timeout(std::time::Duration::from_secs(1), control_rx.recv()) + .await + .expect("subscription stopped forwarding after the queue drained") + .expect("control channel closed instead of delivering the event"); + assert_eq!(next.method, Methods::Event); + forwarder.abort(); +} + +#[tokio::test] +async fn closed_connection_ends_the_forwarder() { + let (writer, control_rx, _artifact_rx) = test_writer(1); + let (events, events_rx) = tokio::sync::watch::channel(run::Snapshot::Queued { + genvm_id: run::GenVMId(1), + host_genvm_id: None, + }); + let forwarder = tokio::spawn(forward_events(writer, events_rx)); + + drop(control_rx); + events.send_replace(run::Snapshot::Event(started_event(1))); + + tokio::time::timeout(std::time::Duration::from_secs(1), forwarder) + .await + .expect("forwarder must exit once the connection is gone") + .unwrap(); +} + +#[tokio::test] +async fn terminal_event_ends_the_forwarder() { + let (writer, mut control_rx, _artifact_rx) = test_writer(1); + let (events, events_rx) = tokio::sync::watch::channel(run::Snapshot::Queued { + genvm_id: run::GenVMId(1), + host_genvm_id: None, + }); + let forwarder = tokio::spawn(forward_events(writer, events_rx)); + let terminal = terminal_event(1); + let expected = event_frame(terminal.clone()); + + events.send_replace(run::Snapshot::Event(terminal)); + + let event = tokio::time::timeout(std::time::Duration::from_secs(1), control_rx.recv()) + .await + .expect("terminal event was not delivered") + .expect("control channel closed instead of delivering the terminal event"); + assert_eq!(event.method, expected.method); + assert_eq!(event.payload, expected.payload); + tokio::time::timeout(std::time::Duration::from_secs(1), forwarder) + .await + .expect("forwarder must exit after a terminal event") + .unwrap(); +} + +#[tokio::test] +async fn coalescing_keeps_the_terminal_event() { + let (writer, mut control_rx, _artifact_rx) = test_writer(1); + let (events, events_rx) = tokio::sync::watch::channel(run::Snapshot::Queued { + genvm_id: run::GenVMId(1), + host_genvm_id: None, + }); + let terminal = terminal_event(1); + let expected = event_frame(terminal.clone()); + + events.send_replace(run::Snapshot::Event(started_event(1))); + events.send_replace(run::Snapshot::Event(terminal)); + let forwarder = tokio::spawn(forward_events(writer, events_rx)); + + let event = tokio::time::timeout(std::time::Duration::from_secs(1), control_rx.recv()) + .await + .expect("coalesced terminal event was not delivered") + .expect("control channel closed instead of delivering the terminal event"); + assert_eq!(event.method, expected.method); + assert_eq!(event.payload, expected.payload); + tokio::time::timeout(std::time::Duration::from_secs(1), forwarder) + .await + .expect("forwarder must exit after the coalesced terminal event") + .unwrap(); +} diff --git a/implementation/src/manager/versioning.rs b/implementation/src/manager/versioning.rs index 43aa2f24..cbddd641 100644 --- a/implementation/src/manager/versioning.rs +++ b/implementation/src/manager/versioning.rs @@ -17,36 +17,36 @@ pub struct Version { pub patch: u32, } -impl<'de> serde::Deserialize<'de> for Version { - fn deserialize<D>(deserializer: D) -> Result<Self, D::Error> - where - D: serde::Deserializer<'de>, - { - let s = String::deserialize(deserializer)?; +impl Version { + /// Reads a manifest key such as `v0.3.0-rc7`. The prerelease suffix is part + /// of the key but not of the ordering, so it is dropped here. + pub fn parse(s: &str) -> Result<Self, &'static str> { let parts: Vec<&str> = s.split('.').collect(); if parts.len() != 3 { - return Err(serde::de::Error::custom("Invalid version format")); + return Err("Invalid version format"); } let part0 = parts[0].strip_prefix("v").unwrap_or(parts[0]); let patch_str = parts[2].split('-').next().unwrap_or(parts[2]); - let major = part0 - .parse() - .map_err(|_| serde::de::Error::custom("Invalid major version"))?; - let minor = parts[1] - .parse() - .map_err(|_| serde::de::Error::custom("Invalid minor version"))?; - let patch = patch_str - .parse() - .map_err(|_| serde::de::Error::custom("Invalid patch version"))?; Ok(Version { - major, - minor, - patch, + major: part0.parse().map_err(|_| "Invalid major version")?, + minor: parts[1].parse().map_err(|_| "Invalid minor version")?, + patch: patch_str.parse().map_err(|_| "Invalid patch version")?, }) } } +impl<'de> serde::Deserialize<'de> for Version { + fn deserialize<D>(deserializer: D) -> Result<Self, D::Error> + where + D: serde::Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + + Version::parse(&s).map_err(serde::de::Error::custom) + } +} + #[derive(Clone)] pub struct Manifest { pub executor_versions: std::collections::BTreeMap<Version, ExecutorVersion>, @@ -138,34 +138,89 @@ impl Ctx { timestamp: chrono::DateTime<chrono::Utc>, ) -> Option<ResolvedVersion> { let lock = self.manifest.read().await; + resolve_version(&lock, timestamp, |ver, _| ver.major == major) + } - let mut ver = lock - .executor_versions - .iter() - .filter(|(ver, ev)| ver.major == major && ev.available_after <= timestamp) - .map(|(ver, _)| *ver) - .max()?; - - // Walk forward to the newest contiguous patch, but only adopt a patch - // whose time gate has already opened. Skipping `available_after` here - // would activate a future (or rc) patch prematurely, which during a - // rolling update with staged manifests causes consensus divergence. - loop { - let mut next = ver; - next.patch += 1; - match lock.executor_versions.get(&next) { - Some(ev) if ev.available_after <= timestamp => { - ver = next; - } - _ => break, + /// Newest line available at `timestamp`, whatever its major. + /// + /// The fallback for a major no installed line provides: the run still + /// reaches an executor, which rejects the contract with a canonical + /// `invalid_contract major_mismatch` instead of never starting. + pub async fn get_newest_version( + &self, + timestamp: chrono::DateTime<chrono::Utc>, + ) -> Option<ResolvedVersion> { + let lock = self.manifest.read().await; + resolve_version(&lock, timestamp, |_, _| true) + } + + /// Newest line whose manifest key matches `pattern`, by the same rules a + /// major goes through. + pub async fn get_matching_version( + &self, + pattern: ®ex::Regex, + timestamp: chrono::DateTime<chrono::Utc>, + ) -> Option<ResolvedVersion> { + let lock = self.manifest.read().await; + resolve_version(&lock, timestamp, |_, key| pattern.is_match(key)) + } +} + +/// The line an exact version string names. +/// +/// No manifest lookup and no rules: the string is the executor directory. The +/// version it reports back is read off the name for logging alone, and a name +/// that is not a version at all still runs -- an exact pin is a statement by a +/// party we trust, not a request to be resolved. +pub fn exact_version(version: &str) -> ResolvedVersion { + ResolvedVersion { + version: Version::parse(version).unwrap_or(Version { + major: 0, + minor: 0, + patch: 0, + }), + orig_key: version.to_owned(), + } +} + +fn resolve_version( + manifest: &Manifest, + timestamp: chrono::DateTime<chrono::Utc>, + matches: impl Fn(&Version, &str) -> bool, +) -> Option<ResolvedVersion> { + let matches = |ver: &Version| match manifest.version_orig_keys.get(ver) { + Some(key) => matches(ver, key), + None => false, + }; + + let mut ver = manifest + .executor_versions + .iter() + .filter(|(ver, ev)| matches(ver) && ev.available_after <= timestamp) + .map(|(ver, _)| *ver) + .max()?; + + // Walk forward to the newest contiguous patch, but only adopt a patch + // whose time gate has already opened. Skipping `available_after` here + // would activate a future (or rc) patch prematurely, which during a + // rolling update with staged manifests causes consensus divergence. The + // patch must satisfy the same predicate: a major cannot change under a + // patch bump, but a pattern the caller wrote can stop matching. + loop { + let mut next = ver; + next.patch += 1; + match manifest.executor_versions.get(&next) { + Some(ev) if ev.available_after <= timestamp && matches(&next) => { + ver = next; } + _ => break, } - - Some(ResolvedVersion { - version: ver, - orig_key: lock.version_orig_keys.get(&ver)?.clone(), - }) } + + Some(ResolvedVersion { + version: ver, + orig_key: manifest.version_orig_keys.get(&ver)?.clone(), + }) } pub async fn detect_major_spec( @@ -234,3 +289,7 @@ pub async fn detect_major_spec( detected_version.map(|v| v.min(possible_major)) } + +#[cfg(test)] +#[path = "versioning_test.rs"] +mod tests; diff --git a/implementation/src/manager/versioning_test.rs b/implementation/src/manager/versioning_test.rs new file mode 100644 index 00000000..0d76af3a --- /dev/null +++ b/implementation/src/manager/versioning_test.rs @@ -0,0 +1,66 @@ +use super::*; + +/// The manifest the manager ships: two live lines, both semver major zero. +fn live_manifest() -> Manifest { + serde_yaml::from_str( + "executor_versions:\n \"v0.2.17\":\n available_after: 2024-09-01T00:00:00Z\n \"v0.3.0-rc7\":\n available_after: 2024-09-01T00:00:00Z\n", + ) + .unwrap() +} + +fn now() -> chrono::DateTime<chrono::Utc> { + "2025-01-01T00:00:00Z".parse().unwrap() +} + +#[test] +fn a_major_cannot_separate_two_zero_major_lines() { + let manifest = live_manifest(); + + // Both keys are major 0, so the only major matching anything matches both + // and the newest wins. This is why a version string exists. + let resolved = resolve_version(&manifest, now(), |ver, _| ver.major == 0).unwrap(); + assert_eq!(resolved.orig_key, "v0.3.0-rc7"); + + assert!(resolve_version(&manifest, now(), |ver, _| ver.major == 2).is_none()); +} + +#[test] +fn a_pattern_selects_a_line_a_major_cannot() { + let manifest = live_manifest(); + let pattern = regex::Regex::new(r"^v0\.2\..*$").unwrap(); + + let resolved = resolve_version(&manifest, now(), |_, key| pattern.is_match(key)).unwrap(); + + assert_eq!(resolved.orig_key, "v0.2.17"); +} + +#[test] +fn a_pattern_matching_nothing_resolves_to_nothing() { + let manifest = live_manifest(); + let pattern = regex::Regex::new(r"^v9\..*$").unwrap(); + + assert!(resolve_version(&manifest, now(), |_, key| pattern.is_match(key)).is_none()); +} + +#[test] +fn the_patch_walk_stops_where_the_predicate_stops() { + let manifest: Manifest = serde_yaml::from_str( + "executor_versions:\n \"v0.3.0\":\n available_after: 2024-09-01T00:00:00Z\n \"v0.3.1\":\n available_after: 2024-09-01T00:00:00Z\n \"v0.3.2\":\n available_after: 2024-09-01T00:00:00Z\n", + ) + .unwrap(); + let through_v1 = regex::Regex::new(r"^v0\.3\.[01]$").unwrap(); + + // Without the predicate on the walk this slides to v0.3.2, which the caller + // asked not to run. + let resolved = resolve_version(&manifest, now(), |_, key| through_v1.is_match(key)).unwrap(); + + assert_eq!(resolved.orig_key, "v0.3.1"); +} + +#[test] +fn an_exact_version_is_the_directory_name_itself() { + // Not in any manifest, and it still runs: the host is trusted to name a + // line, so this is the one selector that never consults the manifest. + assert_eq!(exact_version("my-dev-build").orig_key, "my-dev-build"); + assert_eq!(exact_version("v0.2.17").version.minor, 2); +} diff --git a/implementation/src/scripting/mod.rs b/implementation/src/scripting/mod.rs index b0847002..4b61b5b7 100644 --- a/implementation/src/scripting/mod.rs +++ b/implementation/src/scripting/mod.rs @@ -162,7 +162,8 @@ impl<T, R, E> UserVM<T, R, E> { | StdLib::IO | StdLib::STRING | StdLib::MATH - | StdLib::PACKAGE; + | StdLib::PACKAGE + | StdLib::OS; let vm = mlua::Lua::new_with(lua_libs, mlua::LuaOptions::default())?; diff --git a/install/config/genvm-manager.yaml b/install/config/genvm-manager.yaml index 466b4cd5..a33cdd78 100644 --- a/install/config/genvm-manager.yaml +++ b/install/config/genvm-manager.yaml @@ -8,3 +8,5 @@ log_level: info manifest_path: ${exeDir}/../data/manifest.yaml permits: null # if null will be auto-discovered; full VM takes 2, sync VM takes 1 +execution_retention: 5m # finished executions are retained until ack or this TTL +max_message_bytes: 67108864 # 64 MiB manager websocket message cap diff --git a/support/tools/genvm-tool/genvm_tool/cmd_configure.py b/support/tools/genvm-tool/genvm_tool/cmd_configure.py index 41fef014..25bcbce0 100644 --- a/support/tools/genvm-tool/genvm_tool/cmd_configure.py +++ b/support/tools/genvm-tool/genvm_tool/cmd_configure.py @@ -302,10 +302,17 @@ def main(ctx: common.Context, args) -> int: codegen_phony = n.build('phony', 'codegen') - # Manager-global generated files (test fixtures + docs) have a single output, - # so generate them from the primary line's codegen data. + # Manager-global generated files have a single output. Host protocol data is + # shared; public ABI data still comes from the primary line. p_data = primary_exec_root / 'executor' / 'codegen' / 'data' + shared_data = source_dir / 'crates/modules-interfaces/codegen/data' + host_fns_rs = source_dir / 'crates/modules-interfaces/src/host_fns.rs' host_fns_py = source_dir / 'tests/runner/origin/host_fns.py' + manager_api_rs = source_dir / 'crates/modules-interfaces/src/manager_api.rs' + manager_api_py = source_dir / 'tests/runner/origin/manager_api.py' + manager_socket_consts_rst = ( + source_dir / 'docs/website/src/impl-spec/appendix/manager-socket-consts.rst' + ) public_abi_py = source_dir / 'tests/runner/origin/public_abi.py' constants_rst = source_dir / 'docs/website/src/spec/appendix/constants.rst' # Pending public-abi constants (ADR-012): documented in their own appendix @@ -315,7 +322,11 @@ def main(ctx: common.Context, args) -> int: source_dir / 'docs/website/src/spec/appendix/constants-pending.rst' ) - n.codegen(host_fns_py, 'python', p_data / 'host-fns.json') + n.codegen(host_fns_rs, 'rust', shared_data / 'host-fns.json') + n.codegen(host_fns_py, 'python', shared_data / 'host-fns.json') + n.codegen(manager_api_rs, 'rust', shared_data / 'manager-api.json') + n.codegen(manager_api_py, 'python', shared_data / 'manager-api.json') + n.codegen(manager_socket_consts_rst, 'rst', shared_data / 'manager-api.json') n.codegen(public_abi_py, 'python', p_data / 'public-abi.json') n.codegen(constants_rst, 'rst', p_data / 'public-abi.json') n.codegen( @@ -324,7 +335,16 @@ def main(ctx: common.Context, args) -> int: p_data / 'public-abi-pending.json', ['--rst-anchor-ns=pending'], ) - for out in (host_fns_py, public_abi_py, constants_rst, constants_pending_rst): + for out in ( + host_fns_rs, + host_fns_py, + manager_api_rs, + manager_api_py, + manager_socket_consts_rst, + public_abi_py, + constants_rst, + constants_pending_rst, + ): codegen_phony.add_dependency(out) cargo_cmd = [common.command_to_executable('cargo')] diff --git a/tests/runner/genvm_tool_plugins/genvm.py b/tests/runner/genvm_tool_plugins/genvm.py index 1bfc5c5c..5c72890d 100644 --- a/tests/runner/genvm_tool_plugins/genvm.py +++ b/tests/runner/genvm_tool_plugins/genvm.py @@ -170,6 +170,8 @@ async def start(self) -> ManagerHandle: 'log_level: info', f'manifest_path: {json.dumps(str(manifest_path))}', 'permits: null', + 'execution_retention: 5m', + 'max_message_bytes: 67108864', '', ] ) diff --git a/tests/runner/genvm_tool_plugins/integration.py b/tests/runner/genvm_tool_plugins/integration.py index b2c24198..de2db945 100644 --- a/tests/runner/genvm_tool_plugins/integration.py +++ b/tests/runner/genvm_tool_plugins/integration.py @@ -10,7 +10,6 @@ import difflib import gzip import hashlib -import io import itertools import json import os @@ -219,6 +218,18 @@ def _unfold_conf(x: typing.Any, vars: dict[str, str]) -> typing.Any: return x +def _routing_payload(route: int | str) -> bytes: + """Where a host places a callee: a major, or a version naming a line. + + A major is resolved by the manifest's rules, which cannot separate the live + lines while every one of them is semver major 0 β€” a version string is what + names one of those. + """ + if isinstance(route, str): + return gvm_calldata.encode({'kind': 'version', 'version': route}) + return gvm_calldata.encode({'kind': 'major', 'major': route}) + + def _flatten_tree( entries: list[dict], ) -> list[tuple[str, str | None, dict]]: @@ -450,15 +461,6 @@ def _get_diffs[T](exp: T, got: T, dump: typing.Callable[[T], str]) -> dict | Non } -def _calldata_to_fancy_str(calldata: bytes) -> str: - cd = gvm_calldata.decode(calldata) - buf = io.StringIO() - genvm_tool.formatter.TextFormatter(genvm_tool.formatter.NoLockTextIO(buf)).dump( - genvm_tool.formatter.Formatter.Level.ERROR, 'calldata', calldata=cd - ) - return buf.getvalue() - - class Context(base_host.Context): logger: base_host.Logger @@ -471,12 +473,8 @@ def on_genvm_failure(self): ... def add_stat(self, key: str, value: typing.Any): self.logger.debug('stat', key=key, val=value) - def get_timeout( - self, action: base_host.TimeoutAction, type: base_host.TimeoutType - ) -> float | None: - if type == base_host.TimeoutType.CONNECT_S: - return 5.0 - return None + def get_manager_connect_timeout(self) -> float | None: + return 5.0 class IntegrationSingleStep(genvm_tool.tests.exec.step.Python): @@ -693,12 +691,27 @@ async def _run_single_step( # Create mock host running_address = single_conf['message']['contract_address'] + ctx = Context(logger) + # A case may declare `executor_routes` to send a callee across a major + # boundary; without one the hook answers null and every call stays + # in-process. `hook_cross_contract_calls` asks for the hook on its own, + # so a case can cover a host that is consulted and declines to route. + # A route is a major (int) or a version string (`re:` prefixed to match + # manifest keys rather than name a directory). + routes = {Address(k): v for k, v in single_conf.get('executor_routes', {}).items()} + hook_calls = routes != {} or single_conf.get('hook_cross_contract_calls', False) host = MockHost( path=str(mock_sock_path), storage_path_post=post_storage, storage_path_pre=pre_storage, balances={Address(k): v for k, v in single_conf.get('balances', {}).items()}, running_address=running_address, + ctx=ctx, + resolve_callcontract_executor_hook=( + lambda address, _state, _major: ( + _routing_payload(routes[address]) if address in routes else None + ) + ), ) host.balances.setdefault(running_address, 0) @@ -726,6 +739,8 @@ async def _run_single_step( case_permissions = single_conf.get('permissions') if case_permissions is not None: request_extra['permissions'] = case_permissions + if hook_calls: + request_extra['hook_cross_contract_calls'] = True dflt_bucket = 2**200 bucket_totals: list[int] = single_conf.get('bucket_totals', [dflt_bucket] * 10) @@ -742,26 +757,37 @@ async def _run_single_step( if not single_conf['message'].get('is_init', False): code = None mode = single_conf.get('mode', 'l') - ctx = Context(logger) - res = await base_host.run_genvm( - mock_host, - manager_uri=manager_uri, - message=single_conf['message'], - timeout=single_conf.get('deadline', 10 * 60), - is_sync=(mode == 's'), - host_data=host_data, - ctx=ctx, - host='unix://' + mock_host.path, - debug_mode=self._debug_mode, - code=code, - calldata=calldata_bytes, - bucket_totals=bucket_totals, - gas_data=single_conf.get('gas_data'), - leader_nondet_results=leader_nondet, - message_fee_allocation=message_fee_allocation, - reroute_to=reroute_to, - request_extra=request_extra, - ) + async with base_host.ManagerClient( + manager_uri, + connect_timeout=ctx.get_manager_connect_timeout(), + ) as manager_client: + res = await base_host.run_genvm( + mock_host, + manager_uri=manager_uri, + manager_client=manager_client, + message=single_conf['message'], + timeout=single_conf.get('deadline', 10 * 60), + is_sync=(mode == 's'), + host_data=host_data, + ctx=ctx, + host='unix://' + mock_host.path, + debug_mode=self._debug_mode, + code=code, + calldata=calldata_bytes, + bucket_totals=bucket_totals, + gas_data=single_conf.get('gas_data'), + leader_nondet_results=leader_nondet, + message_fee_allocation=message_fee_allocation, + unsafe_overrides=base_host.UnsafeOverrides(reroute_to=reroute_to), + request_extra=request_extra, + # A case may state the major the host claims, for the + # cases whose subject is what the executor does with a + # major the manager would otherwise refuse to route. + major=single_conf.get('major'), + ) + # Nested executors dial the same listener; wind their loops down + # so a failure on that side surfaces here. + await mock_host.stop_connections() return_part = '' if res.result_kind == public_abi.ResultCode.RETURN: return_part += ( @@ -841,27 +867,17 @@ async def _run_single_step( }, } - result_events: list[list[bytes]] = [] - messages_content = [] for em in res.result_emissions: - if em['type'] == 'EmitEvent': - tem = typing.cast(base_host.EmitEventInner, em) - blob = gvm_calldata.encode(tem['blob']) - result_events.append([*tem['topics'], blob]) messages_content.append(gvm_calldata.to_str(em)) messages_content.append('\n') - # Write hash file (calldata-encoded deterministic result fields) - hash_data = gvm_calldata.encode( - [ - int(res.result_kind), - res.result_data, - res.result_fingerprint, - res.result_storage_changes, - result_events, - ] - ) + # The executor's own execution hash, i.e. the value consensus actually + # compares. It folds kind, data, backtrace, storage changes, sub-VM + # hashes, wasm store hashes and both fee buckets. Emissions are not in + # it; those are covered by the `messages` semantics component, which + # only runs in the main mode. + hash_data = res.execution_hash semantics_parts = { 'stdout': res.stdout, @@ -952,9 +968,9 @@ async def _run_single_step( if expected_hash_path.exists(): original_hash = base64.b64decode(expected_hash_path.read_text().strip()) - diff = _get_diffs(original_hash, hash_data, _calldata_to_fancy_str) - - if diff is not None: + # An opaque digest: there is nothing to diff, only the two + # values and the semantics/stdout goldens to reason from. + if original_hash != hash_data: return { 'passed': False, 'context': { @@ -966,7 +982,8 @@ async def _run_single_step( else 'this run of the main mode', 'hash_path': str(expected_hash_path), 'tree_path': tree_path, - **diff, + 'exp': original_hash.hex(), + 'got': hash_data.hex(), }, } elif not compares_golden: @@ -1386,9 +1403,9 @@ def _single_integration_test( f'{jsonnet_file}: unknown debug_mode {debug_mode!r}, expected one of {_DEBUG_MODES}' ) if debug_mode == 'disabled': - # `reroute_to` is honored only from `safe` up, and that is what points a - # case at its own line's executor. Below it the case would silently run - # whatever the manifest resolves to. + # `unsafe_overrides.reroute_to` is honored only from `safe` up, and that + # is what points a case at its own line's executor. Below it the case + # would silently run whatever the manifest resolves to. raise ValueError(f'{jsonnet_file}: debug_mode "disabled" cannot reroute') # Recompute needed_services after tags update diff --git a/tests/runner/genvm_tool_plugins/ninja.py b/tests/runner/genvm_tool_plugins/ninja.py index 8ed58707..4d34c31d 100644 --- a/tests/runner/genvm_tool_plugins/ninja.py +++ b/tests/runner/genvm_tool_plugins/ninja.py @@ -435,11 +435,6 @@ def register_standard_codegen(self) -> None: 'python', data / 'public-abi.json', ) - self.codegen( - self.exec_root / 'executor/crates/common/src/host_fns.rs', - 'rust', - data / 'host-fns.json', - ) pending_abi = self.exec_root / 'executor/codegen/data/public-abi-pending.json' if pending_abi.exists(): self.codegen( diff --git a/tests/runner/gvm_extra/mock_host.py b/tests/runner/gvm_extra/mock_host.py index db2f3adc..b19b7110 100644 --- a/tests/runner/gvm_extra/mock_host.py +++ b/tests/runner/gvm_extra/mock_host.py @@ -1,12 +1,25 @@ import asyncio import collections.abc +import contextlib import pickle import socket from pathlib import Path -from origin.base_host import HostException, IHost, host_fns, public_abi +from origin.base_host import ( + Context, + HostException, + IHost, + host_fns, + host_loop_on, + public_abi, +) from origin.calldata import Address +type ResolveCallcontractExecutorHook = collections.abc.Callable[ + [Address, public_abi.StorageType, int], + bytes | None, +] + class MockStorage: _storages: dict[Address, dict[bytes, bytearray]] @@ -33,6 +46,33 @@ def write( memoryview(res)[index : index + len(what)] = what +_STOP_CONNECTIONS_TIMEOUT_S = 10.0 +""" +Cap on how long ``MockHost.stop_connections`` waits for a cancelled task to +actually finish. + +Cancellation only requests a ``CancelledError`` at the next await point; a +task stuck outside the event loop, or one that swallows it, would otherwise +hang the test teardown indefinitely. +""" + + +def _close_watched(sock: socket.socket) -> None: + """ + Closes a socket that an asyncio task may still be reading from. + + ``Task.cancel`` only takes effect on the next loop iteration, so a task + blocked in ``sock_accept``/``sock_recv`` deregisters its reader *after* a + synchronous ``__exit__`` has closed the file descriptor -- by which point the + number may already belong to someone else's socket, whose reader it then + silently removes. Dropping the watch first keeps that from happening. + """ + with contextlib.suppress(Exception): + asyncio.get_event_loop().remove_reader(sock.fileno()) + with contextlib.suppress(OSError): + sock.close() + + class MockHost(IHost): sock: socket.socket | None storage: MockStorage | None @@ -45,6 +85,9 @@ def __init__( storage_path_post: Path, balances: dict[Address, int], running_address: Address, + ctx: Context, + expected_hello_data: bytes = b'', + resolve_callcontract_executor_hook: ResolveCallcontractExecutorHook | None = None, ): self.running_address = running_address self.path = path @@ -52,9 +95,16 @@ def __init__( self.storage_path_post = storage_path_post self.storage = None self.sock = None + self.sock_listener = None self.thread = None self.balances = balances self.nondet_disagreement_call_no = None + self.expected_hello_data = expected_hello_data + self.resolve_callcontract_executor_hook = resolve_callcontract_executor_hook + self.ctx = ctx + self._accept_task: asyncio.Task | None = None + self._connection_tasks: list[asyncio.Task] = [] + self._accepted_sockets: list[socket.socket] = [] def __enter__(self): self.created = False @@ -66,7 +116,9 @@ def __enter__(self): self.sock_listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) self.sock_listener.bind(self.path) self.sock_listener.setblocking(False) - self.sock_listener.listen(1) + # A run that delegates across a major boundary spawns nested executors, + # and each dials this same listener. + self.sock_listener.listen(8) return self @@ -75,8 +127,19 @@ def __exit__(self, *_args): with open(self.storage_path_post, 'wb') as f: pickle.dump(self.storage, f) self.storage = None - if self.sock is not None: - self.sock.close() + if self._accept_task is not None: + self._accept_task.cancel() + self._accept_task = None + for task in self._connection_tasks: + task.cancel() + self._connection_tasks.clear() + for accepted in self._accepted_sockets: + _close_watched(accepted) + self._accepted_sockets.clear() + self.sock = None + if self.sock_listener is not None: + _close_watched(self.sock_listener) + self.sock_listener = None Path(self.path).unlink(missing_ok=True) async def notify_nondet_disagreement(self, call_no: int) -> None: @@ -84,25 +147,104 @@ async def notify_nondet_disagreement(self, call_no: int) -> None: pass async def loop_enter(self, cancellation: asyncio.Event): + sock = await self._accept(cancellation) + if sock is None: + raise Exception('Program failed') + self.sock = sock + if self._accept_task is None: + # Serve every later connection ourselves. Accepting stops when the + # run ends, which is when `run_genvm` sets the cancellation event. + self._accept_task = asyncio.create_task(self._accept_connections(cancellation)) + return sock + + async def _accept(self, cancellation: asyncio.Event) -> socket.socket | None: + """ + Accepts one connection, or returns ``None`` once the run is over. + """ async_loop = asyncio.get_event_loop() assert self.sock_listener is not None interesting = asyncio.ensure_future(async_loop.sock_accept(self.sock_listener)) canc = asyncio.ensure_future(cancellation.wait()) - done, pending = await asyncio.wait( - [canc, interesting], return_when=asyncio.FIRST_COMPLETED - ) + try: + done, _pending = await asyncio.wait( + [canc, interesting], return_when=asyncio.FIRST_COMPLETED + ) + finally: + # Also runs when this task is cancelled mid-accept, which is the + # normal way the background acceptor ends. + for task in (interesting, canc): + if not task.done(): + task.cancel() if canc in done: - raise Exception('Program failed') - cancellation.set() - canc.cancel() + return None - self.sock, _addr = interesting.result() - self.sock.setblocking(False) - self.sock_listener.close() - self.sock_listener = None - return self.sock + sock, _addr = interesting.result() + sock.setblocking(False) + self._accepted_sockets.append(sock) + if self.expected_hello_data: + actual = bytearray(len(self.expected_hello_data)) + view = memoryview(actual) + pos = 0 + while pos < len(actual): + read_len = await async_loop.sock_recv_into(sock, view[pos:]) + if read_len == 0: + raise ConnectionResetError() + pos += read_len + if actual != self.expected_hello_data: + raise AssertionError( + f'host hello mismatch: expected {self.expected_hello_data!r}, got {bytes(actual)!r}' + ) + return sock + + async def _accept_connections(self, cancellation: asyncio.Event) -> None: + while True: + sock = await self._accept(cancellation) + if sock is None: + return + self._connection_tasks.append( + asyncio.create_task(host_loop_on(self, sock, ctx=self.ctx)) + ) + + async def stop_connections(self) -> None: + """ + Winds down the nested-connection loops, surfacing whatever they raised. + + Call it once the run is over and before leaving the ``with`` block, so a + nested-side failure becomes a test failure instead of a cancelled task + nobody awaited. + + Bounded by ``_STOP_CONNECTIONS_TIMEOUT_S`` via ``asyncio.wait`` -- + deliberately not ``wait_for``, which keeps re-awaiting a task that + swallows ``CancelledError`` past its own timeout instead of returning. + """ + if self._accept_task is not None: + task = self._accept_task + task.cancel() + _done, pending = await asyncio.wait([task], timeout=_STOP_CONNECTIONS_TIMEOUT_S) + if pending: + raise TimeoutError('accept task did not stop within stop_connections timeout') + with contextlib.suppress(asyncio.CancelledError): + task.result() + self._accept_task = None + for task in self._connection_tasks: + if not task.done(): + task.cancel() + pending_tasks = self._connection_tasks + self._connection_tasks = [] + if pending_tasks: + done, still_pending = await asyncio.wait( + pending_tasks, timeout=_STOP_CONNECTIONS_TIMEOUT_S + ) + if still_pending: + raise TimeoutError( + f'{len(still_pending)} nested host connection(s) did not stop ' + 'within stop_connections timeout' + ) + for task in done: + with contextlib.suppress(asyncio.CancelledError, ConnectionResetError): + task.result() async def storage_read( self, mode: public_abi.StorageType, account: bytes, slot: bytes, index: int, le: int @@ -110,6 +252,21 @@ async def storage_read( assert self.storage is not None return self.storage.read(Address(account), slot, index, le) + async def resolve_callcontract_executor( + self, + contract_address: Address, + state_mode: public_abi.StorageType, + advisory_major: int, + /, + ) -> bytes | None: + if self.resolve_callcontract_executor_hook is None: + return None + return self.resolve_callcontract_executor_hook( + contract_address, + state_mode, + advisory_major, + ) + async def remaining_fuel_as_gen(self) -> int: return 2**32 diff --git a/tests/runner/origin/base_host.py b/tests/runner/origin/base_host.py index 9082ffea..2b2cd4bb 100644 --- a/tests/runner/origin/base_host.py +++ b/tests/runner/origin/base_host.py @@ -1,11 +1,16 @@ import abc import asyncio -import enum +import base64 +import collections.abc +import contextlib +import json import math import socket import time +import types import typing -from dataclasses import dataclass +import urllib.parse +from dataclasses import dataclass, field import aiohttp @@ -15,6 +20,7 @@ from . import ( fees, host_fns, + manager_api, public_abi, ) from .calldata import Address @@ -23,6 +29,24 @@ ACCOUNT_ADDR_SIZE = 20 SLOT_ID_SIZE = 32 +ZERO_SLOT = b'\x00' * SLOT_ID_SIZE +"""The root `SlotID`, all zeroes.""" + +ROOT_OFFSET_MAJOR = 0 +"""Offset of the single-octet public-ABI `major` within the root slot.""" + +UNDEPLOYED_MAJOR = 0 +""" +Major to declare when there is no deployed contract to read one from. + +Every line released so far has semver major `0`, and the manager matches `0` +against all of them and picks the newest, so this is a de-facto "any line". +A deploy is the honest case for it: the real value comes from the contract +package, which this harness does not parse, and the harness pins its executor +with `unsafe_overrides.reroute_to` regardless. It is also why the manager cannot +yet reject `major == 0`. +""" + # Mirrors the executor's `DebugMode` enum (crates/common/src/debug_mode.rs). DebugMode = typing.Literal[ 'disabled', @@ -52,19 +76,6 @@ } -class TimeoutAction(enum.StrEnum): - VMErrorDescribe = 'vm-error/describe' - GenVMGet = '/genvm/{id}' - GenVMRun = '/genvm/run' - GenVMDelete = 'DELETE /genvm/{id}' - - -class TimeoutType(enum.StrEnum): - TOTAL_S = 'HTTP_TIMEOUT_TOTAL_S' - CONNECT_S = 'HTTP_TIMEOUT_CONNECT_S' - SOCK_READ_S = 'HTTP_TIMEOUT_SOCK_READ_S' - - class Context(typing.Protocol): logger: Logger @@ -73,26 +84,12 @@ def on_genvm_failure(self): ... def add_stat(self, key: str, value: typing.Any, /): ... - def get_timeout(self, action: TimeoutAction, type: TimeoutType, /) -> float | None: - return None - - def retry_delay(self, action: TimeoutAction, attempt_no: int, /) -> float | None: - """Returns delay before next retry, or None if no retries are left.""" + def get_manager_connect_timeout(self) -> float | None: return None -def _http_timeout( - ctx: Context, - action: TimeoutAction, -) -> aiohttp.ClientTimeout: - """ - Explicit aiohttp timeout to avoid wedging consensus when the local GenVM manager - accepts a connection but never responds. - """ - total_s = ctx.get_timeout(action, TimeoutType.TOTAL_S) - connect_s = ctx.get_timeout(action, TimeoutType.CONNECT_S) - sock_read_s = ctx.get_timeout(action, TimeoutType.SOCK_READ_S) - return aiohttp.ClientTimeout(total=total_s, connect=connect_s, sock_read=sock_read_s) +def _http_timeout(ctx: Context) -> aiohttp.ClientTimeout: + return aiohttp.ClientTimeout(connect=ctx.get_manager_connect_timeout()) class HostException(Exception): @@ -103,6 +100,25 @@ def __init__(self, error_code: host_fns.Errors, message: str = ''): super().__init__(message or f'GenVM error: {error_code}') +@dataclass(frozen=True) +class UnsafeOverrides: + """Request overrides that reach boundaries production traffic cannot. + + Each member states the `debug_mode` the manager requires before it applies: + `reroute_to` from `safe`, `initial_recursion` from `unsafe`. With debugging + disabled none of them take effect. + """ + + reroute_to: str = '' + initial_recursion: int | None = None + + def as_request_field(self) -> dict[str, typing.Any]: + return { + 'reroute_to': self.reroute_to, + 'initial_recursion': self.initial_recursion, + } + + class Message(typing.TypedDict): contract_address: Address sender_address: Address @@ -195,6 +211,15 @@ async def storage_read( /, ) -> bytes: ... + async def resolve_callcontract_executor( + self, + contract_address: Address, + state_mode: public_abi.StorageType, + advisory_major: int, + /, + ) -> bytes | None: + return None + @abc.abstractmethod async def consume_gas(self, gas: int, /) -> None: ... @abc.abstractmethod @@ -207,14 +232,40 @@ async def remaining_fuel_as_gen(self, /) -> int: ... async def notify_nondet_disagreement(self, call_no: int, /) -> None: ... +async def read_contract_major(handler: IHost, message: Message) -> int: + """ + Reads the public-ABI major the run's contract was deployed against. + + The major is octet `ROOT_OFFSET_MAJOR` of the root slot, written at deploy + time and read on every load. A deploy has nothing to read, so it declares + `UNDEPLOYED_MAJOR` instead. + + The read uses the storage view a top-level run itself uses, so a run cannot + take its major from a state it would not otherwise observe. An address with + no contract reads back `0`, which is indistinguishable from major `0`; that + stays a fallback for now. + """ + if message.get('is_init', False): + return UNDEPLOYED_MAJOR + octet = await handler.storage_read( + public_abi.StorageType.LATEST_NON_FINAL, + message['contract_address'].as_bytes, + ZERO_SLOT, + ROOT_OFFSET_MAJOR, + 1, + ) + return octet[0] + + async def host_loop( handler: IHost, cancellation: asyncio.Event, *, ctx: Context, ) -> None: - async_loop = asyncio.get_event_loop() - + """ + Accepts one connection through the handler, then serves it. + """ logger = ctx.logger logger.trace('entering loop') @@ -227,6 +278,26 @@ async def host_loop( round((host_loop_entered_s - loop_enter_wait_start) * 1000), ) logger.trace('entered loop') + + await host_loop_on(handler, sock, ctx=ctx) + + +async def host_loop_on( + handler: IHost, + sock: socket.socket, + *, + ctx: Context, +) -> None: + """ + Serves an already accepted connection. + + A run that spawns nested executors produces several connections to one + listener, so the accept step has to be separable from the protocol it feeds. + """ + async_loop = asyncio.get_event_loop() + + logger = ctx.logger + accept_time = time.perf_counter() first_method_name: str | None = None first_method_received_s: float | None = None @@ -284,6 +355,22 @@ async def read_slice() -> memoryview: call_counts = {} meth_id: host_fns.Methods | None = None + def emit_host_loop_stats(): + if first_method_name is not None: + ctx.add_stat('host_first_method', first_method_name) + ctx.add_stat('host_total_handling_time_ms', round(total_handling_time * 1000)) + ctx.add_stat( + 'host_time_per_method_ms', + {k: round(v * 1000) for k, v in time_per_method.items()}, + ) + ctx.add_stat('call_counts', call_counts) + logger.debug( + 'handling time', + total=total_handling_time, + by_method=time_per_method, + call_counts=call_counts, + ) + handling_start = time.time() while True: cur_delta = time.time() - handling_start @@ -293,7 +380,11 @@ async def read_slice() -> memoryview: await flush_socket_buffer() - meth_id = host_fns.Methods(await recv_int(1)) + try: + meth_id = host_fns.Methods(await recv_int(1)) + except ConnectionResetError: + emit_host_loop_stats() + return None if first_method_name is None: first_method_name = meth_id.name first_method_received_s = time.perf_counter() @@ -322,35 +413,28 @@ async def read_slice() -> memoryview: else: await send_all(bytes([host_fns.Errors.OK])) await send_all(res) + case host_fns.Methods.RESOLVE_CALLCONTRACT_EXECUTOR: + contract_address = Address(await read_exact(ACCOUNT_ADDR_SIZE)) + state_mode = public_abi.StorageType((await read_exact(1))[0]) + advisory_major = await recv_int(1) + + try: + res = await handler.resolve_callcontract_executor( + contract_address, + state_mode, + advisory_major, + ) + except HostException as e: + await send_all(bytes([e.error_code])) + else: + encoded_res = gvm_calldata.encode(res) + await send_all(bytes([host_fns.Errors.OK])) + await send_int(len(encoded_res)) + await send_all(encoded_res) case host_fns.Methods.CONSUME_RESULT: raise Exception( 'CONSUME_RESULT is not supported in this host loop implementation, use manager provided one' ) - case host_fns.Methods.NOTIFY_FINISHED: - logger.debug( - 'handling time', - total=total_handling_time, - by_method=time_per_method, - call_counts=call_counts, - ) - await send_all(bytes([0])) - await flush_socket_buffer() - - if first_method_name is not None: - ctx.add_stat('host_first_method', first_method_name) - ctx.add_stat('host_total_handling_time_ms', round(total_handling_time * 1000)) - ctx.add_stat( - 'host_time_per_method_ms', - {k: round(v * 1000) for k, v in time_per_method.items()}, - ) - ctx.add_stat('call_counts', call_counts) - logger.debug( - 'handling time', - total=total_handling_time, - by_method=time_per_method, - call_counts=call_counts, - ) - return None case host_fns.Methods.CONSUME_FUEL: gas = await recv_int(32) await handler.consume_gas(gas) @@ -392,6 +476,80 @@ async def read_slice() -> memoryview: raise Exception(f'unknown method {x}') +class ConsumedResultDecodeError(Exception): + """`consumed_result` bytes did not parse into a `ConsumedResult` at all. + + Distinct from `ConsumedResult.internal_error(...)`, which is itself a + valid (if unhappy) result value produced from bytes that *did* parse: + this means the bytes never parsed, so there is no result to hand back. + Callers must treat it the same as any other post-terminal failure -- see + `run_genvm`'s `TerminalResultUnavailable` wrapping -- never as grounds to + start a new run. + """ + + +@dataclass +class ConsumedResult: + """The decoded `consumed_result` blob: a `ResultCode` byte plus calldata.""" + + execution_hash: bytes + result_kind: public_abi.ResultCode + result_data: gvm_calldata.Decoded + result_fingerprint: ResultFingerprint | None = None + result_storage_changes: list[tuple[bytes, bytes]] = field(default_factory=list) + result_emissions: list[ResultEmission] = field(default_factory=list) + result_nondet_results: list[bytes] = field(default_factory=list) + data_fees_remaining: list[int] = field(default_factory=list) + + @classmethod + def internal_error(cls, message: str) -> 'ConsumedResult': + return cls( + execution_hash=b'', + result_kind=public_abi.ResultCode.INTERNAL_ERROR, + result_data=message, + ) + + @classmethod + def decode(cls, raw: typing.Any) -> 'ConsumedResult': + if raw is None: + # The manager never attempted to report a result at all. + return cls.internal_error('no_result') + empty = False + try: + # The socket sends bytes, the deprecated http shim sends base64. + as_bytes = ( + base64.b64decode(raw, validate=True) if isinstance(raw, str) else bytes(raw) + ) + empty = not as_bytes + if not empty: + result_kind = public_abi.ResultCode(as_bytes[0]) + decoded = gvm_calldata.decode(as_bytes[1:]) + except Exception as exc: + # Unreadable bytes are a protocol violation rather than a result, so + # raise instead of returning an `internal_error(...)` value that a + # caller could mistake for a real (if unhappy) execution outcome. + raise ConsumedResultDecodeError( + f'malformed consumed_result ({raw!r:.80}): {exc}' + ) from exc + if empty: + # Distinct from `None`: the manager did send a `consumed_result`, + # but it carries no `ResultCode` byte to read. Still "no usable + # result data" from the caller's point of view. + return cls.internal_error('empty_result') + if not isinstance(decoded, dict): + return cls.internal_error('result is not a mapping') + return cls( + execution_hash=decoded.get('execution_hash', b''), + result_kind=result_kind, + result_data=decoded.get('data'), + result_fingerprint=decoded.get('fingerprint'), + result_storage_changes=decoded.get('storage_changes', []), + result_emissions=decoded.get('emissions', []), + result_nondet_results=decoded.get('nondet_results', []), + data_fees_remaining=decoded.get('data_fees_remaining', []), + ) + + @dataclass class RunHostAndProgramRes: stdout: str @@ -408,48 +566,583 @@ class RunHostAndProgramRes: result_storage_changes: list[tuple[bytes, bytes]] result_emissions: list[ResultEmission] result_nondet_results: list[bytes] + data_fees_remaining: list[int] + metrics: dict[str, typing.Any] | None = None vm_error_description: str | None = None -async def _send_timeout( - manager_uri: str, - genvm_id: str, - ctx: Context, -): - try: - async with aiohttp.request( - 'DELETE', - f'{manager_uri}/genvm/{genvm_id}', - timeout=_http_timeout(ctx, TimeoutAction.GenVMDelete), - ) as resp: - ctx.add_stat('delete_genvm_status', resp.status) - if resp.status != 200: - ctx.add_stat('delete_genvm_failed', True) - ctx.add_stat('delete_genvm_body', await resp.text()) - except (aiohttp.ClientError, asyncio.TimeoutError) as exc: - ctx.add_stat('delete_genvm_request_failed', True) - ctx.add_stat('delete_genvm_request_error', str(exc)) - - -async def _await_first_cancel_others(*it): - _done, pending = await asyncio.wait( - [asyncio.ensure_future(x) for x in it], - return_when=asyncio.FIRST_COMPLETED, - ) - for task in pending: - task.cancel() - for task in pending: +class Frame(typing.NamedTuple): + """One manager socket message: a method, a request id and a calldata payload.""" + + method: manager_api.Methods + request_id: int + payload: typing.Any + + +class ManagerSocketError(Exception): + def __init__(self, code: manager_api.Errors, message: str): + self.code = code + self.message = message + super().__init__(f'{code.name}: {message}') + + +_MANAGER_RECONNECT_ATTEMPTS = 3 +_MANAGER_RECONNECT_BACKOFF_S = 0.05 + + +class ManagerConnectionLost(Exception): + """ + The manager socket dropped. + + `generation` is the connection this happened on, so a waiter that wakes up + after somebody else already reconnected can tell that its loss is stale + rather than reconnecting a second time. + """ + + def __init__(self, message: str, generation: int = -1): + super().__init__(message) + self.generation = generation + + +class ManagerRunLost(Exception): + """ + The manager process that owned a run is gone, so the run is unrecoverable. + + Deliberately not a `ManagerConnectionLost`: that one means "reconnect and keep + waiting", and a waiter that treats this as one waits forever for a run no + reconnect can bring back. + """ + + +class ManagerRunNotStarted(Exception): + """A run never reached execution, so retrying it re-executes nothing. + + Raised for the two ways the manager can refuse before the genvm runs -- a + rejected RUN request and a `failed_to_start` terminal. Retry policy belongs + to the caller; `retryable` tells it whether the refusal is transient (the + manager is still bringing modules up, or the socket bounced) rather than a + permanent rejection (malformed request, absent runner, bad calldata). This + is deliberately distinct from a returned `RETURN`/`*_ERROR` result, which + means the genvm did run and must never be blindly retried. + """ + + def __init__(self, message: str, *, retryable: bool, reason: str): + self.retryable = retryable + self.reason = reason + super().__init__(message) + + +class TerminalResultUnavailable(Exception): + """A run reached a terminal `finished` state, but its result could not be + retrieved or decoded -- artifact transfer failed, or `consumed_result` was + empty/malformed. + + The genvm already executed exactly once for this `genvm_id`; the caller + must never treat this the way it treats a plain `Exception` from + `run_genvm` (which starts a brand new run on retry). Always non-retryable + for that reason -- there is no `retryable` flag to override. + """ + + def __init__(self, message: str, *, genvm_id: int): + self.genvm_id = genvm_id + super().__init__(message) + + +# Transient manager refusals worth retrying. The manager reports these with the +# generic `Errors.INTERNAL` code (no dedicated variant yet), so the message is +# the only discriminator -- kept here so callers never have to string-match. +_RETRYABLE_RUN_REFUSAL_MARKERS: typing.Final = ( + 'modules are required but not running', + 'modules are required but not all are running', +) + + +def _classify_run_refusal(message: str) -> tuple[bool, str]: + for marker in _RETRYABLE_RUN_REFUSAL_MARKERS: + if marker in message: + return True, 'manager_modules_not_running' + return False, 'manager_refused' + + +@dataclass +class RunState: + boot_id: int + genvm_id: int + host_genvm_id: str | None + events: asyncio.Queue[dict[str, typing.Any] | BaseException] + terminal: dict[str, typing.Any] | None = None + acked: bool = False + + +class ManagerClient: + def __init__( + self, + manager_uri: str, + *, + connect_timeout: float | None = None, + max_msg_size: int = 64 * 1024 * 1024, + ): + self.manager_uri = manager_uri + self.connect_timeout = connect_timeout + self.max_msg_size = max_msg_size + self.boot_id: int | None = None + self._session: aiohttp.ClientSession | None = None + self._ws: aiohttp.ClientWebSocketResponse | None = None + self._reader_task: asyncio.Task | None = None + self._request_id = 1 + self._pending: dict[int, asyncio.Future[Frame]] = {} + # The protocol identifies a run as (boot_id, genvm_id): ids restart at 1 + # with the manager process, so genvm_id alone aliases across a restart. + self._runs: dict[tuple[int, int], RunState] = {} + self._orphan_events: dict[tuple[int, int], list[dict[str, typing.Any]]] = {} + self._connect_lock = asyncio.Lock() + self._send_lock = asyncio.Lock() + self._reconnect_lock = asyncio.Lock() + # Bumped on every successful connect, so a disconnect can be attributed + # to the connection it happened on. + self._generation = 0 + + @property + def run_states(self) -> dict[tuple[int, int], RunState]: + return self._runs + + def _key(self, genvm_id: int) -> tuple[int, int]: + assert self.boot_id is not None, 'no hello received yet' + return (self.boot_id, genvm_id) + + async def __aenter__(self): + await self._ensure_connected() + return self + + async def __aexit__(self, *_args): + await self._close() + + async def disconnect(self) -> None: + await self._close() + + async def run(self, payload: dict[str, typing.Any]) -> RunState: + response = await self._request( + manager_api.Methods.RUN, + {'run': payload}, + retry_on_disconnect=True, + ) + genvm_id = int(response['genvm_id']) + key = self._key(genvm_id) + state = self._runs.get(key) + if state is None: + state = RunState( + boot_id=key[0], + genvm_id=genvm_id, + host_genvm_id=payload.get('host_genvm_id'), + events=asyncio.Queue(), + ) + self._runs[key] = state + for event in self._orphan_events.pop(key, []): + self._queue_event(state, event) + return state + + async def attach(self, boot_id: int, genvm_id: int) -> RunState: + response = await self._request( + manager_api.Methods.ATTACH, + {'attach': {'boot_id': boot_id, 'genvm_id': genvm_id}}, + retry_on_disconnect=True, + ) + key = (boot_id, genvm_id) + state = self._runs.get(key) + if state is None: + state = RunState( + boot_id=boot_id, + genvm_id=genvm_id, + host_genvm_id=None, + events=asyncio.Queue(), + ) + self._runs[key] = state + self._queue_event(state, response['snapshot']) + return state + + def _require_current_generation( + self, boot_id: int, genvm_id: int, *, op: str + ) -> None: + """Refuse to act on a run from a manager generation we have since left. + + `CANCEL`/`ACK`/`GET_ARTIFACT` identify a run to the manager by + `genvm_id` alone -- the wire protocol has no `boot_id` field for them + (unlike `ATTACH`, which the manager itself rejects on a mismatch). If + the manager restarted and reused this `genvm_id` for an unrelated run, + sending one of these blind would act on that run instead of a stale + reference to the one this call actually means. + """ + if self.boot_id is not None and boot_id != self.boot_id: + raise ManagerRunLost( + f'refusing to {op} genvm {genvm_id}: it belonged to manager ' + f'boot {boot_id}, but the client is now talking to boot ' + f'{self.boot_id} -- the id may have been reused by the new ' + 'process' + ) + + async def cancel(self, boot_id: int, genvm_id: int) -> None: + self._require_current_generation(boot_id, genvm_id, op='cancel') + await self._request( + manager_api.Methods.CANCEL, + {'cancel': {'genvm_id': genvm_id}}, + retry_on_disconnect=True, + ) + + async def ack(self, boot_id: int, genvm_id: int) -> None: + self._require_current_generation(boot_id, genvm_id, op='ack') + await self._request( + manager_api.Methods.ACK, + {'ack': {'genvm_id': genvm_id}}, + retry_on_disconnect=True, + ) + key = (boot_id, genvm_id) + if state := self._runs.get(key): + state.acked = True + self._runs.pop(key, None) + self._orphan_events.pop(key, None) + + async def get_artifact(self, boot_id: int, genvm_id: int, field: str) -> bytes: + self._require_current_generation(boot_id, genvm_id, op='get_artifact') + offset = 0 + out = bytearray() + while True: + response = await self._request( + manager_api.Methods.GET_ARTIFACT, + { + 'get_artifact': { + 'genvm_id': genvm_id, + 'field': field, + 'offset': offset, + 'max_len': 256 * 1024, + } + }, + retry_on_disconnect=True, + ) + data = bytes(response['data']) + out.extend(data) + offset += len(data) + if offset >= int(response['total_len']): + return bytes(out) + if not data: + raise ManagerConnectionLost('artifact transfer made no progress') + + async def wait_terminal(self, state: RunState) -> dict[str, typing.Any]: + if state.terminal is not None: + return state.terminal + while True: + event = await state.events.get() + if isinstance(event, ManagerConnectionLost): + await self._reconnect_live_runs(event.generation) + continue + if isinstance(event, BaseException): + raise event + for variant in ('failed_to_start', 'finished'): + if variant in event: + state.terminal = event + return event + + async def _request( + self, + method: manager_api.Methods, + payload: typing.Any, + *, + retry_on_disconnect: bool, + ) -> typing.Any: + last_error: BaseException | None = None + for attempt in range(_MANAGER_RECONNECT_ATTEMPTS if retry_on_disconnect else 1): + generation = self._generation + try: + return await self._request_once(method, payload) + except ManagerConnectionLost as exc: + last_error = exc + if attempt + 1 < _MANAGER_RECONNECT_ATTEMPTS and retry_on_disconnect: + await asyncio.sleep(_MANAGER_RECONNECT_BACKOFF_S * (2**attempt)) + await self._reconnect_live_runs(generation) + continue + raise + assert last_error is not None + raise last_error + + async def _request_once( + self, + method: manager_api.Methods, + payload: typing.Any, + ) -> typing.Any: + await self._ensure_connected() + assert self._ws is not None + async with self._send_lock: + request_id = self._request_id + self._request_id += 1 + future = asyncio.get_running_loop().create_future() + self._pending[request_id] = future + body = ( + int(method).to_bytes(2, 'big') + + request_id.to_bytes(8, 'big') + + gvm_calldata.encode(payload) + ) + try: + await self._ws.send_bytes(body) + except ( + BrokenPipeError, + ConnectionError, + aiohttp.ClientConnectionError, + ) as exc: + self._pending.pop(request_id, None) + self._mark_disconnected(ManagerConnectionLost(str(exc))) + raise ManagerConnectionLost(str(exc)) from exc + try: + response = await future + finally: + self._pending.pop(request_id, None) + if response.method != method: + raise ManagerConnectionLost( + f'manager replied with {response.method} for {method}' + ) + return response.payload + + async def _ensure_connected(self) -> None: + if self._ws is not None and not self._ws.closed: + return + async with self._connect_lock: + await self._reconnect_locked(-1) + + async def _connect(self) -> None: + timeout = aiohttp.ClientTimeout(connect=self.connect_timeout) + if self.manager_uri.startswith('unix://'): + connector = aiohttp.UnixConnector(path=self.manager_uri.removeprefix('unix://')) + self._session = aiohttp.ClientSession(connector=connector, timeout=timeout) + ws_url = 'http://localhost/ws' + else: + self._session = aiohttp.ClientSession(timeout=timeout) + ws_url = urllib.parse.urljoin(self.manager_uri.rstrip('/') + '/', 'ws') try: - await task + self._ws = await self._session.ws_connect( + ws_url, + max_msg_size=self.max_msg_size, + ) + frame = await self._read_frame() + except BaseException: + await self._close() + raise + if frame.method != manager_api.Methods.HELLO or frame.request_id != 0: + await self._close() + raise ManagerConnectionLost('manager did not send hello') + hello = frame.payload['hello'] + if hello['protocol_major'] != manager_api.CURRENT_MAJOR: + await self._close() + raise ManagerConnectionLost( + f'unsupported manager socket protocol {hello["protocol_major"]}' + ) + hello_boot_id = int(hello['boot_id']) + # Always adopt the generation we are actually talking to. Keeping the old + # one because a run is still around is what let stale state outlive a + # manager restart and alias a reused id. + if self.boot_id is not None and self.boot_id != hello_boot_id: + self._retire_generation(self.boot_id) + self.boot_id = hello_boot_id + self._generation += 1 + self._reader_task = asyncio.create_task(self._reader_loop()) + + def _retire_generation(self, boot_id: int) -> None: + """ + Drops every run belonging to a manager process that is gone. + + Their ids are about to be handed out again by the new process, so a + waiter must learn its run died with its manager rather than silently + binding to an unrelated run of the same number. + """ + for key in [k for k in self._runs if k[0] == boot_id]: + state = self._runs.pop(key) + if not state.acked and state.terminal is None: + state.events.put_nowait( + ManagerRunLost( + f'manager restarted; run {state.genvm_id} of boot {boot_id} is gone' + ) + ) + for key in [k for k in self._orphan_events if k[0] == boot_id]: + del self._orphan_events[key] + + async def _reader_loop(self) -> None: + try: + while True: + frame = await self._read_frame() + if frame.request_id == 0: + self._handle_notification(frame.method, frame.payload) + continue + future = self._pending.pop(frame.request_id, None) + if future is None or future.done(): + continue + if frame.method == manager_api.Methods.ERROR: + code = manager_api.Errors(frame.payload['code']) + future.set_exception(ManagerSocketError(code, frame.payload['message'])) + else: + future.set_result(frame) except asyncio.CancelledError: - pass + raise + except BaseException as exc: + self._mark_disconnected(ManagerConnectionLost(str(exc))) + + async def _read_frame(self) -> Frame: + assert self._ws is not None + msg = await self._ws.receive() + if msg.type == aiohttp.WSMsgType.BINARY: + body = bytes(msg.data) + elif msg.type in ( + aiohttp.WSMsgType.CLOSE, + aiohttp.WSMsgType.CLOSED, + aiohttp.WSMsgType.ERROR, + ): + raise ManagerConnectionLost('manager websocket closed') + else: + raise ManagerConnectionLost(f'unexpected websocket message {msg.type}') + if len(body) < 10: + raise ManagerConnectionLost('manager sent a short message') + method = manager_api.Methods(int.from_bytes(body[:2], 'big')) + request_id = int.from_bytes(body[2:10], 'big') + payload = gvm_calldata.decode(body[10:]) + return Frame(method, request_id, payload) + + def _handle_notification( + self, method: manager_api.Methods, payload: typing.Any + ) -> None: + if method != manager_api.Methods.EVENT: + return + genvm_id = None + for event in payload.values(): + if isinstance(event, dict) and 'genvm_id' in event: + genvm_id = int(event['genvm_id']) + break + if genvm_id is None: + return + # Events arrive on the live connection, so they belong to its generation. + key = self._key(genvm_id) + state = self._runs.get(key) + if state is None: + self._orphan_events.setdefault(key, []).append(payload) + return + self._queue_event(state, payload) + + def _queue_event(self, state: RunState, event: dict[str, typing.Any]) -> None: + if 'failed_to_start' in event or 'finished' in event: + state.terminal = event + state.events.put_nowait(event) + + def _mark_disconnected(self, exc: ManagerConnectionLost) -> None: + exc.generation = self._generation + self._ws = None + for future in list(self._pending.values()): + if not future.done(): + future.set_exception(exc) + for state in self._runs.values(): + if not state.acked and state.terminal is None: + state.events.put_nowait(exc) + + async def _reconnect_locked(self, generation: int) -> None: + """ + Rebuilds the connection unless someone already did. Holds `_connect_lock`. + + Every teardown-and-reconnect goes through here, so the two entry points + (`_ensure_connected` and `_reconnect_live_runs`) cannot run `_close()` + and `_connect()` against each other and orphan a reader task, a session + or a socket the other one is installing. + + `generation < 0` means "any live connection will do"; a non-negative one + names the generation the caller saw die, so a connection newer than it + is already the replacement and is left alone. + """ + assert self._connect_lock.locked() + if self._ws is not None and not self._ws.closed: + if generation < 0 or self._generation > generation: + return + await self._close() + await self._connect() + + async def _reconnect_live_runs(self, generation: int) -> None: + """ + Reconnects the connection generation `generation` was lost on. + + Several waiters observe the same disconnect, so whoever gets the lock + second finds a newer generation already connected and has nothing to do. + Reconnecting again there would tear down the working socket the first + waiter just built. + """ + async with self._reconnect_lock: + async with self._connect_lock: + await self._reconnect_locked(generation) + # Re-attaching runs its own requests, which take `_connect_lock` + # themselves, so it happens outside of it. `_reconnect_lock` still + # keeps a second waiter out until this generation is whole again. + # + # A restart already retired the previous generation's runs during + # hello, so whatever is left here belongs to the manager we are now + # talking to and re-attaches under its own boot id. + for state in list(self._runs.values()): + if state.acked: + continue + try: + response = await self._request( + manager_api.Methods.ATTACH, + { + 'attach': { + 'boot_id': state.boot_id, + 'genvm_id': state.genvm_id, + } + }, + retry_on_disconnect=False, + ) + except ManagerSocketError as exc: + state.events.put_nowait(exc) + else: + self._queue_event(state, response['snapshot']) + + async def _close(self) -> None: + ws = self._ws + self._ws = None + if self._reader_task is not None: + self._reader_task.cancel() + with contextlib.suppress(BaseException): + await self._reader_task + self._reader_task = None + if ws is not None: + with contextlib.suppress(BaseException): + await ws.close() + if self._session is not None: + with contextlib.suppress(BaseException): + await self._session.close() + self._session = None + + +def _duration_string(seconds: float | None) -> str | None: + if seconds is None: + return None + if seconds < 1: + millis = max(1, math.ceil(seconds * 1000)) + return f'{millis}ms' + if float(seconds).is_integer(): + return f'{int(seconds)}s' + return f'{seconds:g}s' + + +def _decode_genvm_log(data: bytes) -> list[dict[str, typing.Any]]: + """Decodes the genvm log artifact, which is json lines. + + Split on newline bytes only. `str.splitlines` also breaks on U+2028, U+2029 + and U+0085, none of which json escapes and all of which therefore appear raw + inside a string value -- splitting there cuts a record in half and raises + `Unterminated string`. Model output reaches this log, so those codepoints do + turn up in practice. + """ + if not data: + return [] + return [json.loads(line) for line in data.split(b'\n') if line.strip()] async def run_genvm( handler: IHost, *, - timeout: float | None = None, + timeout: float | None = None, # noqa: ASYNC109 manager_uri: str = 'http://127.0.0.1:3999', + manager_client: ManagerClient, ctx: Context, is_sync: bool, debug_mode: DebugMode = 'disabled', @@ -457,371 +1150,211 @@ async def run_genvm( host_data: str = '', gas_data: dict[str, str] | None = None, host: str, - extra_args: list[str] = [], + extra_args: collections.abc.Sequence[str] = (), # default config fee buckets use bucket_no 0 and 1 bucket_totals: list[int], code: bytes | None = None, calldata: bytes, leader_nondet_results: list[bytes] | None = None, - message_fee_allocation: list[fees.MessageAllocationNode] = [], - reroute_to: str = '', - request_extra: dict[str, gvm_calldata.Encodable] = {}, + message_fee_allocation: collections.abc.Sequence[fees.MessageAllocationNode] = (), + unsafe_overrides: UnsafeOverrides | None = None, + request_extra: collections.abc.Mapping[ + str, gvm_calldata.Encodable + ] = types.MappingProxyType({}), shutdown_early: asyncio.Event | None = None, + host_hello_data: collections.abc.Sequence[bytes] = (), + major: int | None = None, ) -> RunHostAndProgramRes: logger = ctx.logger - # `node` fee constants are an ExecutionData field (no longer in host_data). + # `node` fee constants are an ExecutionData field. effective_gas_data = DEFAULT_GAS_DATA if gas_data is None else gas_data - - perf_timeline: dict[str, typing.Any] = { - 'run_started_s': time.perf_counter(), - } - genvm_id_cell: list[str | None] = [None] - status_cell: list[dict | Exception | None] = [None] - timeout_task_cell: list[asyncio.Task | None] = [None] cancellation_event = asyncio.Event() - - started_at = [time.time()] - - async def wrap_proc_body(attempt: int): + host_task = asyncio.create_task(host_loop(handler, cancellation_event, ctx=ctx)) + + client = manager_client + genvm_id: int | None = None + boot_id: int | None = None + cancel_task: asyncio.Task | None = None + terminal_task: asyncio.Task | None = None + terminal_received = False + try: max_exec_mins = 20 if timeout is not None: max_exec_mins = int(max(max_exec_mins, (timeout * 1.5 + 59) // 60)) - timestamp = message.get('datetime', '2024-11-26T06:42:42.424242Z') - - async with ( - aiohttp.request( - 'POST', - f'{manager_uri}/genvm/run', - data=gvm_calldata.encode( - { - 'major': 0, # FIXME - 'message': message, - 'is_sync': is_sync, - 'debug_mode': debug_mode, - 'host_data': host_data, - 'max_execution_minutes': max_exec_mins, # this parameter is needed to prevent zombie genvms - 'timestamp': timestamp, - 'host': host, - 'extra_args': extra_args, - 'code': code, - 'calldata': calldata, - 'leader_nondet_results': leader_nondet_results, - 'bucket_totals': bucket_totals, - 'gas_data': effective_gas_data, - 'message_fee_allocation': message_fee_allocation, - 'initial_time_units_allocation': math.ceil(timeout or 10 * 60), - # Debug-only executor-dir override; honored by the manager only - # under debug_mode >= safe (tests run with safe/unsafe). - 'reroute_to': reroute_to, - **request_extra, - } + deadline = _duration_string(timeout) + host_genvm_id = typing.cast(str | None, request_extra.get('host_genvm_id')) + if host_genvm_id is None: + host_genvm_id = f'{time.time_ns()}-{id(asyncio.current_task())}' + request_payload: dict[str, typing.Any] = { + 'selector': { + 'kind': 'major', + 'major': ( + await read_contract_major(handler, message) if major is None else major ), - timeout=_http_timeout(ctx, TimeoutAction.GenVMRun), - ) as resp - ): - logger.debug('post /genvm/run', status=resp.status, attempt=attempt) - data = await resp.json() - logger.trace('post /genvm/run', body=data) - if resp.status != 200: - logger.error('genvm manager /genvm/run failed', status=resp.status, body=data) - raise Exception(f'genvm manager /genvm/run failed: {resp.status} {data}') - else: - genvm_id = data['id'] - logger.debug('genvm manager /genvm', genvm_id=genvm_id, status=resp.status) - genvm_id_cell[0] = genvm_id - perf_timeline['genvm_id_obtained_s'] = time.perf_counter() - timeout_task_cell[0] = asyncio.ensure_future(wrap_timeout(genvm_id)) - ctx.on_genvm_success() - - async def wrap_proc(): - attempt = 0 - while True: - attempt_start = time.perf_counter() - try: - await wrap_proc_body(attempt) - ctx.add_stat( - 'manager_run_attempt_success', - { - 'attempt': attempt, - 'duration_ms': round((time.perf_counter() - attempt_start) * 1000), - }, - ) - break - except (aiohttp.ClientError, asyncio.TimeoutError) as exc: - delay = ctx.retry_delay(TimeoutAction.GenVMRun, attempt) - ctx.add_stat( - f'manager_run_attempt_{attempt}_error', - { - 'attempt': attempt, - 'error_type': type(exc).__name__, - 'duration_ms': round((time.perf_counter() - attempt_start) * 1000), - 'will_retry': delay is not None, - }, - ) - if delay is None: - logger.error( - 'genvm manager request failed after all retries', - error=str(exc), - attempt=attempt, - ) - ctx.on_genvm_failure() - cancellation_event.set() - raise - logger.warning( - 'genvm manager request failed, retrying', - error=str(exc), - attempt=attempt, - retry_delay_s=delay, - ) - except Exception: - ctx.add_stat( - f'manager_run_attempt_{attempt}_error', - { - 'attempt': attempt, - 'outcome': 'fatal_error', - 'duration_ms': round((time.perf_counter() - attempt_start) * 1000), - }, - ) - raise - finally: - if genvm_id_cell[0] is not None: - logger.debug('proc started', genvm_id=genvm_id_cell[0]) - attempt += 1 - started_at[0] = time.time() - - async def wrap_host(): - r = await host_loop(handler, cancellation_event, ctx=ctx) - logger.debug('host loop finished') - return r - - timeout_fired = asyncio.Event() - - async def wrap_timeout(genvm_id: str): - reason = 'unknown' - if timeout is None: - if shutdown_early is None: - return # nothing to do - else: - await shutdown_early.wait() - reason = 'shutdown_early event set' - else: - if shutdown_early is None: - await asyncio.sleep(timeout) - reason = 'timeout reached' - else: - try: - await asyncio.wait_for(shutdown_early.wait(), timeout=timeout) - reason = 'shutdown_early event set' - except asyncio.TimeoutError: - reason = 'timeout reached' - logger.debug('timeout reached', genvm_id=genvm_id, reason=reason) - timeout_fired.set() - await _send_timeout( - manager_uri, - genvm_id, - ctx=ctx, - ) - - poll_status_mutex = asyncio.Lock() - - async def poll_status(genvm_id: str): - async with poll_status_mutex: - old_status = status_cell[0] - if old_status is not None: - return old_status - try: - async with aiohttp.request( - 'GET', - f'{manager_uri}/genvm/{genvm_id}', - timeout=_http_timeout(ctx, TimeoutAction.GenVMGet), - ) as resp: - logger.debug('get /genvm', genvm_id=genvm_id, status=resp.status) - body = await resp.json() - logger.trace('get /genvm', genvm_id=genvm_id, body=body) - if resp.status != 200: - new_res = Exception(f'genvm manager /genvm failed: {resp.status} {body}') - elif body['status'] is None: - return None - else: - new_res = typing.cast(dict, body['status']) - except (aiohttp.ClientError, asyncio.TimeoutError) as exc: - new_res = Exception(f'genvm manager /genvm request failed: {exc}') - status_cell[0] = new_res - return new_res - - async def prob_died(): - await _await_first_cancel_others(asyncio.sleep(1), cancellation_event.wait()) - - genvm_id = genvm_id_cell[0] - if genvm_id is None: - return - status = await poll_status(genvm_id) - if status is not None and not cancellation_event.is_set(): - logger.error('genvm died without connecting', genvm_id=genvm_id, status=status) - cancellation_event.set() - - fut_host = asyncio.ensure_future(wrap_host()) - fut_proc = asyncio.ensure_future(wrap_proc()) - fut_prob = asyncio.ensure_future(prob_died()) - - # Map futures to names for debugging - task_names = { - id(fut_host): 'host_loop', - id(fut_proc): 'genvm_run', - id(fut_prob): 'prob_died', - } - - # IMPORTANT: if proc setup fails (e.g., manager accepts TCP but never replies), - # don't wait forever on host_loop. - done, pending = await asyncio.wait( - [fut_host, fut_proc, fut_prob], return_when=asyncio.FIRST_EXCEPTION - ) - - # Log which tasks completed/failed for debugging - done_names = [task_names.get(id(t), 'unknown') for t in done] - pending_names = [task_names.get(id(t), 'unknown') for t in pending] - logger.debug( - 'asyncio.wait returned', - done_tasks=done_names, - pending_tasks=pending_names, - genvm_id=genvm_id_cell[0], - ) + }, + 'message': message, + 'is_sync': is_sync, + 'debug_mode': debug_mode, + 'host_data': host_data, + 'max_execution_minutes': max_exec_mins, + 'timestamp': timestamp, + 'host': host, + 'extra_args': list(extra_args), + 'code': code, + 'calldata': calldata, + 'leader_nondet_results': leader_nondet_results, + 'bucket_totals': bucket_totals, + 'gas_data': effective_gas_data, + 'message_fee_allocation': list(message_fee_allocation), + 'initial_time_units_allocation': math.ceil(timeout or 10 * 60), + 'unsafe_overrides': (unsafe_overrides or UnsafeOverrides()).as_request_field(), + 'host_genvm_id': host_genvm_id, + 'host_hello_data': list(host_hello_data), + **request_extra, + } + if deadline is not None: + request_payload['deadline'] = deadline - # If anything errored, stop the host loop. - for task in done: - exc = task.exception() - if exc is not None: - task_name = task_names.get(id(task), 'unknown') - logger.error( - 'task raised exception', - task_name=task_name, - exception_type=type(exc).__name__, - exception_msg=str(exc), - genvm_id=genvm_id_cell[0], + attempt_start = time.perf_counter() + try: + state = await client.run(request_payload) + except Exception as exc: + if isinstance(exc, ManagerConnectionLost): + retryable, reason = True, 'manager_connection_lost' + elif isinstance(exc, ManagerSocketError): + retryable, reason = _classify_run_refusal(exc.message) + else: + retryable, reason = False, 'manager_run_error' + ctx.add_stat( + 'manager_run_attempt_0_error', + { + 'attempt': 0, + 'error_type': type(exc).__name__, + 'duration_ms': round((time.perf_counter() - attempt_start) * 1000), + 'retryable': retryable, + 'reason': reason, + }, ) + ctx.on_genvm_failure() cancellation_event.set() + raise ManagerRunNotStarted(str(exc), retryable=retryable, reason=reason) from exc + genvm_id = state.genvm_id + boot_id = state.boot_id + ctx.add_stat( + 'manager_run_attempt_success', + { + 'attempt': 0, + 'duration_ms': round((time.perf_counter() - attempt_start) * 1000), + }, + ) + logger.debug('genvm manager socket run', genvm_id=genvm_id) + ctx.on_genvm_success() + started_at = time.time() - # Cancel any pending tasks to prevent leaks - for task in pending: - task.cancel() - try: - await task - except asyncio.CancelledError: - pass - - # Cancel the timeout task if it's still pending - timeout_task = timeout_task_cell[0] - if timeout_task is not None and not timeout_task.done(): - timeout_task.cancel() - try: - await timeout_task - except asyncio.CancelledError: - pass - - # Collect exceptions from all tasks, including CancelledError - # Note: CancelledError inherits from BaseException, not Exception - exceptions: list[BaseException] = [] - cancelled_tasks: list[str] = [] - - try: - fut_host.result() - except asyncio.CancelledError: - cancelled_tasks.append('host_loop') - except ConnectionResetError as e: - if not timeout_fired.is_set(): - logger.warning('connection reset without timeout', error=e) - except BaseException as e: - if not timeout_fired.is_set(): - exceptions.append(e) - else: - logger.warning('host handler failed after timeout', error=e) + async def cancel_on_shutdown(): + assert genvm_id is not None + if shutdown_early is None: + return + await shutdown_early.wait() + logger.debug('shutdown_early event set', genvm_id=genvm_id) + await client.cancel(boot_id, genvm_id) - try: - fut_proc.result() - except asyncio.CancelledError: - cancelled_tasks.append('genvm_run') - except BaseException as e: - exceptions.append(e) - - # Log if tasks were cancelled (helps debug root cause) - if cancelled_tasks: - logger.debug( - 'tasks were cancelled', - cancelled_tasks=cancelled_tasks, - exception_count=len(exceptions), - genvm_id=genvm_id_cell[0], - ) + if shutdown_early is not None: + cancel_task = asyncio.create_task(cancel_on_shutdown()) - if len(exceptions) > 0: - # Include cancelled tasks info in the exception message for debugging - error_details = { - 'exceptions': [f'{type(e).__name__}: {e}' for e in exceptions], - 'cancelled_tasks': cancelled_tasks, - 'genvm_id': genvm_id_cell[0], - } - logger.error('genvm execution failed', **error_details) - raise Exception(f'genvm execution failed: {error_details}') from exceptions[0] - - # If all tasks were cancelled but no exceptions, something went wrong - if cancelled_tasks and len(exceptions) == 0: - error_msg = f'all genvm tasks cancelled without error: cancelled={cancelled_tasks}, genvm_id={genvm_id_cell[0]}' - logger.error(error_msg) - raise Exception(error_msg) - - genvm_id = genvm_id_cell[0] - if genvm_id is not None: - await _send_timeout( - manager_uri, - genvm_id, - ctx=ctx, - ) + terminal_task = asyncio.create_task(client.wait_terminal(state)) + while True: + done, _pending = await asyncio.wait( + [terminal_task, host_task], + return_when=asyncio.FIRST_COMPLETED, + ) + if terminal_task in done: + terminal = terminal_task.result() + terminal_received = True + break + if host_task.done(): + host_exc = host_task.exception() + if host_exc is not None: + await client.cancel(boot_id, genvm_id) + cancellation_event.set() + raise host_exc + terminal = await terminal_task + terminal_received = True + break - status = await poll_status(genvm_id) - if status is None: - exceptions.append(Exception('execution failed: no status')) - elif isinstance(status, Exception): - exceptions.append(status) - if len(exceptions) > 0: - final_exception = Exception('execution failed', exceptions[1:]) - raise final_exception from exceptions[0] - - # Result was sent to manager via consume_result, get it from status - consumed_result_raw = ( - status.get('consumed_result') if isinstance(status, dict) else None - ) - if consumed_result_raw is not None: - consumed_result_bytes = bytes(consumed_result_raw) - result_kind = public_abi.ResultCode(consumed_result_bytes[0]) - decoded = gvm_calldata.decode(consumed_result_bytes[1:]) - execution_hash = decoded.get('execution_hash', b'') - result_data = decoded.get('data') - result_fingerprint = decoded.get('fingerprint') - result_storage_changes = decoded.get('storage_changes', []) - result_emissions = decoded.get('emissions', []) - nondet_results = decoded.get('nondet_results', []) + cancellation_event.set() + if cancel_task is not None: + cancel_task.cancel() + with contextlib.suppress(BaseException): + await cancel_task + with contextlib.suppress(BaseException): + await host_task + + if 'finished' in terminal: + status = terminal['finished'] + sizes = status.get('artifact_sizes') or {} + try: + stdout = ( + (await client.get_artifact(boot_id, genvm_id, 'stdout')).decode() + if sizes.get('stdout', 0) + else '' + ) + stderr = ( + (await client.get_artifact(boot_id, genvm_id, 'stderr')).decode() + if sizes.get('stderr', 0) + else '' + ) + genvm_log = ( + _decode_genvm_log(await client.get_artifact(boot_id, genvm_id, 'genvm_log')) + if sizes.get('genvm_log', 0) + else [] + ) + consumed = ConsumedResult.decode(status.get('consumed_result')) + except Exception as exc: + # The run already executed and finished; a failure retrieving or + # decoding its result must never look like an attempt that + # never happened. That distinction is what keeps the outer + # retry loop (`run_genvm_host`) from starting a second + # execution for a result that already exists. + raise TerminalResultUnavailable( + f'failed to retrieve/decode result for finished run: {exc}', + genvm_id=genvm_id, + ) from exc + if ( + status.get('cause') == 'deadline' + and consumed.result_kind != public_abi.ResultCode.RETURN + ): + consumed.result_kind = public_abi.ResultCode.VM_ERROR + consumed.result_data = str(public_abi.VmError.timeout()) else: - execution_hash = b'' - result_kind = public_abi.ResultCode.INTERNAL_ERROR - result_data = 'no_result' - result_fingerprint = None - result_storage_changes = [] - result_emissions = [] - nondet_results = [] - - if timeout_fired.is_set() and result_kind != public_abi.ResultCode.RETURN: - result_kind = public_abi.ResultCode.VM_ERROR - result_data = str(public_abi.VmError.timeout()) + # A failed_to_start terminal means the genvm was accepted but never + # ran, so surface it through the same not-started seam as a rejected + # RUN -- never as an INTERNAL_ERROR result the caller could mistake + # for a run that executed. + error = terminal['failed_to_start']['error'] + retryable, reason = _classify_run_refusal(error) + ctx.add_stat( + 'manager_run_start_failed', + {'reason': reason, 'retryable': retryable}, + ) + # Symmetric with the client.run() rejection branch (and with the + # INTERNAL_ERROR result this used to return): a run that failed to + # start is a manager failure for health tracking. + ctx.on_genvm_failure() + raise ManagerRunNotStarted(error, retryable=retryable, reason=reason) vm_error_description: str | None = None - if result_kind == public_abi.ResultCode.VM_ERROR and isinstance(result_data, str): + if consumed.result_kind == public_abi.ResultCode.VM_ERROR and isinstance( + consumed.result_data, str + ): try: async with aiohttp.request( 'GET', f'{manager_uri}/vm-error/describe', - params={'error': result_data}, - timeout=_http_timeout(ctx, TimeoutAction.VMErrorDescribe), + params={'error': consumed.result_data}, + timeout=_http_timeout(ctx), ) as resp: if resp.status == 200: body = await resp.json() @@ -830,18 +1363,49 @@ async def prob_died(): logger.warning('failed to get vm error description', error=str(e)) return RunHostAndProgramRes( - stdout=status['stdout'], - stderr=status['stderr'], - genvm_log=status.get('genvm_log') or [], - execution_hash=execution_hash, - result_kind=result_kind, - result_data=result_data, - result_fingerprint=result_fingerprint, - result_storage_changes=result_storage_changes, - result_emissions=result_emissions, - result_nondet_results=nondet_results, + stdout=stdout, + stderr=stderr, + genvm_log=genvm_log, + metrics=status.get('metrics'), + execution_hash=consumed.execution_hash, + result_kind=consumed.result_kind, + result_data=consumed.result_data, + result_fingerprint=consumed.result_fingerprint, + result_storage_changes=consumed.result_storage_changes, + result_emissions=consumed.result_emissions, + result_nondet_results=consumed.result_nondet_results, + data_fees_remaining=consumed.data_fees_remaining, vm_error_description=vm_error_description, - execution_time=time.time() - started_at[0], + execution_time=time.time() - started_at, ) - - raise Exception('Execution failed') + finally: + if cancel_task is not None and not cancel_task.done(): + cancel_task.cancel() + with contextlib.suppress(BaseException): + await cancel_task + if terminal_task is not None and not terminal_task.done(): + terminal_task.cancel() + with contextlib.suppress(BaseException): + await terminal_task + if genvm_id is not None and terminal_received: + # ack only releases the manager's retention of a run that already + # produced its result and artifacts. Letting it fail out of here + # would turn a finished execution into an attempt error and send + # the caller back through the retry loop to run the contract again. + try: + await client.ack(boot_id, genvm_id) + except Exception as exc: + logger.warning( + 'failed to ack a finished genvm run', + genvm_id=genvm_id, + error=str(exc), + ) + elif genvm_id is not None: + with contextlib.suppress(BaseException): + await client.cancel(boot_id, genvm_id) + if not cancellation_event.is_set(): + cancellation_event.set() + if not host_task.done(): + host_task.cancel() + with contextlib.suppress(BaseException): + await host_task diff --git a/tests/runner/origin/host_fns.py b/tests/runner/origin/host_fns.py index 24024ae2..316d11e8 100644 --- a/tests/runner/origin/host_fns.py +++ b/tests/runner/origin/host_fns.py @@ -15,7 +15,8 @@ class Methods(IntEnum): REMAINING_FUEL_AS_GEN = 4 NOTIFY_NONDET_DISAGREEMENT = 5 CONSUME_RESULT = 6 - NOTIFY_FINISHED = 7 + RESOLVE_CALLCONTRACT_EXECUTOR = 7 + RUN_NESTED = 8 class Errors(IntEnum): diff --git a/tests/runner/origin/manager_api.py b/tests/runner/origin/manager_api.py new file mode 100644 index 00000000..6dbf41f3 --- /dev/null +++ b/tests/runner/origin/manager_api.py @@ -0,0 +1,31 @@ +# This file is auto-generated. Do not edit! + +# fmt: off +# ruff: noqa + +import typing +from enum import IntEnum, StrEnum + + +class Methods(IntEnum): + ERROR = 0 + HELLO = 1 + EVENT = 2 + RUN = 3 + ATTACH = 4 + CANCEL = 5 + ACK = 6 + GET_ARTIFACT = 7 + + +class Errors(IntEnum): + INTERNAL = 0 + MALFORMED_FRAME = 1 + UNKNOWN_METHOD = 2 + UNKNOWN_ID = 3 + BOOT_ID_MISMATCH = 4 + BAD_REQUEST_ID = 5 + NOT_FINISHED = 6 + + +CURRENT_MAJOR: typing.Final[int] = 0 diff --git a/tests/system/cross-major-observability/test.py b/tests/system/cross-major-observability/test.py new file mode 100644 index 00000000..99df3864 --- /dev/null +++ b/tests/system/cross-major-observability/test.py @@ -0,0 +1,674 @@ +"""Cross-major CallContract observability and determinism integration tests. + +This case reuses the real-manager harness from ``tests/system/cross-major`` +without adding more concerns to that already-large case. +""" + +import importlib.util +import json +import pickle +import sys +import typing +from dataclasses import dataclass +from pathlib import Path + +import genvm_tool.tests +import genvm_tool.tests.stage.collection +from gvm_extra.mock_host import MockStorage +from origin import public_abi +from origin.calldata import Address + + +def _load_cross_major_harness(): + path = Path(__file__).parents[1] / 'cross-major' / 'test.py' + spec = importlib.util.spec_from_file_location('cross_major_system_harness', path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +base = _load_cross_major_harness() + +ADDR_CALLER_V02 = Address('0x' + '41' * 20) +ADDR_OBSERVER_V03 = Address('0x' + '42' * 20) +ADDR_HELPER_V03 = Address('0x' + '43' * 20) +ADDR_SELF_V03 = Address('0x' + '44' * 20) +ADDR_TRAP_V03 = Address('0x' + '45' * 20) +ADDR_USER_ERROR_V03 = Address('0x' + '46' * 20) +ADDR_CHAIN_V03 = Address('0x' + '47' * 20) +ADDR_CHAIN_V02 = Address('0x' + '48' * 20) +ABSENT = Address('0x' + '99' * 20) + + +CALLER_V02 = base._contract_source( + 2, + """ + from genlayer import * + + + class Contract(gl.Contract): + def __init__(self): + pass + + @gl.public.view + def context(self, target: Address): + return gl.get_contract_at(target).view().context() + + @gl.public.view + def permissions(self, target: Address, helper: Address): + return gl.get_contract_at(target).view().permissions(helper) + + @gl.public.view + def debug_alias(self, target: Address): + return gl.get_contract_at(target).view().debug_alias() + + @gl.public.view + def read(self, target: Address) -> int: + return gl.get_contract_at(target).view().read() + + @gl.public.view + def answer(self, target: Address) -> int: + return gl.get_contract_at(target).view().answer() + """, +) + +OBSERVER_V03 = base._contract_source( + 3, + f""" + import genlayer as gl + import _genlayer_wasi as wasi + from genlayer.types import Address, u32 + + + class Contract(gl.contract.Contract): + value: u32 + + def __init__(self): + self.value = 777 + + @gl.public.view + def context(self): + return {{ + 'contract': gl.message.contract_address.as_hex, + 'sender': gl.message.sender_address.as_hex, + 'origin': gl.message.origin_address.as_hex, + 'signer': gl.message.signer_address.as_hex, + 'stack': [address.as_hex for address in gl.message.stack], + 'value': int(gl.message.value), + 'is_init': gl.message.is_init, + }} + + @gl.public.view + def read(self) -> int: + return self.value + + @gl.public.view + def permissions(self, helper: Address): + def attempt(action): + try: + value = action() + return {{'kind': 'allowed', 'value': str(value)}} + except OSError as exc: + return {{'kind': 'oserror', 'errno': exc.errno}} + except Exception as exc: + return {{'kind': type(exc).__name__, 'value': str(exc)}} + + write = attempt( + lambda: wasi.storage_write(b'\\x77' * 32, 0, b'forbidden') + ) + send = attempt( + lambda: gl.contract.get_at(Address(b'\\x66' * 20)).emit().ping() + ) + nondet = attempt( + lambda: gl.vm.run_nondet(lambda: 1, lambda _result: True) + ) + + registered = gl.vm.register_runner( + b'# {{ "Depends": "py-genlayer:{base.PY_GENLAYER_V03}" }}\\n' + b'print("registered")\\n' + ) + called = gl.contract.get_at(helper).view().answer() + return {{ + 'write': write, + 'send': send, + 'nondet': nondet, + 'registered': registered, + 'called': called, + }} + + @gl.public.view + def debug_alias(self) -> int: + result = gl.vm.spawn_sandbox(lambda: 1, runner='py-genlayer:test') + return gl.vm.unpack_result(result) + """, +) + +HELPER_V03 = base._contract_source( + 3, + """ + import genlayer as gl + + + class Contract(gl.contract.Contract): + def __init__(self): + pass + + @gl.public.view + def answer(self) -> int: + return 11 + """, +) + +SELF_V03 = base._contract_source( + 3, + """ + import genlayer as gl + + + class Contract(gl.contract.Contract): + def __init__(self): + pass + + @gl.public.view + def recurse(self, depth: int) -> int: + if depth == 0: + return 1 + return 1 + gl.contract.get_at(self.address).view().recurse(depth - 1) + + @gl.public.view + def observe_recursion(self, depth: int) -> str: + try: + return 'ok:' + str(self.recurse(depth)) + except gl.vm.UserError as exc: + return 'caught:' + str(exc.data) + """, +) + +TRAP_V03 = base._contract_source( + 3, + """ + import genlayer as gl + + + class Contract(gl.contract.Contract): + def __init__(self): + pass + + @gl.public.view + def answer(self) -> int: + raise RuntimeError('callee trap') + """, +) + +USER_ERROR_V03 = base._contract_source( + 3, + """ + import genlayer as gl + + + class Contract(gl.contract.Contract): + def __init__(self): + pass + + @gl.public.view + def answer(self) -> int: + gl.vm.UserError.immediate('nested user error') + """, +) + +CHAIN_V03 = base._contract_source( + 3, + """ + import genlayer as gl + from genlayer.types import Address + + + class Contract(gl.contract.Contract): + def __init__(self): + pass + + @gl.public.view + def hop(self, depth: int, other: Address) -> int: + if depth <= 0: + return 3 + return 3 + gl.contract.get_at(other).view().hop( + depth - 1, gl.message.contract_address + ) + """, +) + +CHAIN_V02 = base._contract_source( + 2, + """ + from genlayer import * + + + class Contract(gl.Contract): + def __init__(self): + pass + + @gl.public.view + def hop(self, depth: int, other: Address) -> int: + if depth <= 0: + return 2 + return 2 + gl.get_contract_at(other).view().hop( + depth - 1, gl.message.contract_address + ) + """, +) + + +@dataclass +class ObservabilityCase(genvm_tool.tests.test.Case): + description: genvm_tool.tests.test.Description + shared: genvm_tool.tests.SharedContext + + async def into_steps(self) -> list[genvm_tool.tests.exec.step.Step]: + return [ObservabilityStep(self)] + + +class ObservabilityStep(base.CrossMajorStep): + async def _run_all(self): + root = self.case.shared.root_dir + build_info = json.loads((root / 'build' / 'info.json').read_text()) + self.build_dir = Path(build_info['build_dir']) + self.versions = { + 2: build_info['executor_versions']['v0.2'], + 3: build_info['executor_versions']['v0.3'], + } + work_dir = self.case.shared.artifacts_dir / 'cross-major-observability' + self.storage_path = work_dir / 'storage.pickle' + work_dir.mkdir(parents=True, exist_ok=True) + with open(self.storage_path, 'wb') as storage_file: + pickle.dump(MockStorage(), storage_file) + + async with base.ManagerProc( + build_dir=self.build_dir, + work_dir=work_dir / 'manager', + ) as manager: + self.manager = manager + + self.phase = 'deploy observability fixtures' + for line, address, code in ( + (2, ADDR_CALLER_V02, CALLER_V02), + (3, ADDR_OBSERVER_V03, OBSERVER_V03), + (3, ADDR_HELPER_V03, HELPER_V03), + (3, ADDR_SELF_V03, SELF_V03), + (3, ADDR_TRAP_V03, TRAP_V03), + (3, ADDR_USER_ERROR_V03, USER_ERROR_V03), + (3, ADDR_CHAIN_V03, CHAIN_V03), + (2, ADDR_CHAIN_V02, CHAIN_V02), + ): + await self._deploy(line, address, code) + + self.phase = 'leader and validator modes are genuine' + await self._assert_validator_mode() + + self.phase = 'callee observes the preserved nested message' + await self._assert_context() + + self.phase = 'nested permissions and disabled debug are observable' + await self._assert_permissions_and_debug() + + self.phase = 'read-only roots and nested storage reads agree' + await self._assert_read_only_storage() + + self.phase = 'self-call recursion budgets agree in lvs' + await self._assert_self_recursion() + + self.phase = 'deep alternating nesting agrees in lvs' + await self._assert_deep_nesting() + + self.phase = 'contract-controlled nested errors are not internal' + await self._assert_error_propagation() + + self.phase = 'imported deterministic fuel agrees in lvs' + await self._assert_fuel() + + async def _execute( + self, + *, + permissions: str = 'wscn', + **kwargs, + ): + name = typing.cast(str, kwargs['name']) + address = typing.cast(Address, kwargs['address']) + resolve_hook = kwargs.get('resolve_hook') + read_log = kwargs.get('read_log') + host_fuel = kwargs.get('host_fuel') + host = await self._new_host( + name, + address, + resolve_hook, + read_log=read_log, + host_fuel=host_fuel, + ) + ctx = host.ctx + with host as mock_host: + try: + async with base.base_host.ManagerClient(self.manager.uri) as manager_client: + result = await base.base_host.run_genvm( + mock_host, + manager_uri=self.manager.uri, + manager_client=manager_client, + ctx=ctx, + is_sync=kwargs.get('is_sync', True), + leader_nondet_results=kwargs.get('leader_nondet_results'), + message=base._message( + address, is_init=typing.cast(bool, kwargs['is_init']) + ), + host_data='{"node_address":"test","tx_id":"cross-major-observe"}', + host='unix://' + mock_host.path, + code=kwargs.get('code'), + calldata=typing.cast(bytes, kwargs['calldata']), + timeout=kwargs.get('timeout', 30), + debug_mode='unsafe', + unsafe_overrides=base.base_host.UnsafeOverrides( + reroute_to=self.versions[typing.cast(int, kwargs['line'])], + initial_recursion=kwargs.get('debug_initial_recursion'), + ), + request_extra={ + 'permissions': permissions, + 'no_modules': True, + 'hook_cross_contract_calls': kwargs.get( + 'hook_cross_contract_calls', True + ), + }, + bucket_totals=[2**200] * 20, + ) + if ( + kwargs.get('apply_changes', True) + and result.result_kind == public_abi.ResultCode.RETURN + ): + assert mock_host.storage is not None + base._apply_storage_changes( + mock_host.storage, + address, + result.result_storage_changes, + ) + return result + finally: + await host.stop_connections() + + async def _lvs_extended(self, *, name: str, **kwargs): + async def one(suffix: str, **mode): + return await self._execute( + name=f'{name}-{suffix}', + code=None, + is_init=False, + apply_changes=False, + **kwargs, + **mode, + ) + + leader = await one('leader', is_sync=False) + validator = await one( + 'validator', + is_sync=False, + leader_nondet_results=leader.result_nondet_results, + ) + sync = await one('sync', is_sync=True) + assert leader.execution_hash == validator.execution_hash, ( + name, + leader.execution_hash.hex(), + validator.execution_hash.hex(), + ) + assert leader.execution_hash == sync.execution_hash, ( + name, + leader.execution_hash.hex(), + sync.execution_hash.hex(), + ) + for label, result in ( + ('leader', leader), + ('validator', validator), + ('sync', sync), + ): + assert result.result_kind != public_abi.ResultCode.INTERNAL_ERROR, ( + name, + label, + result, + ) + return leader, validator, sync + + def _route_v03(self, address, _state, _major): + if address in { + ADDR_OBSERVER_V03, + ADDR_SELF_V03, + ADDR_TRAP_V03, + ADDR_USER_ERROR_V03, + }: + return self._route(3) + return None + + def _route_chain(self, address, _state, _major): + if address == ADDR_CHAIN_V03: + return self._route(3) + if address == ADDR_CHAIN_V02: + return self._route(2) + return None + + async def _assert_validator_mode(self): + # A validator presented with a leader result that this deterministic call + # never consumes must reject it. This prevents l/v agreement below from + # being vacuous because both requests accidentally ran as sync. + bogus = await self._execute( + name='validator-mode-is-real', + line=3, + address=ADDR_CHAIN_V03, + calldata=base._calldata('hop', 1, ADDR_CHAIN_V02), + code=None, + is_init=False, + resolve_hook=self._route_chain, + apply_changes=False, + is_sync=False, + leader_nondet_results=[base.gvm_calldata.encode({})], + ) + assert bogus.result_kind == public_abi.ResultCode.VM_ERROR, bogus + + async def _assert_context(self): + routes: list[Address] = [] + + def resolve(address, state, major): + routes.append(address) + return self._route_v03(address, state, major) + + leader, validator, sync = await self._lvs_extended( + name='context', + line=2, + address=ADDR_CALLER_V02, + calldata=base._calldata('context', ADDR_OBSERVER_V03), + resolve_hook=resolve, + ) + expected = { + 'contract': ADDR_OBSERVER_V03.as_hex, + 'sender': base.SENDER.as_hex, + 'origin': base.SENDER.as_hex, + 'signer': base.SENDER.as_hex, + 'stack': [ADDR_CALLER_V02.as_hex], + 'value': 0, + 'is_init': False, + } + for result in (leader, validator, sync): + assert result.result_kind == public_abi.ResultCode.RETURN, result + assert result.result_data == expected, (result.result_data, expected) + assert routes == [ADDR_OBSERVER_V03] * 3, routes + self.notes.append(('nested context', expected)) + + async def _assert_permissions_and_debug(self): + leader, validator, sync = await self._lvs_extended( + name='permissions', + line=2, + address=ADDR_CALLER_V02, + calldata=base._calldata('permissions', ADDR_OBSERVER_V03, ADDR_HELPER_V03), + resolve_hook=self._route_v03, + permissions='wscnu', + ) + for result in (leader, validator, sync): + assert result.result_kind == public_abi.ResultCode.RETURN, result + observed = result.result_data + for denied in ('write', 'send', 'nondet'): + assert observed[denied]['kind'] == 'SystemError', observed + assert observed[denied]['value'] == '6: forbidden', observed + assert observed['registered'].startswith('custom:'), observed + assert observed['called'] == 11, observed + self.notes.append(('nested permission observations', leader.result_data)) + + top_level = await self._execute( + name='debug-alias-top-level', + line=3, + address=ADDR_OBSERVER_V03, + calldata=base._calldata('debug_alias'), + code=None, + is_init=False, + resolve_hook=self._route_v03, + apply_changes=False, + ) + # Unsafe mode resolves the alias and gets as far as running it. The + # deliberately mismatched sandbox payload is then an ordinary user-visible + # invalid-contract result, rather than the absent/malformed runner emitted + # before startup under disabled debug. + assert top_level.result_kind == public_abi.ResultCode.USER_ERROR, top_level + assert top_level.result_data == 'vm error: invalid_contract', top_level + + leader, validator, sync = await self._lvs_extended( + name='debug-alias-nested', + line=2, + address=ADDR_CALLER_V02, + calldata=base._calldata('debug_alias', ADDR_OBSERVER_V03), + resolve_hook=self._route_v03, + ) + for result in (leader, validator, sync): + assert result.result_kind == public_abi.ResultCode.VM_ERROR, result + assert result.result_data == str( + public_abi.VmError.invalid_contract().malformed_runner() + ), result + self.notes.append( + ( + 'nested debug observation', + ':test resolves at unsafe top level and is malformed_runner when nested', + ) + ) + + async def _assert_read_only_storage(self): + for permissions in ('c', 'wscn'): + reads: list[tuple[Address, public_abi.StorageType]] = [] + leader, validator, sync = await self._lvs_extended( + name=f'storage-{permissions}', + line=2, + address=ADDR_CALLER_V02, + calldata=base._calldata('read', ADDR_OBSERVER_V03), + resolve_hook=self._route_v03, + permissions=permissions, + read_log=reads, + ) + for result in (leader, validator, sync): + assert result.result_kind == public_abi.ResultCode.RETURN, result + assert result.result_data == 777, result + assert any(account == ADDR_OBSERVER_V03 for account, _mode in reads), reads + self.notes.append(('read-only nested storage', 'c and wscn both returned 777')) + + async def _assert_self_recursion(self): + expected_error = str(public_abi.VmError.out_of().vm_recursion()) + outcomes = [] + for budget in (0, 1, 2, 4, 5, 6): + leader, validator, sync = await self._lvs_extended( + name=f'self-recursion-{budget}', + line=3, + address=ADDR_SELF_V03, + calldata=base._calldata('observe_recursion', 4), + resolve_hook=self._route_v03, + debug_initial_recursion=budget, + ) + for result in (leader, validator, sync): + if budget < 5: + assert result.result_kind == public_abi.ResultCode.VM_ERROR, result + assert result.result_data == expected_error, ( + budget, + result.result_data, + expected_error, + ) + else: + assert result.result_kind == public_abi.ResultCode.RETURN, result + assert result.result_data == 'ok:5', (budget, result) + outcomes.append( + ( + budget, + str(leader.result_kind), + str(leader.result_data), + leader.execution_hash.hex(), + ) + ) + self.notes.append(('self recursion outcomes', outcomes)) + + async def _assert_deep_nesting(self): + outcomes = [] + for depth in (1, 4, 8): + leader, validator, sync = await self._lvs_extended( + name=f'deep-{depth}', + line=3, + address=ADDR_CHAIN_V03, + calldata=base._calldata('hop', depth, ADDR_CHAIN_V02), + resolve_hook=self._route_chain, + ) + expected = sum(3 if index % 2 == 0 else 2 for index in range(depth + 1)) + for result in (leader, validator, sync): + assert result.result_kind == public_abi.ResultCode.RETURN, result + assert result.result_data == expected, (depth, result, expected) + outcomes.append((depth, expected, leader.execution_hash.hex())) + self.notes.append(('deep alternating hashes', outcomes)) + + async def _assert_error_propagation(self): + cases = ( + ('undeployed', ABSENT), + ('non-contract', base.SENDER), + ('trap', ADDR_TRAP_V03), + ('user-error', ADDR_USER_ERROR_V03), + ) + observed = [] + for label, target in cases: + leader, validator, sync = await self._lvs_extended( + name=f'error-{label}', + line=2, + address=ADDR_CALLER_V02, + calldata=base._calldata('answer', target), + resolve_hook=lambda address, state, major, target=target: ( + self._route(3) + if address == target + else self._route_v03(address, state, major) + ), + ) + for result in (leader, validator, sync): + assert result.result_kind in { + public_abi.ResultCode.VM_ERROR, + public_abi.ResultCode.USER_ERROR, + }, (label, result) + observed.append((label, str(leader.result_kind), str(leader.result_data)[:120])) + self.notes.append(('nested error outcomes', observed)) + + async def _assert_fuel(self): + outcomes = [] + for fuel in (0, 1, 1000, 2**32): + leader, validator, sync = await self._lvs_extended( + name=f'fuel-{fuel}', + line=2, + address=ADDR_CALLER_V02, + calldata=base._calldata('read', ADDR_OBSERVER_V03), + resolve_hook=self._route_v03, + host_fuel=fuel, + ) + for result in (leader, validator, sync): + assert result.result_kind == public_abi.ResultCode.RETURN, result + assert result.result_data == 777, result + outcomes.append((fuel, leader.execution_hash.hex())) + self.notes.append(('nested deterministic fuel hashes', outcomes)) + + +def collect(ctx: genvm_tool.tests.stage.collection.Context) -> None: + desc = genvm_tool.tests.test.Description( + name='tests/system/cross-major-observability', + tags=frozenset({'integration', 'stable', 'cross-major'}), + console_pool=True, + ) + ctx.add_case(ObservabilityCase(description=desc, shared=ctx.shared)) diff --git a/tests/system/cross-major/test.py b/tests/system/cross-major/test.py new file mode 100644 index 00000000..5636ecbb --- /dev/null +++ b/tests/system/cross-major/test.py @@ -0,0 +1,1459 @@ +"""Cross-major CallContract system test. + +The case owns its manager because it exercises manager cancellation and +deadlines while executor processes are nested beneath it. +""" + +import asyncio +import contextlib +import json +import os +import pickle +import socket +import textwrap +import typing +from dataclasses import dataclass +from pathlib import Path + +import aiohttp +import genvm_tool.tests +import genvm_tool.tests.stage.collection +import origin.base_host as base_host +import origin.calldata as gvm_calldata +from gvm_extra.mock_host import MockHost, MockStorage +from origin import public_abi +from origin.calldata import Address + +PY_GENLAYER_V03 = '9b8kjyda2ycxyq4ea6g4yfpnydxhd52gqba5rb8dw7krkh5mn9p0' +PY_GENLAYER_V02 = '1jb45aa8ynh2a9c9xn3b7qqh8sm5q93hwfp7jqmwsfhh8jpz09h6' + +ADDR_CALLER_V03 = Address('0x' + '31' * 20) +ADDR_CALLER_V02 = Address('0x' + '21' * 20) +ADDR_CALLEE_V03 = Address('0x' + '32' * 20) +ADDR_CALLEE_V02 = Address('0x' + '22' * 20) +ADDR_BUSY_V02 = Address('0x' + '23' * 20) +ADDR_LOOP_V03 = Address('0x' + '33' * 20) +ADDR_LOOP_V02 = Address('0x' + '24' * 20) +ADDR_SIGNER_V03 = Address('0x' + '34' * 20) +ADDR_SIGNER_CALLER_V02 = Address('0x' + '25' * 20) +ADDR_STRUCTURED_V03 = Address('0x' + '35' * 20) +ADDR_BLOB_V03 = Address('0x' + '36' * 20) +ADDR_BLOB_CALLER_V02 = Address('0x' + '26' * 20) +ADDR_UERR_V03 = Address('0x' + '37' * 20) +ADDR_UERR_V02 = Address('0x' + '27' * 20) +ADDR_STATE_CALLER_V03 = Address('0x' + '38' * 20) +ADDR_CHAIN_V03 = Address('0x' + '39' * 20) +ADDR_CHAIN_V02 = Address('0x' + '28' * 20) + +SENDER = Address('0x' + '01' * 20) +# Larger than the manager cap the test configures below, so the nested envelope +# cannot fit through the socketpair. +"""Client frame cap for `/ws`; kept low so a nested envelope can exceed it.""" +MANAGER_MAX_MESSAGE_BYTES = 1 * 1024 * 1024 +BLOB_BYTES = 2 * 1024 * 1024 +TIMESTAMP = '2024-11-26T06:42:42.424242Z' + + +def _contract_source(line: int, body: str) -> bytes: + runner = PY_GENLAYER_V03 if line == 3 else PY_GENLAYER_V02 + return ( + f'# {{ "Depends": "py-genlayer:{runner}" }}\n' + textwrap.dedent(body).lstrip() + ).encode() + + +CALLER_V03 = _contract_source( + 3, + """ + import genlayer as gl + from genlayer.types import Address + + + class Contract(gl.contract.Contract): + def __init__(self): + pass + + @gl.public.view + def call(self, target: Address) -> int: + return gl.contract.get_at(target).view().answer() + """, +) + +CALLER_V02 = _contract_source( + 2, + """ + from genlayer import * + + + class Contract(gl.Contract): + def __init__(self): + pass + + @gl.public.view + def call(self, target: Address) -> int: + return gl.get_contract_at(target).view().answer() + """, +) + +STATE_CALLER_V03 = _contract_source( + 3, + """ + import genlayer as gl + from genlayer.types import Address + + + class Contract(gl.contract.Contract): + def __init__(self): + pass + + @gl.public.view + def call(self, target: Address, final: bool) -> int: + mode = ( + gl.vm.public_abi.StorageType.LATEST_FINAL + if final + else gl.vm.public_abi.StorageType.LATEST_NON_FINAL + ) + return gl.contract.get_at(target).view(state=mode).answer() + """, +) + +CALLEE_V03 = _contract_source( + 3, + """ + import genlayer as gl + + + class Contract(gl.contract.Contract): + def __init__(self): + pass + + @gl.public.view + def answer(self) -> int: + return 303 + """, +) + +CALLEE_V02 = _contract_source( + 2, + """ + from genlayer import * + + + class Contract(gl.Contract): + def __init__(self): + pass + + @gl.public.view + def answer(self) -> int: + return 202 + """, +) + +SIGNER_V03 = _contract_source( + 3, + """ + import genlayer as gl + + + class Contract(gl.contract.Contract): + def __init__(self): + pass + + @gl.public.view + def signer(self) -> str: + return gl.message.signer_address.as_hex + """, +) + +SIGNER_CALLER_V02 = _contract_source( + 2, + """ + from genlayer import * + + + class Contract(gl.Contract): + def __init__(self): + pass + + @gl.public.view + def signer(self, target: Address) -> str: + return gl.get_contract_at(target).view().signer() + """, +) + +STRUCTURED_ERROR_V03 = _contract_source( + 3, + """ + import genlayer as gl + + + class Contract(gl.contract.Contract): + def __init__(self): + pass + + @gl.public.view + def answer(self) -> int: + gl.vm.UserError.immediate({'code': 7}) + """, +) + +BLOB_V03 = _contract_source( + 3, + """ + import genlayer as gl + + + class Contract(gl.contract.Contract): + def __init__(self): + pass + + @gl.public.view + def sink(self, blob: str) -> int: + return len(blob) + """, +) + +BLOB_CALLER_V02 = _contract_source( + 2, + """ + from genlayer import * + + + class Contract(gl.Contract): + def __init__(self): + pass + + @gl.public.view + def send(self, target: Address, size: int) -> int: + return gl.get_contract_at(target).view().sink('x' * size) + """, +) + +USER_ERROR_V03 = _contract_source( + 3, + """ + import genlayer as gl + + + class Contract(gl.contract.Contract): + def __init__(self): + pass + + @gl.public.view + def answer(self) -> int: + gl.vm.UserError.immediate('boom') + """, +) + +USER_ERROR_V02 = _contract_source( + 2, + """ + from genlayer import * + + + class Contract(gl.Contract): + def __init__(self): + pass + + @gl.public.view + def answer(self) -> int: + raise gl.vm.UserError('boom') + """, +) + +CHAIN_V03 = _contract_source( + 3, + """ + import genlayer as gl + from genlayer.types import Address + + + class Contract(gl.contract.Contract): + def __init__(self): + pass + + @gl.public.view + def hop(self, depth: int, other: Address) -> int: + if depth <= 0: + return 3 + me = gl.message.contract_address + return 3 + gl.contract.get_at(other).view().hop(depth - 1, me) + """, +) + +CHAIN_V02 = _contract_source( + 2, + """ + from genlayer import * + + + class Contract(gl.Contract): + def __init__(self): + pass + + @gl.public.view + def hop(self, depth: int, other: Address) -> int: + if depth <= 0: + return 2 + me = gl.message.contract_address + return 2 + gl.get_contract_at(other).view().hop(depth - 1, me) + """, +) + +BUSY_V02 = _contract_source( + 2, + """ + from genlayer import * + + + class Contract(gl.Contract): + def __init__(self): + pass + + @gl.public.view + def answer(self) -> int: + while True: + pass + """, +) + +LOOP_V03 = _contract_source( + 3, + """ + import genlayer as gl + from genlayer.types import Address + + + class Contract(gl.contract.Contract): + def __init__(self): + pass + + @gl.public.view + def loop(self, remaining: int, first: Address, second: Address) -> int: + if remaining > 0: + return gl.contract.get_at(first).view().loop( + remaining - 1, first, second + ) + return gl.contract.get_at(second).view().loop(first, second) + """, +) + +LOOP_V02 = _contract_source( + 2, + """ + from genlayer import * + + + class Contract(gl.Contract): + def __init__(self): + pass + + @gl.public.view + def loop(self, first: Address, second: Address) -> int: + return gl.get_contract_at(first).view().loop(0, first, second) + """, +) + + +def _wat_data(data: bytes) -> str: + return ''.join(f'\\{byte:02x}' for byte in data) + + +def _loop_wat(target: Address) -> str: + call = gvm_calldata.encode( + { + 'CallContract': { + 'address': target, + 'calldata': {}, + 'state': public_abi.StorageType.LATEST_NON_FINAL, + } + } + ) + depth = gvm_calldata.encode({'Return': 'depth'}) + failure = gvm_calldata.encode({'UserError': 'unexpected nested result'}) + depth_offset = len(call) + failure_offset = depth_offset + len(depth) + return f""" + (module + (func $gl_call (import "genlayer_sdk" "gl_call") + (param i32 i32 i32) (result i32)) + (func $fd_read (import "wasi_snapshot_preview1" "fd_read") + (param i32 i32 i32 i32) (result i32)) + (memory (export "memory") 1) + (data (i32.const 0) "{_wat_data(call)}") + (data (i32.const {depth_offset}) "{_wat_data(depth)}") + (data (i32.const {failure_offset}) "{_wat_data(failure)}") + + (func (export "_start") + ;; The iovec at 4096 reads the child result into 8192. + (i32.store (i32.const 4096) (i32.const 8192)) + (i32.store (i32.const 4100) (i32.const 1024)) + (if + (call $gl_call + (i32.const 0) (i32.const {len(call)}) (i32.const 4108)) + (then unreachable)) + (if + (call $fd_read + (i32.load (i32.const 4108)) + (i32.const 4096) (i32.const 1) (i32.const 4112)) + (then unreachable)) + (if + (i32.or + (i32.eq (i32.load8_u (i32.const 8192)) (i32.const 0)) + (i32.eq (i32.load8_u (i32.const 8192)) (i32.const 2))) + (then + (drop + (call $gl_call + (i32.const {depth_offset}) + (i32.const {len(depth)}) + (i32.const 4108))) + unreachable) + (else + (drop + (call $gl_call + (i32.const {failure_offset}) + (i32.const {len(failure)}) + (i32.const 4108))) + unreachable) + ) + ) + ) + """ + + +class _TestContext(base_host.Context): + def __init__(self, logger: base_host.Logger): + self.logger = logger + self.stats: dict[str, typing.Any] = {} + + def on_genvm_success(self): ... + def on_genvm_failure(self): ... + + def add_stat(self, key: str, value: typing.Any, /): + self.stats[key] = value + + +class ManagerProc: + def __init__(self, *, build_dir: Path, work_dir: Path): + self.build_dir = build_dir + self.work_dir = work_dir + self.process: asyncio.subprocess.Process | None = None + self.log_file = None + self.port = self._unused_port() + + @staticmethod + def _unused_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(('127.0.0.1', 0)) + return sock.getsockname()[1] + + @property + def uri(self) -> str: + return f'http://127.0.0.1:{self.port}' + + async def __aenter__(self): + self.work_dir.mkdir(parents=True, exist_ok=True) + config_path = self.work_dir / 'genvm-manager.yaml' + config_path.write_text( + '\n'.join( + [ + 'threads: 4', + 'blocking_threads: 48', + 'log_disable: tracing*,polling*,tungstenite*,tokio_tungstenite*', + 'log_level: info', + f'manifest_path: {self.build_dir / "out" / "data" / "manifest.yaml"}', + 'permits: 2', + 'execution_retention: 30s', + f'max_message_bytes: {MANAGER_MAX_MESSAGE_BYTES}', + '', + ] + ) + ) + self.log_file = open(self.work_dir / 'manager.log', 'wb') + self.process = await asyncio.subprocess.create_subprocess_exec( + str(self.build_dir / 'out' / 'bin' / 'genvm-modules'), + 'manager', + '--port', + str(self.port), + '--die-with-parent', + '--config', + str(config_path), + stdin=asyncio.subprocess.DEVNULL, + stdout=self.log_file, + stderr=self.log_file, + start_new_session=True, + ) + await self._wait_healthy() + return self + + async def __aexit__(self, *_args): + if self.process is not None: + try: + self.process.terminate() + await asyncio.wait_for(self.process.wait(), timeout=5) + except asyncio.TimeoutError: + self.process.kill() + await self.process.wait() + if self.log_file is not None: + self.log_file.close() + + async def _wait_healthy(self): + last_error: BaseException | None = None + for _ in range(80): + if self.process is not None and self.process.returncode is not None: + raise RuntimeError(f'manager exited early with {self.process.returncode}') + try: + async with aiohttp.request( + 'GET', + f'{self.uri}/status', + timeout=aiohttp.ClientTimeout(total=1), + ) as response: + if response.status == 200: + return + except BaseException as exc: + last_error = exc + await asyncio.sleep(0.1) + raise TimeoutError(f'manager did not become healthy: {last_error}') + + +def _message(address: Address, *, is_init: bool) -> base_host.Message: + return { + 'contract_address': address, + 'sender_address': SENDER, + 'origin_address': SENDER, + 'signer_address': SENDER, + 'chain_id': 61999, + 'value': 0, + 'is_init': is_init, + 'datetime': TIMESTAMP, + } + + +def _calldata(method: str, *args: typing.Any) -> bytes: + return gvm_calldata.encode({'': method, 'args': list(args)}) + + +def _apply_storage_changes( + storage: MockStorage, + address: Address, + changes: list[tuple[bytes, bytes]], +) -> None: + for key, value in changes: + assert len(key) == 36 + storage.write( + address, + key[:32], + int.from_bytes(key[32:], byteorder='big') * 32, + value, + ) + + +def _descendants(pid: int) -> set[int]: + children: dict[int, set[int]] = {} + for stat_path in Path('/proc').glob('[0-9]*/stat'): + try: + raw = stat_path.read_text() + parent = int(raw[raw.rfind(')') + 2 :].split()[1]) + child = int(stat_path.parent.name) + except (FileNotFoundError, ProcessLookupError, ValueError): + continue + children.setdefault(parent, set()).add(child) + result: set[int] = set() + pending = [pid] + while pending: + parent = pending.pop() + for child in children.get(parent, ()): + if child not in result: + result.add(child) + pending.append(child) + return result + + +async def _wait_for_descendants( + pid: int, expected: int, timeout: float = 10 +) -> set[int]: + async with asyncio.timeout(timeout): + while True: + current = _descendants(pid) + if len(current) >= expected: + return current + await asyncio.sleep(0.05) + + +async def _wait_for_no_descendants(pid: int, timeout: float = 10) -> None: + async with asyncio.timeout(timeout): + while _descendants(pid): + await asyncio.sleep(0.05) + + +@dataclass +class CrossMajorCase(genvm_tool.tests.test.Case): + description: genvm_tool.tests.test.Description + shared: genvm_tool.tests.SharedContext + + async def into_steps(self) -> list[genvm_tool.tests.exec.step.Step]: + return [CrossMajorStep(self)] + + +class CrossMajorStep(genvm_tool.tests.exec.step.Python): + def __init__(self, case: CrossMajorCase): + self.case = case + self.phase = 'start' + self.notes: list[typing.Any] = [] + + def to_str(self) -> str: + return '<cross-major CallContract system test>' + + async def run( + self, previous_results: list[typing.Any] + ) -> genvm_tool.tests.test.Result: + try: + await self._run_all() + return genvm_tool.tests.test.Result( + passed=True, context={'notes': self.notes}, elapsed_seconds=0 + ) + except BaseException as exc: + return genvm_tool.tests.test.Result( + passed=False, + context={'phase': self.phase, 'error': repr(exc)}, + elapsed_seconds=0, + ) + + async def _run_all(self): + root = self.case.shared.root_dir + build_info = json.loads((root / 'build' / 'info.json').read_text()) + self.build_dir = Path(build_info['build_dir']) + self.versions = { + 2: build_info['executor_versions']['v0.2'], + 3: build_info['executor_versions']['v0.3'], + } + work_dir = self.case.shared.artifacts_dir / 'cross-major' + self.storage_path = work_dir / 'storage.pickle' + work_dir.mkdir(parents=True, exist_ok=True) + with open(self.storage_path, 'wb') as storage_file: + pickle.dump(MockStorage(), storage_file) + + async with ManagerProc( + build_dir=self.build_dir, + work_dir=work_dir / 'manager', + ) as manager: + self.manager = manager + + self.phase = 'deploy fixtures' + await self._deploy(3, ADDR_CALLER_V03, CALLER_V03) + await self._deploy(2, ADDR_CALLER_V02, CALLER_V02) + await self._deploy(3, ADDR_CALLEE_V03, CALLEE_V03) + await self._deploy(2, ADDR_CALLEE_V02, CALLEE_V02) + await self._deploy(2, ADDR_BUSY_V02, BUSY_V02) + await self._deploy(3, ADDR_LOOP_V03, LOOP_V03) + await self._deploy(2, ADDR_LOOP_V02, LOOP_V02) + await self._deploy(3, ADDR_SIGNER_V03, SIGNER_V03) + await self._deploy(2, ADDR_SIGNER_CALLER_V02, SIGNER_CALLER_V02) + await self._deploy(3, ADDR_STRUCTURED_V03, STRUCTURED_ERROR_V03) + await self._deploy(3, ADDR_BLOB_V03, BLOB_V03) + await self._deploy(2, ADDR_BLOB_CALLER_V02, BLOB_CALLER_V02) + await self._deploy(3, ADDR_UERR_V03, USER_ERROR_V03) + await self._deploy(2, ADDR_UERR_V02, USER_ERROR_V02) + await self._deploy(3, ADDR_STATE_CALLER_V03, STATE_CALLER_V03) + await self._deploy(3, ADDR_CHAIN_V03, CHAIN_V03) + await self._deploy(2, ADDR_CHAIN_V02, CHAIN_V02) + loop_v03 = await self._compile_wat( + _loop_wat(ADDR_LOOP_V02), work_dir / 'loop-v03' + ) + loop_v02 = await self._compile_wat( + _loop_wat(ADDR_LOOP_V03), work_dir / 'loop-v02' + ) + self._replace_code(ADDR_LOOP_V03, LOOP_V03, loop_v03) + self._replace_code(ADDR_LOOP_V02, LOOP_V02, loop_v02) + + self.phase = 'v0.3 to v0.2' + forward = await self._call( + 3, + ADDR_CALLER_V03, + ADDR_CALLEE_V02, + lambda address, _state, _major: ( + self._route(2) if address == ADDR_CALLEE_V02 else None + ), + ) + assert forward.result_kind == public_abi.ResultCode.RETURN, forward + assert forward.result_data == 202, forward + assert len(forward.execution_hash) == 32, forward.execution_hash + assert forward.execution_hash.hex() == ( + 'c0e5932da9b2e9bf79579079f1fec46f74b83f02114bcd2f711a6798ee20bdd6' + ), forward.execution_hash.hex() + + self.phase = 'v0.2 to v0.3' + backward = await self._call( + 2, + ADDR_CALLER_V02, + ADDR_CALLEE_V03, + lambda address, _state, _major: ( + self._route(3) if address == ADDR_CALLEE_V03 else None + ), + ) + assert backward.result_kind == public_abi.ResultCode.RETURN, backward + assert backward.result_data == 303, backward + assert len(backward.execution_hash) == 32, backward.execution_hash + assert backward.execution_hash.hex() == ( + '6aad898ed66c94e7bc9fc1f6bb7d615b9b4cf01d1e1205ca0c640712a8c6792d' + ), backward.execution_hash.hex() + + self.phase = 'v0.2 forwards the signer it never exposes' + await self._assert_signer_preserved() + + self.phase = 'a nested envelope is not bounded by the websocket cap' + await self._assert_envelope_ignores_websocket_cap() + + self.phase = 'a structured user error survives the boundary' + await self._assert_structured_user_error() + + self.phase = 'null preserves same-major behavior' + resolved: list[tuple[Address, int]] = [] + + def resolve_null(address, _state, advisory_major): + resolved.append((address, advisory_major)) + return None + + same_major = await self._call( + 3, + ADDR_CALLER_V03, + ADDR_CALLEE_V03, + resolve_null, + ) + assert same_major.result_kind == public_abi.ResultCode.RETURN, same_major + assert same_major.result_data == 303, same_major + assert resolved == [(ADDR_CALLEE_V03, 0)], resolved + + self.phase = 'null preserves mismatch behavior' + mismatch = await self._call( + 3, + ADDR_CALLER_V03, + ADDR_CALLEE_V02, + lambda _address, _state, _major: None, + ) + assert mismatch.result_kind == public_abi.ResultCode.VM_ERROR, mismatch + assert str(mismatch.result_data).startswith( + str(public_abi.VmError.invalid_contract().absent_runner_comment()) + ), mismatch.result_data + + self.phase = 'a host that does not hook is never asked' + await self._assert_unhooked_stays_in_process() + + self.phase = 'leader, validator and sync agree on the nested hash' + await self._assert_lvs_agrees() + + self.phase = 'the route a host picks does not change the hash' + await self._assert_route_is_hash_invariant() + + self.phase = 'a plain user error survives the boundary' + await self._assert_plain_user_error() + + self.phase = 'the state mode a caller picks crosses the boundary' + await self._assert_state_mode_crosses() + + self.phase = 'a deep alternating chain stays deterministic' + await self._assert_deep_chain() + + self.phase = 'a starved fuel budget never becomes an internal error' + await self._assert_starved_fuel() + + self.phase = 'a contract cannot manufacture an internal error' + await self._assert_child_failures_are_canonical() + + self.phase = 'cancel kills process chain' + await self._assert_chain_stopped('cancel') + + self.phase = 'deadline kills process chain' + await self._assert_chain_stopped('deadline') + + self.phase = 'a nested chain that runs out of time reports timeout' + await self._assert_nested_deadline_is_timeout() + + self.phase = 'resolve loop bottoms out on the recursion budget' + await self._assert_resolve_loop_recursion() + + async def _new_host( + self, + name: str, + running_address: Address, + resolve_hook=None, + read_log: list[tuple[Address, public_abi.StorageType]] | None = None, + host_fuel: int | None = None, + ) -> MockHost: + ctx = _TestContext(self.case.shared.logger) + host_path = ( + Path('/tmp') + / f'gvm-cross-major-{os.getpid()}' + / f'{name[:20]}-{os.urandom(4).hex()}.sock' + ) + host_path.parent.mkdir(parents=True, exist_ok=True) + host = MockHost( + ctx=ctx, + path=str(host_path), + storage_path_pre=self.storage_path, + storage_path_post=self.storage_path, + balances={}, + running_address=running_address, + resolve_callcontract_executor_hook=resolve_hook, + ) + if read_log is not None: + original = host.storage_read + + async def logged(mode, account, slot, index, le, /): + read_log.append((Address(account), mode)) + return await original(mode, account, slot, index, le) + + host.storage_read = logged # type: ignore[method-assign] + if host_fuel is not None: + + async def remaining_fuel_as_gen() -> int: + return host_fuel + + host.remaining_fuel_as_gen = remaining_fuel_as_gen # type: ignore[method-assign] + return host + + async def _execute( + self, + *, + name: str, + line: int, + address: Address, + calldata: bytes, + code: bytes | None, + is_init: bool, + resolve_hook=None, + timeout: float = 30, + debug_initial_recursion: int | None = None, + is_sync: bool = True, + leader_nondet_results: list[bytes] | None = None, + apply_changes: bool = True, + read_log: list[tuple[Address, public_abi.StorageType]] | None = None, + host_fuel: int | None = None, + hook_cross_contract_calls: bool = True, + ): + host = await self._new_host( + name, address, resolve_hook, read_log=read_log, host_fuel=host_fuel + ) + ctx = host.ctx + with host as mock_host: + try: + async with base_host.ManagerClient(self.manager.uri) as manager_client: + result = await base_host.run_genvm( + mock_host, + manager_uri=self.manager.uri, + manager_client=manager_client, + ctx=ctx, + is_sync=is_sync, + leader_nondet_results=leader_nondet_results, + message=_message(address, is_init=is_init), + host_data='{"node_address":"test","tx_id":"cross-major"}', + host='unix://' + mock_host.path, + code=code, + calldata=calldata, + timeout=timeout, + debug_mode='unsafe', + unsafe_overrides=base_host.UnsafeOverrides( + reroute_to=self.versions[line], + initial_recursion=debug_initial_recursion, + ), + request_extra={ + 'no_modules': True, + 'hook_cross_contract_calls': hook_cross_contract_calls, + }, + bucket_totals=[2**200] * 20, + ) + if apply_changes and result.result_kind == public_abi.ResultCode.RETURN: + assert mock_host.storage is not None + _apply_storage_changes( + mock_host.storage, + address, + result.result_storage_changes, + ) + return result + finally: + await host.stop_connections() + + async def _deploy(self, line: int, address: Address, code: bytes): + result = await self._execute( + name=f'deploy-{line}-{address.as_bytes.hex()}', + line=line, + address=address, + calldata=gvm_calldata.encode({}), + code=code, + is_init=True, + ) + assert result.result_kind == public_abi.ResultCode.RETURN, result + assert len(result.execution_hash) == 32, result.execution_hash + + async def _compile_wat(self, wat: str, path: Path) -> bytes: + path.mkdir(parents=True, exist_ok=True) + wat_path = path / 'contract.wat' + wasm_path = path / 'contract.wasm' + wat_path.write_text(wat) + process = await asyncio.create_subprocess_exec( + 'wat2wasm', + '--enable-annotations', + '-o', + str(wasm_path), + str(wat_path), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, stderr = await process.communicate() + assert process.returncode == 0, (stdout, stderr) + return wasm_path.read_bytes() + + def _replace_code(self, address: Address, old_code: bytes, new_code: bytes) -> None: + with open(self.storage_path, 'rb') as storage_file: + storage: MockStorage = pickle.load(storage_file) + slots = storage._storages[address] + code_slots = [ + slot + for slot, data in slots.items() + if int.from_bytes(data[:4], byteorder='little') == len(old_code) + and bytes(data[4 : 4 + len(old_code)]) == old_code + ] + assert len(code_slots) == 1, code_slots + storage.write( + address, + code_slots[0], + 0, + len(new_code).to_bytes(4, byteorder='little') + new_code, + ) + with open(self.storage_path, 'wb') as storage_file: + pickle.dump(storage, storage_file) + + def _route(self, line: int) -> bytes: + """Routing payload placing a callee on `line`'s executor. + + It names the directory rather than the major, because a major cannot + pick between the live lines: every manifest key is semver major 0, so + `{'kind': 'major'}` resolves to the newest line whatever it asks for. + """ + return gvm_calldata.encode({'kind': 'version', 'version': self.versions[line]}) + + async def _call(self, line: int, caller: Address, target: Address, resolve_hook): + return await self._execute( + name=f'call-{line}-{caller.as_bytes.hex()}-{target.as_bytes.hex()}', + line=line, + address=caller, + calldata=_calldata('call', target), + code=None, + is_init=False, + resolve_hook=resolve_hook, + ) + + def _raw_request( + self, + *, + line: int, + address: Address, + host_path: str, + deadline: str, + ) -> dict[str, typing.Any]: + return { + 'selector': {'kind': 'major', 'major': base_host.UNDEPLOYED_MAJOR}, + 'message': _message(address, is_init=False), + 'is_sync': True, + 'debug_mode': 'unsafe', + 'host_data': '{"node_address":"test","tx_id":"cross-major-lifecycle"}', + 'max_execution_minutes': 20, + 'timestamp': TIMESTAMP, + 'host': 'unix://' + host_path, + 'extra_args': [], + 'code': None, + 'calldata': _calldata('call', ADDR_BUSY_V02), + 'leader_nondet_results': None, + 'bucket_totals': [2**200] * 20, + 'gas_data': base_host.DEFAULT_GAS_DATA, + 'message_fee_allocation': [], + 'initial_time_units_allocation': 60, + 'no_modules': True, + 'hook_cross_contract_calls': True, + 'unsafe_overrides': base_host.UnsafeOverrides( + reroute_to=self.versions[line] + ).as_request_field(), + 'deadline': deadline, + 'host_hello_data': [], + } + + async def _assert_chain_stopped(self, mode: typing.Literal['cancel', 'deadline']): + nested_resolved = asyncio.Event() + + def resolve(address, _state, _major): + if address == ADDR_BUSY_V02: + nested_resolved.set() + return self._route(2) + return None + + host = await self._new_host(f'lifecycle-{mode}', ADDR_CALLER_V03, resolve) + cancellation = asyncio.Event() + assert self.manager.process is not None + manager_pid = self.manager.process.pid + with host as mock_host: + host_task = asyncio.create_task( + base_host.host_loop(mock_host, cancellation, ctx=host.ctx) + ) + try: + async with base_host.ManagerClient(self.manager.uri) as client: + state = await client.run( + self._raw_request( + line=3, + address=ADDR_CALLER_V03, + host_path=mock_host.path, + deadline='30s' if mode == 'cancel' else '3s', + ) + ) + await asyncio.wait_for(nested_resolved.wait(), timeout=10) + await _wait_for_descendants(manager_pid, 2) + if mode == 'cancel': + await client.cancel(state.boot_id, state.genvm_id) + terminal = await asyncio.wait_for(client.wait_terminal(state), timeout=10) + assert terminal['finished']['cause'] == ( + 'cancelled' if mode == 'cancel' else 'deadline' + ), terminal + await _wait_for_no_descendants(manager_pid) + await client.ack(state.boot_id, state.genvm_id) + finally: + cancellation.set() + with contextlib.suppress(BaseException): + await host_task + await host.stop_connections() + + async def _assert_signer_preserved(self): + """This line's contract-facing message has no signer, but the executor + still receives one and must hand it to a callee that does expose it.""" + result = await self._execute( + name='signer-v02-to-v03', + line=2, + address=ADDR_SIGNER_CALLER_V02, + calldata=_calldata('signer', ADDR_SIGNER_V03), + code=None, + is_init=False, + resolve_hook=lambda address, _state, _major: ( + self._route(3) if address == ADDR_SIGNER_V03 else None + ), + ) + assert result.result_kind == public_abi.ResultCode.RETURN, result + assert result.result_data == SENDER.as_hex, (result.result_data, SENDER.as_hex) + + async def _assert_structured_user_error(self): + """A callee's user error is contract data, so it must stay contract-visible + on the caller. v0.2 user errors are strings, so a structured payload cannot + be carried across as one and degrades to the vm error naming the boundary + that could not represent it β€” never an internal error.""" + result = await self._execute( + name='structured-error-v02-to-v03', + line=2, + address=ADDR_CALLER_V02, + calldata=_calldata('call', ADDR_STRUCTURED_V03), + code=None, + is_init=False, + resolve_hook=lambda address, _state, _major: ( + self._route(3) if address == ADDR_STRUCTURED_V03 else None + ), + ) + assert result.result_kind == public_abi.ResultCode.VM_ERROR, result + expected = str(public_abi.VmError.invalid_contract().major_mismatch()) + assert result.result_data == expected, (result.result_data, expected) + + async def _assert_envelope_ignores_websocket_cap(self): + """A contract picks the calldata it forwards, so nothing an operator + configures may decide whether the call goes through. This manager runs + with `max_message_bytes` deliberately below the envelope this builds: the + cap bounds client frames on `/ws`, never the executor socketpair.""" + assert BLOB_BYTES > MANAGER_MAX_MESSAGE_BYTES + result = await self._execute( + name='large-envelope-v02-to-v03', + line=2, + address=ADDR_BLOB_CALLER_V02, + calldata=_calldata('send', ADDR_BLOB_V03, BLOB_BYTES), + code=None, + is_init=False, + resolve_hook=lambda address, _state, _major: ( + self._route(3) if address == ADDR_BLOB_V03 else None + ), + timeout=60, + ) + assert result.result_kind == public_abi.ResultCode.RETURN, result + assert result.result_data == BLOB_BYTES, result + + async def _assert_nested_deadline_is_timeout(self): + """The chain dies on a signal, so no executor reports anything. What the + caller must still observe is a timeout, not the internal error the dead + boundary would otherwise produce.""" + result = await self._execute( + name='deadline-v03-to-v02', + line=3, + address=ADDR_CALLER_V03, + calldata=_calldata('call', ADDR_BUSY_V02), + code=None, + is_init=False, + resolve_hook=lambda address, _state, _major: ( + self._route(2) if address == ADDR_BUSY_V02 else None + ), + timeout=10, + ) + assert result.result_kind == public_abi.ResultCode.VM_ERROR, result + assert result.result_data == str(public_abi.VmError.timeout()), result + + async def _lvs( + self, *, name: str, line: int, address: Address, calldata: bytes, resolve_hook + ): + """Run the same step as leader, validator and sync run. + + The three must agree on the execution hash; anything else is a + determinism violation an honest validator would see as a disagreement. + """ + + async def one(suffix: str, **kwargs): + return await self._execute( + name=f'{name}-{suffix}', + line=line, + address=address, + calldata=calldata, + code=None, + is_init=False, + resolve_hook=resolve_hook, + apply_changes=False, + **kwargs, + ) + + leader = await one('leader', is_sync=False) + validator = await one( + 'validator', is_sync=False, leader_nondet_results=leader.result_nondet_results + ) + sync = await one('sync', is_sync=True) + return leader, validator, sync + + async def _assert_lvs_agrees(self): + # A leader/validator run that silently degraded to a sync one would make + # the agreement below vacuous, so first pin that the validator mode is + # real: handing it a leader result nothing consumes must be rejected. + bogus = await self._execute( + name='lvs-mode-is-real', + line=3, + address=ADDR_CALLER_V03, + calldata=_calldata('call', ADDR_CALLEE_V02), + code=None, + is_init=False, + resolve_hook=lambda address, _state, _major: ( + self._route(2) if address == ADDR_CALLEE_V02 else None + ), + apply_changes=False, + is_sync=False, + leader_nondet_results=[gvm_calldata.encode({})], + ) + assert bogus.result_kind == public_abi.ResultCode.VM_ERROR, bogus + + for suffix, line, caller, callee, major in ( + ('fwd', 3, ADDR_CALLER_V03, ADDR_CALLEE_V02, 2), + ('bwd', 2, ADDR_CALLER_V02, ADDR_CALLEE_V03, 3), + ): + target = callee + leader, validator, sync = await self._lvs( + name=f'lvs-{suffix}', + line=line, + address=caller, + calldata=_calldata('call', callee), + resolve_hook=lambda address, _state, _major, target=target, major=major: ( + self._route(major) if address == target else None + ), + ) + for label, run in (('leader', leader), ('validator', validator), ('sync', sync)): + assert run.result_kind == public_abi.ResultCode.RETURN, (suffix, label, run) + assert len(run.execution_hash) == 32, (suffix, label, run.execution_hash) + assert leader.execution_hash == validator.execution_hash, ( + suffix, + leader.execution_hash.hex(), + validator.execution_hash.hex(), + ) + assert leader.execution_hash == sync.execution_hash, ( + suffix, + leader.execution_hash.hex(), + sync.execution_hash.hex(), + ) + + async def _assert_unhooked_stays_in_process(self): + """Without `hook_cross_contract_calls` the manager answers the resolve + method itself, so a host is never consulted even when the callee really + does live on another major, and the call stays in-process.""" + asked: list[Address] = [] + + def resolve(address, _state, _major): + asked.append(address) + return self._route(2) + + result = await self._execute( + name='unhooked-cross-major', + line=3, + address=ADDR_CALLER_V03, + calldata=_calldata('call', ADDR_CALLEE_V02), + code=None, + is_init=False, + resolve_hook=resolve, + apply_changes=False, + hook_cross_contract_calls=False, + ) + assert asked == [], asked + # The in-process outcome for a v0.2 callee, identical to answering null. + assert result.result_kind == public_abi.ResultCode.VM_ERROR, result + assert str(result.result_data).startswith( + str(public_abi.VmError.invalid_contract().absent_runner_comment()) + ), result.result_data + + async def _assert_route_is_hash_invariant(self): + """A host picks the route, a contract does not. A same-major call that a + host chooses to send through the nested boundary must hash exactly as the + in-process one, otherwise the route is consensus-visible.""" + in_process = await self._execute( + name='route-in-process', + line=3, + address=ADDR_CALLER_V03, + calldata=_calldata('call', ADDR_CALLEE_V03), + code=None, + is_init=False, + resolve_hook=lambda _address, _state, _major: None, + apply_changes=False, + ) + nested = await self._execute( + name='route-nested', + line=3, + address=ADDR_CALLER_V03, + calldata=_calldata('call', ADDR_CALLEE_V03), + code=None, + is_init=False, + resolve_hook=lambda address, _state, _major: ( + self._route(3) if address == ADDR_CALLEE_V03 else None + ), + apply_changes=False, + ) + assert in_process.result_kind == public_abi.ResultCode.RETURN, in_process + assert nested.result_kind == public_abi.ResultCode.RETURN, nested + assert in_process.result_data == nested.result_data == 303, (in_process, nested) + # The parent's own execution hash folds the callee's small hash, which + # commits to the outcome and to nothing about where the callee ran. + assert in_process.execution_hash == nested.execution_hash, ( + 'the route is consensus-visible', + in_process.execution_hash.hex(), + nested.execution_hash.hex(), + ) + + async def _assert_plain_user_error(self): + """A string user error is representable on both lines, so it must reach + the caller as a user error in either direction.""" + for suffix, line, caller, callee, major in ( + ('v02-to-v03', 2, ADDR_CALLER_V02, ADDR_UERR_V03, 3), + ('v03-to-v02', 3, ADDR_CALLER_V03, ADDR_UERR_V02, 2), + ): + target = callee + result = await self._execute( + name=f'plain-error-{suffix}', + line=line, + address=caller, + calldata=_calldata('call', callee), + code=None, + is_init=False, + resolve_hook=lambda address, _state, _major, target=target, major=major: ( + self._route(major) if address == target else None + ), + apply_changes=False, + ) + assert result.result_kind == public_abi.ResultCode.USER_ERROR, (suffix, result) + + async def _assert_state_mode_crosses(self): + """The caller's state mode selects which chain view the callee reads, so + the boundary must hand the callee exactly the mode the in-process route + would have used.""" + for final, expected in ( + (False, public_abi.StorageType.LATEST_NON_FINAL), + (True, public_abi.StorageType.LATEST_FINAL), + ): + modes: dict[bool, set[public_abi.StorageType]] = {} + for nested in (False, True): + resolved: list[public_abi.StorageType] = [] + read_log: list[tuple[Address, public_abi.StorageType]] = [] + + def resolve(address, state, _major, nested=nested, resolved=resolved): + if address != ADDR_CALLEE_V03: + return None + resolved.append(state) + return self._route(3) if nested else None + + result = await self._execute( + name=f'state-mode-{"final" if final else "nonfinal"}-{nested}', + line=3, + address=ADDR_STATE_CALLER_V03, + calldata=_calldata('call', ADDR_CALLEE_V03, final), + code=None, + is_init=False, + resolve_hook=resolve, + apply_changes=False, + read_log=read_log, + ) + assert result.result_kind == public_abi.ResultCode.RETURN, ( + final, + nested, + result, + ) + assert result.result_data == 303, (final, nested, result) + assert resolved == [expected], (final, nested, resolved) + modes[nested] = { + mode for account, mode in read_log if account == ADDR_CALLEE_V03 + } + assert modes[False] == modes[True], (final, modes) + assert modes[True] == {expected}, (final, modes) + + def _chain_resolve(self, address, _state, _major): + if address == ADDR_CHAIN_V03: + return self._route(3) + if address == ADDR_CHAIN_V02: + return self._route(2) + return None + + async def _assert_deep_chain(self): + """Every hop of an alternating chain crosses the boundary, so the whole + chain is a stack of nested processes. Its answer and its hash must not + depend on which of the three runs produced them.""" + depth = 6 + leader, validator, sync = await self._lvs( + name='deep-chain', + line=3, + address=ADDR_CHAIN_V03, + calldata=_calldata('hop', depth, ADDR_CHAIN_V02), + resolve_hook=self._chain_resolve, + ) + # The chain alternates v0.3, v0.2, ..., so it sums 3 and 2 in turn. + expected = sum(3 if index % 2 == 0 else 2 for index in range(depth + 1)) + for label, run in (('leader', leader), ('validator', validator), ('sync', sync)): + assert run.result_kind == public_abi.ResultCode.RETURN, (label, run) + assert run.result_data == expected, (label, run.result_data, expected) + assert leader.execution_hash == validator.execution_hash, ( + leader.execution_hash.hex(), + validator.execution_hash.hex(), + ) + assert leader.execution_hash == sync.execution_hash, ( + leader.execution_hash.hex(), + sync.execution_hash.hex(), + ) + + async def _assert_starved_fuel(self): + """The caller hands the callee whatever fuel the host says is left. A + callee that cannot finish on it must report a canonical vm error, and two + hosts reporting the same figure must agree on the outcome.""" + outcomes = [] + for fuel in (0, 1, 1000, 10**6): + result = await self._execute( + name=f'starved-fuel-{fuel}', + line=3, + address=ADDR_CALLER_V03, + calldata=_calldata('call', ADDR_CALLEE_V02), + code=None, + is_init=False, + resolve_hook=lambda address, _state, _major: ( + self._route(2) if address == ADDR_CALLEE_V02 else None + ), + apply_changes=False, + host_fuel=fuel, + ) + assert result.result_kind != public_abi.ResultCode.INTERNAL_ERROR, (fuel, result) + outcomes.append((fuel, result.result_kind, result.result_data)) + repeat = await self._execute( + name=f'starved-fuel-{fuel}-again', + line=3, + address=ADDR_CALLER_V03, + calldata=_calldata('call', ADDR_CALLEE_V02), + code=None, + is_init=False, + resolve_hook=lambda address, _state, _major: ( + self._route(2) if address == ADDR_CALLEE_V02 else None + ), + apply_changes=False, + host_fuel=fuel, + ) + assert repeat.execution_hash == result.execution_hash, ( + fuel, + result.execution_hash.hex(), + repeat.execution_hash.hex(), + ) + self.notes.append(('starved fuel outcomes', [str(entry) for entry in outcomes])) + + async def _assert_child_failures_are_canonical(self): + """A contract must not be able to manufacture an + internal error. Every failure a contract can pick the input for β€” a + method the callee lacks, an address nothing was ever deployed to, a + callee routed to a line its stored major disagrees with β€” must land on a + user or vm error instead.""" + absent = Address('0x' + '99' * 20) + cases = ( + # The callee has `sink`, not `answer`. + ('missing-method', 2, ADDR_CALLER_V02, ADDR_BLOB_V03, 3), + ('absent-callee', 3, ADDR_CALLER_V03, absent, 2), + # The callee's stored major is 2; the host routes it to v0.3 anyway. + ('routed-to-wrong-line', 3, ADDR_CALLER_V03, ADDR_CALLEE_V02, 3), + ) + observed = [] + for label, line, caller, callee, major in cases: + target = callee + result = await self._execute( + name=f'canonical-{label}', + line=line, + address=caller, + calldata=_calldata('call', callee), + code=None, + is_init=False, + resolve_hook=lambda address, _state, _major, target=target, major=major: ( + self._route(major) if address == target else None + ), + apply_changes=False, + ) + assert result.result_kind != public_abi.ResultCode.INTERNAL_ERROR, (label, result) + observed.append((label, str(result.result_kind), str(result.result_data)[:80])) + self.notes.append(('child failure outcomes', [str(entry) for entry in observed])) + + async def _assert_resolve_loop_recursion(self): + resolved: list[tuple[Address, int]] = [] + + def resolve(address, _state, advisory_major): + resolved.append((address, advisory_major)) + if address == ADDR_LOOP_V02: + return self._route(2) + if address == ADDR_LOOP_V03: + return self._route(3) + return None + + result = await self._execute( + name='depth-loop', + line=2, + address=ADDR_LOOP_V02, + calldata=gvm_calldata.encode({}), + code=None, + is_init=False, + resolve_hook=resolve, + timeout=30, + # One executor process per unit of budget, so the real limit would + # cost 511 spawns to reach; this hits the same code path in two. + debug_initial_recursion=2, + ) + assert result.result_kind == public_abi.ResultCode.VM_ERROR, ( + result, + len(resolved), + resolved[-3:], + ) + assert result.result_data == str(public_abi.VmError.out_of().vm_recursion()), result + assert len(resolved) == 2, len(resolved) + # Neither line can report a meaningful major: v0.3 forwards the raw root + # byte and v0.2 has no such byte at all, so both send the unwritten + # default and the host decides on the address alone. + assert resolved == [ + (ADDR_LOOP_V03, 0), + (ADDR_LOOP_V02, 0), + ], resolved + + +def collect(ctx: genvm_tool.tests.stage.collection.Context) -> None: + desc = genvm_tool.tests.test.Description( + name='tests/system/cross-major', + tags=frozenset({'integration', 'stable', 'cross-major'}), + console_pool=True, + ) + ctx.add_case(CrossMajorCase(description=desc, shared=ctx.shared)) diff --git a/tests/system/manager-socket/test.py b/tests/system/manager-socket/test.py new file mode 100644 index 00000000..a489cc41 --- /dev/null +++ b/tests/system/manager-socket/test.py @@ -0,0 +1,1034 @@ +import asyncio +import contextlib +import json +import os +import pickle +import socket +import stat +import typing +from dataclasses import dataclass +from pathlib import Path + +import aiohttp +import genvm_tool.tests +import genvm_tool.tests.stage.collection +import origin.base_host as base_host +import origin.calldata as gvm_calldata +from gvm_extra.mock_host import MockHost, MockStorage +from origin.calldata import Address +from origin.manager_api import CURRENT_MAJOR, Errors, Methods + +HELLO_WORLD = Path( + 'executors/v0.3.x/tests/integration/hello-world/hello_world/hello_world_trivial.py' +) +BUSY_CONTRACT = Path('tests/system/permits/busy_contract.py') + +FAKE_MESSAGE: base_host.Message = { + 'contract_address': Address('0x' + '00' * 20), + 'sender_address': Address('0x' + '01' * 20), + 'origin_address': Address('0x' + '01' * 20), + 'signer_address': Address('0x' + '01' * 20), + 'chain_id': 0, + 'value': 0, + 'is_init': True, +} + + +class _TestContext(base_host.Context): + def __init__(self, logger: base_host.Logger): + self.logger = logger + self.stats: dict[str, typing.Any] = {} + + def on_genvm_success(self): ... + def on_genvm_failure(self): ... + + def add_stat(self, key: str, value: typing.Any, /): + self.stats[key] = value + + +class ManagerWsClient: + def __init__(self, manager_uri: str, *, max_msg_size: int = 64 * 1024 * 1024): + self.manager_uri = manager_uri + self.max_msg_size = max_msg_size + self.session: aiohttp.ClientSession | None = None + self.ws: aiohttp.ClientWebSocketResponse | None = None + + async def __aenter__(self): + if self.manager_uri.startswith('unix://'): + connector = aiohttp.UnixConnector(path=self.manager_uri.removeprefix('unix://')) + self.session = aiohttp.ClientSession(connector=connector) + ws_url = 'http://localhost/ws' + else: + self.session = aiohttp.ClientSession() + ws_url = self.manager_uri.rstrip('/') + '/ws' + try: + self.ws = await self.session.ws_connect( + ws_url, + max_msg_size=self.max_msg_size, + ) + except BaseException: + await self.close() + raise + return self + + async def __aexit__(self, *_args): + await self.close() + + async def read_frame(self) -> tuple[int, int, typing.Any]: + assert self.ws is not None + msg = await self.ws.receive() + if msg.type != aiohttp.WSMsgType.BINARY: + raise ConnectionResetError(f'websocket closed: {self.ws.close_code}') + body = bytes(msg.data) + if len(body) < 10: + raise ConnectionResetError('websocket message shorter than protocol header') + method = int.from_bytes(body[:2], 'big') + request_id = int.from_bytes(body[2:10], 'big') + payload = gvm_calldata.decode(body[10:]) + return method, request_id, payload + + async def send(self, method: Methods, request_id: int, payload: typing.Any): + assert self.ws is not None + encoded = gvm_calldata.encode(payload) + body = int(method).to_bytes(2, 'big') + request_id.to_bytes(8, 'big') + encoded + await self.ws.send_bytes(body) + + async def send_raw_body(self, body: bytes): + assert self.ws is not None + await self.ws.send_bytes(body) + + async def send_oversized(self, method: Methods, request_id: int, payload_len: int): + assert self.ws is not None + header = int(method).to_bytes(2, 'big') + request_id.to_bytes(8, 'big') + await self.ws.send_bytes(header + b'\x00' * payload_len) + + async def wait_closed(self) -> int | None: + assert self.ws is not None + close_code = None + while not self.ws.closed: + msg = await self.ws.receive() + if isinstance(msg.data, int): + close_code = msg.data + if msg.type in ( + aiohttp.WSMsgType.CLOSE, + aiohttp.WSMsgType.CLOSED, + aiohttp.WSMsgType.ERROR, + ): + break + return self.ws.close_code or close_code + + async def close(self) -> None: + if self.ws is not None: + await self.ws.close() + self.ws = None + if self.session is not None: + await self.session.close() + self.session = None + + +class ManagerProc: + def __init__( + self, + *, + root: Path, + build_dir: Path, + artifacts_dir: Path, + name: str, + socket_path: Path, + max_message_bytes: int = 67108864, + execution_retention: str = '30s', + use_unix_listener: bool = False, + ): + self.root = root + self.build_dir = build_dir + self.work_dir = artifacts_dir / name + self.socket_path = socket_path + self.max_message_bytes = max_message_bytes + self.execution_retention = execution_retention + self.use_unix_listener = use_unix_listener + self.process: asyncio.subprocess.Process | None = None + self.port = self._unused_port() + self.log_file = None + + @staticmethod + def _unused_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(('127.0.0.1', 0)) + return sock.getsockname()[1] + + @property + def uri(self) -> str: + if self.use_unix_listener: + return f'unix://{self.socket_path}' + return f'http://127.0.0.1:{self.port}' + + async def __aenter__(self): + self.work_dir.mkdir(parents=True, exist_ok=True) + config_path = self.work_dir / 'genvm-manager.yaml' + config_path.write_text( + '\n'.join( + [ + 'threads: 4', + 'blocking_threads: 48', + 'log_disable: tracing*,polling*,tungstenite*,tokio_tungstenite*', + 'log_level: info', + f'manifest_path: {self.build_dir / "out" / "data" / "manifest.yaml"}', + 'permits: 2', + f'execution_retention: {self.execution_retention}', + f'max_message_bytes: {self.max_message_bytes}', + '', + ] + ) + ) + listener_args = ( + ['--socket', str(self.socket_path)] + if self.use_unix_listener + else ['--port', str(self.port)] + ) + self.log_file = open(self.work_dir / 'manager.log', 'wb') + self.process = await asyncio.subprocess.create_subprocess_exec( + str(self.build_dir / 'out' / 'bin' / 'genvm-modules'), + 'manager', + *listener_args, + '--die-with-parent', + '--config', + str(config_path), + stdin=asyncio.subprocess.DEVNULL, + stdout=self.log_file, + stderr=self.log_file, + start_new_session=True, + ) + await self._wait_healthy() + return self + + async def __aexit__(self, *_args): + if self.process is not None: + try: + self.process.terminate() + await asyncio.wait_for(self.process.wait(), timeout=5) + except asyncio.TimeoutError: + self.process.kill() + await self.process.wait() + if self.log_file is not None: + self.log_file.close() + + async def _wait_healthy(self): + last_error: BaseException | None = None + for _ in range(80): + if self.process is not None and self.process.returncode is not None: + raise RuntimeError(f'manager exited early with {self.process.returncode}') + try: + if self.use_unix_listener: + async with aiohttp.ClientSession( + connector=aiohttp.UnixConnector(path=str(self.socket_path)) + ) as session: + async with session.get( + 'http://localhost/status', + timeout=aiohttp.ClientTimeout(total=1), + ) as resp: + if resp.status == 200: + return + else: + async with aiohttp.request( + 'GET', + f'{self.uri}/status', + timeout=aiohttp.ClientTimeout(total=1), + ) as resp: + if resp.status == 200: + return + except BaseException as exc: + last_error = exc + await asyncio.sleep(0.1) + raise TimeoutError(f'manager did not become healthy: {last_error}') + + +def _run_request( + root: Path, + host_path: Path, + *, + extra_args: list[str] | None = None, + reroute_to: str = '', +) -> dict[str, typing.Any]: + return { + 'run': { + 'selector': {'kind': 'major', 'major': base_host.UNDEPLOYED_MAJOR}, + 'message': FAKE_MESSAGE, + 'is_sync': True, + 'debug_mode': 'unsafe', + 'host_data': '{"node_address":"test","tx_id":"test"}', + 'max_execution_minutes': 20, + 'timestamp': '2024-11-26T06:42:42.424242Z', + 'host': 'unix://' + str(host_path), + 'extra_args': extra_args or [], + 'code': (root / HELLO_WORLD).read_bytes(), + 'calldata': gvm_calldata.encode({}), + 'leader_nondet_results': None, + 'bucket_totals': [2**200] * 20, + 'gas_data': base_host.DEFAULT_GAS_DATA, + 'message_fee_allocation': [], + 'initial_time_units_allocation': 60, + 'no_modules': True, + 'unsafe_overrides': base_host.UnsafeOverrides( + reroute_to=reroute_to + ).as_request_field(), + 'deadline': '30s', + 'host_hello_data': [], + } + } + + +async def _read_hello(client: ManagerWsClient) -> int: + method, request_id, payload = await client.read_frame() + assert method == Methods.HELLO + assert request_id == 0 + assert payload['hello']['protocol_major'] == CURRENT_MAJOR + return payload['hello']['boot_id'] + + +async def _read_error(client: ManagerWsClient, request_id: int, code: Errors): + method, got_request_id, payload = await client.read_frame() + assert method == Methods.ERROR, (method, got_request_id, payload) + assert got_request_id == request_id, (method, got_request_id, payload) + assert payload['code'] == code, payload + return payload + + +async def _wait_event(client: ManagerWsClient, variant: str, timeout: float = 10.0): + async with asyncio.timeout(timeout): + while True: + method, request_id, payload = await client.read_frame() + if method == Methods.EVENT and request_id == 0 and variant in payload: + return payload[variant] + + +async def _wait_terminal(client: ManagerWsClient, timeout: float = 10.0): + """Wait for whichever terminal event the run produces. + + Which one it is depends on how far the run got, and the point of the test is + that *some* terminal event arrives without the host having to time out. + """ + async with asyncio.timeout(timeout): + while True: + method, request_id, payload = await client.read_frame() + if method != Methods.EVENT or request_id != 0: + continue + for variant in ('failed_to_start', 'finished'): + if variant in payload: + return variant, payload[variant] + + +async def _terminal_from_snapshot_or_events( + client: ManagerWsClient, + snapshot: dict[str, typing.Any], + timeout: float = 10.0, +): + for variant in ('failed_to_start', 'finished'): + if variant in snapshot: + return variant, snapshot[variant] + return await _wait_terminal(client, timeout=timeout) + + +async def _http_json(method: str, uri: str, path: str, **kwargs): + async with aiohttp.request(method, uri + path, **kwargs) as resp: + return resp.status, await resp.json() + + +async def _read_reply( + client: ManagerWsClient, + method: Methods, + request_id: int, +) -> typing.Any: + got_method, got_request_id, payload = await client.read_frame() + assert got_method == method, (got_method, got_request_id, payload) + assert got_request_id == request_id, (got_method, got_request_id, payload) + return payload + + +async def _get_artifact(client: ManagerWsClient, genvm_id: int, field: str) -> bytes: + offset = 0 + out = bytearray() + request_id = 1000 + while True: + await client.send( + Methods.GET_ARTIFACT, + request_id, + { + 'get_artifact': { + 'genvm_id': genvm_id, + 'field': field, + 'offset': offset, + 'max_len': 64 * 1024, + } + }, + ) + artifact = await _read_reply(client, Methods.GET_ARTIFACT, request_id) + data = bytes(artifact['data']) + out.extend(data) + offset += len(data) + request_id += 1 + if offset >= artifact['total_len']: + return bytes(out) + + +def _make_mock_host( + manager: ManagerProc, + name: str, + ctx: _TestContext, + running_address: Address | None = None, +) -> tuple[MockHost, Path]: + tmp_dir = manager.work_dir / name + tmp_dir.mkdir(parents=True, exist_ok=True) + storage_path = tmp_dir / 'storage.pickle' + with open(storage_path, 'wb') as f: + pickle.dump(MockStorage(), f) + host_path = ( + Path('/tmp') / f'gvm-ms-{os.getpid()}' / f'{manager.work_dir.name}-{name}.sock' + ) + host_path.parent.mkdir(parents=True, exist_ok=True) + host = MockHost( + path=str(host_path), + storage_path_pre=storage_path, + storage_path_post=storage_path, + balances={}, + running_address=running_address or Address('0x' + '00' * 20), + ctx=ctx, + ) + return host, host_path + + +@dataclass +class ManagerSocketCase(genvm_tool.tests.test.Case): + description: genvm_tool.tests.test.Description + shared: genvm_tool.tests.SharedContext + + async def into_steps(self) -> list[genvm_tool.tests.exec.step.Step]: + return [ManagerSocketStep(self)] + + +class ManagerSocketStep(genvm_tool.tests.exec.step.Python): + def __init__(self, case: ManagerSocketCase): + self.case = case + self.phase = 'start' + + def to_str(self) -> str: + return '<manager socket protocol test>' + + async def run( + self, previous_results: list[typing.Any] + ) -> genvm_tool.tests.test.Result: + try: + await self._run_all() + return genvm_tool.tests.test.Result(passed=True, context={}, elapsed_seconds=0) + except BaseException as exc: + return genvm_tool.tests.test.Result( + passed=False, + context={'phase': self.phase, 'error': repr(exc)}, + elapsed_seconds=0, + ) + + async def _manager( + self, + name: str, + *, + max_message_bytes: int = 67108864, + use_unix_listener: bool = False, + ): + artifacts = self.case.shared.artifacts_dir / 'manager_socket' + socket_path = Path('/tmp') / f'gvm-ms-{os.getpid()}' / f'{name}.sock' + return ManagerProc( + root=self.case.shared.root_dir, + build_dir=Path( + json.loads((self.case.shared.root_dir / 'build' / 'info.json').read_text())[ + 'build_dir' + ] + ), + artifacts_dir=artifacts, + name=name, + socket_path=socket_path, + max_message_bytes=max_message_bytes, + use_unix_listener=use_unix_listener, + ) + + async def _manager_with_retention(self, name: str, retention: str): + artifacts = self.case.shared.artifacts_dir / 'manager_socket' + socket_path = Path('/tmp') / f'gvm-ms-{os.getpid()}' / f'{name}.sock' + return ManagerProc( + root=self.case.shared.root_dir, + build_dir=Path( + json.loads((self.case.shared.root_dir / 'build' / 'info.json').read_text())[ + 'build_dir' + ] + ), + artifacts_dir=artifacts, + name=name, + socket_path=socket_path, + execution_retention=retention, + ) + + async def _run_all(self): + self.phase = 'protocol errors' + await self._malformed_unknown_and_survival() + self.phase = 'oversized close' + await self._oversized_closes_connection() + self.phase = 'startup failures' + await self._startup_failure_events_and_permits() + self.phase = 'happy path' + await self._happy_path_artifact_ack_attach() + self.phase = 'reconnect mid-run' + await self._disconnect_mid_run_reconnect_attach() + self.phase = 'finish before ack reconnect' + await self._disconnect_after_finish_before_ack() + self.phase = 'multi attach' + await self._two_clients_receive_terminal() + self.phase = 'ack and ttl' + await self._ack_and_ttl_drop_results() + self.phase = 'boot mismatch' + await self._manager_restart_boot_mismatch() + self.phase = 'restart loses run' + await self._manager_restart_fails_waiter() + self.phase = 'cancel after start' + await self._cancel_after_start_single_terminal() + self.phase = 'deadline event' + await self._deadline_pushes_terminal() + self.phase = 'ttl drain' + await self._client_never_returns_ttl_drains() + self.phase = 'http socket parity' + await self._http_and_socket_same_fixture() + self.phase = 'unix stale' + await self._unix_parent_and_stale_socket() + + async def _malformed_unknown_and_survival(self): + async with await self._manager('protocol-errors') as manager: + async with ManagerWsClient(manager.uri) as client: + await _read_hello(client) + unknown_body = (65535).to_bytes(2, 'big') + (1).to_bytes(8, 'big') + await client.send_raw_body(unknown_body) + await _read_error(client, 1, Errors.UNKNOWN_METHOD) + await client.send(Methods.RUN, 2, b'not calldata') + await _read_error(client, 2, Errors.MALFORMED_FRAME) + await client.send(Methods.CANCEL, 3, {'cancel': {'genvm_id': 999}}) + await _read_error(client, 3, Errors.UNKNOWN_ID) + + async def _oversized_closes_connection(self): + async with await self._manager('oversized', max_message_bytes=32) as manager: + async with ManagerWsClient(manager.uri) as client: + await _read_hello(client) + await client.send_oversized(Methods.CANCEL, 7, payload_len=1024 * 1024) + # The cap is enforced by tungstenite's reader, which reports it as a + # plain read error rather than performing a close handshake, so the + # connection drops and the client sees 1006. The property under test + # is that the manager refuses the message instead of buffering it, + # not which code it refuses with. + close_code = await client.wait_closed() + assert close_code in ( + aiohttp.WSCloseCode.ABNORMAL_CLOSURE, + aiohttp.WSCloseCode.MESSAGE_TOO_BIG, + ), close_code + + async def _startup_failure_events_and_permits(self): + async with await self._manager('startup-failures') as manager: + async with ManagerWsClient(manager.uri) as client: + await _read_hello(client) + host_path = manager.work_dir / 'unused-host.sock' + await client.send( + Methods.RUN, + 10, + _run_request(self.case.shared.root_dir, host_path, extra_args=['--bad-flag']), + ) + method, request_id, payload = await client.read_frame() + assert method == Methods.RUN + assert request_id == 10 + genvm_id = payload['genvm_id'] + # The executor rejects the flag and exits on its own, so this is + # a `finished` with its exit code rather than `failed_to_start`. + variant, event = await _wait_terminal(client, timeout=5) + assert event['genvm_id'] == genvm_id + assert variant == 'finished', variant + assert event['exit_code'] not in (0, None), event + + async with ManagerWsClient(manager.uri) as client: + await _read_hello(client) + status, permits_before = await _http_json('GET', manager.uri, '/permits') + assert status == 200 + await client.send( + Methods.RUN, + 20, + _run_request( + self.case.shared.root_dir, + manager.work_dir / 'unused-host-2.sock', + reroute_to='missing-executor-version', + ), + ) + method, request_id, _payload = await client.read_frame() + assert method == Methods.RUN + assert request_id == 20 + await _wait_event(client, 'failed_to_start', timeout=5) + for _ in range(30): + status, permits_after = await _http_json('GET', manager.uri, '/permits') + assert status == 200 + if permits_after['permits'] == permits_before['permits']: + break + await asyncio.sleep(0.1) + else: + raise AssertionError('permit count was not restored') + + async def _happy_path_artifact_ack_attach(self): + async with await self._manager('happy') as manager: + tmp_dir = manager.work_dir / 'host' + tmp_dir.mkdir(parents=True, exist_ok=True) + storage_path = tmp_dir / 'storage.pickle' + with open(storage_path, 'wb') as f: + pickle.dump(MockStorage(), f) + host_path = tmp_dir / 'host.sock' + ctx = _TestContext(self.case.shared.logger) + host = MockHost( + path=str(host_path), + storage_path_pre=storage_path, + storage_path_post=storage_path, + balances={}, + running_address=Address('0x' + '00' * 20), + ctx=ctx, + ) + cancel_host = asyncio.Event() + with host as mock_host: + host_task = asyncio.create_task( + base_host.host_loop(mock_host, cancel_host, ctx=ctx) + ) + async with ManagerWsClient(manager.uri) as client: + boot_id = await _read_hello(client) + await client.send( + Methods.RUN, + 30, + _run_request(self.case.shared.root_dir, host_path), + ) + method, request_id, payload = await client.read_frame() + assert method == Methods.RUN + assert request_id == 30 + genvm_id = payload['genvm_id'] + started = await _wait_event(client, 'started') + assert started['genvm_id'] == genvm_id + finished = await _wait_event(client, 'finished') + assert finished['genvm_id'] == genvm_id + assert finished['artifact_sizes']['stdout'] > 0 + await client.send( + Methods.GET_ARTIFACT, + 31, + { + 'get_artifact': { + 'genvm_id': genvm_id, + 'field': 'stdout', + 'offset': 0, + 'max_len': 64, + } + }, + ) + method, request_id, artifact = await client.read_frame() + assert method == Methods.GET_ARTIFACT + assert request_id == 31 + assert b'1' in artifact['data'] + await client.send(Methods.ACK, 32, {'ack': {'genvm_id': genvm_id}}) + method, request_id, payload = await client.read_frame() + assert method == Methods.ACK + assert request_id == 32 + assert payload == {} + await client.send( + Methods.ATTACH, + 33, + {'attach': {'boot_id': boot_id, 'genvm_id': genvm_id}}, + ) + await _read_error(client, 33, Errors.UNKNOWN_ID) + + cancel_host.set() + try: + await asyncio.wait_for(host_task, timeout=5) + except asyncio.TimeoutError: + host_task.cancel() + await host_task + + async def _disconnect_mid_run_reconnect_attach(self): + async with await self._manager('reconnect-mid-run') as manager: + ctx = _TestContext(self.case.shared.logger) + host, host_path = _make_mock_host(manager, 'host', ctx) + cancel_host = asyncio.Event() + with host as mock_host: + host_task = asyncio.create_task( + base_host.host_loop(mock_host, cancel_host, ctx=ctx) + ) + async with ManagerWsClient(manager.uri) as client: + boot_id = await _read_hello(client) + req = _run_request(self.case.shared.root_dir, host_path) + req['run']['code'] = (self.case.shared.root_dir / BUSY_CONTRACT).read_bytes() + req['run']['deadline'] = '1s' + await client.send(Methods.RUN, 40, req) + genvm_id = (await _read_reply(client, Methods.RUN, 40))['genvm_id'] + started = await _wait_event(client, 'started') + assert started['genvm_id'] == genvm_id + await client.close() + + async with ManagerWsClient(manager.uri) as client: + await _read_hello(client) + await client.send( + Methods.ATTACH, + 41, + {'attach': {'boot_id': boot_id, 'genvm_id': genvm_id}}, + ) + snapshot = (await _read_reply(client, Methods.ATTACH, 41))['snapshot'] + variant, terminal = await _terminal_from_snapshot_or_events(client, snapshot) + assert variant == 'finished' + assert terminal['genvm_id'] == genvm_id + assert terminal['cause'] == 'deadline' + await client.send(Methods.ACK, 42, {'ack': {'genvm_id': genvm_id}}) + await _read_reply(client, Methods.ACK, 42) + cancel_host.set() + with contextlib.suppress(BaseException): + await host_task + + async def _disconnect_after_finish_before_ack(self): + async with await self._manager('finish-before-ack') as manager: + ctx = _TestContext(self.case.shared.logger) + host, host_path = _make_mock_host(manager, 'host', ctx) + cancel_host = asyncio.Event() + with host as mock_host: + host_task = asyncio.create_task( + base_host.host_loop(mock_host, cancel_host, ctx=ctx) + ) + async with ManagerWsClient(manager.uri) as client: + boot_id = await _read_hello(client) + await client.send( + Methods.RUN, + 50, + _run_request(self.case.shared.root_dir, host_path), + ) + genvm_id = (await _read_reply(client, Methods.RUN, 50))['genvm_id'] + _finished = await _wait_event(client, 'finished') + await client.close() + + async with ManagerWsClient(manager.uri) as client: + await _read_hello(client) + await client.send( + Methods.ATTACH, + 51, + {'attach': {'boot_id': boot_id, 'genvm_id': genvm_id}}, + ) + snapshot = (await _read_reply(client, Methods.ATTACH, 51))['snapshot'] + variant, terminal = await _terminal_from_snapshot_or_events(client, snapshot) + assert variant == 'finished' + assert terminal['artifact_sizes']['stdout'] > 0 + stdout = await _get_artifact(client, genvm_id, 'stdout') + assert b'1' in stdout + await client.send(Methods.ACK, 52, {'ack': {'genvm_id': genvm_id}}) + await _read_reply(client, Methods.ACK, 52) + cancel_host.set() + with contextlib.suppress(BaseException): + await host_task + + async def _two_clients_receive_terminal(self): + async with await self._manager('two-clients') as manager: + ctx = _TestContext(self.case.shared.logger) + host, host_path = _make_mock_host(manager, 'host', ctx) + cancel_host = asyncio.Event() + with host as mock_host: + host_task = asyncio.create_task( + base_host.host_loop(mock_host, cancel_host, ctx=ctx) + ) + async with ( + ManagerWsClient(manager.uri) as first, + ManagerWsClient(manager.uri) as second, + ): + boot_id = await _read_hello(first) + await _read_hello(second) + await first.send( + Methods.RUN, + 60, + _run_request(self.case.shared.root_dir, host_path), + ) + genvm_id = (await _read_reply(first, Methods.RUN, 60))['genvm_id'] + await second.send( + Methods.ATTACH, + 61, + {'attach': {'boot_id': boot_id, 'genvm_id': genvm_id}}, + ) + snapshot = (await _read_reply(second, Methods.ATTACH, 61))['snapshot'] + first_finished = await _wait_event(first, 'finished') + _second_variant, second_finished = await _terminal_from_snapshot_or_events( + second, snapshot + ) + assert first_finished['genvm_id'] == genvm_id + assert second_finished['genvm_id'] == genvm_id + await first.send(Methods.ACK, 62, {'ack': {'genvm_id': genvm_id}}) + await _read_reply(first, Methods.ACK, 62) + cancel_host.set() + with contextlib.suppress(BaseException): + await host_task + + async def _ack_and_ttl_drop_results(self): + async with await self._manager_with_retention('ttl-drop', '500ms') as manager: + ctx = _TestContext(self.case.shared.logger) + host, host_path = _make_mock_host(manager, 'ack-host', ctx) + cancel_host = asyncio.Event() + with host as mock_host: + host_task = asyncio.create_task( + base_host.host_loop(mock_host, cancel_host, ctx=ctx) + ) + async with ManagerWsClient(manager.uri) as client: + boot_id = await _read_hello(client) + await client.send( + Methods.RUN, + 70, + _run_request(self.case.shared.root_dir, host_path), + ) + genvm_id = (await _read_reply(client, Methods.RUN, 70))['genvm_id'] + await _wait_event(client, 'finished') + await client.send(Methods.ACK, 71, {'ack': {'genvm_id': genvm_id}}) + await _read_reply(client, Methods.ACK, 71) + await client.send( + Methods.ATTACH, + 72, + {'attach': {'boot_id': boot_id, 'genvm_id': genvm_id}}, + ) + await _read_error(client, 72, Errors.UNKNOWN_ID) + cancel_host.set() + with contextlib.suppress(BaseException): + await host_task + + async def _manager_restart_boot_mismatch(self): + manager = await self._manager('boot-mismatch') + async with manager: + async with ManagerWsClient(manager.uri) as client: + boot_id = await _read_hello(client) + genvm_id = 1 + async with manager: + async with ManagerWsClient(manager.uri) as client: + await _read_hello(client) + await client.send( + Methods.ATTACH, + 80, + {'attach': {'boot_id': boot_id, 'genvm_id': genvm_id}}, + ) + await _read_error(client, 80, Errors.BOOT_ID_MISMATCH) + + async def _manager_restart_fails_waiter(self): + manager = await self._manager('restart-run-lost') + ctx = _TestContext(self.case.shared.logger) + host, host_path = _make_mock_host(manager, 'host', ctx) + cancel_host = asyncio.Event() + with host as mock_host: + host_task = asyncio.create_task( + base_host.host_loop(mock_host, cancel_host, ctx=ctx) + ) + await manager.__aenter__() + try: + async with base_host.ManagerClient( + manager.uri, + connect_timeout=ctx.get_manager_connect_timeout(), + ) as client: + req = _run_request(self.case.shared.root_dir, host_path)['run'] + req['code'] = (self.case.shared.root_dir / BUSY_CONTRACT).read_bytes() + state = await client.run(req) + # The manager that owns the run dies and a fresh one takes its + # place, handing out the same ids again -- the run cannot be + # re-attached, so the waiter has to fail rather than reconnect. + await manager.__aexit__() + await manager.__aenter__() + try: + terminal = await asyncio.wait_for(client.wait_terminal(state), timeout=15) + except base_host.ManagerRunLost: + pass + else: + raise AssertionError(f'run outlived its manager: {terminal}') + finally: + await manager.__aexit__() + cancel_host.set() + with contextlib.suppress(BaseException): + await host_task + + async def _cancel_after_start_single_terminal(self): + async with await self._manager('cancel-after-start') as manager: + ctx = _TestContext(self.case.shared.logger) + host, host_path = _make_mock_host(manager, 'host', ctx) + cancel_host = asyncio.Event() + with host as mock_host: + host_task = asyncio.create_task( + base_host.host_loop(mock_host, cancel_host, ctx=ctx) + ) + async with ManagerWsClient(manager.uri) as client: + await _read_hello(client) + req = _run_request(self.case.shared.root_dir, host_path) + req['run']['code'] = (self.case.shared.root_dir / BUSY_CONTRACT).read_bytes() + req['run']['deadline'] = '30s' + await client.send(Methods.RUN, 90, req) + genvm_id = (await _read_reply(client, Methods.RUN, 90))['genvm_id'] + await _wait_event(client, 'started') + await client.send(Methods.CANCEL, 91, {'cancel': {'genvm_id': genvm_id}}) + await _read_reply(client, Methods.CANCEL, 91) + variant, terminal = await _wait_terminal(client) + assert variant == 'finished' + assert terminal['cause'] == 'cancelled' + try: + extra = await asyncio.wait_for(client.read_frame(), timeout=0.5) + except asyncio.TimeoutError: + extra = None + assert extra is None or not ( + extra[0] == Methods.EVENT + and any(k in extra[2] for k in ('failed_to_start', 'finished')) + ) + await client.send(Methods.ACK, 92, {'ack': {'genvm_id': genvm_id}}) + await _read_reply(client, Methods.ACK, 92) + cancel_host.set() + with contextlib.suppress(BaseException): + await host_task + + async def _deadline_pushes_terminal(self): + async with await self._manager('deadline') as manager: + ctx = _TestContext(self.case.shared.logger) + host, host_path = _make_mock_host(manager, 'host', ctx) + cancel_host = asyncio.Event() + with host as mock_host: + host_task = asyncio.create_task( + base_host.host_loop(mock_host, cancel_host, ctx=ctx) + ) + async with ManagerWsClient(manager.uri) as client: + await _read_hello(client) + req = _run_request(self.case.shared.root_dir, host_path) + req['run']['code'] = (self.case.shared.root_dir / BUSY_CONTRACT).read_bytes() + req['run']['deadline'] = '1s' + await client.send(Methods.RUN, 100, req) + genvm_id = (await _read_reply(client, Methods.RUN, 100))['genvm_id'] + variant, terminal = await _wait_terminal(client, timeout=5) + assert variant == 'finished' + assert terminal['cause'] == 'deadline' + await client.send(Methods.ACK, 101, {'ack': {'genvm_id': genvm_id}}) + await _read_reply(client, Methods.ACK, 101) + cancel_host.set() + with contextlib.suppress(BaseException): + await host_task + + async def _client_never_returns_ttl_drains(self): + async with await self._manager_with_retention('ttl-drain', '500ms') as manager: + ctx = _TestContext(self.case.shared.logger) + host, host_path = _make_mock_host(manager, 'host', ctx) + cancel_host = asyncio.Event() + with host as mock_host: + host_task = asyncio.create_task( + base_host.host_loop(mock_host, cancel_host, ctx=ctx) + ) + async with ManagerWsClient(manager.uri) as client: + boot_id = await _read_hello(client) + await client.send( + Methods.RUN, + 110, + _run_request(self.case.shared.root_dir, host_path), + ) + genvm_id = (await _read_reply(client, Methods.RUN, 110))['genvm_id'] + with contextlib.suppress(BaseException): + await host_task + for _ in range(330): + status, body = await _http_json('GET', manager.uri, '/status') + assert status == 200 + if str(genvm_id) not in body['executions']: + break + await asyncio.sleep(0.2) + else: + raise AssertionError('retained execution did not drain after TTL') + async with ManagerWsClient(manager.uri) as client: + await _read_hello(client) + await client.send( + Methods.ATTACH, + 111, + {'attach': {'boot_id': boot_id, 'genvm_id': genvm_id}}, + ) + await _read_error(client, 111, Errors.UNKNOWN_ID) + + async def _http_and_socket_same_fixture(self): + async with await self._manager('http-socket-parity') as manager: + http_res = await self._run_fixture_over_http(manager, 'http') + socket_res = await self._run_fixture_over_socket(manager, 'socket') + assert http_res['stdout'] == socket_res.stdout + assert http_res['stderr'] == socket_res.stderr + assert http_res['result_kind'] == socket_res.result_kind + assert http_res['result_data'] == socket_res.result_data + + async def _run_fixture_over_http(self, manager: ManagerProc, name: str): + ctx = _TestContext(self.case.shared.logger) + host, host_path = _make_mock_host(manager, name, ctx) + cancel_host = asyncio.Event() + with host as mock_host: + host_task = asyncio.create_task( + base_host.host_loop(mock_host, cancel_host, ctx=ctx) + ) + req = _run_request(self.case.shared.root_dir, host_path)['run'] + async with aiohttp.request( + 'POST', + f'{manager.uri}/genvm/run', + data=gvm_calldata.encode(req), + ) as resp: + body = await resp.json() + assert resp.status == 200, body + genvm_id = body['id'] + with contextlib.suppress(BaseException): + await host_task + for _ in range(30): + status, body = await _http_json('GET', manager.uri, f'/genvm/{genvm_id}') + assert status == 200 + if body['status'] is not None: + result = body['status'] + break + await asyncio.sleep(0.2) + else: + raise AssertionError('HTTP shim result did not become available') + assert isinstance(result['consumed_result'], str) + consumed = base_host.ConsumedResult.decode(result.get('consumed_result')) + return { + 'stdout': result['stdout'], + 'stderr': result['stderr'], + 'result_kind': consumed.result_kind, + 'result_data': consumed.result_data, + } + + async def _run_fixture_over_socket(self, manager: ManagerProc, name: str): + ctx = _TestContext(self.case.shared.logger) + host, host_path = _make_mock_host(manager, name, ctx) + with host as mock_host: + async with base_host.ManagerClient( + manager.uri, + connect_timeout=ctx.get_manager_connect_timeout(), + ) as manager_client: + return await base_host.run_genvm( + mock_host, + manager_uri=manager.uri, + manager_client=manager_client, + ctx=ctx, + is_sync=True, + message=FAKE_MESSAGE, + host_data='{"node_address":"test","tx_id":"test"}', + host='unix://' + str(host_path), + code=(self.case.shared.root_dir / HELLO_WORLD).read_bytes(), + calldata=gvm_calldata.encode({}), + timeout=30, + debug_mode='unsafe', + request_extra={'no_modules': True}, + bucket_totals=[2**200] * 20, + ) + + async def _unix_parent_and_stale_socket(self): + artifacts = Path('/tmp') / f'gvm-ms-{os.getpid()}' + socket_path = artifacts / 'unix-stale.sock' + socket_path.parent.mkdir(parents=True, exist_ok=True) + socket_path.write_text('stale') + async with await self._manager('unix-stale', use_unix_listener=True) as manager: + assert manager.socket_path == socket_path + mode = os.stat(socket_path).st_mode + assert stat.S_ISSOCK(mode) + + +def collect(ctx: genvm_tool.tests.stage.collection.Context) -> None: + # console_pool: this case starts, restarts and kills managers of its own, and + # waits out retention sweeps. Sharing the runner with other cases makes their + # manager unreachable partway through, which they report as a connection + # refused far away from the cause. + desc = genvm_tool.tests.test.Description( + name='tests/system/manager-socket', + tags=frozenset({'integration', 'stable', 'manager-socket'}), + console_pool=True, + ) + ctx.add_case(ManagerSocketCase(description=desc, shared=ctx.shared)) diff --git a/tests/system/permits/test.py b/tests/system/permits/test.py index 97685f15..b5ab128a 100644 --- a/tests/system/permits/test.py +++ b/tests/system/permits/test.py @@ -78,100 +78,121 @@ async def run( bg_tasks: list[asyncio.Task] = [] shut_downer = asyncio.Event() + async with base_host.ManagerClient(manager_uri) as manager_client: + try: + # Set permits to N * 2 + await self._set_permits(manager_uri, num_permits) + logger.debug('permits set', permits=num_permits) + + # Start N*2 background genvms (each sync, 1 permit) + tmp_dir = shared.artifacts_dir / 'permits_test' + tmp_dir.mkdir(parents=True, exist_ok=True) + + code = BUSY_CONTRACT.read_bytes() + + for i in range(num_permits): + sock_path = f'/tmp/genvm-test-permits-{i}.sock' + storage_path = tmp_dir / f'storage_{i}.pickle' + + # Create empty storage + with open(storage_path, 'wb') as f: + pickle.dump(MockStorage(), f) + + ctx = _TestContext(logger) + host = MockHost( + path=sock_path, + storage_path_pre=storage_path, + storage_path_post=storage_path, + balances={}, + running_address=Address('0x' + '00' * 20), + ctx=ctx, + ) - try: - # Set permits to N * 2 - await self._set_permits(manager_uri, num_permits) - logger.debug('permits set', permits=num_permits) - - # Start N*2 background genvms (each sync, 1 permit) - tmp_dir = shared.artifacts_dir / 'permits_test' - tmp_dir.mkdir(parents=True, exist_ok=True) - - code = BUSY_CONTRACT.read_bytes() - - for i in range(num_permits): - sock_path = f'/tmp/genvm-test-permits-{i}.sock' - storage_path = tmp_dir / f'storage_{i}.pickle' - - # Create empty storage - with open(storage_path, 'wb') as f: - pickle.dump(MockStorage(), f) - - host = MockHost( - path=sock_path, - storage_path_pre=storage_path, - storage_path_post=storage_path, - balances={}, - running_address=Address('0x' + '00' * 20), - ) - - ctx = _TestContext(logger) - task = asyncio.create_task( - self._run_background_genvm( - host, manager_uri, ctx, code, sock_path, genvm_ids, shut_downer + task = asyncio.create_task( + self._run_background_genvm( + host, + manager_uri, + manager_client, + ctx, + code, + sock_path, + genvm_ids, + shut_downer, + ) ) - ) - bg_tasks.append(task) - - # Wait for permits to be consumed - for _ in range(30): - status = await self._get_status(manager_uri) - current = status['permits']['current'] - logger.trace('polling permits', current=current) - if current == 0: - break - await asyncio.sleep(0.5) - else: - return genvm_tool.tests.test.Result( - passed=False, - context={'reason': 'permits were not consumed within timeout'}, - elapsed_seconds=0, - ) + bg_tasks.append(task) + + # Wait for permits to be consumed + for _ in range(30): + status = await self._get_status(manager_uri) + current = status['permits']['current'] + logger.trace('polling permits', current=current) + if current == 0: + break + await asyncio.sleep(0.5) + else: + return genvm_tool.tests.test.Result( + passed=False, + context={'reason': 'permits were not consumed within timeout'}, + elapsed_seconds=0, + ) + + logger.debug('all permits consumed, testing blocking behavior') - logger.debug('all permits consumed, testing blocking behavior') + # Try to start another genvm; it should block on semaphore. + blocked = await self._assert_blocks(manager_uri, code) + if not blocked: + return genvm_tool.tests.test.Result( + passed=False, + context={'reason': 'POST /genvm/run did not block when permits exhausted'}, + elapsed_seconds=0, + ) - # Try to start another genvm β€” should block on semaphore - blocked = await self._assert_blocks(manager_uri, code) - if not blocked: - return genvm_tool.tests.test.Result( - passed=False, - context={'reason': 'POST /genvm/run did not block when permits exhausted'}, - elapsed_seconds=0, + socket_cancelled = await self._assert_socket_cancel_queued( + manager_uri, manager_client, code ) + if not socket_cancelled: + return genvm_tool.tests.test.Result( + passed=False, + context={ + 'reason': 'socket cancel while queued consumed a permit or missed terminal event' + }, + elapsed_seconds=0, + ) - logger.info('permits test passed') - return genvm_tool.tests.test.Result(passed=True, context={}, elapsed_seconds=0) - - finally: - shut_downer.set() - # Shut down running genvms - for gid in genvm_ids: - try: - async with aiohttp.request( - 'DELETE', - f'{manager_uri}/genvm/{gid}', - timeout=aiohttp.ClientTimeout(total=10), - ): + logger.info('permits test passed') + return genvm_tool.tests.test.Result(passed=True, context={}, elapsed_seconds=0) + + finally: + shut_downer.set() + # Shut down running genvms + for gid in genvm_ids: + try: + async with aiohttp.request( + 'DELETE', + f'{manager_uri}/genvm/{gid}', + timeout=aiohttp.ClientTimeout(total=10), + ): + pass + except Exception as e: + logger.warning('failed to delete genvm', genvm_id=gid, error=e) pass - except Exception as e: - logger.warning('failed to delete genvm', genvm_id=gid, error=e) - pass - for task in bg_tasks: - try: - await task - except (asyncio.CancelledError, Exception): - pass + for task in bg_tasks: + try: + await task + except (asyncio.CancelledError, Exception): + pass - # Restore permits - await self._set_permits(manager_uri, original_permits) - logger.debug('permits restored', permits=original_permits) + # Restore permits + await self._set_permits(manager_uri, original_permits) + logger.debug('permits restored', permits=original_permits) async def _run_background_genvm( self, host: MockHost, manager_uri: str, + manager_client: base_host.ManagerClient, ctx: _TestContext, code: bytes, sock_path: str, @@ -185,6 +206,7 @@ async def _run_background_genvm( await base_host.run_genvm( mock_host, manager_uri=manager_uri, + manager_client=manager_client, ctx=ctx, is_sync=True, message=FAKE_MESSAGE, @@ -194,7 +216,9 @@ async def _run_background_genvm( calldata=gvm_calldata.encode({}), timeout=300, debug_mode='unsafe', - reroute_to=getattr(self, '_reroute_to', ''), + unsafe_overrides=base_host.UnsafeOverrides( + reroute_to=getattr(self, '_reroute_to', ''), + ), request_extra={'no_modules': True}, shutdown_early=shut_downer, bucket_totals=[2**200] * 20, @@ -206,7 +230,7 @@ async def _assert_blocks(self, manager_uri: str, code: bytes) -> bool: """Make a raw POST /genvm/run and assert it blocks (times out).""" data = gvm_calldata.encode( { - 'major': 0, + 'selector': {'kind': 'major', 'major': base_host.UNDEPLOYED_MAJOR}, 'message': FAKE_MESSAGE, 'is_sync': True, 'host_data': '{"node_address":"test","tx_id":"test"}', @@ -248,6 +272,50 @@ async def _assert_blocks(self, manager_uri: str, code: bytes) -> bool: except (asyncio.TimeoutError, TimeoutError): return True + async def _assert_socket_cancel_queued( + self, + manager_uri: str, + client: base_host.ManagerClient, + code: bytes, + ) -> bool: + payload = { + 'selector': {'kind': 'major', 'major': base_host.UNDEPLOYED_MAJOR}, + 'message': FAKE_MESSAGE, + 'is_sync': True, + 'debug_mode': 'unsafe', + 'host_data': '{"node_address":"test","tx_id":"test"}', + 'max_execution_minutes': 20, + 'timestamp': '2024-11-26T06:42:42.424242Z', + 'host': 'unix:///tmp/genvm-test-permits-socket-cancel.sock', + 'extra_args': [], + 'code': code, + 'calldata': gvm_calldata.encode({}), + 'bucket_totals': [2**200] * 20, + 'no_modules': True, + 'leader_nondet_results': [], + 'initial_time_units_allocation': 60, + 'unsafe_overrides': base_host.UnsafeOverrides( + reroute_to=self._reroute_to, + ).as_request_field(), + 'deadline': '30s', + } + state = await client.run(payload) + before = await self._get_status(manager_uri) + if before['permits']['current'] != 0: + return False + await client.cancel(state.boot_id, state.genvm_id) + terminal = await client.wait_terminal(state) + after = await self._get_status(manager_uri) + await client.ack(state.boot_id, state.genvm_id) + if 'finished' not in terminal: + return False + finished = terminal['finished'] + return ( + finished['cause'] == 'cancelled' + and finished['exit_code'] is None + and after['permits']['current'] == 0 + ) + async def _get_permits(self, manager_uri: str) -> int: async with aiohttp.request('GET', f'{manager_uri}/permits') as resp: data = await resp.json() From 04e75971702d10b200faca8d7f1c69f6deb95bc9 Mon Sep 17 00:00:00 2001 From: kp2pml30 <kp2pml30@gmail.com> Date: Tue, 18 Aug 2026 16:06:46 +0900 Subject: [PATCH 09/45] fix(ci): harden PR automation --- .coderabbit.yaml | 12 +-- .github/workflows/branch_pr_checklist.yaml | 6 +- .../workflows/incl_release_build_test.yaml | 5 +- .github/workflows/queue.yaml | 94 +++++++++++++++---- .github/workflows/release.yaml | 5 +- executors/v0.2.x | 2 +- executors/v0.3.x | 2 +- support/ci/tools/genvm_merge_into_dev.py | 4 +- support/ci/tools/pr_action_panel.py | 12 ++- support/ci/unit_tests/test_merge_gates.py | 46 +++++++++ 10 files changed, 146 insertions(+), 42 deletions(-) create mode 100644 support/ci/unit_tests/test_merge_gates.py diff --git a/.coderabbit.yaml b/.coderabbit.yaml index 5959a757..3e5e11cd 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -22,12 +22,10 @@ reviews: - "!**/*.hash" - "!**/*.lock" - "!docs/website/src/_static/**" - - "!executors/v0.3.x/runners/py-libs/pure-py/**" - - "!executors/v0.3.x/runners/models/**" - - "!executors/v0.3.x/runners/softfloat/berkeley-softfloat-3/**" - - "!**/fuzz/inputs*/**" - # Root-level system-test fixtures (contracts, expected outputs); crate and - # runner test code under */tests/ is still reviewed. - - "!tests/**" + - "!**/fuzz/**" + - "!**/*_test.rs" + - "!**/*.json" + - "!**/tests/**" + - "!**/test/**" # genvm-tool is internal dev tooling; not worth reviewing. - "!support/tools/genvm-tool/**" diff --git a/.github/workflows/branch_pr_checklist.yaml b/.github/workflows/branch_pr_checklist.yaml index 9454209b..a25b2256 100644 --- a/.github/workflows/branch_pr_checklist.yaml +++ b/.github/workflows/branch_pr_checklist.yaml @@ -1,6 +1,6 @@ name: branch / post PR action panel -# When a PR is opened against a dev branch (v<X>-dev) we: +# When a PR is opened against main or a dev branch (v<X>-dev) we: # - post the action panel (checklist comment) that replaces the old # /genvm-* slash commands; # - auto-add `ci-safe` if the PR author has write access, so their PR can @@ -13,7 +13,7 @@ name: branch / post PR action panel on: pull_request_target: - branches: ['v*-dev'] + branches: [main, 'v*-dev'] # `edited` fires when the PR base branch is changed, so a PR retargeted # onto a dev branch also gets the panel. types: [opened, edited] @@ -72,7 +72,7 @@ jobs: - linear history β€” 0 commits behind base - green full GenVM CI and green cross-repo E2E - Full CI runs only when **`run-full-tests`** or **`rtm`** is set; "Force run full tests" is a sticky toggle for the former. + Full CI starts only through "Force" or "Rerun full tests" above. "Force" also sets **`run-full-tests`**, making future pushes run the full suite; **`rtm`** only authorizes Merge. Every repo lands ONE squashed commit, subject `<PR title> (#N)`. </details> diff --git a/.github/workflows/incl_release_build_test.yaml b/.github/workflows/incl_release_build_test.yaml index c379db2a..dd4a1885 100644 --- a/.github/workflows/incl_release_build_test.yaml +++ b/.github/workflows/incl_release_build_test.yaml @@ -5,8 +5,9 @@ name: Build and test release artifacts # # Callers see four artifacts, a tag and the executor lines; the matrices that # produce them are an implementation detail. `release.yaml` is this plus a -# publisher; `queue.yaml` runs it on a PR (label `test-release-pipeline`) to -# exercise the pipeline itself without releasing anything. +# publisher; `queue.yaml` runs it on a PR carrying `test-release-pipeline` when +# CI starts from the action panel or a push, exercising the pipeline itself +# without releasing anything. on: workflow_call: diff --git a/.github/workflows/queue.yaml b/.github/workflows/queue.yaml index b65ea9a3..554125ef 100644 --- a/.github/workflows/queue.yaml +++ b/.github/workflows/queue.yaml @@ -16,12 +16,9 @@ name: GenVM full # - without a marker `validate-end` fails, so the CI check stays red # until a full run has happened. # -# The action panel ("Force"/"Rerun full tests") triggers a run directly via -# `workflow_dispatch` rather than relying on a label edit: a label applied by -# the bot's GITHUB_TOKEN does NOT emit a `labeled` event (GitHub suppresses -# recursive runs), but a `workflow_dispatch` from that same token DOES run. -# The dispatch targets the PR head branch, so the run's head_sha equals the -# PR head and the Merge gate (which matches by head_sha) counts it. +# Labels are passive: adding `rtm`, `run-full-tests`, or any unrelated label +# never starts or cancels CI. "Force" and "Rerun full tests" on the action panel +# dispatch this workflow explicitly; a push still starts the always-on checks. on: pull_request: branches: ['v*-dev'] @@ -30,27 +27,34 @@ on: # matrix-tests cell (over an hour of work). The PR title can therefore be # edited to something invalid after `initial / pr-title` went green β€” the # Merge action re-checks it against the title it is about to land. - types: [opened, synchronize, reopened, ready_for_review, labeled] + types: [opened, synchronize, reopened, ready_for_review] workflow_dispatch: inputs: pr: description: >- - PR number this manual run validates. Without it the PR gates below - have nothing to read and are skipped, leaving a green run that only - the heavy tests earned β€” pass it to authorize a merge. + PR number supplied by the action panel and bound to the dispatch's + exact repository head before any CI job runs. type: string + required: true + release_pipeline_test: + description: Exercise the release pipeline without publishing + type: boolean required: false + default: false run-name: >- GenVM full${{ github.event_name == 'workflow_dispatch' && format(' (manual, PR #{0})', inputs.pr) || '' }} +permissions: + contents: read + pull-requests: read + # One in-flight full run per PR head branch; a newer push cancels the stale run. # `github.head_ref` is the PR head branch on pull_request events. Cancelling a # superseded head's run is safe: the Merge gate only consumes CI that is green on -# the exact head it fast-forwards, which is always the newest one. The panel's -# workflow_dispatch has no head_ref (its `github.ref_name` is `<n>/merge` on PRs, -# not a branch), so fall back to the unique run id β€” a manual rerun then gets its -# own group and never cancels (or is cancelled by) another PR's run. +# the exact head it fast-forwards, which is always the newest one. Panel dispatches +# have no head_ref, so each falls back to its unique run id and never cancels (or +# is cancelled by) another PR's run. concurrency: group: genvm-full-${{ github.head_ref || github.run_id }} cancel-in-progress: true @@ -60,7 +64,51 @@ defaults: shell: bash -exo pipefail {0} jobs: + validate-trigger: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Validate CI trigger + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + EVENT_NAME: ${{ github.event_name }} + DISPATCH_ACTOR: ${{ github.actor }} + TRIGGERING_ACTOR: ${{ github.triggering_actor }} + RUN_ATTEMPT: ${{ github.run_attempt }} + PR: ${{ inputs.pr }} + RUN_SHA: ${{ github.sha }} + run: | + if [ "$RUN_ATTEMPT" != 1 ]; then + echo "::error::CI reruns must be started from the PR action panel" + exit 1 + fi + if [ "$EVENT_NAME" != workflow_dispatch ]; then + exit 0 + fi + if [ "$DISPATCH_ACTOR" != 'github-actions[bot]' ] || \ + [ "$TRIGGERING_ACTOR" != 'github-actions[bot]' ]; then + echo "::error::full CI must be started from the PR action panel, not the Actions UI" + exit 1 + fi + if [[ ! "$PR" =~ ^[1-9][0-9]*$ ]]; then + echo "::error::invalid PR number: $PR" + exit 1 + fi + + head="$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR" \ + --jq '[.head.repo.full_name, .head.sha] | @tsv')" + IFS=$'\t' read -r head_repo head_sha <<<"$head" + if [ "$head_repo" != "$GITHUB_REPOSITORY" ]; then + echo "::error::the action panel cannot dispatch CI for a fork PR" + exit 1 + fi + if [ "$head_sha" != "$RUN_SHA" ]; then + echo "::error::dispatch ref is $RUN_SHA, but PR #$PR is at $head_sha" + exit 1 + fi + initial: + needs: validate-trigger uses: ./.github/workflows/incl_initial.yaml with: # Resolved from whichever event triggered this run, so the PR-scoped @@ -89,10 +137,12 @@ jobs: # pull_request GitHub already runs this workflow file from the PR, so a same-repo # author controls it regardless.) check-executor-prs: + needs: validate-trigger if: > - (github.event_name == 'workflow_dispatch' && inputs.pr != '') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name == github.repository) + needs.validate-trigger.result == 'success' && + ((github.event_name == 'workflow_dispatch' && inputs.pr != '') || + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository)) runs-on: ubuntu-latest timeout-minutes: 15 steps: @@ -111,7 +161,7 @@ jobs: run: ./support/ci/run.sh tool pr-branches check gate-check: - needs: [initial, check-executor-prs] + needs: [validate-trigger, initial, check-executor-prs] runs-on: ubuntu-latest timeout-minutes: 15 if: ${{ always() }} @@ -120,6 +170,10 @@ jobs: steps: - name: Check gate results run: | + if [ "${{ needs.validate-trigger.result }}" != "success" ]; then + echo "::error::CI trigger validation failed" + exit 1 + fi if [ "${{ needs.initial.result }}" != "success" ]; then echo "::error::initial checks failed" exit 1 @@ -165,7 +219,7 @@ jobs: # failure. release-pipeline-test: needs: [initial, gate-check] - if: ${{ contains(github.event.pull_request.labels.*.name, 'test-release-pipeline') }} + if: ${{ (github.event_name == 'workflow_dispatch' && inputs.release_pipeline_test) || contains(github.event.pull_request.labels.*.name, 'test-release-pipeline') }} uses: ./.github/workflows/incl_release_build_test.yaml secrets: inherit @@ -191,7 +245,7 @@ jobs: # No run-full-tests marker -> heavy tests were skipped; keep the # gate red so the PR is not mergeable until they run. if [ "$HAS_MARKER" != "true" ]; then - echo "::error::full tests have not run β€” check 'Force run full tests' on the action panel or add the 'run-full-tests' or 'rtm' label" + echo "::error::full tests have not run β€” tick 'Force run full tests' or 'Rerun full tests' on the action panel" exit 1 fi results="${{ needs.matrix-tests.result }}" diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 771fddf1..dabab419 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -3,8 +3,9 @@ name: GenVM release # incl_release_build_test.yaml plus a publisher: it plans, builds every artifact # and tests every platform; this workflow attaches what it produced to a # genvm-manager release tagged with `version` from .genvm-monorepo-root. -# `queue.yaml` runs the same pipeline on a PR (label `test-release-pipeline`), -# so everything below is the only release-only part. +# `queue.yaml` runs the same pipeline on a PR carrying `test-release-pipeline` +# when CI starts from the action panel or a push, so everything below is the +# only release-only part. # # genvm-<arch>-<os>.tar.xz β€” manager + every active executor line for that # platform (bin/, lib/, config/, data/, executor/) diff --git a/executors/v0.2.x b/executors/v0.2.x index e3e1ffa8..5b23854c 160000 --- a/executors/v0.2.x +++ b/executors/v0.2.x @@ -1 +1 @@ -Subproject commit e3e1ffa8d2aa7560a1d0ca4a3f8788ea5e287b42 +Subproject commit 5b23854c49cc978995b5e1d4ac50c64f1c576736 diff --git a/executors/v0.3.x b/executors/v0.3.x index ef7dd2fb..8a3a80cc 160000 --- a/executors/v0.3.x +++ b/executors/v0.3.x @@ -1 +1 @@ -Subproject commit ef7dd2fb9bab428827972ca67e5e92a18be8d82b +Subproject commit 8a3a80cc93c3433c9d34690171d8fbd2e2540738 diff --git a/support/ci/tools/genvm_merge_into_dev.py b/support/ci/tools/genvm_merge_into_dev.py index e5926a14..01cdc24a 100755 --- a/support/ci/tools/genvm_merge_into_dev.py +++ b/support/ci/tools/genvm_merge_into_dev.py @@ -468,8 +468,8 @@ def check_gates(pr): check_approval(head_sha, pr['labels']) # 3. full GenVM CI (queue.yaml) green on the head commit. The same head may - # carry skipped runs (label events that didn't run full tests) alongside the - # real one, and the run may have been started by either a push + # carry a failed markerless push run alongside the full panel run, and the + # successful run may have been started by either a push # (event=pull_request) or the action panel (event=workflow_dispatch) β€” so we # query all events and require ANY completed run on this exact commit to have # succeeded, not just the most recent. diff --git a/support/ci/tools/pr_action_panel.py b/support/ci/tools/pr_action_panel.py index fd95817c..0bedcb85 100644 --- a/support/ci/tools/pr_action_panel.py +++ b/support/ci/tools/pr_action_panel.py @@ -160,10 +160,10 @@ def head_branch(): ).strip() -def dispatch_full_tests(): +def dispatch_full_tests(*, release_pipeline_test: bool): # Start queue.yaml on the PR head branch so the run's head_sha matches the - # PR head (the Merge gate keys off head_sha). A workflow_dispatch fired with - # GITHUB_TOKEN does create a run, unlike a bot-applied `labeled` event. + # PR head (the Merge gate keys off head_sha). Label edits intentionally do not + # trigger queue.yaml; workflow_dispatch is the only panel-driven entry point. branch = head_branch() if not branch: print('could not resolve PR head branch; cannot dispatch full tests') @@ -178,6 +178,8 @@ def dispatch_full_tests(): branch, '-f', f'pr={pr_number()}', + '-f', + f'release_pipeline_test={str(release_pipeline_test).lower()}', ) print(f'dispatched queue.yaml on `{branch}`') @@ -353,7 +355,9 @@ def run_panel(): # otherwise queue two runs for the same head, and the second cancels the # first through the `queue.yaml` concurrency group. if dispatch: - dispatch_full_tests() + dispatch_full_tests( + release_pipeline_test='test-release-pipeline' in current, + ) # Provision: momentary -> force-push executor mirror branches and open their PRs. if ACTION_PROVISION in actions: diff --git a/support/ci/unit_tests/test_merge_gates.py b/support/ci/unit_tests/test_merge_gates.py new file mode 100644 index 00000000..a97166e1 --- /dev/null +++ b/support/ci/unit_tests/test_merge_gates.py @@ -0,0 +1,46 @@ +from types import SimpleNamespace + +import pytest +from tools import genvm_merge_into_dev as merge + + +class MergeBlocked(Exception): + pass + + +def test_force_merge_still_requires_rebased(monkeypatch): + head_sha = 'a' * 40 + pr = { + 'baseRefName': 'v0.6-dev', + 'headRefOid': head_sha, + 'isDraft': False, + 'state': 'OPEN', + 'title': 'fix(ci): keep the rebase gate', + } + monkeypatch.setenv('FORCE_ACTOR', 'repo-admin') + monkeypatch.setattr(merge, 'pr_view', lambda *fields: pr) + monkeypatch.setattr(merge, 'pr_number', lambda: '7') + monkeypatch.setattr(merge, 'check_title', lambda value: None) + monkeypatch.setattr(merge, 'announce_force', lambda actor, sha: None) + + def git(*args, **kwargs): + if args[0] == 'fetch': + return SimpleNamespace(stdout='') + if args == ('rev-parse', 'refs/prhead'): + return SimpleNamespace(stdout=f'{head_sha}\n') + raise AssertionError(f'unexpected git call after rebase gate: {args}') + + def behind(base, head): + assert base == 'origin/v0.6-dev' + assert head == 'refs/prhead' + return 2 + + def block(message): + raise MergeBlocked(message) + + monkeypatch.setattr(merge, 'git', git) + monkeypatch.setattr(merge.behind_lib, 'behind_by_git', behind) + monkeypatch.setattr(merge, 'block', block) + + with pytest.raises(MergeBlocked, match='2 commit.* behind'): + merge.merge_into_dev() From 164d50889ab47c7381b86bec1f6fa7bb9ec26c30 Mon Sep 17 00:00:00 2001 From: kp2pml30 <kp2pml30@gmail.com> Date: Fri, 14 Aug 2026 17:45:58 +0900 Subject: [PATCH 10/45] =?UTF-8?q?fix:=20harden=20VM=20execution=20without?= =?UTF-8?q?=20runner=20hash=20changes=20=F0=9F=94=92=EF=B8=8F=E2=9C=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .claude/settings.local.json | 3 + .claude/skills/agentic-fuzzing/SKILL.md | 4 +- .claude/skills/commit-style/SKILL.md | 3 + .claude/skills/rust-test-style/SKILL.md | 2 +- .claude/skills/spec/SKILL.md | 8 +- .claude/skills/test/SKILL.md | 11 + .genvm-monorepo-root | 1 + .genvm-tool.py | 135 +- .../incl_release_build_test_cell_build.yaml | 1 + AGENTS.md | 54 + CLAUDE.md | 54 +- SECURITY.md | 13 +- crates/fuzzing/Cargo.lock | 177 +++ crates/fuzzing/Cargo.toml | 17 + crates/fuzzing/preload/Cargo.lock | 7 + crates/fuzzing/preload/Cargo.toml | 10 + crates/fuzzing/preload/src/lib.rs | 78 ++ crates/fuzzing/src/depth_limit.rs | 408 ++++++ crates/fuzzing/src/lib.rs | 177 +++ crates/modules-interfaces/Cargo.lock | 7 - crates/modules-interfaces/Cargo.toml | 2 - .../codegen/data/host-fns.json | 20 +- crates/modules-interfaces/src/abi_stub.rs | 36 - crates/modules-interfaces/src/domain.rs | 10 +- .../modules-interfaces/src/domain/fees/mod.rs | 10 +- crates/modules-interfaces/src/host_fns.rs | 87 +- crates/modules-interfaces/src/manager_api.rs | 2 +- crates/modules-interfaces/src/nested.rs | 6 +- docs/adr/012. runner memory accounting.md | 2 +- ...pre-validating untrusted decoded inputs.md | 96 +- docs/adr/015. cross-major contract calls.md | 2 +- docs/contributing/explanation/README.md | 1 + docs/contributing/explanation/fuzz.md | 39 + docs/contributing/howto/README.md | 2 +- docs/contributing/howto/genvm-tool.md | 23 +- docs/contributing/howto/testing/README.md | 9 +- docs/contributing/howto/testing/fuzzing.md | 162 ++- .../contributing/howto/testing/integration.md | 22 +- docs/contributing/howto/testing/rust.md | 21 + docs/schemas/default-config.json | 34 +- docs/schemas/runners.json | 38 +- docs/website/generate.py | 6 +- docs/website/merge_txts.py | 3 +- .../01-core-architecture/01-components.rst | 2 +- .../src/impl-spec/02-vm/03-consensus.rst | 4 +- .../src/impl-spec/appendix/manager-api.yaml | 33 +- .../spec/02-execution-environment/01-wasm.rst | 22 +- .../02-execution-environment/02-wasip1.rst | 127 +- .../03-wasi_genlayer_sdk/02-gl_call.rst | 34 +- .../03-wasi_genlayer_sdk/03-schemas.rst | 29 +- .../02-execution-environment/04-runners.rst | 79 +- docs/website/src/spec/03-vm/01-startup.rst | 13 +- .../src/spec/03-vm/03-ram-limiting.rst | 29 +- docs/website/src/spec/03-vm/05-result.rst | 67 +- .../src/spec/04-contract-interface/02-abi.rst | 8 +- .../src/spec/appendix/constants-pending.rst | 31 - docs/website/src/spec/appendix/constants.rst | 181 +-- docs/website/src/spec/appendix/index.rst | 1 + .../src/spec/appendix/internal-constants.rst | 156 +++ docs/website/src/spec/changelog.rst | 54 + docs/website/src/spec/index.rst | 1 + executors/v0.2.x | 2 +- executors/v0.3.x | 2 +- flake.nix | 6 +- implementation/.ya-test-config.json | 9 +- implementation/Cargo.lock | 17 - implementation/default.nix | 20 +- .../1 | 0 .../2 | 3 + ...3ga150hdc2vahv7sek5vhev60wgxrqmchnxqg3tkgj | 1 + ...sba677v1ndtj0816kwhjph61a9mtjqrhqp45j4pn8j | 54 + ...6tt8xf34nkv5f4jvcv7t5qnbvq8e6vspmf0we96ydt | 3 + ...3bt3yk923gfp1a4cnxp8n9k0nqjsd6003qrspy8ng0 | 4 + ...gtrmf50h7edgb1jvdpp767qs4j6vjfp33ty9rr9w1t | 1 + ...698ck5t9h0fn9dbv9x26q7gxekqbn9chzzbskfp53c | 1 + ...1ay1bw7ngb8zmtkdfkyvrka6qbjs9jwv46mt3cbgde | 1 + ...7425x8bd6nvkk4kgq4f2499n09vpv1paeafe025c6c | 1 + ...658bdw9cssdkncjmxvzbhmff4n1j4pwhntw0n9mj2t | 3 + ...7nh5jet076pmv9f8pqe6swkjsqdqc9yxw67105wrnc | 1 + ...4bxzvvbkk9zdf1wh22spsbdwbqeqerm0e43pv2ngct | 2 + ...2mzaj1hv1tb2txtrjrs0ejszt3fgn469604sqnqj6w | 1 + ...d5brepkxdgktr1f7xwgdg5swstym8vsx6fg3ncrmp4 | 1 + ...fwrht8gf6bz9tf3ag2ah4sdtaexyeeaatfy86cfbse | 2 + ...cwmc836hmvcx4tcsgxjt88gfqgr6crja3s08qjb2em | 2 + ...zfjthc54aqca5f5q9776emq6rtmc4psze4xt987swm | 1 + ...bwhymbmwe8yfb2af3mdmd6sreg4w60t8km5ctnghr8 | 16 + ...2aedsc1y7xgr695dmyvyf4zm587ab3yd1acdhrvr9a | 2 + ...wbfz1w1wys0ckdwxg5y4tvhshzwmq46h9mh79qw1ae | 4 + ...dp76k6cfjc5zkzec3cn0t5h0t58w96yyzeyhzsm6he | 3 + ...9xg8cfb3xb62hhx33xhhd813c2qrfrwjjgj5ebsw7w | 5 + ...rse79871djv54arat6a564d9wk6b32et8xfaatccqr | 19 + ...xqyra4jsre5cw5fmk3fg3y4gx74fc32dxwggnkqz6} | 9 +- ...26d1454cv5768t5j9se7txfecpxxsm9x8s7r91qf1e | 1 + ...a3j6967xv5d1a2074nyqtnbbtv76pvbpzgevzfgy54 | 1 + ...jncdjcy75m9m6q0qk6ezw8zj1596rhj36e402ew754 | 2 + ...2kcn21t453s1b2ngz23bwnbzkm23e5ks6x0v2z5td0 | 7 + ...kxs54nn6ex33a27wg1ppmthvkknkp3mep8f5b98k50 | 2 + ...jkt9cpff1d660acdtdzdpfv43mfctgbpy3he9beysw | 1 + ...ka8ved58jyqtjbr88vpkg80h3sbf3pjyznxq7j1904 | 1 + ...4j1rns05r9jsbwe40h08etjq468myc7zyqnr06d3et | 1 + ...j4895jyafkpw0jnvzmxdnhkv2p5jz9p6kc483sgk50 | 1 + ...z7rfp162bvpwtqdcsjfcte6gnaqhbn0brtbqmrfeyy | 17 + ...1440bpgmpf20twrkek8mv0d8005q3q499gz8acnzmm | 1 + ...cmt9ms89v020yrryvpa8jvmc20h657s4n8p4ypamy0 | 1 + ...yfsbfxsjvhq4h4px6vne04dr3tedqcnk65kx1h0x5a | 2 + ...sp9r1b3r2ws1hk69wtkfwp9rrxedhcyss8bdcnvaha | 17 + ...p4vcf7xkp37ac1718tt0rxvgts1agh64p5aw1fhxbj | 1 + ...f0t7k419d92gz7xhsav1gcmq07n07wydnnwsrgphym | 20 + ...z2hbxc3cj4tw3tje2j8d5ty7kc6kfpsmfte6p2twm2 | 1 + ...jstbv6j02k45pq0cvc2v78est9zdsmbnrffr0r7nv2 | 2 + ...p9v5s7zg0r44q1q10j4cw9xczdd69pvt8v340a6f8c | 1 + ...aehj72a3q4n0svg1mw6sr73c755f4fs585a33prhzp | 22 + ...0wvbm2cwde34147rf5ha82ry271c5rc0agxjrc47x4 | 1 + ...gxbt71nyw23zrd1qy323jtwynep5d2pdaav2tffxp8 | 1 + ...evrr1ecfjkt9k5kg0tk3xh25s33wx4ww96573naghw | 1 + ...h9wc8w0a8z0w2028vtcx5xs4dgyzf5w1s7jdppr3xy | 1 + ...2s7zs5t4wmrbvk0yg71wxveqe55ztfpg8jbsa3bqmg | 3 + ...hdv1x7bnfg9pavb755sf0sc40qmhsbcwqf5nrs7ae4 | 15 + ...cx4dk7z22dava03ptrv3ymfjycm4m4ffx7j0d98m4p | 1 + ...15a1rq85qvscnan6she76mdje40y78df623nha4fyj | 3 + ...r491j8kg26xvpnfe3a0yb3ve9wvy43c3n09dgv7ctw | 1 + ...ejvhs0c6k96dxf9zspqgns0xwvzn3ntmnc3m7hfgbw | 1 + ...myw0naed8mgex7ss57qmqbfwapn427xst8jsqe3ngj | 15 + ...m9pj961n1pq2vsy22xpbxw0nhyq93px4r60hahzjma | 2 + ...4137gha4gmqm24trscfjqc7jm4479w6qj1mex11fry | 1 + ...k1nwfyggr3b0wc35azwddz7t89e408jvrrapd5edrw | 12 + ...xfq59pf73na9hbcz0b6vdm97rwqjegsmyfg1vqsm82 | 17 + ...4f8x8hn228h57pkdye6wysbhsrrhjw49z4gepbqzvj | 1 + ...qg1fmemq3efa31gttwkc9ks89mdx2dbzc5qhgj92s4 | 1 + ...xn03hxa23gxe1md3gbn3zv7cngzcerj1j675n9780a | 1 + ...3w7v3kk9wzmhsayk4at70qkrajx55jgnepk7z6hr0r | 1 + ...xq1kcqh6mwx1jv8e4m8p45ep59sdmv28pmzy4pw1a4 | 2 + ...a1p19ay28psnn5f4zhyd2t4kfvda8a4y3dpc14v5f4 | 5 + ...z35x95rw4mrq1a5fxzy55qjks164pc29znrc45cdtp | 1 + ...2p7k1zb2gkvyfdaz725jw76qrm1wncjwqkqc4kk1m8 | 2 + ...a4awjzxj0n7cxbv03m54mfedmavr2f6m01vv3zvtte | 1 + ...p9v5282spx6x5aagjzrpyypvmtpkq3r0rrfdg4m67c | 3 + ...bs58b5a87gfxksxtm649ypqgmkacmejzry836t92k0 | 1 + ...5ns6xkaakcy132nmxmq4x2zgqr90x0dmsnz627ed66 | 2 + ...qwhcyefn0frp5r1ca0x7c5fx7z73799v9jgh5kznrm | 1 + ...6zv22vbtxd1h5fyy6aqs1709dm2kcner8xgmdjw3gy | 10 + ...tt4mqq71y2pvf4twdsffeyvc71wp3h2hvs5ahme1nr | 1 + ...g17dez65x6wswjp25acskmrx7b5tcmq386s35axwnc | 1 + ...fqq0x171fhs9f654mhmhzdam101sx78y1pg4xvzj2e | 1 + ...xsdkrcj75gxj5bh515daqdr49gxas5gq4w3mne8fxe | 2 + ...jwfv96hh6722m749ghsxbesbbkgqgajvx8czn183fa | 1 + ...t3zk62tgnhnhvqgzvfw352g9yy662b1fn2087cqc18 | 1 + ...k398b7qp548psbch15wvnbqehqgkz4pf2gtj1tpb7m | 1 + ...3p5fxq3fpxt5g0j5tpje93fn9r6c4b4qz0zmfn1df2 | 1 + ...95hbtgbyx52kbb43mdbrmt6gzsc3693yd49g1j2jga | 1 + ...tbeedk3mnayqb6z0dj77583dh0arvgz4456d2yp2wy | 1 + ...as110gqppmf984s9600v742ztpwrbf4gkgkrayw2q0 | 3 + ...ftjj6197wjb7dqch5ammz2f5esnh0zrw4fh1vzk35e | 1 + ...6p7whcvzg482hfct6tzax39zggc7bt3qqxj393awqr | 2 + ...502ckstahndqb4bqhxdwq7hzsxbw4a3dh9008n5grr | 2 + ...bambjhke293td0wnyhmqnprz3dr3kckw2t81vdeq32 | Bin 0 -> 129 bytes ...p4rpamny067vhq88ke789xdnyt23zn2ywjq7sbxy6p | 1 + ...zarnhgndp91dxsf8m775r96x1hb59bpaa1942fe22p | 1 + ...131f7jpbsdzp0t3f3ry9shwecm2mbn3sfxqy4vxqrt | 1 + ...had75c83y5s8a6z9ftdbnnmatgktgsasr18av2cj36 | 6 + ...vkc6r7q84h3g63whr2hdc69chys6bafjqyr21vd556 | 16 + ...c7343p2qsgg1vr2c8cfxf29rxjte6xaf1j3kqrthp2 | 2 + ...vz3ffjf0rsaayrf4drj4x6a87162a5dxa9x5gbyy40 | 4 + ...qkpa6a6a55kg78v14wk3yyesena6v3304gw7mp792t | 7 + ...w3bbq8bsesg4jf189jf9ard8cnp9m4sxsxsb96q3fj | 21 + ...z17sfvejgr7dfczk6w4qbt77qwanv1307w4b8ja0ma | 1 + ...4w84w1cnqb7cr41p52saatagrqsxk8wj2ehhbpc7fw | Bin 0 -> 127 bytes ...37jnwf7qj4e3jy7dngv203sb8da6za3tmdtaz8m71y | 5 + ...d5p3vpd27tw2r15btrpq06gt1b708brb74z1z6k5f8 | 1 + ...hwstm4ezxvnsem3n9rkbwe5971cq5r9eyvt51n1ca4 | 2 + ...23ehxrzsjx4h98n1t8tzxdeez2q54n4ghv7y87h99y | 3 + ...pan24avchbwj139h8gwrgrn168hpdn6wctp4w3kvnw | 2 + ...9f41n2ynf9n1r6yq9wv9h9zygjpz50ghzy7qzfxb4e | 1 + ...hye51bpxdznqxjc3n7txbqtdxw8tk999evs6pe6pty | 13 + ...snsywt1yg9608yk9dtkzcsw8j226873xmgrbtx5e2w | 7 + ...93yvqp1d1zes49sk9vwyf8enqcmkfectpyjg700p5r | 1 + ...357mm22rrbcdb29bv4883ajrnye3p05wh3p4e9vvde | 29 + ...gjt8z5jp6gss3nfv19ht36jhan7xc4x46dh9wxxb14 | 16 + ...nm2369k6hgrghespqr1pe0qdxrwv8yqrwjpevdvk68 | 16 + ...drvgpzq462t4wsnd5k20xxfvpwckw3b1gg9077ey4t | 2 + ...w69mppz8dkd8kkysh4gc00j3021v0zv38sy6nhq02p | 3 + ...p0893yw33esbfnsx3vtpn03jw2f2kav5261w8tk8fj | 1 + ...z0gcyv9ydzwdszy7q1c4ktka3rfn4d8pk9e4318etr | 10 + ...8nxj26hfm5ez2vrp38j59q0rpgvxffcvd8x0vme6n4 | 4 + ...xxf03629wvddwgvezn1r428tykhd7spzdd5hn2fcmm | 22 + ...v6wqxrtgkh6rbsmf4n7ta1xyndvfaedd6rbk08peap | 1 + ...fgm5c9jtkw8ae8gwh01s6tkk18my2ag0y7epc1ckd2 | 1 + ...fn4qcz970rxcapmbg7axet37yk0c30pden7zfnbrcp | 1 + ...2a7qvwfc8df5hzpjcyhkc65hvjwfk4b9v649x59n46 | 1 + ...w2k51qnmw7z4j79g0dngx0x3xv81fqx11rmktjvd5w | 1 + ...na3xrtgxct2en27tz1pb3v7dvtt5qh2fynhyvmhwew | 4 + ...9y0m4gwtzm6ybeecvxq6d297w7h1h3zwfpqeqwejp6 | 1 + ...kpme1rg6chpv2jg81r2mp6a1pqeb7jnpxkrj1zvvz0 | 18 + ...xhvm8f01c4cdcaymqg3nmjsbc91sxz2xryj3nmng3a | 1 + ...kn4y353ss52zenyq9e8gr7vcarz4y36dtff3nd9af8 | 9 + ...pdwe2qc8f6ck1zfkd4czrtxzbgf92qywfftqgxjhr0 | 1 + ...zz84st57awtf74kqg6x1ph1nc1b550hxnn8ky5e46p | 3 + ...556f8ke1g4dtjahxs94mzecqzw1n3twh8m9vtxdk5c | 1 + ...21mjw0h89fdx7t27e856cxrbv2ahsh3xybqjfqrq36 | 2 + ...enj50bwcb52ts5me0mptej88cabpzew64b22ncw82m | 1 + ...dxfswt8ax6ccwdq71k2wzyr2pv2sd7hcpcb3yes5rm | 2 + ...rdf9z0bqhr2s4f0s7yw5gawhafpd26csq0xrn78ehe | 2 + ...m98q123vvxhqx0aagr7fptytjjm2sesvzstq7vnjd4 | 2 + ...76gsq2b7p3hqv2h0fssazfg0mbppj9hr8yxyhbykwg | 8 + ...38ym1f018m8ctq0gbb06cgkamh1yb5yhztnt2sbgwm | 2 + ...m73vd7pqnte9f4dmahm7eddc8n3nb78hf0f9d14b9j | 1 + ...2mbvdzrxewmjfyk3g5xg07xn8j49tc92n845861rty | 1 + ...nmrht3baacq1spykfq06esnj3smp0rd5e7jp79eyc} | Bin 48 -> 35 bytes ...2107rfefkdjabs9rhdsw49zx8y2gx8p2c0qgt21926 | 1 + ...8k38eg2vs9bccsw8mnjcnx5te4y1g11rgme8h3gybg | 4 + ...js7hxbkmv5bb8nk2ey2d0wjx8k82f81c6768pc92ge | 1 + ...2afa7b9e3hmb9et592k41sh0f15patxjf2v5y5ax28 | 1 + ...kbm0kf3s0n4adszk893w6w8wnyjvf1e76ye91gg6tw | 1 + ...4fhxdd75yrzwqrt2e1e9wewtthq7qdsxqqdng0b060 | 2 + ...hs07vxp9p01tfz7pvd0jraxfs0jee1nw0cwkehxx94 | 2 + ...y7y6s511635spzptgtg0zhbcsa537xqby99h0nv250 | 1 + ...f3sj3zza00020bnq6r03axvz1vksraj1573jebfd6g | 17 + ...bbrs4vynrjpt7pmjj4897n9mfpbe50b7rwn7x9pgk2 | 2 + ...z1xs6nywx9v858m9bnd8ast8y973mrmt6s2kgtp9we | 1 + ...ask5rah2rmhtk8w53w5dvfptr9anhetwce58yktk1e | 1 + ...stbc50pvj2vw2nave0q2ry661kggrh2xrzfbnabzeg | 2 + ...2hjx36v13jedse38grc2h9jcqcrq3panpry2b2cq5p | 1 + ...m4s9k7pjvttjg3vmhhwkvgtp5rt9rren3884mtychm | 1 + ...8fgk28sjsw9kc4ct5vaw0nscda5yjqb3wxn81bjdpj | 1 + ...0p1x6sm52v6sgcw3k06afpbfxmb9d3xhzjretpzxhy | 1 + ...7gt19ynvbezapg0t7ercqeqbyetjdb4h3f7xy7s28y | 1 + ...0e8msvtg6fh2rgj738kh0zrmsnr65xhp8sknvdaf0g | 20 + ...6jqed3mz3m6eaqe529dqf6qrbr9pzx13mer6gwwnvy | 1 + ...82thpv82fgbs86s3dtzsz5wnxt0yqz4dvhhwz3r1ye | 1 + ...7bygx5f8z6r1s2a2519a6c625ytcfwmz4q9nt641qr | 3 + ...z0981gspjax26nkh652wc7tcb54w7fy5csjms0d66e | 1 + ...g0chm9b3g0gtyv3vwa5125zj6h9wpewa1d9ttasjcc | 3 + ...sncn3nmnesyx8kpmqmvfkvkj44xzebssp7xnczq6hr | 1 + ...yksckw2vp4xrca0xy5xnsa8sy9ytjw2wt1zhhndx36 | Bin 0 -> 32 bytes ...ca34p8jvyyv4r4t9aq8fqz0wyr6c5w6a5j1cns3wtj | 5 + ...3t0nyge0w12tkmwt21zv8gphpckwv79ky7syc8fz04 | 1 + ...q1wankjerhv937773xend4eakwq3xfyaescec2yy54 | 1 + ...msy3nbk3n9x2eezrgbjj21j80xj62n9c8svwt2weva | 1 + ...kgwrwycj09hfrjtmyfhpgmnerkapbh6xhp55gskj9w | 1 + ...611whwhv6gmbtrtwm09b0cbpnjr0mcnrvfas0ky1ma | 1 + ...hmvfczvhj7ba6jaxjqhx7d44drdy8nv5ypw3przvme | 2 + ...mmjcchvtx07q0s65jd41npa39zadh6pdfavdebrk3a | 1 + ...9vnef87wq5077p90tfrqwq4zwa7seh3erqrztx5xt} | Bin 47 -> 42 bytes ...51hcwchb3mvj75w58m2grm3dm3n82nvv3c0e6mwv0m | 1 + ...rg5k6rz6vg0cmfwbrnmbzybvxptypmp4ghactbwkzm | 4 + ...drhkddcaqa06fhrjy51knjj279hx2a87rtc9rwgs7c | 1 + ...vwve9j7bc3f8x0jb6z5qtff5tqedkwkad3zr0g7y5t | 1 + ...k3vxza1nxqkpp32q8r5027bbbv4qbfy37ad1knt6w4 | 2 + ...xpcxncfjgm9e5bff4v1wbh7pny1gd26hqybg84sabg | 6 + ...n879zp21139g3x4ra4m07sh9s2gn1fwwnn1m1sn1br | 1 + ...gmc6sv8n8tzcz1s8j5cpzq2n7a26ermg60hyz4t1h4 | 1 + ...srznj4t6bk5w9x4pcr0zwxmh6avsnt7gpn0fm7ak3c | 2 + ...g3wx92025zvx1dnn197sr2sxja0yvrdy1p97xdxthw | 1 + ...ta1s9373nq89nvskpew6nssr5b9gvk6dyv1r0jrcz0 | 2 + ...n716s2sn0567k9wwcq1vs380t3ap6m3526mkcgdhtt | 1 + ...48mftv5xr6nej0mnpbg4eygjz8mqgfgnt0qsve1pt4 | 16 + ...ak2fjfc2q7p4515s3gt8c42n443gqq58qt9hwcnzs0 | 1 + ...3kq1c31cejcj9qtn0rz5k7pzevbjyah0t1jsj84cbt | 1 + ...h0f2sd99x7d18esapxv0x0gvqmzzz8r3fw4sg1gvw0 | 1 + ...a1qexyqvggrsm11r04zrg8bt4ygm1c5jr683y1xvk6 | 1 + ...cfc2vbwxfdptecap1mxfgd5xeprwj203yq0fj9smhp | 1 + ...qj9g5qak95p6mw2p3p7814bgwesxeceqwd97f83464 | 21 + ...r9rqrptwd0md4bppt1jr9vpf9r4b77a74wzqc1e0pt | 1 + ...wq1t7yq53h7rpn35n04399zqvqkar42gebmqmbefmc | 2 + ...krjdvvw5pmgfnbbh5tyhw5hfpbtfpk72x35bcvyb0c | 12 + ...21spbt5fm52anbs9a3b7dj3s86f40gg6cz81acrt7t | 23 + ...dx92zmptj4h2j7zf3r3e42w3zh6bvbrz5z2p2e3h2c | 1 + ...g78eczceasp09agz77gav04w2cmqttcb1a1z89k7rg | 2 + ...xkpezne5pp3m6z8age604q7cpv1x920qnkvz1bkwyp | 8 + ...gag5enzvbfj1jj3p1h9vnbqe748dq2ppejheajd5ng | 1 + ...bsr64qxtbfwjkrca2mcwecqydf7daxy28pc0vfe7ga | 1 + ...78x0s7vpt01c4jm93bcz1b34cy3fr0tqwv7tr549er | 5 + ...deg4erafb4pzb2901wkjqtdds2c1xdb7fv3hxmfy08 | 1 + ...35dbxwq0p8n3ms3tvtmkqeb60fef4pnkwz0fvagavg | 7 + ...5gqqwqynzs9x0hw765emeg89cewjz87h01btpt6hsm | 15 + ...wzpqkyc3m2nervjxc7xrff5by8m334by2ehynfa566 | 17 + ...2d0dafx3tczd8ba0d3q08tdben51y533v9nfbp9tt} | 4 +- ...xjzt801waqf7pynga8dtztvf33cxav6gnakf3drksj | 1 + ...qet48avcywyfmwdxfve36a9ghbr9wsa51d35cydp2c | 3 + ...1rznx8wf9tt6fyrps4510fq9zj0d5d4634jyyevnq0 | 2 + ...3r8mjwycqk3cc19tk6vz6v0mpx5b1skh4619860kcy | 1 + ...rd7rbzfmpqy99m87rakngp591kxkjb6npwzhyetr0g | 2 + ...4fjc4187d9vgnra9sgq9mms214txwpsp3kvqgd3cvj | 2 + ...t19scjh8kgqscps8x2f1rz3e4jbc9aasct5hsk1s72 | 1 + ...a57zfe11k63pk5mdzqwqh9q4wnhr3rjsqsvpsd56b2 | 3 + ...x97mnxq79jxx3p6v1ash40kmtc2sg8rb0pkxga505p | 1 + ...7xgc4drd8e67mde018kypayx07c8vb8vh8j7vpd57r | 1 + ...hxytcf4491x4ydx46vqpyacj3205hsb3brs7p7n4ty | 6 + ...e6g98xw74xt2m44tza0kca6t24qsaqm5cmrgw0xf86 | 1 + ...p7skkfw1b68nrs3sq5zxtr9axxaxd01nrkkgbqebw0 | 4 + ...bhyqcknm6pyt1f7j8jd61q4yrzhmvhy353v23257tr | 1 + ...swdcbj9e7djcah0pmjvtxsd7m588966stts93xg9yg | 17 + ...y027n4v93kp4s0qh20fx4gkved3fw15n7r18yq07ha | 1 + ...bt1he8ntbggfw9tse48emwb2kbycj77jgz76ddeyfg | 1 + ...77qz80hpxtgnynsqmqj74byzc144gtxq76kwyrmad6 | 2 + ...jbvhqx45fv2n0r972app5mqy1hrq1xdk2bnq3he5qj | 2 + ...vedsdq879bn98wspeyrsn6dvzagxmryh2y5r1d1c9w | 2 + ...5fmyk9fhwvtzx0qcm8p3hc0z75ys0dtvtpqnvcyw52 | 1 + ...cdbfdsm22fw79j8b0h99d158d59af312qf7xembhzr | 2 + ...rja2fjxnemgfffhcbwms91t3nct5hha8x28py326y8 | 1 + ...ygmpq420qqgrshs10nenpk20mnzqtqz3mt22jhm9zw | 2 + ...tmxbfh43pa7xerzq5gzg15ad9hz007etcbz5fm0g7c | 1 + ...z7fcaqtq179b6m9pjb88yd9ge48fcq98mz0z7z04jm | 1 + ...41e9f692tk3awzkp4y3mfm8tz4tzk9mqdrydn4jwh8 | Bin 0 -> 33 bytes ...bar4weepxqm36gjtmngd0s4s87t0zs71gras1azkja | 1 + ...wync6tykyywar2shdyswn9r1re8ta4x7kxc21586p0 | 1 + ...bc8ad66w2zzn4rpff9ntkjw5r6g7rsqa16hgy3at1g | 1 + ...bwdmzca793m5jvdh96qbx9kmsq4ydvyg3e6yqny8k2 | 1 + ...hes7r132gs7dr83w6w7q2x33ce0h82vffctfn6eqv0 | 1 + ...rvjnahynn25ac14cwvt9t6jzvggz372ydp8fqswv2a | 2 + ...3v2zfg6c9n6n90t48dxem8619svqp59wxtvej6c5ha | 2 + ...2ghfrhxs1ahgdn3fw4m2y492x836wf7k46sr5err0m | 1 + ...btm14hgyxeyn8ey5dgnxtggr212adw4kwvzbnybdct | 1 + ...pzscez3h4q8sxq7dmxcqvnfn5pgkdfxrqq1gr9csvt | 1 + ...y8ft3qegkfnscpfrx977aq71fvfkfggktv1q8f7f78 | 5 + ...a62pfbntw5ftd5xs2v8rrcdsmt2s84vc608cfn9efy | 1 + ...5edm56x6y688pfpkvrm2m67q4j9xxaw2twbsbkbyec | 16 + ...ck26hk0rmgzpwed603cmmqm6wzhp1e3aarwg34r7xg | 6 + ...ah1hcnmr8captbmxmrvjxs1dn8gbn0v7a10jf9msfp | 16 + ...zvhamg4x4p2dc22wt60w0stt38wpk0re71drs2dajr | 1 + ...98q0hhqgn55z6vczfxgp42qb5awzyjd131cz51zwbg | 1 + ...j3fy8nbnk194bxprz80a7yfmdhdkqtgtd6s8zk138w | 1 + ...qdka9j6w1fg7mg0wn0tb355jkbatrnzv6n7nmw4skt | 1 + ...hrxsbftvpfmavg3smrgjc4hh6ezkawrfcepxk2jmde | 2 + ...w56dbhmxvhk9fhkmk8y0p0gwa88vszgnchgenysxh6 | 1 + ...3zk4k8vtbbkpxa5q0sp8fch71gka8tv8bny9kdktm2 | 2 + ...hp2xf71j0b3sxjm9mkbwvpyf8jff8n64n6wkjr6368 | 2 + ...mh4hmmgpgm7dxwht1cp1amvsd0qxnc23tdgfdh8k9g | 1 + ...d98pan04nx6q9q9n5mkqtz86cg7hnkm1mkcq44qjmt | 4 + ...r3qc3h1anzm3hkwr6j3r6rmad853tc0ds25hrc2fb2 | 21 + ...r6yembj5xj2bm2cyg20q57cj4hrf1p8x80ndymj4aw | 1 + ...643cjwk8wfh0004edbbst8wa1b73aywaqfc0mcf0kp | 5 + ...j6pprejy2v5shedeb0xq7dqfchsdcjy9tt523dz5mc | 1 + ...r1bp9z55bqng865rp8s6wx7w7hc6n2jk0wanp61062 | 1 + ...w61hvabmpf0a96abnadvjsnn6q53zwxatx2t7d0tw6 | 1 + ...ard33ab13sqxf820qr4xb9cgscj55spqa0neg7epe4 | 1 + ...amzadf8rnaf6j7g1t39dft2r9tm3b9rkj28d3edhhp | 2 + ...54yqfez4decm5rhrdrs2bxfdjmcge4tgd9wygtgh10 | 1 + ...c9e1wemy51xq2xpwm52cc7pjda593acq5vtwd1bfd4 | 1 + ...qrp6srq0pd6vccjfkfymafvrh4qg32cx1br2exnd5a | 20 + ...8e15aj1a7yp124h139794c8we0wdnmp7jrtepj9zzt | 5 + ...6vx30habsa70mpy6c1mtqscca4q0q70695yqxx3578 | 1 + ...gssx6eqdc4ktnadb6t92gz3g3veep8a5fqkeve8jsy | 1 + ...b5xm7mmt0whe7p2yqrr46d84j7tphrj202tazvzgej | 1 + ...7me5gr10qg7qmhs49aqy7ymgpef4b6y8j5tp56p6ke | 1 + ...2pz5srn6gthdq019taegmadr7pfwxv24e5qajd5cbe | 1 + ...zvf0969q6r8nwnmr7yhwyegx8m8vffgnbjh8darjcj | 1 + ...crh1twhkb756ves35p72ydzvds4v9a58ckxv4ccjhy | Bin 0 -> 20 bytes ...cjbaa7kxnvt61ysr5aa9a1htsgsc5akc4w41k3y6ht | 1 + ...eytr2fzvg7ep4x96tzbq20tv1b9mfv6q6q9bbrakrr | 16 + ...y4weyyekrmefx9xx4jcsfnf1dj0vy1e5vp1zsyyzny | 1 + ...jhwp88r17fndf8a6mw93spgxbzcv3bgyvnhv70ca5g | 3 + ...8bc75k3bzqhjeqpwkyv29kzt7p8bazzx3ty7artsk8 | 1 + ...w4120f7nv6dvvhmsyb1khrxmwbqkgs22k838edwedj | 1 + ...bvcxcy4p2925nytewgyq3wh3tjbn6mp8d0cap1n7jw | 17 + ...tbsr7qeshkz69p2kgshe0xsa45ntjw6ypx6a3142by | 2 + ...e52cvp0d2ebjhvbb9gfcyjss12qtv2kr0cdjkcang4 | 1 + ...2apb41f7ev2vzwfh5002wp8jq1t9hgycz9b913znx2 | 5 + ...fg7md78205facm1zrbcwazzg87wrxkg6nhhyc7242p | 1 + ...bx5dcssbk9fgs3250tt06s57txd90ddpdevz64y7kt | 21 + ...wvhjaq25kg1r9489qdcrbdp051mxnyexd4r98zcxw8 | 1 + ...d3q0q8nty2kamzc30yd3yrsca6cv5p1qwzb5jkkh2m | 1 + ...jjd8tqccyt151zdc50zv32f9v7e9x39qmxrf50pswy | 1 + ...64smhx7gpf3pf0ztfjdp4cqr4fjdcgdwjzgdzxnbf2 | 2 + ...dsw5z4095vy5a9nrzbbmctqj1s60t8a4nx4z5masbw | 2 + ...v323xdcfz69fkq3hdepy0014265agv95f8v9685n56 | 1 + ...f2j2cp06v3pxabfw8a6qfz1gzxpvnzm06d4rt8qtmw | 2 + ...16nf5284aa24r0spdgrph37mpb4d7ws7ez4qwqzt3t | 1 + ...gmyzxnswhedm95ey6jmpa2egzn9jceyd74zjj8ep8t | 2 + ...gep5rgg42hdagzzcaqcwkq8dnq5xywq41p8yzhjhxr | 1 + ...3jgfgacqbyp0e0hw5w3bw7ne6fqnsbymaw99qvym9a | 1 + ...rh55v0g8j79j5bb7267tzqgdjrqzhpbvde5qp98sxt | 2 + ...dkkddzftkjddb7ztsdbjz6dmpjjn6x3y295gq39n98 | 1 + ...r0t9ghya12j6htrh77pjte2jeq23ftqyw4b2qjej52 | 1 + ...12egc4y89qq05pep6z4f2y6mxw2xrc5yw0q9b25nd4 | 13 + ...dhzv05hc0sxkg8vgjmvymaqw3f6q26mwmty8vygs48 | 31 + ...gc8zcqrbh0f5yzpjrqts7ganxrw1b6bpcvm9d3g1wm | 1 + ...pzxjkayxkqzjsefhdxc96qbm94etg252p41fx214pt | 1 + ...knp9mtyb13s8xzb43p22w4g6hasm6s9xn96z93hhh0 | 2 + ...adfvkr5t4afqhfb6tbtgg4heqcv34z860ftazx8f1e | 1 + ...7204a9gvnrwhsfs6gw3wd9dgp53k6wcj5v7t5vt39g | 2 + ...khvdfpwcw9t9dhmdggn32aycqwaftsqjc93ph6f9mr | 1 + ...dm8vrs55qdq3mbmh9s1rr4x1qyhfb9hz91w0fg6am6 | 1 + ...2ade597qyvp0q44d1pnbj9yrxc2aegmjtzxw2g6cwr | 1 + ...2m26p9tjmqpcgjmm15a87y3q6ge5vr12vfd5cegjza | 2 + ...jk64ptxd1rqjsh0g7yg9bwc3vr6jay0rqjnb2cgq3g | 7 + ...nahc097sf7wfdz84yntp9h8xhb1cyntp5mtfbh0bcr | 1 + ...pdd7znb1h88w52fyf4ss16j6g5x6y1xdrbch3f2498 | 1 + ...m55ahtbz3bstsb7wa1zc5cwe8j0kf1ac6kn2b3c9e0 | 1 + ...ptbbyxjbtzx5r683vg13k1gv4c428jbfk27m58vwca | 2 + ...pj4v790fdb76zgnp1h74skr5nv7mcty01stnyj86dc | 21 + ...jkqvqr89xn9f94n91n25rqzs58s7gkq12c4qb19n82 | 1 + ...hcrg11qye6fygr8ve1jrv75z6vxzgwpc10ykare34p | 1 + ...5jcbq1bh9rmwq2arjsd3nbkfj9cabtkjc6rytb707y | 1 + ...8y3t1ypcbv2r80jsymm5yqtkbx41kqb7yax0dag4rp | 4 + ...tg4v8q8489zy60xa9kjwfn9dtnf6f5841aw9mbz84j | 1 + ...kydcqpd8k65kkc18gahhkr8ckmc2tfs4fsc3rx54mw | 1 + ...h1asmc9q9qptsthtjgw247pyeasgpcy17gx32ka5kc | 1 + ...1bfa4kh47929dmgnee4d1f05hgv6js583hxnzacqat | 1 + ...te430cppsj8f0n1tr69zgme8zm3zdctwkts89tqkgj | 2 + ...d1xq1cbjw4bv28vrdaptenxn31d03qfycsxd94dztw | 1 + ...p7ykhgax3jbh9pm9v832kxgfpc8yj2wdqnj0dnbshc | 1 + ...nbfz83cjp98eh9qrz4e7baqbcfr8w07c9mr8h8h76r | 1 + ...d8pnyd810hns19dkvp0pm6mbctydr6nxh2h5bj28f2 | 1 + ...dcsdt837x02eq88kajab91nq0ctkmc5fja6s14x49g | 19 + ...vf31j6shg87ajmc19gh3f36tc4vr6b7v81n73f21cg | 1 + ...04cjxa879cfr3mkp8j5v99ewqdh1ka3bqptc3r7248 | 1 + ...d75zm9sz89vs3pnwfs94sn7rxdxg25tyqgx5jbdbp0 | 1 + ...fgsxp69zhyth69xdj5wvrsj2v898bb12r1xvcpg390 | 2 + ...4rfdgtkkcy6efj3fyz1excf7pww2x15239qk3etjpy | 1 + ...zvzad8fpgnp25t9ea96c4nbkmts8gxsdkq301t1sbr | 1 + ...bgpf2hrga2ja9gan85nve6vs8ndygv9atc4gvx9zr} | Bin 2 -> 2 bytes ...qzg4fntq5azg6h5f1n3k89v7c3zkd79a68gvqpa9jy | 1 + ...jgg28j8qmpnrxvgwqwhy9rsa3qzrnqmzj9ppdafjag | 1 + ...5t75s30qs8f06japgnm7sc9vq1m1qwrqs634edemx6 | 3 + ...jcc0akvqmxres0ak235a7j1g5x8af42gvewk0w0ary | 5 + ...wa6f4rx04dgwghpam5zkxp9c4yvnry1a9dm6708hcr | 16 + ...q5vbgw31nc3nvdrve6xhx519nntpfp466y772cd562 | 1 + ...wctsh5htv0fhkw46ab49ws2dqrng17h3panefejg3c | 15 + ...826zbpsn5r4wex6spd3043c4y5gcnbvyxmfpgj97w2 | 1 + ...v9q3a817bjvgqk19qhsrmre8hr82mhrjc2zq2m19gr | 7 + ...mhak8fj2023hsbnd500aknqbzpvx1gghaxdfbmx9mr | 14 + ...d3zk59t022ennk6sne1h1m5jzkv16e7tcqaaeh4ngc | 1 + ...62jc16td8pgmwkmpb887sccg89r05ph3n0k2emj4jw | 1 + ...ndvgd652rw5bfgenmkatrxbypkxm658ykfnq29h4g0 | 1 + ...3f07z6bfh43433v3t4sfrg22ybgsyfy42fqm6x1v3r | 18 + ...41vh54xd59bpg48hmrjgz890q69jmejh756a0q8nnt | 1 + ...thskpwwnmrm8g6khj3j8hxs5yfvhxgf086pq30jhtr | 1 + ...cfp9bdvcqkqg5tm08ggrc5bda7zmqz8xq3k3d6ysam | 2 + ...v9r7a7eysaxd0fpm6d58tbdg981d7z1jdjvcy9xmj6 | 1 + ...33vsd7h36xx5pzk5wzyk4hnn9ha7y87eh90vszdtwc | 1 + ...pyftwdby1bddzawz9ee6bavbfagjwem3n3yvsa5936 | 1 + ...fsvp41g06qv09d387sz4kknemgat7pv72apmrvav5r | 2 + ...vbtddm9r6x9jq3wdgpckfe4xzxgp19bh2x9ncjqwnc | 2 + ...9ctz9pyrpym136n3yra4fhxqvzjks22hjm9y2errxy | 16 + ...grm16kjp4rfzwbzk3p7sda804ykfma4wmb53ar4998 | 3 + ...ttf0zq9pvmgs3qjpdegznawqw1ndzbtbvs40the3h6 | 1 + ...qgr1jgbkmebh826wq0heyv6071h8j84qw4xmjdcrap | 4 + ...sbgnjd6mtrdw1nsn4p31s2arbkw7gy8manf8vp4tz4 | 2 + ...hxbrtkmkz7zt2vg3zq107ez9g017egpmcac6w50vbm | 2 + ...jv8nn4e18xh44q9kswsggmy91vy11ykx85ttpc6a7g | 1 + ...mqvmwkazrrm0gzftmzw2604x4pkpner8a4g52rgnhc | 53 + ...pvb2zt2hrkg19mkfqc9h9sk8yz3rxj95z6y2n6tbee | 1 + ...0hek7tx48svhkcmgtr4jp6cp0nmzsgzqdp0qkrbjz0 | 3 + ...bqrqshc4a4e544gy7bra0cc7ngwb8nk98zwexz4ek2 | 1 + ...6b8jk1e8d8gg6n97s15b4h6y9q8n0r5yp0r9wsmzgc | 1 + ...ye8hftb4hf441w1rsnrk9zhps2tfwfm6xdz6v8497a | 1 + ...a0gfw61g6810w4p3njyk0d0wh0xf498g53ntcj4rjt | 1 + ...henq6tnsbsncefwrtcxcj2zc3h3jkdmcb7f95skgny | Bin 0 -> 47 bytes ...nvqbr2pm9vaagx1nkz146334pgeaqhwkvdsw3pv9wy | 2 + ...4ywfh2ctkm2tm83xfhj4nv0a6hph0220eznt31rf9t | 10 + ...fw3gwg4st7g6jsby4tqa9b6b8qz8yfsy3wxcqa8m9m | 3 + ...jhhgnmdp394wyrsrdhhaarc93anpyxdb722st357r0 | 1 + ...tqa29astpjxqb9qb9nqy7a9p2yp7jff7ewbyqj4jjt | 1 + ...djb7j6ykqpqp5xkznbzx0fx4j0fv9eeb86t21ngqc8 | 1 + ...11rn82hs4z3c99f5kvs8e08skq9frctwpbtgj6vxzw | 1 + ...6az1bt52tr38crbmgm9ffvsaqm9me6x8w8z62qca6w | 1 + ...zes6wrn2rt75rrgmvsx54ep3zb3e2psa7fvyghhd28 | 1 + ...40rh8bnnba51n7mg5cgxjhd6m83qazmq7w4yp7b13r | 1 + ...vsbkhabpazvdv8bwezwexe2e88n4dn8ve59wy3dkfp | 1 + ...7785fpwp0z6r9he4bbkpk3xqjndx2yanmy2d4qdvcp | 2 + ...pvtd5v0g86azye9vbwtdfeq00p1sseg096m3j9cs80 | 2 + ...mg4qtd0ezmtb4bf6beemmaqh9h76hkfx8th8y94mey | 2 + ...857bs4stbmpedvjgn6avmv6weje9x8gk386v9jrspt | 1 + ...xf7j5hfttj5g4d8e2hzb90pmtfb16brhwm4v6gx11p | 19 + ...jccxftrntwr3se574eq9e7ykcpcapyewr1rsa5kf6g | 2 + ...046tvy72ss2g7vv16ppqtgy8nrq349ajkt5esk7j60 | 1 + ...d5j0jb6rj91rnf8vdfv5mj7frgnh6b3q6ty1nff452 | 1 + ...4svg8jmemyt4177w80xen3bs5fstgc8jbkb6ttcrqg | 1 + ...zw9ba8amkm74msqq0012txz5aqq9q9qe99f8csqr1c | 1 + ...x24jfe7r8jg011g5j6z871zjjqc0hvtcsf8bksf9rp | 10 + ...rj3me5g573cnrzgk30d8350x3e8tdtkqn6banswm2y | 1 + ...8n6g0e3b9b31ptzy2zdmf8q7xp9txsqhqhkhj78hp8 | 3 + ...4tg672179s8epq9vjc1t8xc7p426bjcz5arcz9p9py | 1 + ...27pt6r50fwsjxvtcvtgrqqyjskvzraqnmbz5c4nhs8 | 1 + ...pt36hmn1raq16m83ywr2v1hfksbc677j05jz3a5cta | 1 + ...503723z2emxs30rr989mgg7jcw5597bgjzdtbsn43y | 9 + ...0ny12mz2f64ah46mqt4h1244vn25gtjdh4rcp40yxr | 14 + ...n23yd7cbe5zk7fasp1c8h7h6r9pqx84kf59kkq9t6j | 1 + ...dpx0fdp3t2b9f0dq8tkyjz7bb17wp7qek6c5j2wbpy | 1 + ...fma1cype1bcfa17gp8y8mzvsqb7r09f6trh09sv72y | 1 + ...2jxcnb53f4y97p9a8rgypqezdfap98c3wgneg7k3fg | 3 + ...wnq7kzhv187n0eefn36fxjqqkh811hpe5mschmt682 | 12 + ...sza8ysntt59aawddf8nrjfz11yr5avv3mfzjpa4cct | 2 + ...fmbd7qgdmfcmx9gnpypcabepgxw28cyk07c90xqzkg | 1 + ...hb1cez1ecc2v4ft3g9br7hgm9pn19295v93vj7xd98 | 2 + ...nv6b036px26c1pfxdyetr2ke0t5k19q9x0697h0620 | 1 + ...1vwn92g566pqhj4psd63yh1t2mmrty1xvfba4v2dmg | 27 + ...8e6jejmqxgs5feynsczrd46hhyhh4w8kr21xx78fs2 | 1 + ...32e17fbw05ezyr972xx9kp3yv30jgk5fk31yhwhhye | 1 + ...venxzh25x3nhg28jjsvkz3r9vdwrmn3mp51n9hjscc | 1 + ...fpyb5ted5r0ydph71yvwk92w9ad8jc7cwdey5d1wt8 | 1 + ...zxzy6c0se61kh6fw955s9q7c74r6dqrn29pq7j7nva | 1 + ...g0z2n7emt92qnkhvn9e40mre0cm7fj6cavcg3fa8pt | 1 + ...ebm6z9e8603wab4pna1mf1cdec3wrp32jem3aw64zy | Bin 0 -> 291 bytes ...frz98yn558h2wzg6a1q9m9mna6h73tfa0bazs7sn8j | 2 + ...p1rtsrfr2e8b3kkp860q87jekvsmh09s5yh461kebm | 2 + ...kjt3y2enqdjd79k21h1c0knams8se5fehqmjz3ezmj | 12 + ...1wp4f4g2j3eev35nwd693ztg9rywxbf2xz2sw97efc | 2 + ...77sr15mekm241hw6307vdxw6jfpvtzcv96e0kpb30t | 1 + ...w7jkye43zq3m8tcjfavsyxb0ydpfvhecr88cct7qqy | 1 + ...xash0dv9x92eban9n4bja4a9f02re6s0cfdvgxggsg | 1 + ...2yfy9gpzwpkyd441q5bxa9d6q0znw8cn5c38yakb8j | 2 + ...4kk1aszm9zrt7r2rk33emns5jh7mt8v9ma5atbwd3t | 1 + ...838tn0zrencezbxme6vs2k4ehm2m25vpvaakhz7ger | 1 + ...j2956qabnb1858sgp1x5fqec9f81j796pe84cd0fnr | 62 + ...5975x2xbt7q49nejc75g2gkgq1sazyfz2g4s7dbza0 | 1 + ...rbet0j38wtdpacqvcedcp5nskf95r02gek39rmg8qw | 1 + ...w7s5sstzbwkzhcmrd55mrgw5ydy3kwys6cpx1znspy | 34 + ...08rz29x6xjyhzyt3cdtz1btr6tgatnyhnc07a2d5vg | 1 + ...bam1yxkm107he6rdbwpthe7npazfph718gv3802qy8 | 2 + ...br34j2epdjf3hm6gs0bn44dkv64q71s8vxrwn4ghfy | 17 + ...qwc9ng2a7y42dae5ax8bv7zvms6s4jqb3ms9q1062a | 1 + ...01nxpr25fjkpb44v57kfxy22jbhre9jk9xmz903st6 | 1 + ...2tx60pfkvq45nsmz9b71pp579pdqd5g9njtv0bnw90 | 3 + ...35tsas864bn4tkwtkbxf623k66fnffb27x07czte6t | 2 + ...d11vja9g02rct4rgk4sbyz2pq8g6ce8sv4hc7wwbtr | 19 + ...42ng8jwjq002dmsatnmn21as2fd1g5h4srjywym9g2 | 1 + ...nwfm2teefsk6cqhaahqvmaeppjcdzgfyph7y64cs06 | 6 + ...0dxa6py99hk9vmne6g5w317jh1spjt90vf5pkq904c | 2 + ...sd04pp0ek735a5d3fzbxnxhpvbafw9hvh99kvbwz8p | 2 + ...tj4zjgtrxfvw4hmnbkzdp6bdh1j0x9rrj0804xz2g6 | 1 + ...2wz64dne91ntf7e90dhbf9srktb5p29aq26jzwk0bg | Bin 0 -> 76 bytes ...q0tjf2yp04axpjztgdrg2q1m1rvzsvp351nc80dpam | 19 + ...jkd2759bpn5f461dkws3h2j15yy7keerbyspd857b0 | 1 + ...21bx76yg9g3vd6kmxcnkh2dg6q0pctj5bhngfannba | 1 + ...74w7s5gapgnshxyn15401mxhzm5hh3fve4cdmjwncr | 16 + ...z13a134phq13pmsjfmm4a8jzb5n2nbzjaq0bvj78ja | 1 + ...wdm2hqypr8xfsgwq6cgekw1wqsrzpww5nwscttq46p | 1 + ...3at38nnnj5pbjqbfs84vekef7cz8w0hppz1egw5dwg | 1 + ...a3ry1jg09wd47m2xwpvgae90wnxtr2rvwf8e146qg6 | 1 + ...tzypr7b7x9c00zgpnw90pch381j378ttszp1jfb658 | 3 + ...2dbqk4z17473cs581byf9df8452h86tz7r7vjja3er | 9 + ...q6gtnpe7dtegw8qsney5km59ma0yn4690y7qg2hq04 | 1 + ...wmvp49h1dt7259q310ypjasyxkq3fxs1svg1rr3c18 | 1 + ...t3ewgwe2zgrax02f99a2mz17c7mzej0zkhmc5sxz60 | 1 + ...prghszwxv6ppp3mrranea2ptkgwenp6h2gpyn03rsm | 1 + ...zy0ph9dzdwvdf02pvmmnb3z71fxqdzmk5tfcd95016 | 1 + ...dhbg2n4jtfvgpzkdrvag402km42qm3cvykssn1f7c2 | 1 + ...zrf6rdq7jav2mc6v518jtcs63dr389zfqj1awnmx82 | 1 + ...j4521be0c23ktwdyqs75esqbby40y7tc4ee40mg33e | 5 + ...61457rhjg8q1gtgwrxwmhv7fe4hd36n16x15e3zjmm | 4 + ...z8wygwa45hanm7s90tvaw890psc9tg15712zb3p7j4 | 1 + ...391srmekajtdzkg6xvyr4rd6ndvp5279vwc0xn2bcp | 1 + ...fwzkf15t5ab0xfzq7kwvzf9c3ej1vk3sdz9jpnykm8 | 2 + ...xy4531pmxv9847jz7perpg0cpz4wnkdkxgtydg3v60 | 17 + ...7p1xkxz9brkee1ea2ywawfgxqn5x8t9svnqt9244ea | 1 + ...baa8yz1ty6dywpg4xjeawgmehkk287m40ygkx6f6zp | 3 + ...8f4rk68hcfbeqdx4pk7bzjz07vg6e7a3cxzx3v9wft | 2 + ...g8stb3a070a7d4pg61a7rveqv0ccr5bhfpn7a4rstg | 1 + ...gq9ge6v81dz4ah04p7s2wmgpxxvzqwytpzbmwz7b3a | 4 + ...084c49th6y4ryysvh0027yvt77cfmxxn79qcyyhzgg | 1 + ...bayrg290j0f6myh3tt1p63zzjx1mf29k0rwewqqchw | 3 + ...3vzjvnvfx83c7abfycjh14x57afa1ceyznkx4tg0k8 | 2 + ...cbzae9g1dae8mbrgtcph3phzvn6xs0p4hgwcxnzeea | 20 + ...p3m70qztxm8dd5ftste6jege5zgy5t6j021833j7ej | 1 + ...hyy4pf9dk9sm960mm91qpnw8draq2q4mwvdngp56d2 | 1 + ...xz58hzw7pmvdgyt09dyarfk10n5g1rfxf5nep1531r | Bin 0 -> 260 bytes ...x0c4w7p7jsrz1h1ezka109ryjqetjtm9n6bth6ea4e | 1 + ...k8bqhfqc4qy61zpzberwejg3b5evkeq3dma3js2t0e | 1 + ...egm2adakfzwk3dh4hwkepwt67gpkyfw5x6ywrrs9jg | 1 + ...754wqsxw871nt24v192x2eck1jz9w9s8bk89m7657m | 20 + ...k3z20wf9ahpv6ttewhna2m0251hx8tvtsk13pdazb8 | 1 + ...grvyam74hae6x8wsw3an148qa9caw8cjx2jwxpyznr | 1 + ...wg34atrg7afqky1ajnxprcdzbbdqfg09fb92m063r2 | 2 + ...1j3r352wvahzbgnh8t449j3pm9kb4d1ax0ezky2b7j | 1 + ...6n5jgrsxe3aydj10cvq35ma4wsg7qczhz1r64d7hn0 | 1 + ...kz92b2h54dsc8s1vzbhfzenxw9r7hx740z41fexja0 | 1 + ...ddf8e6nbae84a2vgwbs0dcm847zgw3vctbsjd6j2ag | 2 + ...kfkn00xvw37f3t551t5tcemzfqe9jegt9wgftas6c2 | 16 + ...r8ycqfat197v4k1hxtkt0zcyrvhgx8tmzx326zrd0r | 2 + ...gy0dcn4srykfn7x9vabhne57j2x7y8dg4mhrjdw7wj | 1 + ...qfpe5y3st6etwykvvb01v697hvpsbzgpq98e44xvpy | 1 + ...mj82p5zpjpyzzvxe1ztgte2spnpdqet0gm82frhmda | 5 + ...5ncgkjyx1j6h3h5vw9x9m3nw6f6h5rnkv1e1qx55qp | 2 + ...njzsw0nb75195g8emmhk4tjw68n44xwvgkz4w8063p | Bin 0 -> 74 bytes ...xm7ybsmr740a6hbpeaavxczg1yq6eza16jvst2ppg0 | 27 + ...nzbxvyc2zgevg0344280fmx3ahkk1hnjzr1715g3j6 | 1 + ...v0zqe12ekgy8jzscfbwprhaj8nwpz70p11k1zwsmzg | 6 + ...gvt6d2g6613m3w203h288dzd2k6d92wn4kfvzbd7je | 2 + ...888ywyvy5fr83bdyj7dxjeq62mc10s4xshxsqhw7wp | 5 + ...04cp3v0gt2pjkkzmqd84nx5qfv1d90x6qtre0n9qcg | 3 + ...hadzgnrkbdch4ja8rjvc6yc6fdk5canytgzgr7qhs2 | 1 + ...vajs26edvpejqkc27syy0dqxttg0epvf1dbzfvce18 | 2 + ...zmgpapfbw9q5gd1r99hzchwqmnc130bv3e102xeekw | 14 + ...gfdkbdfjrbhyctrkkdwzyy6tv0xx7xdjd3syvq9pte | 1 + ...2m63pssmgd46emw3s4mrk88agm1pr3ykwkpxx5nhna | 3 + ...c0kzcgk2v3hq0ehgh1fcs2272ryyan83wm7p3gxa78 | 2 + ...hfvekz2y5g1jhg1ykyemphdhfyarhmwg7smgvy28w8 | 7 + ...b6d6578z22xtepbzrc67xhp4cr6c5azxex2p3m841y | 2 + ...tdmjfsm33pdkx1vneefqterd9ajy4w4caztg49m49r | 2 + ...e27vb4h8m64gn1qepw3kgrzm6ygdtgrr03nb685w3y | 10 + ...nh283dae76dtmhfkvq7nmd6qt8rva7y2hbz51fmdzy | 3 + ...ts9sxdjzpxdfkb3kape5azt6yde0se4f664h6tefke | 1 + ...keac33py9haf2ervq2ggmcwhen40aj4sc291x22xv6 | 2 + ...hqt7jp74v9e149zrv089cb812zcdqz9nsrvzyn54p4 | 1 + ...jbergqyrxdm309dnn4x1t085j601kvppjycgfam982 | 1 + ...wdf80q75qrbb8zbzgycmahpcbtyjzgm502764ee8tp | 1 + ...fx60td314ksyxv666k9awxntamx4zcnbxbyw8pvsrc | 1 + ...0zpc4dd7hc1cac6jydf05dyf59kv9g86pyygsmq76m | 17 + ...x8wb9rbc02j98tz450a1xkst7j3v185atgmtsbcvfm | 1 + ...53xadg9jt8grsvsh9a8bkceye8zr1rxx4wn1qh5gwp | 2 + ...7y97hwf2hespz771jydy0mxe1gbgqqj86q67z3bxw8 | 44 + ...4ta37vnr37zdj46hcvkxg3tzdbvv9hrkcmj98rhhy2 | 1 + ...rd5e0gr5evfwqz6n983xmpr1zqcpg53ap1gftvkx3p | 1 + ...dwkz0s5qa16xke0n9d4zkaefmp349kbwyjpd6nj1ry | 1 + ...tv9rzs0ewfa9c71aqc4t7qym7mjyywwxb0cjkm39tc | 4 + ...djkygzy5vt8e1yhtb9r0dhg19hxzxcdtp5whmpe6a0 | 19 + ...9ekbqz6ts2av7eb8n3yjv4wytdaarys3ed9gvnz5b4 | 4 + ...km3rtc76y4amxbq6a3bvj28wt28t9127hf7cverh92 | 3 + ...jtsqeqb0xm894bjt7vy73prmyhyhzpe9hg1nk349n4 | 1 + ...z7cgm8ra1sarnn2w689q42sed8xb9ve78mvj85knry | 4 + ...38jmv5xazrbrknbzv2w01bzfh74j3s7pw5wntdgzcg | 1 + ...ka9cctyna6x83rethzyb8d7vy6915xmv1xa90snb2c | 2 + ...gwrajz3mzbq960z8vb6sq1mf72n39dbja6a93mfqsy | 1 + ...q2vdertx5s0530v4cc7hf5932h40dhsz5wbkh6ac38 | 1 + ...xbt4vz15rxtvy75a2c7zrcqq80ywr682e36936shdy | 3 + ...dfh3hfn0b7cn92xd8fyrx6jx96e1sp3pwdq64wvpc8 | 1 + ...gas0n2fctkfdx4m1bevmrk333drnreerk4scwtmb0t | 17 + ...c8w3t0sa6bgt4hqfmk0s702fhw3aj2s23vrf113vhj | 17 + ...d1aeh2c5te4wzhqg19vgrt8xay96svegd8190h53fa | 2 + ...j8p56a7r8jrt40152sd7fcc4c90j1y2890rqat47kw | 2 + ...gxgmb7s36z7t5w0c13v2kz5m7bf4qw7bgcys380f08 | 33 + ...5phm9g9r65k34nf2bnbvktjcxarpnshyhss1hgzrcm | 2 + ...qsn694mxs46wv9pxk5wr0v7w32h7yq1am1qxnrq3a8 | 2 + ...z2tx173gmjdhbh0tkeg62s8n7kcsv9gktyt864mhnm | 2 + ...51mdw2h6myk3rettmc5nhmye9h8k67d8rd5s747qx2 | 26 + ...szcfjdy98emzg60xkmccxk3qdcsk6ads62zsf4vyqm | 1 + ...apxmj9qkwvszvqsrktmkcrkvss8wq7dfm9p462p064 | 2 + ...7nrbmhdc6556cy33hd5d3fqcw518m68q4gcbsmzpma | 1 + ...z5qnj4nkj1esk3m2v3tw4hzgcvd4aq2zcv0a0vv90a | 1 + ...jxhewcm04e4px2m2znvdxjbr686pmewxrstvaxz9he | Bin 0 -> 139 bytes ...4hj84agz7txk2hsapr4r7gbpxac5ttbdefme657ysg | 1 + ...dnet3e2y8ztax53q7hddaej4xqjgaz5kveh5eeyxbw | 1 + ...wttecepmpdjfnc0ht0x0gwb6a1jq03s3yj444kqjcy | 15 + ...fsc8tmnxj0cxxmr3jes8gz76p54fs8wzfqh7pz9274 | 1 + ...p59y4jd3yq8j6gz8zz668capv25c5ya0s58qydxcst | 1 + ...mbkfs19k1fmkfhx1ej4jqns8h1v8dyj58eg6j33dmt | 1 + ...t24ensy38wm2t0at68frg52g4g812hm20ws9ne1mde | 19 + ...60vyjpbs117mf2gz69h6enprvzm0hcwt3z641kf5xe | 1 + ...m9qrp2a5axfa9jy5h8ewtxanwygafwhx4r5fwbztce | 5 + ...mr0cmdm54pr7gsbxyskvymrvkdyh003phfftngp9dj | 3 + ...hqarrsg7mnv6d49q2ksgb6q2afqe8z16ct7dkxy9aw | 1 + ...t8fg331gpkvmxp9mkgyxyk8r1ndbqtfxxh7904mkda | 21 + ...hpq4mgferty6rhpdp79xvte4cfjmzwtf53kmsxaz4g | 3 + ...zkfk06j8qbanxvsdd0ctez8e8js8qq86w1n8s85kzj | 1 + ...3qx6671gfk42fjcpzapxgjv932b4najhcdcvb87vcy | 5 + ...72ey1t36ymxs03rc2yz80xavs5dfjfc8gy1zgsm2yc | 1 + ...has31kcvzdmf3rmzmc54tja0yk4ebs21kt9v1htm16 | Bin 0 -> 20 bytes ...bvdrjj88h71m40pamp9x2ty7p1v0h5afjcscnhfjvm | 21 + ...pad9g37xfp5826mapya0j9fjfasbja8fsz721k7hse | 1 + ...6a08a240xtex9khqvm8zkj6v4ane2yqvajq13a64q8 | 2 + ...pnkqd0fd7b89g1r8zw2sn6t1ffhc34chtkb4zt4g6p | 1 + ...k4cpr62tte8fpee0h93ej82ebadtdmzev02ankgxxc | 4 + ...qw3wvss3wv90ty3689ayg9bh3hm1yj1v81d7mmkmrm | 2 + ...6cytj7h2etyzm73q23gzwb3r3zzn46a26147c3ngtt | 20 + ...g13cjhwvkmnjkzhap1wv65s0d6mrsfnpvgqjd4d43t | 1 + ...p61x7qvhm54q7venkq2j0rxvcj8pkz54tktsmz696m | 18 + ...nvsew9f6adkessshmv0jp07cs9t1ammp05wfapy95w | 2 + ...bn1tkad4dcz38gynvv3b9vjjxa753a6smp4w0gq8fw | 1 + ...555xm1czdmsz55emnt639nhve81rb305236qf5cvw2 | 1 + ...4jnm2pgnprzney40fatb7pp46t42kb5fk5bwagzksy | 1 + ...ybwf9g3rm5g31dmygq0ksw3hdxwpmr2msqhz46581g | 20 + ...f0w63svmgw0k720qpmarnn3b7x1jfd6sxzfjrv0qsj | 3 + ...9vhjanjap5bqrg5n0y4awn9ey3b9hv04y0505ycf8m | 2 + ...kk8x1hczb24rycg0wk93k0n71vrkd129shee2kb770 | 2 + ...1caxe8khbx3femfbaeeqyf8rtjxxnvh7peyzvr6nr8 | 1 + ...w64nvaq0ek0d3n4trpsnkrx13bh73eexhqy3qz1n50 | 1 + ...qvqw6nxfk7ccsbfqbphcbtjm3chag6qdy7t667dvxw | 4 + ...7vyjk6cj3yh3k8f0rdzt65b7j6tp7vzhht4e8g3ed6 | 2 + ...bwkxt1sgswfqnxb2ksghfs930jn8wt94e4ftsrh84g | 1 + ...5h82x0xmdasfnqp69gzrv0kag7fyytg72g7wsqp1qe | 30 + ...wy5wqq8a78r9krk8bdrkc6dgy4rfwcjjjm0d6n04zm | 3 + ...s9rn0bks5a8stzkxj5z2e617qr56dn9r6y4amjf6aw | 1 + ...tcpmdyg4mv2echk5hmx1gw435cy74yzqe63rv3w2xe | 1 + ...5h883zt39y2k6qb2s6t6aj88pdr0f3vfs1fw18y5gr | 15 + ...7ztryqh0pqzj368wwbxqkvm7qbvs33n66hxx7nrqwm | 1 + ...mwrdvrc4nsc74704jejejj7axdre87g6zpggvrbd3r | 3 + ...89f9aqmhsv9bnyeqzn91bpdt7b98hfef1h55f3svva | 1 + ...9vf59rm5hmm5f7jxhc5st3mtygp36p0x9wwj2qtnbw | 2 + ...59m6kqnn4x3frndv6zm7acsqtt2abaknc1zk5px8ce | 2 + ...y1dsabxynmvt3x0c6gehwykk90p57sm0dw4yv30wn2 | 1 + ...y3d9a13g5dmxd7kahqes3eqy60fcv6v0xd2dhd783j | 1 + ...30495hyfzbhes4m5b24jh013b6wkyrb1nv59takz1e | 1 + ...fwk9mv069s48jzh27mv334kwn483b343g05psgg1h0 | 2 + ...dswqsnt0nz180w1njgh84rsevdd04qvzx737tqp4ny | 1 + ...y0bw3s81dq313y936cngh5v3vbxst6c4s91v444jr2 | 1 + ...6cm42qnjr54s0gwv2gehm9t0v2kq8p3fb84fdbq6wm | 2 + ...94atgazn4sqp0y06bfer99f5n9d0z47251b0t1nbrc | 1 + ...918nmttxgteyaykpskyqswf27byd6dv7v3c6mr9ce8 | 1 + ...srgc94jph3xbps9f1zmwdyb5z68rtbqmhx2j43rv54 | 6 + ...c3ayxsqq26r2y7ehk3t0x139cmem9fyjmx7q07yr70 | 1 + ...tmawhwhs2ha6c76dxh02gswf2h4vh6vham7m3sfdct | 2 + ...935cet85vqftahbkvazp0hy57ee73y1gccm9hf8gqc | 1 + ...3r5tp52qea2p58k13t20fp1y8z4y6sctkjf8bhmyec | 1 + ...v67fs6raebm8yy7tnxdrfn4nstrdcc7xtwj7ec4864 | 1 + ...ndd6wvxxe9pztv02yrhqsk0qhkk03hhsb6z4xqpqg6 | 1 + ...z15vspeftz6hvkdstfgdnmv7q9bh5v5tpasqshkx74 | 1 + ...naj2py5tjqqj394c5ndb89ycet34x94fw8dygjd4ew | 16 + ...01kbbdv8hwahc86r3tjncbce3gynjtrz5t02p99bqy | 1 + ...3c0xzs6tr5c3y2meepqx344ra5nnwzj5k4p3w197yw | 17 + ...szrpddaqfpf849d2mmygc3pa2w2j7canbcmvbvns8y | 2 + ...7mydv35s2xwtnrdnyc3pd56awqzh67157x8y6mpw9e | 2 + ...x30snm4efr2k1x8mqpej6xdv2hpwc13vv2njapmt3c | 1 + ...zpavsm0j6fx9mbn21b8126993m8mc8fkpmheapshac | 1 + ...8t8wtgrt740a0aeyfjjts65pyaw6w3sjpazhpt2h94 | 3 + ...4pv72mn0h9ekhe92x8zhn6r03n7j6fh0w6vz1spe9g | 1 + ...wkgjqjyja1vak6tv7yxyaygm73g4psv9pexeqr9v1c | 1 + ...8kv3f7ap1y7frhkfzjcxs3p4ja3za0e46vj4e245g6 | 3 + ...xxt64rh1qcbhctjpsbnhx1s8h1kt28aejhvs4nt71a | 1 + ...f6wcyfjdk2c23myaqx4sqbef6wsy8j3dvz3m449fzr | 1 + ...jrk8cbm11vcgfv0k9vbc3v6mhntmfevse94ddtx87j | 1 + ...78ck9a0kyxemegbagc2rjfedsha7z80gt6fztyht2w | 2 + ...74myh8qrgwatp0tvksqb3r6rze4am7zcerdfpbt95g | 1 + ...wz4qs3hsbf428b1mkem5f51drw08afybzybxz4kz24 | 3 + ...h16tn6nzr6s54fqys8twzwvt4nf3r1jnpjz1abaj6y | 21 + ...k1hwdpd06mahek2rzzrfqsmhng70qv6ttv0zfkva50 | 6 + ...81h7q6wx5z6s7b4mrma2mj26awddz79ve8mttxh7fg | 1 + ...5key1yebs99qhj5qr9paqvy0ck0w2y7ph9e63n4xyc | 15 + ...rsjzm5ffxrb7tjz9g2ttmbsg79bd399fgh3qffaxr0 | 15 + ...m0t56pfry6xsf9mvc4w1rkrzy7crwff5srqf7zz2a6 | 1 + ...0xew56vpdtmecqyd965zxgzke9tmb0tgj5nsbr6hke | 20 + ...1zkyxmscw8gfnprjyjz3s4mq07nb1ndg0bf4ptr5x6 | 16 + ...s76p2mngwb2g0s36t87nfw9771gnkbh88x64v3b9za | 1 + ...m2140kkdd3y52kv6da4r8r7vk6bhxk2nhhj2wrrx90 | 2 + ...kt9mh86k8rq8ksknbzsvptc3274n2n1zjgswg1ehqm | 1 + ...f8cpayztamyr7evqjfy6wga3zngxhk4kvx0dza4mv8 | 1 + ...rpmnjbk9d5ymty9jcby1093s57dkm118qcs250k05y | 4 + ...3epw20qqgp7shh42j9kbbeq79znh82r03bmv9jq832 | 16 + ...690xw3m4yzy85b9vb4t4cq42158d96rfd923m8aa2p | 32 + ...my1w9wtefsgpzdk6nvsz98z23k9wmz63w1ej6xe766 | 1 + ...f95x81dj1gn9tpbj8w8r0ht7c3w60hskcn6mgjtrme | 1 + ...ebjxz3vwfj4pd0dc1kerm6gq0egmy1mxr4g82zjg64 | 1 + ...59mn46kmhq246znnmh6wzajqcmngexwn2qhbepg02a | 1 + ...9n5fkfwsfdery7z1e7gsh5a9qdqmt0vfmkf185rp7p | 16 + ...1emsm4enja8d8gdxb0173t52zah1pm9rbbatghtg6a | 17 + ...dg1yxdy9fbywxyn689e3n8s17jqvs8vj3dm4c09f6p | 1 + ...8fxxg0886x5118vcn9hn9re95kjbm64m1mnv1v2yv8 | 1 + ...dzxmff201w13fv07e00kgx7y6xzk80dmf6h0n6e8he | 8 + ...rjc9gm33ymfwkpsw3pdm4bm7e38hyakp6m5rd3hdf6 | 1 + ...jdjb5ztvry0ky10ryyx6j3a0v84f4jcfwa8v3swn3j | 1 + ...c2ds26vf511s9ev9e0ymjbmw01an7s43nypap1yd2j | 2 + ...xyqbzsmnv7z2r12gtre42jtnn0evnz8sghsbzfbvty | 1 + ...a29n6vmqy1cmsdgn259sa2gear6ansyytb7dsnj380 | 1 + ...5gj9bemrbyaptz86xrsvkgrz43e5fd766v8sptck92 | 1 + ...qzbe79qkxy5zv5daw6m1x7kh2f68gnxz4k8rxr7bet | Bin 0 -> 256 bytes ...ws2y6pk25cgmf8fg74p3z1e8tw9g7h72m2x5am92tc | 18 + ...t88k7rhyt3jyq9174j4k059espkf5nnnvt1wn254ea | 1 + ...cheaygp5n2htj123hvsgfyfjdpmwb46v7yv8k9kbsp | 1 + ...v320wcgc0ydng7frvmm58x4az6r1w8j5knk9c8ejma | 1 + ...5wtb7jsahx35v0fgnd0d5cp3gnas1fv24jsbyt7fet | 3 + ...4z4m42s02q4ab3gezsvp1ytyfhhmxag9sdxdrsh3cr | 1 + ...dd04qajrp85sczjm8s2y26c7xyns3mwtypkx1n19y8 | 2 + ...4rs232s782j37a38v7f3cbgmxr8gw2450eq26f9732 | 1 + ...f3etsn2vgy2342p8ezq2bbbn5mgwqx5mb67eyre9hg | 2 + ...mdfagf8w7x7tfzqsgb5k4abcfmk3p1kmggc8ya37dg | 3 + ...803bhd7qf17s9aq8q5rxrh6384f5q48h7e3pbmn94y | 2 + ...wr3jct03x1hcdfyr9zc5de8kjzhqdftatb9p0txjy2 | 4 + ...m6axcx8mpe1qp1whakh1y0t0wgfahsvgwth5p7mwy4 | 1 + ...vb980qjve6cak095mg75awgrd5p14s3bqnccs3d9vp | 1 + ...68phs41hzs9gfadgbhm8jp91n9zbyb73cbmvhc8mhy | 1 + ...b48n0xygtvkx3nckbc9p4ky0kphkxj1b0zzpfa9bg4 | 1 + ...j6z0sr8yz063zzaxameqvq8kntp05bh1cp3tnefr06 | Bin 0 -> 11 bytes ...pj4jk38xv6fhpf1bbn2398bkvcgmkqq1crd55ja9ke | 1 + ...q6t50jp1c9gst4ek2j8r245e2hw5a7gwvvpgnfhf1t | 1 + ...mpfv58qes02wt4ktjxrwveypfwns9hne6hvk9mrd44 | 1 + ...1xj66vxs4qczheqy8ne3rtf5jsa0e84d787cfeh8xe | 2 + ...8yw6t9nqwhn9ygqqjfn42egqbmz9tvxwjj4wtvvp7j | Bin 0 -> 278 bytes ...m6vptf0pxp94z7dseek6j01pcsaxa669xmnj0gpy68 | 1 + ...ntzfd0n1t66793erxeqbgbp8etqkyfsg0m56gb19xe | 1 + ...m0dbv2tezpv8wtgzehd84x35bbqecq4r3pp1ss5s7y | 1 + ...qjmqw0ersskqb60kag18j2xj99gcz1caszeapaxv82 | 7 + ...qfegw5ybpp8qakdfc3k8gwf3jwx1e9jpm8nvyjaa2g | 1 + ...7j344ypcyv8zrwr98g3g5n8h3pr19p8mcmmg6n1avc | 21 + ...j1zs8cr9sxh03r64qr0y9rkmatt2qs2772k49fk6qt | 1 + ...6nrpbnsfjd33qy67epbwhnvf8rjr75dhcz9w85t8ta | 7 + ...x9h9xjstjgzmmj18r8g515kwvre8wrb9zr8btxhpyj | 1 + ...hk88m8twgs79ndsypq12yb3ab7ae4n2t3z45gy2ey8 | 4 + ...phk58zbbyr4dcgzymj9fvdwpyazrc9f0qkpgb17p5w | 8 + ...6bs5fkzz4rmvkf6p1sv8mhyafa5bgaar290ae37kxc | 3 + ...s82sf0h9mheve4rtr7et2yahhaapjhjfajk33b8kbc | 1 + ...tdgqz67sfj5ykcwv0fr8e3ssaesj6zxnfbb2pgjczw | 3 + ...cecva3rt0z8852gz1f9tgq7a7eebcfbs5dvhbp3hr4 | 1 + ...3smqnjfb2way9n0n9praqmjxjz6j87a2t6n04gzxha | 1 + ...s9nevz7ys78r8cyz6jhd7z2e50c8ve5npey393prya | 34 + ...pj46txknxehtjbtm2mf8xe42m810vdfw9ngf5e2p6j | 19 + ...z172q8pprzf70wxcd76nvqz816abpercv1m6d0tsje | 1 + ...jvn9nfee3kzs0w03f45814khxz0m70hns9mf3kbhrt | 2 + ...21yexdccy2rhr3w32q832kk62dbwj6rmj7wnnfsyj0 | 1 + ...5nna4pvs9azyhqsq3rqz5dxsr2m9n1a557zm3ejx1a | 1 + ...wj19zc6xv7kmsb087yrcrhdzb4hhvf3w65pxkmbkh8 | 1 + ...pevrk0tzwspssqan4ycb1pk0scekrjak5y8t6jm9e6 | 3 + ...1y9pevygw3shngyczkcfcxbmjv4fmf0tqzcgy9w5wy | 3 + ...1bx4mpgstvnv1fns60d9hrzswpt6s8fgmvr6pjtzd6 | 2 + ...yp22kyv11cm1njcbsxr0d0p9cnyaqjn8cpheqapv4c | 1 + ...pv417w2tcwf9p9n6sj8c5aa14eryvjt55q6m61byf0 | 1 + ...zkkv7z5sq8ejty3d85npfsypfnrtktm61ce75wm8gm | 1 + ...4ga1sjg4xj9mvqfbxq5a8g0rczv1p5hmpjj7m25fw6 | 3 + ...1n9487pkkys6qjqnwh3dxwnnfwwd3yrgg6cb49jjdc | 1 + ...d9gvse0wb83r7aqgkstv6gdcck4h94tej49q39tp7m | 1 + ...6k6yx46bcymgfhcmn1hbhej3km6yfr3c9wvh5xe63t | Bin 0 -> 127 bytes ...ydke73da5rnxnmpdn1sywrw0e9qw1z39rm8vw1875p | 15 + ...83sdtasgb2mw9aaaqtd2ag8bd8r8sg02e78e5nrhqr | 4 + ...3bsfmd8z424mnm6vnmqxxk2578bkzztxtxbw7td7a2 | 21 + ...jmb7kcpmjdfbwtqxt8mmfqwacc1ag8yprymdb8snep | 1 + ...c9q6nt579crjpv174n51wpjn43bn909zaxnaayb4sj | 17 + ...4ytb1yy66twvrgjyfvwhfnkxk2kx2b0tdx3pcbsxae | 2 + ...8pqkhmpd4zqgwev6da6th185z93cx5xdwgvjmwjtr8 | 1 + ...bs2ndsbvwv69rf6yrma4478m5tbq1rxaew320spk8c | 1 + ...x73x7d4skxrhkga3978tv1ftdjwa13gf35gxpb4mz6 | 1 + ...mw0n5g8krbh5mfk84gqvbstaw9hxayxjqf9q0zwsra | 4 + ...mye7nza29sez2a9029kn3h0g0f51yvm5ertx0cppyr | 16 + ...kg31h97m32mqp6qrs2p8f41cvr6y7f6qfkr7bcpzke | 1 + ...c5jyr3pe6v9bs5qeaf4khz36sn00g713mypkjvb7me | 1 + ...ehabpnf1n5b1znxw3tssraag8cb1z8ff0zscdtsgwt | 1 + ...0kr0xyk4tgzr9xmzgk85mw4abc3f7qw7hjrgyjhv9r | Bin 0 -> 284 bytes ...p7d81dhz89yhk5yk0pxn4jxj751g9yh1kvqwtpd5ww | Bin 0 -> 32 bytes ...0b0s3mn5gg6e6jjxn7fs8khj3jh9de89q75yjp2tt2 | 1 + ...8ppt850729b51y9nq4wkbkeczhyr1qbxdyekh7ggcr | 3 + ...d4c43m17kehn6f4gbvjv81512b7ma1ffaq1xm5e4vt | 33 + ...nrkkqzmrd17eh91gmtanm2hqs9jva7xed2a54cayd0 | 1 + ...dv403zxegm5pgexxxknh3xzd09f9s1zw488xfzab4r | 18 + ...t7a2mr57vzp3k7ydqxm7jdqg2xykadr6zbjkq1qys4 | 1 + ...rjfrctqnbnhkwyyjgkcfx3e93q9g42xfx6prq8mqg0 | 6 + ...vavtjyjphjx3r37e97cpexy5871p899xet1x7m3ht4 | 2 + ...krf36h9dg6rwsac79b4k61gw1vqv6kw8n84ambexj4 | 1 + ...9ftrgnt21xexwhtaznm62y4d0n39sgcey08s4pqn32 | Bin 0 -> 3 bytes ...szdz6rkktpz6712rj3jfh5x4by23nm5tyrkgabbxqr | 2 + ...sxtx5gy8fg6qqxy4fp89bd6j5p43r7k2rd53s0546m | 1 + ...hzsh2sy3ddcsg40ew1mwzk7t3ndxnx76b7rk8pcbza | 4 + ...m05gz4tfbkzy9d0x40p1cj1mpvv18yx53kpwk0hajm | 1 + ...gy5jc4ey7xqhd0pqgv9peh363c6rtzpqgzxr2j370j | 9 + ...9tph355x87s5cafd7358xfetzmna2jaje9tn63hnze | 1 + ...7yr8tr3r56kpmyh06mgde04kmepy0y78fxwkqbvya2 | Bin 0 -> 13 bytes ...b6ayrwvh5vxfdq3xv5m2qvx77r48zezsjedz6mmae2 | 1 + ...70yp56sq147e0zedx9kpbkqgceykbz4v2y6ga8wvjw | 2 + ...ccemyszf7rt08ae038h1y4j3eangvrjm1f0qgfzw18 | 2 + ...7wk4wrch8ce8vdw81g5azp09v462g422pxzsxn13jp | 1 + ...gkzzk97gb87zw8n0jxt3e1tm0af7ga1s127eg8y6g2 | 1 + ...hpm7n0mr8w9zvfjj77ja3dds7d3t4eykavvjyb9m18 | 21 + ...pmhw2nvq116a2mq66w5ac0t0n0rh9n803ws906ezt6 | 2 + ...4ehya912q5endeyjvppc7xk4a3p4pywpme4e8cm0f2 | 3 + ...bhk3wqc0516fwh3znyrpb8ccx98nb67ytedg78vz4w | 18 + ...6w6c43jv18cfeb1bnxkjc1dv6yvb16ghhpksdtdqam | 1 + ...dwb8kqxsbsft3xqrjr6ph3ppj99yhenczadg1b6ate | 8 + ...ny0ttw0wsmmhf0tt3qwz2w8n455h6ty27qjq0pwqxe | 20 + ...f8grtvs24scm7eht2f38dwpeffbmnxptb81yszkbh4 | 1 + ...x2wxpjc3ypdq1rfnjhg8jxg94pnfyzdsmkqq10vgz8 | 1 + ...0gwb56jzrre8y93et4mcv9x71s3takyq3bkf8kwagg | 1 + ...xkhpw15t9fyev7w2frdce2kzxtrbmpvrpm77cvz15e | 5 + ...zwvrh6qzg0q78njxekj5xg587hfb3x88dytjc91tqy | 2 + ...rzd9d9nszedymt6vd4b1nfyv5g3ne00k2cqf1d4zxe | 1 + ...ww77ga0fk0cdsxtckn180ensm0ns09dyavgvwky8r4 | 4 + ...kywx2ywv64776g163kf3cefzwz3694bxebr7svcyw8 | 1 + ...m0hhkv5f63jcf4j05zb3n78vd0qp63fex6qeq4xkp2 | 1 + ...3tt0qcnf44gpzmrtys7c6m0payywz0ebwd84xq28ky | 1 + ...a3yyr5zkyk7dg3xpn2rzrn42dgytz80he4m10560xy | 2 + ...rwxqf32axt398a08qrrvz9h90qzhjq4x86evknssbc | 8 + ...kkahbvyvb8sffh3rwmf7eq5bydnv3vx5gtap05a9z8 | 1 + ...8sf0fpgcf5nth6d4xkhjytxynyqpr8kp2z0nmxc0e6 | 1 + ...gxh7cnypmzrfphqxsy7qtxbqtsfnq1ep7ndksgww2e | 2 + ...j5q9vxs4g30xzwvsbz7hg6nb9dn00tpdc6qk9tyhg2 | 4 + ...xzq69qcqef54brey6fwpv7dh8fqmmgmbzq7a6n4j60 | 2 + ...g24hdjy5vwww4x7y5p44azbjrvzjn1717jas35cteg | Bin 0 -> 256 bytes ...wypz73cj0fkgaachvc33598qjb92j8ctd4j4ng3ndr | 16 + ...npjteafpchkcd19tsd63z67d4m1fxbrwy46skjamq0 | 1 + ...fz3gbm7k0mxgbew3vw5n5bmq6jhakqmttp3r5dbgrm | 4 + ...25bd73w9dazbw6jb2pgh84cpw34z4aga4jzn7b82ap | 1 + ...t44ytv2x4cagakbp8vyced6dxaffy35h6t2fsa3nk4 | Bin 0 -> 76 bytes ...29a8rvrkwbhyktew1xb3frek1atfrxzw33kq8ssgh2 | 1 + ...bwa2c3bmhgjnc1qmt5xrhnr96aarma0tk1n2dz8sft | 1 + ...ar75v78qkx3d928h0he832sqbkctbmcrk5fe9ws8ba | 1 + ...0vfcsy4ex7923x16y4wrhh0h70kenfhy215j09jf28 | 1 + ...wmgvasgfdha3k13fyf7h2m6mg00kd77waj719nhbfe | 1 + ...56dm9x1mzfj5g2th5wmys862hry0ryn5rde5xg3be8 | 2 + ...2pz4cpn6fxk22zay0d1g8k7nzbhffyf573ajvg3te6 | 6 + ...3zs63jsevc9x72va1apz2t0ynsktxdxryvsf8bpt10 | 1 + ...6hd1d29xjz910rz9haaxdj81atz1gpmk6a8910w5jj | 1 + ...166zk28n709g67hkqn9e4vb38t749q18srgb226yhy | 1 + ...dqfbtbahx5wngqwwzjrs59v7p10gfcndybp0d96j6j | 2 + ...7ssenrycdbatem6dtavfcxnyqs4qmaca3cnaxk1bya | 2 + ...8wqbx83fgq50mww2ect3a8j08189dy7v8663mrregr | 4 + ...w3wrjhp2ywmqdg8cbgedrjy9apfx30bvm0c4h4w9gm | 9 + ...dm8g9sepcbgchsj5gjfhqnn15r4xgxzv225hd94dqa | 2 + ...t61ess9kww0xvgsq35ej5new78w63pzkhfc252tchg | 3 + ...4mctxj9r2m4f7mp8cdmg1s776c0feyxn27h206yms6 | 5 + ...5qa0n4pp10ejqmt8qrzy6wvwkja36my7rhtc046q14 | 5 + ...3ttywnza76t2k3z4qvrbmyg2b9ptvr7ennmym1n59g | 1 + ...bj1ak3n7ercyw3x0f3nf884dja04kcbg7wey0kfvxc | 31 + ...tvvynb5h7hgm8py35yf9ta0r6psqww75yg99ypm26g | 2 + ...gkggtxf22b7w6d38faw092jf348v67vp78dc7esj1r | 1 + ...e036b3prr4germa9hy89g82j908yd9tn2pg7vds984 | 1 + ...0v8nk3akyvwhn06hwzz1b1ssz04n83wwzc1ex5p0tm | 1 + ...ksqp4rzj5czsyq8ad6dmr0zbw8mwqpzpyt5zgvs638 | 1 + ...fn9cmx58aw16edmv25m9p93zvvp2wx3y9fd0e9ktey | 4 + ...9wfbcss4wzg30xfctg28sang4r84taka353p2j7v3w | 2 + ...5t71dcjjsxgs535bvcjgn24hegspkjfzsdbajznj4c | 1 + ...ags1x4xer51704f72d0ymmk40kabm1eeezg84rf8r6 | 1 + ...st7rt4c9n8pqq0t14s83m0akm8cx594yv0fnc5zmg2 | 2 + ...5adm1pqs5928hf33j2sedc6j7try738exxnh6hypc0 | 2 + ...w4p18yyc3a01apa5qz0w31n0sbgbwj1jxvs3dwh8zc | 1 + ...3t34dgg13219xkkbakdj9wgrshqsg2vqmj82t9hxm8 | 2 + ...sr0jjm2qa355th3w231761y1s3cwy2bqzd7eam3wbp | 2 + ...9bjk8bhnjw5zvg8bmm9tbvfxjk0vp4ejy7yqn51cat | 17 + ...2wxzhvb64j5v9y1wsvw25wskfh6r4r41dhcjx9vf84 | 1 + ...bt75p0d02masjc7bb36dqz3w478hrp838z7vgnmkfc | 2 + ...n1mgbwk5sm1eksq5rdw36x3qs1s410zsd7fyz8tb7e | 1 + ...9y3rfa7r2p2as8e6cb9nd4mcze4ez61hrdc59bj774 | 17 + ...hhbw4nq8hp3bnekcxcgkxvmz213h30h866bw592m50 | 7 + ...bem843j8vaevc0551zcb60f8br7ezhwvyzfh8g9wdw | 1 + ...8061ezanp6hv7db2605w2r8v3r092yfcs6bnj4fxqy | 1 + ...9d9fqq2p5tbxy7bg0kkye1zhj9dg932bfny8h7j412 | 4 + ...wspeerh2t52fm38ngbcv1a0vy24k6j2hykbtf6g7vw | 1 + ...nec3t7xfrezaacqkt64v40xmbapa90ra07h5kqfj12 | 1 + ...nqjd3axjyy7vswtf683s63g7x4qbtynp1xjtpawrx0 | Bin 0 -> 266 bytes ...c2sv4nwbamhn2p6es0n4t1r292078dwe5w9mwr1j8t | 35 + ...re3dg37xp3988dzxnkeaezejnbejvjt93b72g20c5j | 1 + ...r14gk3a7rr65r5tk7hfat39pevmz8why3gn0tvh57c | 15 + ...wqaa2w39myhpa309a3b2hdr202f63jfbr343nzhmza | 1 + ...5ewyq17ta0sadjn489c4dpkgn5bnrmw6be2tmw0dw8 | 1 + ...sqt9zk6h6bt0tq2wf62xb1pck71qvn6r8d0g70mmja | 11 + ...wyb9ad0rfk3yjwtfjha05etx8te3peqbwddheh8fwc | 1 + ...tjskn8hg3ydkemexpfa75fvm46rasg29y7zjkd7za4 | 2 + ...45r3n70nw3wj6d638e448qf6zbrd0mt87j9698pyj0 | 1 + ...jyccjejyze9n80tsqf7xsjpq5nzt1c21g8rmsa1khr | 1 + ...ne8e51c1hch84yxk993218ykvmfp3gq4c66r4x22hw | 1 + ...de5sh1dmy64dvf4jjs30x34d291a6v6r2d03nqnpvj | 1 + ...w7nq6z3pdayphm1vyc5rq1xsd9fadnp8m3y6ykep1y | 2 + ...j9bnaksy94s17nvg4gef25kr6w1p125bq2ypdwv7ac | 1 + ...y1mf9z97wqwnxyrhj1qrt0tydh3cgttqwhpdxyk6by | 2 + ...4nfccdc4nxyk791qsyfeysmt4w73kr0t1xbffvyv76 | 1 + ...0k36mhsd91yaj4gkennz0hng2992z3wc1d2b7xh1b6 | 19 + ...exx9baft7b1mvwk8w7s45arqjmgph7gepxhyjyzfyp | 1 + ...jsfaczgb0t5ef708k00m489v4b8nrkhz4y6nnxcf40 | 1 + ...z6nb3rz8wx7y76cymb95zt6z691adqvfnpv80yvsy2 | 5 + ...k3gmkbgwb9fe497h121av0m23cnd6zk1rr3fyp7hjr | 2 + ...d6rzthp5zxm8jg5f6nh6yqs81fefzxdy874fmh6kg8 | 6 + ...964nbb5yb9ps6jtk72grm9y99dv2q7xc1nbk41h5fj | 1 + ...06rh07tncp10v3ck21ftg8esb1ez461dsrafy4b1cy | 2 + ...g712vt6x1b3zhcj1s1p8jxfen93gtd42fxtjktwykp | 1 + ...2htwvdcpehh55smk7n56fcmz7mmqcpfc2k83q51qkj | 4 + ...5xrkzh6cbr4afqcawsaredtfg9c5ape7seebsaax4m | 2 + ...8a1df23mmj32g64hykewsqzng41a5zkj006089msq8 | 1 + ...ec8gra9kkb9kp03kfsq32en9qf0xnq8e3628mk4msa | 1 + ...6r68778ggn6d8n6npcppzwrkfhzk84jxv5a4ty4rgr | 1 + ...exssab5tdnfx7511f6tk9p893yhkvfe0kw4scj03dw | 2 + ...85j4ftnhe1nvktt9j6dmch9hj2bhwcqzpjxvws2jrg | 3 + ...hkdhwqsd9t5eq7ks3wf602jf3bwt8eczx2vybh6zc0 | 4 + ...hyj7wq34fhptmx4amwpefx3nqsww494c8wyrz37eva | 1 + ...mh1pze8kkvezr7abqjr3tnhpqbbxp48kd2qwv0wk54 | 10 + ...k59m6rrgjabzhxmr32c09fh750e6k03zfa20k2vrxa | 1 + ...jsxa265sxqyb2cwx1thqtexxcar9egnjvnpcvkcrvj | 1 + ...kydgt59x46cdhf6vjhy39pe44ge3rvkhvb1nkwb2zr | 1 + ...qckb2scmz4kjf7qt1yztrsfky9qraatwe03x51h7k6 | 1 + ...vw1ngyxqxp5wn2km4cd12kbm9cye0pz29wcymqgwgp | 1 + ...2bjs2xxz4h1frvp4kewrt6hzem2ksne0k68c6w2478 | 1 + ...8bcx0bhxt4bc16ybcgjxgakyhz6081ej47wwb6yr6g | 1 + ...ryqvfm418005sc8r4bfm6v9ys4bbrmf0x5nnvsn1m0 | 2 + ...1q1k0181fd425hst88zjpc27fcjs81z16vp3f89rm6 | 1 + ...p3tct6s8eqh3eck88e5aw9sqj3ga4yzznm2m2j0604 | 2 + ...vjgwmv5q3vwyd58gkw4gc501v7zkwjx00pkmgkft22 | 18 + ...1ffmaw8crmjm9de5g8zqtarpbwr23epvv20t5bs8x8 | 1 + ...v083f1apdzf1kg8vbg2jjfnvxgz9z00xa1h4mf65sa | 1 + ...0xsh5caxhzqb9z8nwpjzsvr0q8g0fjkz3y4y43kdjg | 1 + ...xey2qy8j4gqm0rd91k8fbx1p23t4nwmc7gabhafhsm | 2 + ...65mn3xv8v3v6zetbedfg88vjcvffa19ebk98pc5t6e | 1 + ...c9yjnew3s8krzbcd9m8b567mmq3d92b7e6henf2nhj | 1 + ...gxjckb10wqsjcp91bmtqq9chq2qq5gpdyym87v6q1j | 1 + ...8204mejxydjsawbrhn7vje3xjgaar0cpk7zy64ws0r | 2 + ...vef7cvxgb2tybnypx352v6tdbpmr49vx29z97z5jtr | 3 + ...953c90pegf5s0v2ehgb286hxrsgzh77cv6mx6zrfa2 | 1 + ...jegx974khs3jmwa42r8h98wypp5gsv1ss4s8je0166 | 1 + ...qhjcy7hbng5ndn9f36zaxab33220jxj2d0q7rgt72j | 11 + ...fa3ehdtsn1djntnwmv4mn2jm17t3t659k2b3mn1ryj | 1 + ...y1chwm64dgy0t105pr72a9se3x4e75xennhgf7fct2 | 14 + ...1d2g9vv3w2s6eh0fgy6esppgtmhary97nfcjqgtfza | 16 + ...w7wxymampccdp97g9rzv76qwxk7vj05z145zenb364 | 2 + ...0apphqhr3g24550pxepzcyb5j2gp8px27fdjj1dt0r | 2 + ...zdf4ndsymnwtkeh8j0tztts0y186jq1bcw3wch8pe} | 0 ...z5en5ybbdkbh9axzqq7c4wgs8rt6mctebe7zf9kzy2 | 1 + ...6faf4876a11a92c2e045c0ad0fcc738399f61fc240 | 1 - ...35d3dfa3145613e296cb645884ffe0b885b63bd99b | 2 - ...6149d6c486e771088809bf02e59aa61ca3d1c3e2e1 | Bin 74 -> 0 bytes ...429fd729ffc1624caf063177229727e526f3dcbf1c | 1 - ...ka9q4c2mgw2zqpxgjpmtbrpcx3q4ywn28z3hh4ss9r | 1 + ...8fveg5getzbynkzmb3t77fh8g40qsq7xp2ebfm6ye6 | 1 + ...37vrcb0qn02pw3an43bzssvwsrc1e4t60fq0dknhc} | Bin 34 -> 33 bytes ...43def521c0bf14f754fa13984e8f6f4b25e3fe19fe | Bin 6 -> 0 bytes ...7fac90f1bababca6ec0095c6d7dc6fdb4680112804 | Bin 21 -> 0 bytes ...42d6aca8346cd197d647ed119cee4a6e5e95785183 | Bin 88 -> 0 bytes ...ph5gamd1qcg99hg1yc9c37h5ej81ny8m647z61vj0} | 0 ...e66a33a33b94a6e6fc11da8e0ac8dc684550637684 | 1 - ...4360e307c4f4b41dcf7a8fe4db7b82dde9103e1351 | 1 - ...265651102ad3f0d53bbada124914d102b92f2be068 | 1 - ...78f7092ae6217152bc42205385d0b5996721c6034b | Bin 22 -> 0 bytes ...5w22w0a3d2c5tddyy9qwdjg55hj6ve1xmfnpct7dj} | Bin 2962 -> 3036 bytes ...a99736fa4eae1ca1a646bf880d8b5894c955033450 | 1 - ...dbef5b5eb38c4c38d887618ff5e94a8d117d4973df | 1 - ...ap88t5te2mew2v7b9ekgkdy86ddj7kyg122q62qyrg | 1 + ...2gqsr02f3mbqpv5sb1arw5zcrvjptra0e18973v30m | 1 + ...0f4bwrk39hybvx8szvyvqyck9ad0aejq1zz34tgcby | 2 + ...wwcdggzxf2s57jye94rmc46zrnd8551hd6gaaf0t0r | 32 + ...d2gzh8zfzp34tyr6eh4jq6nyqev0nbgvtpswcybxrg | Bin 0 -> 8 bytes ...z1gbe2d7tkmrb5jab1m391vjpytdx7rgx8bne9ams6 | 1 + ...dd8596874c5d0b7fa33c9685fb77ac22738dbe9edf | 1 - ...70c99715c2701e115f3023b1e7893d335bf807d4c9 | 1 - ...38f98eed6e39312ebf0b61fa1f053b5cdb3c8869a0 | Bin 67 -> 0 bytes ...eb7973a5c90df1234758b5ee02c97755227a084e3f | 1 - ...e4phj2re52zkjrh1vj0vgh66t73e2870bhrrvey074 | 1 + ...c7bjwbwg1ycpwcfp5dpmmjhvm7jk2smf2eevwcsvjy | 1 + ...8b2017b11348cfd8bd0f591c2abc2bbef7b6e95134 | 1 - ...7a1141b4d6b3ee23fdde1151f799df7c3d14c86ca1 | 1 - ...ffd5ae2c195565e2e058ec518421272eb952dbd429 | 1 - ...0bb3247f783fb8fc33584238baeb1519ab70b426a1 | Bin 1990 -> 0 bytes ...42085ffcc424ebab149f1c85bdb6991de0b4ab86f6 | 1 - ...3816c93c0d9c830264b24ca58c386075072b388372 | 1 - ...f506e2660aa0352a11c83af4f9e58b01169ecb8ac9 | Bin 67 -> 0 bytes ...f697cve2978kqd9jby90x9ekg4m8m5nysp1jngy8jc | Bin 0 -> 18 bytes ...46f077b5751015cea9dc045849d65509918973b62d | 1 - ...8e7b03f3cc4cc9e948192cfd5e7d0cb77b31d069ef | 1 - ...nee7h4x050p29a485e3t21m37hsa5y7a1wn9amb3wc | 1 + ...qs9fgddtc4r3s7axbxz1hr61n8rtqp6nsqc0cd5me6 | Bin 0 -> 79 bytes ...6ksepb7j5k4q8t8e3p230agfvr357se20qa952dn2r | 17 + ...2bb5a2789bedcac8a5bd30820c7aac957d3c6e1684 | 7 - ...e97ecbf0812db63788b4eb8411bcddc4be7267e428 | 1 - ...9abf2fe09932db7ef15b4c939c079cf8fb0d275cbc | 1 - ...c684b6d3af75c26871dd228082e3f3171ebda3b5a4 | Bin 171 -> 0 bytes ...f9ebb138147e03034d9acbe149c2c1e93fba076dad | Bin 43 -> 0 bytes ...a53fbde2113736faefe5e5107d3d7da299369df1b0 | Bin 22 -> 0 bytes ...32338d9af447d0b4e17345b30d0874b8ecccc88535 | Bin 7 -> 0 bytes ...e5j750xjrn3kyfs8n3vgzt7389p5z7rnpxdw20mfrt | 7 + ...4fe2jqv1ktz8h03heqvtybz5zekjr3kzgmg73jyzzr | 164 +++ ...x1w6qq7hdrmawg8d8m32fccy4ft8jk32h57863qgc} | Bin 8832 -> 8759 bytes ...d366b028830ebe2391400b3d6658ac26f5dba36868 | Bin 24 -> 0 bytes ...tsxj7hgjqf2j6yrqfqcbqa85px7mms469p8j7n2bw} | Bin 1057 -> 1043 bytes ...ftywfhe3yc82jp0f4jad4wad9awev9gbq4px62c0gc | 1 + ...r5j8mde696580twqnhkvrkjg1b8b44pf9vvyhbyc7m | 1 + ...30caerhxjrabr6nxxp215bmmnzp576ds3n0fnd5qej | Bin 0 -> 4 bytes ...29ac9bea30c17dff4b027e19ec303d143d82cc3eb6 | 1 - ...cy15k37c4rk914aktt5fxp4z1wdv0sjzp0va05a3s0 | 2 + ...4e0c537a889e6d301c9de358ab1959ec04e80af57f | 1 - ...xhgpacrfh1aq74fkjnkd6qha0rp1z9zr22byttbgm} | Bin ...0dgbx4j5a96gazak55b9ekgvw4f0cgk2zbmtjjjkve | 2 + ...760a116fd4b1e13a3905ae812b4c4434484297ca53 | 1 - ...d50023edf88c026e39d5a407c1b7b9155704add33d | Bin 1195 -> 0 bytes ...4f35c608d9f2d2c8e9320b136bd312cf028400df4c | Bin 274 -> 0 bytes ...a3b4887a2672716b4ef446659f26d1c030acedd63d | Bin 96 -> 0 bytes ...c9cc73174b55c7a826833b26ab01a78cc5304725ea | Bin 63 -> 0 bytes ...8d835171514a94c3b3fee05e4cf9caae20e0d28df0 | Bin 36 -> 0 bytes ...92ckej8yvykkywjmdmz3y8k7062my5rk753watxfj0 | Bin 0 -> 326 bytes ...vwya7erng8g6r1xee9we5cjd7ys97ejb0yc1crfpzr | 3 + ...b8741b2c0cbac5e873876430bc8f6e41b5ad0209c8 | 1 - ...dd92358725c1c5603183da9733bb90a125f85ab9db | Bin 66 -> 0 bytes ...b6a225ef83020029066c04ad009ec529c273de0fa8 | 1 - ...7g71e7k6bcs5q565ze8z0f8xna7kpmt8j8kcp7t4rg | 1 + ...8aaza5he7xhhe0pff01yxcft748ycsfxjkz1p1pzw2 | 1 + ...p1b7ghxs3gbj8e6je6nc4bwqra5kpkd4exvremh62} | 0 ...371s3kcysrjp95yrvxpccg93687d7c8c7ypv3xadvg | 1 + ...e7ds6xt6f0mvqxw2gvqmwnsgb15fas67ge0wb0mbe0 | 1 + ...460ef3d4a1d5d7121d0ff0d8d6cdbc374ae9946c80 | Bin 1804 -> 0 bytes ...0223e06edd28d33945777415f169219fc560a6025b | Bin 23 -> 0 bytes ...04f448e90e33f72f377cf4fcc582fa70c9fbeec75a | Bin 4343 -> 0 bytes ...qrc9ajgg1n1dtv2p2v1qjxjgrh535tb5hwqkngyz00 | 1 + ...3nnr4swa87d0ztyyvvrvqb28vkagw6m9xs5vb9grsm | 4 + ...trgawv1nqn6hg52g9t04zpspzmdrakrktftpejfg4y | 1 + ...3xa25drwh4m46gdt9naav2gzf3yg450m3gtd5xhz4} | Bin ...syahmzmj5v16fek5xs8aa88g7nbe8n3mwbkpy4j1ce | 1 + ...w58ycvjfwcfsv728ky0ym67zwq5ghv97tr3gvs709w | Bin 0 -> 16 bytes ...rnbpw8tdkz7x7d88ksjpvma82a1eesd5kq3ma7cbma | Bin 0 -> 198 bytes ...zts2m7cnj5v0cwbp2tzeez12bya52tf8ff84vcx2a} | 3 +- ...4mt72pk0x462yncar39r2ztrbqa2hz12d6akabk92} | Bin 5544 -> 5004 bytes ...b3dwrfzdq3w7zdrmsf5e5ed4djp29a0a2khgaff4ky | 1 + ...qk5d7x3mfk4eh2r3erw2bcjvbb2a4tx5bmyg5h7gm} | 128 +- ...kv3gm079cr4xbqx5r88g7n89246m27dcckhwdrdn4} | Bin 2029 -> 1814 bytes ...xgdq2vw8r2zch1k1m1j8w8s48dtst8d6ckyj2hzg6} | Bin 4193 -> 4139 bytes ...pjfw6n0r4khj55ptpnx9cayycmwj6rb0hc1q6m8rb4 | 1 + ...4np5n178b82cn37w6wwwew2s3hbz2xgtawd732qm4} | 4 + ...9fnaja49pea92v7ynqbrmtv9fmeqp91kpvb5cxkqj6 | 3 + ...pp6xkygnwpdwe92026bywfvmxgwjmms4xp1p031gne | 1 + ...wpn2bs92fdxdgba32kgfqzy20eqysnv60tc2bdtw00 | 12 + ...n6cfvewh2w64bd0za74k56dm5w120w067p8e7s8zqe | 1 + ...eqw4f9cwxkxm8qjv999ydvxydehy9xfrw5x0pynjg} | Bin 1584 -> 1476 bytes ...1v0f123z425e6e676d6jxpx5depn9zy8mrncp32ky2 | 1 + ...0byn41gzhq0wqwfcs77e4kcmftqh4r11237dqpg8j4 | 1 + ...s2n5jd5yenchgmwk4zny08by0d7qtqq6paysg185h6 | 1 + ...9094vv8hx6rmr13g2ncxp0wjj2pbvd73t02498vdq8 | Bin 0 -> 4 bytes ...vp155d7b5sa8zbgqzc2x6176bgs69qz9g0hx99r79e | 18 + ...m9nh1znjv0nh4qcy272s9brgsz64pz4dq52sx0ajec | 1 + ...j76kpffjnykfqxcgx0925xen98vn3t09atfgxapbmp | 8 + ...0pbmdjev80ps4d0js812gzqa86q9jpa7rj5dn29rrp | Bin 0 -> 17 bytes ...zfk0mrj3zpzt3637hkx9zr4578xrbznyq6e2a7w91g | 1 + ...xj30614zv737b6tjpmtnce0992v52bzadh2rhxzjf0 | 1 + ...9arzd57h63sh653erphygbrm9xnq5t4e4yhs2yq5aj | 1 + ...sf6kzdrj1gay4k3mpkhwen689j0z2191zpt6qw1bm0 | 26 + ...edhhm3tshmxs05mtw2vsqenrw8wv4cvw7wae800jk0 | 1 + ...rp6nrpx9teh9vx8q52bwe8pz3gmzv2320qv3mttqtw | 1 + ...0m1ba2zgfd9y9bqvh13wfm83kqbqpsjwhv4mjmqebj | 1 + ...h60rdd803axv8pw9kh27xe1enmx1nhfhmjrags7jy8 | Bin 0 -> 212 bytes ...8cax2dvjwqrnpxdq5srdezdcfwjfsfx6bsfcf9sk78 | 4 + ...k58tjef594ycch1z76qwvtyms5e4rvj2989her6rey | Bin 0 -> 6 bytes ...1c9vqggbmfxm8shz7vrn24bvpvajg0wpdeym43vbym | Bin 0 -> 344 bytes ...trx14yq823m5wamqwvdrz61sh6vkhc4y3fps0m02w6 | 1 + ...bt1mp5mztw0q4nzhjezfq249br5779q1v51s6hk85e | 1 + implementation/src/lib.rs | 66 +- implementation/src/llm/handler.rs | 20 +- implementation/src/manager/handlers.rs | 6 +- implementation/src/manager/mod.rs | 142 +- implementation/src/manager/run.rs | 101 +- implementation/src/manager/run_test.rs | 47 + implementation/src/manager/socket.rs | 209 +-- implementation/src/manager/socket_test.rs | 3 +- implementation/src/scripting/ctx/dflt.rs | 3 +- implementation/tests/dflt_requests.rs | 2 + implementation/tests/manager_permits.rs | 66 + implementation/tests/merge.rs | 2 + implementation/tests/overloaded.rs | 2 + implementation/tests/policy_dispatch.rs | 2 + implementation/tests/providers.rs | 2 + implementation/tests/request_body_limit.rs | 2 + implementation/tests/request_localhost.rs | 2 + implementation/tests/signing_server.rs | 2 + implementation/tests/timeout.rs | 2 + implementation/tests/web_render_webdriver.rs | 2 + install/bin/genvm-post-install | 3 +- install/config/genvm-manager.yaml | 6 +- install/lib/python/post-install/__main__.py | 18 +- libs/iconv.nix | 2 +- libs/libc.nix | 2 +- libs/liblua.nix | 2 +- libs/lsqlite3.nix | 2 +- ruff.toml | 15 +- runners/default.nix | 3 - support/ci/pipelines/tests.py | 6 +- support/ci/tools/open_executor_prs.py | 2 +- support/manifest-base.yaml | 2 +- support/runner-script.py | 38 +- support/scripts/check-commit-message.py | 6 +- support/scripts/check-source-text.py | 3 +- support/scripts/ci-changes.py | 9 +- support/scripts/get-all-git.py | 21 +- support/scripts/md-local-links.py | 6 +- support/tools/genvm-tool/default.nix | 15 +- support/tools/genvm-tool/flake.lock | 61 + support/tools/genvm-tool/flake.nix | 65 + ...7zeewq0aqkcykz2bb1xwhdr99xh4w8sajwhz3k7t4c | Bin 0 -> 26 bytes ...zdpm0keqes0emct1z75fvxqefmg86rabh7f58b9j1t | Bin 0 -> 76 bytes ...m9gb15nqrbsf0s0yxnwkxw8bvjchzt3g3292nfh89r | Bin 0 -> 22 bytes ...k1b713h3b43wtnpxr3k4szgy779kkpt5fmmgk3k7r8 | Bin 0 -> 68 bytes ...6sfd3ce16mneqnza9a6k44q1q0b630bjyjsr0bay2t | 1 + ...9c6vjcr10k5mct810px4hwtvkjjmfap9g4v7vqw49m | 1 + ...a5kfma2ck2ptfr4vgf7x93035nc52ysw31vcmwz8kw | Bin 0 -> 87 bytes ...she2mqj9vscyc77m0xhsh0mv1syb797gssvxk2qgrg | Bin 0 -> 4 bytes ...n8db9k106dpsxg3p4safeddkwr8vyb46fzb42ycz6m | Bin 0 -> 30 bytes ...gh2n7349b26mbas03fnk883hbd8bfxkk3ta9gfzem8 | 2 + ...d4fq9jcf95wvg4fpk73h0ftkyg9dy285arxvgqft2j | Bin 0 -> 62 bytes ...jjf4c6xgejs3ntw83jbpakfa8y26hrka9d9ykpkt7y | Bin 0 -> 19 bytes ...fkg45vmckx2qj1gpm5tbke0cqedmvsgky5pa5fdhdj | Bin 0 -> 67 bytes ...9w8qmh1wz818jmnwnrd7aqq6f7y39hx1fqn6gret3a | Bin 0 -> 61 bytes ...e9wd637rp52z6bazc1ph9jbacfknb3mpkcwff8y8be | 1 + ...y9xan3xzj1vam3946gqbepe41p9471t8f5gw0gfn1a | 1 + ...dpj58bhnvv9zcqaxg1gyf6ty707k25avczqr13n414 | Bin 0 -> 3 bytes ...e39ergtbmmtvn8dwf5rts4c4k5aaac1t6rf7rypn1y | 1 + ...06nwh64gfav2jxv96hbmhrb1t03yqf5e61s7jf7tfm | 1 + ...bamrjk4e8cm6nc8qmp2jx8dkqqhfs13wkb1hb5rq90 | Bin 0 -> 44 bytes ...23ykykwfjgdmnsfxxtrqzfjx5r0b30frarwecm3rd8 | Bin 0 -> 19 bytes ...hqg28z4ergsgh57jkqfyx2p66pdby1r1zs5wth779p | 1 + ...g00cnvfa0gs77n0y1ww0tdn8908qnja4bxw9a7gpmw | Bin 0 -> 20 bytes ...7021ggw1rmd6qz9p3spjn266k328eh5vr8pg0pm2gm | Bin 0 -> 22 bytes ...dmacngtkjb9y93610yv1zh6a9rtd1s6dt3vjz00zcg | Bin 0 -> 21 bytes ...9117xzpdr731k488aawap9fa25kbg0yez0yxncp8r6 | Bin 0 -> 50 bytes ...t8a0r7d706fa7tnee39qz1rbdc92ekgbxtgejyntrc | 3 + ...wrvwgtw8a0xw2836gdfej6qxws1f66ej3hxtqf480m | 1 + ...7ypke6xjfbx2n9jpex04wy2rd0nh950vrteqpx6v4c | 3 + ...tb59xsy5bkg0vacd8wrqrhm8c5s10zd3xcpc7edtn4 | 1 + ...76h73qn7c9yh8r6xfzehdtvsbn0hkxvz5gz1w8cw6r | 1 + ...1mdgj72234c89gzj6k0cvw7vpb5v2n3c9327rmch3a | Bin 0 -> 5 bytes ...pxmbt1f6axs8fh7kz3mb4xhstcm3erhpx9nnsmb5c8 | 1 + ...19bz3ejma3sk5xyk9bhnz9yna4w7qwc5xvqt12n1nr | Bin 0 -> 30 bytes ...xem18j5vnd0db4gmheccjjrzp6114yjzv0rwpvkarr | 1 + ...56vzwq4xcp9g7ncsqmc1n0bpa2391kkjwa70anx5km | 1 + ...aq332qw2ef9kjhgknt5jgzqvq9959t18a9netpe7n2 | Bin 0 -> 79 bytes ...77sr15mekm241hw6307vdxw6jfpvtzcv96e0kpb30t | 1 + ...jgmhyf89bdtnk33sfphqeqh99a7bgnjm672d7491sp | Bin 0 -> 21 bytes ...9mwm00yyna4dgd0vfdd2xsz9tta24qqnb3a5j2fpb4 | Bin 0 -> 13 bytes ...k3np5dvfnrfhm0mp84qmkpbweyfyqtenpb9tybvv4r | Bin 0 -> 25 bytes ...62a9qzh27m009584enyxjpcgrydm3gpvh2pfgragvp | Bin 0 -> 19 bytes ...tthhttxra0bw9pr03wswx1zca0tasa30ban582d1ry | 1 + ...j8gfaf7y5xn1bd2f55dxj285rfmrv8ja0zypgzdjc2 | Bin 0 -> 46 bytes ...x2w3me566fak1gsq86nytqw7x2egsgqntxh4m811rp | 1 + ...06mw10v4v4hs3dd98gfqnpszmh7s2bfp3bgstpx1sc | Bin 0 -> 42 bytes ...62nwahseh6j9eb81qsbggf2k585jf9t01jf5s49cwt | Bin 0 -> 21 bytes ...xrb5teh8rxxejpttpqzwhthnpctfrnxma8fzf07zqt | 1 + ...d47bpcygzb55b6jsjwh3xb7nq5p5swwg52dv0je7qt | 3 + ...73qwy1mj50y89pkcasgkkm326mykrzx9yvx94ts4kw | Bin 0 -> 10 bytes ...cnqnb3bh8m1dtcd9wh20qxbyhggqhk559d4xafecb8 | Bin 0 -> 64 bytes ...2b2f1cvyd4nnep751pzhdh0fn48ga066nn3d2jpgs4 | 1 + ...0wbyzj59320tt8pbm99abg80w2v0spd0v1g1q2jxaa | 1 + ...wz2fcjyav4ksb0rcj53w0ansjt4gb9yh1ynhapkz0j | 4 + ...w2s5d2rq6ncbzmryxtzcc62jz09j2vzb1cs24t8mcg | 1 + ...e3xvpb0s9d1f7ffrnkwrg2h61dqqkgrn5sap51xkqm | 1 + ...apj6jkdsnskpkhbnpxhq75zq69msynk5mab9mgv54w | Bin 0 -> 21 bytes ...65bgt8wk77gp5pacydweetf153h40chbhpb5advvj8 | 1 + ...gb0ky8p285276zx7gnbyfd8ygy3t6r7vrv2x9y7rjt | Bin 0 -> 46 bytes ...mwgxfve8zjkwyag90ha2k573f1sfja9mcxttfmqkw0 | Bin 0 -> 10 bytes ...9xg001ksft36c10dn085qrsfwqv5q0414rnrpwtzgm | 1 + ...4nd9vqrwfx5hb6ve95gbakraz1e9xmqse6m6vyme7e | Bin 0 -> 12 bytes ...hh7c9j6r51vggk6ab360krd6x252xxgh3tdfp6rksa | Bin 0 -> 20 bytes ...dzpff11mgb0mtkjc166frrkre6smknejpqcpt7f22p | Bin 0 -> 27 bytes ...ykf5cbnv0n2r107km5yh0qqmx2vsjjp1qwybpjgy5t | Bin 0 -> 35 bytes ...exxw1y9681n735pv748t072w0qz7s9x23907cdjj68 | 2 +- ...j52m3a64ws7xxvx7qps9xsnhehfay1k6j88tr3ypke | 1 + ...1968q1gq2y6dbw2nbykrrendnqp1stprbxvd7mt558 | Bin 0 -> 28 bytes ...symeqpec56fp0mrr72150b33memf6nex59dhnbkqfg | Bin 0 -> 14 bytes ...red7xrw335kmfkv4px3sppvkjx8kff9ykcv0nfr1gg | Bin 0 -> 12 bytes ...8wzrej02f2qhyj478hsx7jzj55ge236qr4hz2zajdt | Bin 0 -> 6 bytes ...n5x7sxq15qm7yckys35008w0b0d4mp59gcp0th6hta | Bin 0 -> 36 bytes ...mbh2tkh0ce4nq474f49zrg0egdva3bxyhs2m0jhv7m | Bin 0 -> 51 bytes ...8068b46gp8ajd9xzv810kws9j84q5eb2rj5pgmn9t2 | 1 + ...xq2ds6v5amcy0sv3q7s2c8thhpb4tafjsmawb7f132 | 1 + ...mbw1dr6vxhxvmavvsa3zq0h4mm209am9qteesgyekg | 1 + ...a2rpj7esftmqkevjy248v1p0a13rawsz3cz6jg172a | 1 + ...fkpxc9em4mhhhymes72frtqxkjckgjbvwn5cdnzy4e | 3 + ...dmm6a6rbny7cwfsbxz7p2ccdjg3r42cfxqjaqj0vea | 1 + ...pbrm7ks6yytqpe4eye2n8zbhh15n3fnkng3p5m9a9w | Bin 0 -> 25 bytes ...b5rmw43mb4w66amzywjvy5hrdy2zj8s9wy15g2vfsc | 1 + ...bxpwxedbn894hgq2x212xe7nc1ky9brgnqpt9emw6y | Bin 0 -> 24 bytes ...hw0f8kdytdk9ww8p3pb17seayx23p73ye02px68sra | 1 + ...chygnfyvjc98mxcqa33pxqdtny51d3avkgy5rqrmbc | Bin 0 -> 53 bytes ...97drzaeq8r3a3tsb44s1xhd9rq5w34jceqcf0cnzk2 | Bin 0 -> 64 bytes ...n7080jpjd06ttd23q11f1gvf7e8x2vksq2987e74x6 | Bin 0 -> 8 bytes ...3jgagqpa7t8wm0p1cm563cqzs3t7bnma3b1c15ks70 | 1 + ...2zs7xpq5936mtf5cachrha72j0j1512g1c8hbgvmnm | 2 + ...qhn9hdmjz2f11dxm70nzrwmb4akc4rh536xmx8mkvj | Bin 0 -> 4 bytes ...px2ffwwzkhmysrgp72696z3nwm1b9c0w0rjqs21ge0 | 1 + ...qeb27pkvczyyz2b75c59yaa5c9sghmgrekcr80ea28 | Bin 0 -> 25 bytes ...9y9cs4pzfg1bwv73k271wp240w8gnpgdbs1532n3mp | 2 + ...tvrqmg8a20yn6zjg040fznngbfzqc6wd8hxqjza3de | 1 + ...bj14d4d6jx6t9dwevjz9qjy9hvkm6yq0zsvhq0traa | Bin 0 -> 13 bytes ...vvbrx1tcewc1h1932qkz4qm9yjdbs7g3072ackqg06 | 1 + ...rh5101zazqjygn9mnf19aa05ybkq9hyh1w1as4pbjc | Bin 0 -> 86 bytes .../tools/genvm-tool/fuzz/src/scheduling.py | 346 +++++ .../tools/genvm-tool/genvm_tool/__init__.py | 4 + .../tools/genvm-tool/genvm_tool/__main__.py | 19 +- .../genvm_tool/cmd_build_manifest.py | 3 +- .../genvm-tool/genvm_tool/cmd_codegen.py | 3 +- .../genvm-tool/genvm_tool/cmd_configure.py | 43 +- .../tools/genvm-tool/genvm_tool/cmd_docs.py | 3 +- .../genvm-tool/genvm_tool/cmd_fuzz_corpus.py | 429 +++++++ .../tools/genvm-tool/genvm_tool/cmd_test.py | 3 +- .../genvm-tool/genvm_tool/codegen/__init__.py | 3 +- .../tools/genvm-tool/genvm_tool/codegen/go.py | 24 +- .../genvm-tool/genvm_tool/codegen/model.py | 39 +- .../genvm-tool/genvm_tool/codegen/python.py | 8 +- .../genvm-tool/genvm_tool/codegen/rst.py | 13 +- .../genvm-tool/genvm_tool/codegen/rust.py | 20 +- support/tools/genvm-tool/genvm_tool/common.py | 25 +- .../tools/genvm-tool/genvm_tool/formatter.py | 8 +- .../genvm_tool/git/check_for_push.py | 12 +- .../genvm_tool/git/create_branches.py | 9 +- .../genvm_tool/git/verify_pushed_executors.py | 6 +- support/tools/genvm-tool/genvm_tool/io.py | 82 ++ .../tools/genvm-tool/genvm_tool/manifest.py | 6 +- support/tools/genvm-tool/genvm_tool/misc.py | 71 + .../genvm-tool/genvm_tool/tests/__init__.py | 25 +- .../tools/genvm-tool/genvm_tool/tests/cli.py | 29 +- .../genvm_tool/tests/exec/command.py | 58 +- .../genvm_tool/tests/exec/process.py | 46 + .../genvm_tool/tests/exec/service.py | 56 +- .../genvm-tool/genvm_tool/tests/exec/step.py | 18 +- .../genvm_tool/tests/stage/collection.py | 74 +- .../genvm_tool/tests/stage/configuration.py | 3 +- .../genvm_tool/tests/stage/execution.py | 23 +- .../genvm_tool/tests/stage/filter.py | 8 +- .../genvm_tool/tests/stage/scheduling.py | 429 ++++--- .../tools/genvm-tool/genvm_tool/tests/tags.py | 68 + .../tools/genvm-tool/genvm_tool/tests/test.py | 3 + .../genvm_tool/tests/util/watchdog.py | 70 +- .../unit_tests/test_codegen_detail_suffix.py | 94 ++ .../genvm-tool/unit_tests/test_fuzz_corpus.py | 150 +++ .../unit_tests/test_fuzz_corpus_layout.py | 27 + .../unit_tests/test_process_teardown.py | 90 ++ .../unit_tests/test_service_semaphores.py | 356 +++++ .../unit_tests/test_service_startup.py | 79 ++ .../unit_tests/test_watchdog_ack.py | 55 + tests/presets/python-fuzz.txt | 1 + tests/presets/python.txt | 2 +- tests/runner/genvm_tool_plugins/afl.py | 594 +++++++++ tests/runner/genvm_tool_plugins/cargo.py | 340 +++-- tests/runner/genvm_tool_plugins/docker.py | 84 +- tests/runner/genvm_tool_plugins/genvm.py | 133 +- .../runner/genvm_tool_plugins/integration.py | 150 ++- tests/runner/genvm_tool_plugins/ninja.py | 45 +- tests/runner/genvm_tool_plugins/pytest.py | 152 ++- tests/runner/genvm_tool_plugins/runners.py | 55 + tests/runner/gvm_extra/case_inputs.py | 17 + tests/runner/origin/base_host.py | 36 +- tests/runner/origin/fees.py | 3 +- tests/runner/origin/host_fns.py | 22 +- tests/runner/origin/log_asserts.py | 12 +- tests/runner/origin/public_abi.py | 88 +- .../assets/v0.2/caller.py | 27 + .../assets/v0.2/chain.py | 15 + .../assets/v0.3/chain.py | 16 + .../assets/v0.3/helper.py | 11 + .../assets/v0.3/observer.py | 60 + .../assets/v0.3/self_recursion.py | 20 + .../assets/v0.3/trap.py | 11 + .../assets/v0.3/user_error.py | 11 + .../system/cross-major-observability/test.py | 642 ++++------ .../cross-major/assets/v0.2/blob_caller.py | 11 + tests/system/cross-major/assets/v0.2/busy.py | 12 + .../system/cross-major/assets/v0.2/callee.py | 11 + .../system/cross-major/assets/v0.2/caller.py | 11 + tests/system/cross-major/assets/v0.2/chain.py | 14 + tests/system/cross-major/assets/v0.2/loop.py | 11 + .../cross-major/assets/v0.2/signer_caller.py | 11 + .../cross-major/assets/v0.2/user_error.py | 11 + tests/system/cross-major/assets/v0.3/blob.py | 11 + .../system/cross-major/assets/v0.3/callee.py | 11 + .../system/cross-major/assets/v0.3/caller.py | 12 + tests/system/cross-major/assets/v0.3/chain.py | 15 + tests/system/cross-major/assets/v0.3/loop.py | 14 + .../system/cross-major/assets/v0.3/signer.py | 11 + .../cross-major/assets/v0.3/state_caller.py | 17 + .../assets/v0.3/structured_error.py | 11 + .../cross-major/assets/v0.3/user_error.py | 11 + .../system/cross-major/cases/routing.jsonnet | 95 ++ ..._null-mismatch-callee_null-mismatch.stdout | 2 + ...er_null-same-callee_null-same-major.stdout | 2 + ...ller_plain-error-callee_plain-error.stdout | 2 + ...-reverse-callee_plain-error-reverse.stdout | 2 + ....signer-caller_signer-callee_signer.stdout | 2 + ...tured-error-callee_structured-error.stdout | 2 + ...caller_v02-to-v03-callee_v02-to-v03.stdout | 2 + ...caller_v03-to-v02-callee_v03-to-v02.stdout | 2 + tests/system/cross-major/test.py | 1141 ++++++----------- tests/system/manager-socket/test.py | 374 +++--- tests/system/parse_version/test.py | 5 +- tests/system/permits/test.py | 75 +- tests/tags.json | 148 +++ .../simple_deploy_then_write.jsonnet | 8 +- webdriver/src/prj/package.json | 4 +- webdriver/src/prj/src/duration.test.ts | 40 + webdriver/src/prj/src/duration.ts | 65 + webdriver/src/prj/src/index.ts | 262 +++- webdriver/src/prj/src/semaphore.test.ts | 270 ++++ webdriver/src/prj/src/semaphore.ts | 156 +++ webdriver/src/prj/ts-node-register.mjs | 6 + 1348 files changed, 13438 insertions(+), 2932 deletions(-) create mode 100644 .claude/settings.local.json create mode 100644 AGENTS.md mode change 100644 => 120000 CLAUDE.md create mode 100644 crates/fuzzing/Cargo.lock create mode 100644 crates/fuzzing/Cargo.toml create mode 100644 crates/fuzzing/preload/Cargo.lock create mode 100644 crates/fuzzing/preload/Cargo.toml create mode 100644 crates/fuzzing/preload/src/lib.rs create mode 100644 crates/fuzzing/src/depth_limit.rs create mode 100644 crates/fuzzing/src/lib.rs create mode 100644 docs/contributing/explanation/fuzz.md create mode 100644 docs/website/src/spec/appendix/internal-constants.rst create mode 100644 docs/website/src/spec/changelog.rst rename implementation/fuzz/{inputs-genvm-modules-complete-json => inputs-genvm-modules-complete-json-curated}/1 (100%) create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json-curated/2 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/0/0/00d3ga150hdc2vahv7sek5vhev60wgxrqmchnxqg3tkgj create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/0/0/00esba677v1ndtj0816kwhjph61a9mtjqrhqp45j4pn8j create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/0/0/00m6tt8xf34nkv5f4jvcv7t5qnbvq8e6vspmf0we96ydt create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/0/1/0113bt3yk923gfp1a4cnxp8n9k0nqjsd6003qrspy8ng0 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/0/3/03tgtrmf50h7edgb1jvdpp767qs4j6vjfp33ty9rr9w1t create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/0/4/045698ck5t9h0fn9dbv9x26q7gxekqbn9chzzbskfp53c create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/0/5/05r1ay1bw7ngb8zmtkdfkyvrka6qbjs9jwv46mt3cbgde create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/0/5/05z7425x8bd6nvkk4kgq4f2499n09vpv1paeafe025c6c create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/0/8/082658bdw9cssdkncjmxvzbhmff4n1j4pwhntw0n9mj2t create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/0/9/09j7nh5jet076pmv9f8pqe6swkjsqdqc9yxw67105wrnc create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/0/a/0a34bxzvvbkk9zdf1wh22spsbdwbqeqerm0e43pv2ngct create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/0/a/0a52mzaj1hv1tb2txtrjrs0ejszt3fgn469604sqnqj6w create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/0/a/0a7d5brepkxdgktr1f7xwgdg5swstym8vsx6fg3ncrmp4 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/0/a/0asfwrht8gf6bz9tf3ag2ah4sdtaexyeeaatfy86cfbse create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/0/a/0avcwmc836hmvcx4tcsgxjt88gfqgr6crja3s08qjb2em create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/0/b/0b3zfjthc54aqca5f5q9776emq6rtmc4psze4xt987swm create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/0/d/0dkbwhymbmwe8yfb2af3mdmd6sreg4w60t8km5ctnghr8 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/0/h/0h92aedsc1y7xgr695dmyvyf4zm587ab3yd1acdhrvr9a create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/0/h/0hfwbfz1w1wys0ckdwxg5y4tvhshzwmq46h9mh79qw1ae create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/0/j/0j3dp76k6cfjc5zkzec3cn0t5h0t58w96yyzeyhzsm6he create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/0/j/0j89xg8cfb3xb62hhx33xhhd813c2qrfrwjjgj5ebsw7w create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/0/m/0m4rse79871djv54arat6a564d9wk6b32et8xfaatccqr rename implementation/fuzz/{inputs-genvm-modules-normalize-whitespace/9a70598e97b57c7a6c6d7d77cc64509b13c0caf72203e8cfbe0c6f2e3446c804 => inputs-genvm-modules-complete-json/0/m/0mddxqyra4jsre5cw5fmk3fg3y4gx74fc32dxwggnkqz6} (50%) create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/0/n/0nz26d1454cv5768t5j9se7txfecpxxsm9x8s7r91qf1e create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/0/p/0paa3j6967xv5d1a2074nyqtnbbtv76pvbpzgevzfgy54 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/0/p/0prjncdjcy75m9m6q0qk6ezw8zj1596rhj36e402ew754 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/0/q/0qn2kcn21t453s1b2ngz23bwnbzkm23e5ks6x0v2z5td0 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/0/r/0rtkxs54nn6ex33a27wg1ppmthvkknkp3mep8f5b98k50 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/0/t/0tjjkt9cpff1d660acdtdzdpfv43mfctgbpy3he9beysw create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/0/w/0wqka8ved58jyqtjbr88vpkg80h3sbf3pjyznxq7j1904 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/0/y/0y24j1rns05r9jsbwe40h08etjq468myc7zyqnr06d3et create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/0/y/0ynj4895jyafkpw0jnvzmxdnhkv2p5jz9p6kc483sgk50 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/1/2/12qz7rfp162bvpwtqdcsjfcte6gnaqhbn0brtbqmrfeyy create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/1/2/12v1440bpgmpf20twrkek8mv0d8005q3q499gz8acnzmm create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/1/4/149cmt9ms89v020yrryvpa8jvmc20h657s4n8p4ypamy0 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/1/4/14qyfsbfxsjvhq4h4px6vne04dr3tedqcnk65kx1h0x5a create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/1/5/152sp9r1b3r2ws1hk69wtkfwp9rrxedhcyss8bdcnvaha create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/1/5/15bp4vcf7xkp37ac1718tt0rxvgts1agh64p5aw1fhxbj create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/1/5/15sf0t7k419d92gz7xhsav1gcmq07n07wydnnwsrgphym create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/1/7/17hz2hbxc3cj4tw3tje2j8d5ty7kc6kfpsmfte6p2twm2 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/1/8/184jstbv6j02k45pq0cvc2v78est9zdsmbnrffr0r7nv2 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/1/9/19dp9v5s7zg0r44q1q10j4cw9xczdd69pvt8v340a6f8c create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/1/a/1acaehj72a3q4n0svg1mw6sr73c755f4fs585a33prhzp create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/1/a/1aj0wvbm2cwde34147rf5ha82ry271c5rc0agxjrc47x4 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/1/a/1ajgxbt71nyw23zrd1qy323jtwynep5d2pdaav2tffxp8 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/1/b/1b6evrr1ecfjkt9k5kg0tk3xh25s33wx4ww96573naghw create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/1/c/1cph9wc8w0a8z0w2028vtcx5xs4dgyzf5w1s7jdppr3xy create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/1/f/1f72s7zs5t4wmrbvk0yg71wxveqe55ztfpg8jbsa3bqmg create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/1/f/1fghdv1x7bnfg9pavb755sf0sc40qmhsbcwqf5nrs7ae4 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/1/f/1fjcx4dk7z22dava03ptrv3ymfjycm4m4ffx7j0d98m4p create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/1/j/1jj15a1rq85qvscnan6she76mdje40y78df623nha4fyj create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/1/k/1k3r491j8kg26xvpnfe3a0yb3ve9wvy43c3n09dgv7ctw create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/1/k/1knejvhs0c6k96dxf9zspqgns0xwvzn3ntmnc3m7hfgbw create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/1/m/1msmyw0naed8mgex7ss57qmqbfwapn427xst8jsqe3ngj create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/1/r/1rmm9pj961n1pq2vsy22xpbxw0nhyq93px4r60hahzjma create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/1/s/1sf4137gha4gmqm24trscfjqc7jm4479w6qj1mex11fry create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/1/v/1v0k1nwfyggr3b0wc35azwddz7t89e408jvrrapd5edrw create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/1/v/1vfxfq59pf73na9hbcz0b6vdm97rwqjegsmyfg1vqsm82 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/1/x/1x74f8x8hn228h57pkdye6wysbhsrrhjw49z4gepbqzvj create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/1/y/1ytqg1fmemq3efa31gttwkc9ks89mdx2dbzc5qhgj92s4 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/1/z/1zbxn03hxa23gxe1md3gbn3zv7cngzcerj1j675n9780a create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/2/0/20n3w7v3kk9wzmhsayk4at70qkrajx55jgnepk7z6hr0r create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/2/1/210xq1kcqh6mwx1jv8e4m8p45ep59sdmv28pmzy4pw1a4 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/2/1/21ya1p19ay28psnn5f4zhyd2t4kfvda8a4y3dpc14v5f4 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/2/3/23vz35x95rw4mrq1a5fxzy55qjks164pc29znrc45cdtp create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/2/5/25c2p7k1zb2gkvyfdaz725jw76qrm1wncjwqkqc4kk1m8 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/2/6/26wa4awjzxj0n7cxbv03m54mfedmavr2f6m01vv3zvtte create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/2/9/29tp9v5282spx6x5aagjzrpyypvmtpkq3r0rrfdg4m67c create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/2/b/2babs58b5a87gfxksxtm649ypqgmkacmejzry836t92k0 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/2/c/2c95ns6xkaakcy132nmxmq4x2zgqr90x0dmsnz627ed66 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/2/d/2d8qwhcyefn0frp5r1ca0x7c5fx7z73799v9jgh5kznrm create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/2/e/2ej6zv22vbtxd1h5fyy6aqs1709dm2kcner8xgmdjw3gy create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/2/h/2hmtt4mqq71y2pvf4twdsffeyvc71wp3h2hvs5ahme1nr create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/2/j/2jng17dez65x6wswjp25acskmrx7b5tcmq386s35axwnc create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/2/k/2kqfqq0x171fhs9f654mhmhzdam101sx78y1pg4xvzj2e create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/2/k/2ksxsdkrcj75gxj5bh515daqdr49gxas5gq4w3mne8fxe create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/2/n/2n3jwfv96hh6722m749ghsxbesbbkgqgajvx8czn183fa create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/2/n/2n4t3zk62tgnhnhvqgzvfw352g9yy662b1fn2087cqc18 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/2/p/2p0k398b7qp548psbch15wvnbqehqgkz4pf2gtj1tpb7m create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/2/p/2pm3p5fxq3fpxt5g0j5tpje93fn9r6c4b4qz0zmfn1df2 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/2/q/2q195hbtgbyx52kbb43mdbrmt6gzsc3693yd49g1j2jga create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/2/r/2rmtbeedk3mnayqb6z0dj77583dh0arvgz4456d2yp2wy create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/2/s/2s4as110gqppmf984s9600v742ztpwrbf4gkgkrayw2q0 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/2/t/2t0ftjj6197wjb7dqch5ammz2f5esnh0zrw4fh1vzk35e create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/2/w/2wn6p7whcvzg482hfct6tzax39zggc7bt3qqxj393awqr create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/2/x/2xa502ckstahndqb4bqhxdwq7hzsxbw4a3dh9008n5grr create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/2/y/2ykbambjhke293td0wnyhmqnprz3dr3kckw2t81vdeq32 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/2/z/2z3p4rpamny067vhq88ke789xdnyt23zn2ywjq7sbxy6p create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/2/z/2zazarnhgndp91dxsf8m775r96x1hb59bpaa1942fe22p create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/3/0/30d131f7jpbsdzp0t3f3ry9shwecm2mbn3sfxqy4vxqrt create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/3/0/30hhad75c83y5s8a6z9ftdbnnmatgktgsasr18av2cj36 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/3/2/325vkc6r7q84h3g63whr2hdc69chys6bafjqyr21vd556 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/3/2/32wc7343p2qsgg1vr2c8cfxf29rxjte6xaf1j3kqrthp2 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/3/2/32zvz3ffjf0rsaayrf4drj4x6a87162a5dxa9x5gbyy40 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/3/3/335qkpa6a6a55kg78v14wk3yyesena6v3304gw7mp792t create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/3/3/33ew3bbq8bsesg4jf189jf9ard8cnp9m4sxsxsb96q3fj create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/3/3/33qz17sfvejgr7dfczk6w4qbt77qwanv1307w4b8ja0ma create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/3/4/34j4w84w1cnqb7cr41p52saatagrqsxk8wj2ehhbpc7fw create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/3/5/35237jnwf7qj4e3jy7dngv203sb8da6za3tmdtaz8m71y create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/3/5/35wd5p3vpd27tw2r15btrpq06gt1b708brb74z1z6k5f8 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/3/6/36qhwstm4ezxvnsem3n9rkbwe5971cq5r9eyvt51n1ca4 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/3/7/37223ehxrzsjx4h98n1t8tzxdeez2q54n4ghv7y87h99y create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/3/7/37epan24avchbwj139h8gwrgrn168hpdn6wctp4w3kvnw create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/3/8/38t9f41n2ynf9n1r6yq9wv9h9zygjpz50ghzy7qzfxb4e create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/3/9/39jhye51bpxdznqxjc3n7txbqtdxw8tk999evs6pe6pty create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/3/b/3b5snsywt1yg9608yk9dtkzcsw8j226873xmgrbtx5e2w create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/3/b/3bc93yvqp1d1zes49sk9vwyf8enqcmkfectpyjg700p5r create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/3/c/3cv357mm22rrbcdb29bv4883ajrnye3p05wh3p4e9vvde create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/3/d/3dygjt8z5jp6gss3nfv19ht36jhan7xc4x46dh9wxxb14 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/3/e/3ejnm2369k6hgrghespqr1pe0qdxrwv8yqrwjpevdvk68 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/3/g/3gtdrvgpzq462t4wsnd5k20xxfvpwckw3b1gg9077ey4t create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/3/h/3htw69mppz8dkd8kkysh4gc00j3021v0zv38sy6nhq02p create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/3/j/3jep0893yw33esbfnsx3vtpn03jw2f2kav5261w8tk8fj create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/3/k/3kbz0gcyv9ydzwdszy7q1c4ktka3rfn4d8pk9e4318etr create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/3/k/3kv8nxj26hfm5ez2vrp38j59q0rpgvxffcvd8x0vme6n4 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/3/m/3mdxxf03629wvddwgvezn1r428tykhd7spzdd5hn2fcmm create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/3/q/3qbv6wqxrtgkh6rbsmf4n7ta1xyndvfaedd6rbk08peap create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/3/q/3qtfgm5c9jtkw8ae8gwh01s6tkk18my2ag0y7epc1ckd2 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/3/r/3r1fn4qcz970rxcapmbg7axet37yk0c30pden7zfnbrcp create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/3/t/3tx2a7qvwfc8df5hzpjcyhkc65hvjwfk4b9v649x59n46 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/3/w/3w7w2k51qnmw7z4j79g0dngx0x3xv81fqx11rmktjvd5w create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/3/z/3zrna3xrtgxct2en27tz1pb3v7dvtt5qh2fynhyvmhwew create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/4/0/4039y0m4gwtzm6ybeecvxq6d297w7h1h3zwfpqeqwejp6 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/4/2/42ckpme1rg6chpv2jg81r2mp6a1pqeb7jnpxkrj1zvvz0 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/4/2/42txhvm8f01c4cdcaymqg3nmjsbc91sxz2xryj3nmng3a create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/4/4/44nkn4y353ss52zenyq9e8gr7vcarz4y36dtff3nd9af8 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/4/5/454pdwe2qc8f6ck1zfkd4czrtxzbgf92qywfftqgxjhr0 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/4/6/46hzz84st57awtf74kqg6x1ph1nc1b550hxnn8ky5e46p create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/4/6/46s556f8ke1g4dtjahxs94mzecqzw1n3twh8m9vtxdk5c create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/4/7/47a21mjw0h89fdx7t27e856cxrbv2ahsh3xybqjfqrq36 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/4/7/47aenj50bwcb52ts5me0mptej88cabpzew64b22ncw82m create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/4/7/47zdxfswt8ax6ccwdq71k2wzyr2pv2sd7hcpcb3yes5rm create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/4/8/483rdf9z0bqhr2s4f0s7yw5gawhafpd26csq0xrn78ehe create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/4/8/488m98q123vvxhqx0aagr7fptytjjm2sesvzstq7vnjd4 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/4/9/49076gsq2b7p3hqv2h0fssazfg0mbppj9hr8yxyhbykwg create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/4/9/49438ym1f018m8ctq0gbb06cgkamh1yb5yhztnt2sbgwm create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/4/e/4e6m73vd7pqnte9f4dmahm7eddc8n3nb78hf0f9d14b9j create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/4/f/4f82mbvdzrxewmjfyk3g5xg07xn8j49tc92n845861rty rename implementation/fuzz/{inputs-genvm-modules-normalize-whitespace/b82254c422c4a0de4fde9cf66488007a6f787d972f1ff42a8f0d3a07204fdaff => inputs-genvm-modules-complete-json/4/g/4g0fnmrht3baacq1spykfq06esnj3smp0rd5e7jp79eyc} (50%) create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/4/h/4hn2107rfefkdjabs9rhdsw49zx8y2gx8p2c0qgt21926 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/4/p/4pr8k38eg2vs9bccsw8mnjcnx5te4y1g11rgme8h3gybg create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/4/p/4pyjs7hxbkmv5bb8nk2ey2d0wjx8k82f81c6768pc92ge create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/4/q/4qp2afa7b9e3hmb9et592k41sh0f15patxjf2v5y5ax28 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/4/q/4qqkbm0kf3s0n4adszk893w6w8wnyjvf1e76ye91gg6tw create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/4/r/4rd4fhxdd75yrzwqrt2e1e9wewtthq7qdsxqqdng0b060 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/4/r/4rshs07vxp9p01tfz7pvd0jraxfs0jee1nw0cwkehxx94 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/4/w/4wey7y6s511635spzptgtg0zhbcsa537xqby99h0nv250 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/5/0/50sf3sj3zza00020bnq6r03axvz1vksraj1573jebfd6g create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/5/1/511bbrs4vynrjpt7pmjj4897n9mfpbe50b7rwn7x9pgk2 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/5/1/51dz1xs6nywx9v858m9bnd8ast8y973mrmt6s2kgtp9we create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/5/2/521ask5rah2rmhtk8w53w5dvfptr9anhetwce58yktk1e create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/5/3/53nstbc50pvj2vw2nave0q2ry661kggrh2xrzfbnabzeg create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/5/5/5532hjx36v13jedse38grc2h9jcqcrq3panpry2b2cq5p create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/5/5/55pm4s9k7pjvttjg3vmhhwkvgtp5rt9rren3884mtychm create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/5/5/55y8fgk28sjsw9kc4ct5vaw0nscda5yjqb3wxn81bjdpj create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/5/7/57h0p1x6sm52v6sgcw3k06afpbfxmb9d3xhzjretpzxhy create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/5/7/57q7gt19ynvbezapg0t7ercqeqbyetjdb4h3f7xy7s28y create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/5/7/57x0e8msvtg6fh2rgj738kh0zrmsnr65xhp8sknvdaf0g create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/5/8/58s6jqed3mz3m6eaqe529dqf6qrbr9pzx13mer6gwwnvy create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/5/a/5a982thpv82fgbs86s3dtzsz5wnxt0yqz4dvhhwz3r1ye create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/5/b/5ba7bygx5f8z6r1s2a2519a6c625ytcfwmz4q9nt641qr create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/5/c/5ckz0981gspjax26nkh652wc7tcb54w7fy5csjms0d66e create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/5/h/5h2g0chm9b3g0gtyv3vwa5125zj6h9wpewa1d9ttasjcc create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/5/h/5h3sncn3nmnesyx8kpmqmvfkvkj44xzebssp7xnczq6hr create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/5/j/5j5yksckw2vp4xrca0xy5xnsa8sy9ytjw2wt1zhhndx36 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/5/k/5ksca34p8jvyyv4r4t9aq8fqz0wyr6c5w6a5j1cns3wtj create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/5/m/5m13t0nyge0w12tkmwt21zv8gphpckwv79ky7syc8fz04 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/5/m/5meq1wankjerhv937773xend4eakwq3xfyaescec2yy54 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/5/n/5nxmsy3nbk3n9x2eezrgbjj21j80xj62n9c8svwt2weva create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/5/p/5pzkgwrwycj09hfrjtmyfhpgmnerkapbh6xhp55gskj9w create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/5/q/5q2611whwhv6gmbtrtwm09b0cbpnjr0mcnrvfas0ky1ma create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/5/r/5rhhmvfczvhj7ba6jaxjqhx7d44drdy8nv5ypw3przvme create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/5/s/5sjmmjcchvtx07q0s65jd41npa39zadh6pdfavdebrk3a rename implementation/fuzz/{inputs-genvm-modules-normalize-whitespace/b99e1959cf0de1171606c1971933f6b68cfc618b1937fc3117af71e4877ea0f1 => inputs-genvm-modules-complete-json/5/s/5st59vnef87wq5077p90tfrqwq4zwa7seh3erqrztx5xt} (76%) create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/5/t/5tm51hcwchb3mvj75w58m2grm3dm3n82nvv3c0e6mwv0m create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/5/t/5tmrg5k6rz6vg0cmfwbrnmbzybvxptypmp4ghactbwkzm create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/5/t/5trdrhkddcaqa06fhrjy51knjj279hx2a87rtc9rwgs7c create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/5/w/5wqvwve9j7bc3f8x0jb6z5qtff5tqedkwkad3zr0g7y5t create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/5/y/5y8k3vxza1nxqkpp32q8r5027bbbv4qbfy37ad1knt6w4 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/5/y/5yxxpcxncfjgm9e5bff4v1wbh7pny1gd26hqybg84sabg create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/5/z/5z2n879zp21139g3x4ra4m07sh9s2gn1fwwnn1m1sn1br create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/5/z/5zmgmc6sv8n8tzcz1s8j5cpzq2n7a26ermg60hyz4t1h4 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/6/0/601srznj4t6bk5w9x4pcr0zwxmh6avsnt7gpn0fm7ak3c create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/6/0/606g3wx92025zvx1dnn197sr2sxja0yvrdy1p97xdxthw create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/6/1/61mta1s9373nq89nvskpew6nssr5b9gvk6dyv1r0jrcz0 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/6/2/620n716s2sn0567k9wwcq1vs380t3ap6m3526mkcgdhtt create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/6/2/62148mftv5xr6nej0mnpbg4eygjz8mqgfgnt0qsve1pt4 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/6/2/62kak2fjfc2q7p4515s3gt8c42n443gqq58qt9hwcnzs0 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/6/4/64p3kq1c31cejcj9qtn0rz5k7pzevbjyah0t1jsj84cbt create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/6/5/65jh0f2sd99x7d18esapxv0x0gvqmzzz8r3fw4sg1gvw0 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/6/8/68ka1qexyqvggrsm11r04zrg8bt4ygm1c5jr683y1xvk6 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/6/9/698cfc2vbwxfdptecap1mxfgd5xeprwj203yq0fj9smhp create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/6/c/6c3qj9g5qak95p6mw2p3p7814bgwesxeceqwd97f83464 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/6/c/6car9rqrptwd0md4bppt1jr9vpf9r4b77a74wzqc1e0pt create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/6/e/6ebwq1t7yq53h7rpn35n04399zqvqkar42gebmqmbefmc create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/6/h/6h4krjdvvw5pmgfnbbh5tyhw5hfpbtfpk72x35bcvyb0c create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/6/h/6he21spbt5fm52anbs9a3b7dj3s86f40gg6cz81acrt7t create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/6/h/6hwdx92zmptj4h2j7zf3r3e42w3zh6bvbrz5z2p2e3h2c create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/6/k/6kjg78eczceasp09agz77gav04w2cmqttcb1a1z89k7rg create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/6/m/6m6xkpezne5pp3m6z8age604q7cpv1x920qnkvz1bkwyp create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/6/m/6mqgag5enzvbfj1jj3p1h9vnbqe748dq2ppejheajd5ng create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/6/m/6mrbsr64qxtbfwjkrca2mcwecqydf7daxy28pc0vfe7ga create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/6/n/6nh78x0s7vpt01c4jm93bcz1b34cy3fr0tqwv7tr549er create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/6/n/6nndeg4erafb4pzb2901wkjqtdds2c1xdb7fv3hxmfy08 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/6/p/6pr35dbxwq0p8n3ms3tvtmkqeb60fef4pnkwz0fvagavg create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/6/p/6py5gqqwqynzs9x0hw765emeg89cewjz87h01btpt6hsm create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/6/q/6qywzpqkyc3m2nervjxc7xrff5by8m334by2ehynfa566 rename implementation/fuzz/{inputs-genvm-modules-normalize-whitespace/9e76479946b62a0bcd38c64e6362921ffc3e5c9d88c50c8eb3dd77529793625d => inputs-genvm-modules-complete-json/6/s/6s8c2d0dafx3tczd8ba0d3q08tdben51y533v9nfbp9tt} (85%) create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/6/s/6saxjzt801waqf7pynga8dtztvf33cxav6gnakf3drksj create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/6/s/6srqet48avcywyfmwdxfve36a9ghbr9wsa51d35cydp2c create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/6/s/6ss1rznx8wf9tt6fyrps4510fq9zj0d5d4634jyyevnq0 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/6/t/6tx3r8mjwycqk3cc19tk6vz6v0mpx5b1skh4619860kcy create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/6/v/6vkrd7rbzfmpqy99m87rakngp591kxkjb6npwzhyetr0g create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/6/x/6x04fjc4187d9vgnra9sgq9mms214txwpsp3kvqgd3cvj create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/6/y/6yvt19scjh8kgqscps8x2f1rz3e4jbc9aasct5hsk1s72 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/6/z/6z0a57zfe11k63pk5mdzqwqh9q4wnhr3rjsqsvpsd56b2 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/6/z/6zkx97mnxq79jxx3p6v1ash40kmtc2sg8rb0pkxga505p create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/6/z/6zm7xgc4drd8e67mde018kypayx07c8vb8vh8j7vpd57r create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/7/0/700hxytcf4491x4ydx46vqpyacj3205hsb3brs7p7n4ty create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/7/3/732e6g98xw74xt2m44tza0kca6t24qsaqm5cmrgw0xf86 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/7/4/740p7skkfw1b68nrs3sq5zxtr9axxaxd01nrkkgbqebw0 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/7/6/761bhyqcknm6pyt1f7j8jd61q4yrzhmvhy353v23257tr create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/7/7/779swdcbj9e7djcah0pmjvtxsd7m588966stts93xg9yg create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/7/a/7aey027n4v93kp4s0qh20fx4gkved3fw15n7r18yq07ha create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/7/b/7bpbt1he8ntbggfw9tse48emwb2kbycj77jgz76ddeyfg create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/7/c/7c777qz80hpxtgnynsqmqj74byzc144gtxq76kwyrmad6 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/7/c/7cqjbvhqx45fv2n0r972app5mqy1hrq1xdk2bnq3he5qj create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/7/f/7f6vedsdq879bn98wspeyrsn6dvzagxmryh2y5r1d1c9w create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/7/g/7g15fmyk9fhwvtzx0qcm8p3hc0z75ys0dtvtpqnvcyw52 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/7/g/7gpcdbfdsm22fw79j8b0h99d158d59af312qf7xembhzr create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/7/j/7jhrja2fjxnemgfffhcbwms91t3nct5hha8x28py326y8 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/7/m/7mpygmpq420qqgrshs10nenpk20mnzqtqz3mt22jhm9zw create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/7/n/7n4tmxbfh43pa7xerzq5gzg15ad9hz007etcbz5fm0g7c create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/7/n/7ncz7fcaqtq179b6m9pjb88yd9ge48fcq98mz0z7z04jm create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/7/r/7rm41e9f692tk3awzkp4y3mfm8tz4tzk9mqdrydn4jwh8 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/7/s/7s0bar4weepxqm36gjtmngd0s4s87t0zs71gras1azkja create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/7/t/7t6wync6tykyywar2shdyswn9r1re8ta4x7kxc21586p0 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/7/y/7y7bc8ad66w2zzn4rpff9ntkjw5r6g7rsqa16hgy3at1g create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/8/3/83rbwdmzca793m5jvdh96qbx9kmsq4ydvyg3e6yqny8k2 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/8/5/85rhes7r132gs7dr83w6w7q2x33ce0h82vffctfn6eqv0 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/8/6/867rvjnahynn25ac14cwvt9t6jzvggz372ydp8fqswv2a create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/8/6/86k3v2zfg6c9n6n90t48dxem8619svqp59wxtvej6c5ha create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/8/7/8772ghfrhxs1ahgdn3fw4m2y492x836wf7k46sr5err0m create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/8/7/87qbtm14hgyxeyn8ey5dgnxtggr212adw4kwvzbnybdct create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/8/b/8b7pzscez3h4q8sxq7dmxcqvnfn5pgkdfxrqq1gr9csvt create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/8/b/8bky8ft3qegkfnscpfrx977aq71fvfkfggktv1q8f7f78 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/8/b/8bxa62pfbntw5ftd5xs2v8rrcdsmt2s84vc608cfn9efy create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/8/d/8d55edm56x6y688pfpkvrm2m67q4j9xxaw2twbsbkbyec create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/8/e/8eqck26hk0rmgzpwed603cmmqm6wzhp1e3aarwg34r7xg create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/8/g/8gdah1hcnmr8captbmxmrvjxs1dn8gbn0v7a10jf9msfp create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/8/g/8gezvhamg4x4p2dc22wt60w0stt38wpk0re71drs2dajr create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/8/g/8gt98q0hhqgn55z6vczfxgp42qb5awzyjd131cz51zwbg create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/8/j/8jqj3fy8nbnk194bxprz80a7yfmdhdkqtgtd6s8zk138w create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/8/m/8m2qdka9j6w1fg7mg0wn0tb355jkbatrnzv6n7nmw4skt create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/8/n/8n4hrxsbftvpfmavg3smrgjc4hh6ezkawrfcepxk2jmde create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/8/p/8psw56dbhmxvhk9fhkmk8y0p0gwa88vszgnchgenysxh6 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/8/q/8qk3zk4k8vtbbkpxa5q0sp8fch71gka8tv8bny9kdktm2 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/8/r/8r0hp2xf71j0b3sxjm9mkbwvpyf8jff8n64n6wkjr6368 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/8/r/8rxmh4hmmgpgm7dxwht1cp1amvsd0qxnc23tdgfdh8k9g create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/8/r/8rzd98pan04nx6q9q9n5mkqtz86cg7hnkm1mkcq44qjmt create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/8/s/8s2r3qc3h1anzm3hkwr6j3r6rmad853tc0ds25hrc2fb2 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/8/v/8vdr6yembj5xj2bm2cyg20q57cj4hrf1p8x80ndymj4aw create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/8/w/8wy643cjwk8wfh0004edbbst8wa1b73aywaqfc0mcf0kp create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/8/x/8xbj6pprejy2v5shedeb0xq7dqfchsdcjy9tt523dz5mc create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/8/y/8ykr1bp9z55bqng865rp8s6wx7w7hc6n2jk0wanp61062 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/9/0/900w61hvabmpf0a96abnadvjsnn6q53zwxatx2t7d0tw6 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/9/6/969ard33ab13sqxf820qr4xb9cgscj55spqa0neg7epe4 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/9/6/96samzadf8rnaf6j7g1t39dft2r9tm3b9rkj28d3edhhp create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/9/8/98854yqfez4decm5rhrdrs2bxfdjmcge4tgd9wygtgh10 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/9/a/9ajc9e1wemy51xq2xpwm52cc7pjda593acq5vtwd1bfd4 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/9/a/9arqrp6srq0pd6vccjfkfymafvrh4qg32cx1br2exnd5a create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/9/d/9dt8e15aj1a7yp124h139794c8we0wdnmp7jrtepj9zzt create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/9/f/9f06vx30habsa70mpy6c1mtqscca4q0q70695yqxx3578 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/9/f/9ftgssx6eqdc4ktnadb6t92gz3g3veep8a5fqkeve8jsy create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/9/m/9m5b5xm7mmt0whe7p2yqrr46d84j7tphrj202tazvzgej create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/9/p/9pa7me5gr10qg7qmhs49aqy7ymgpef4b6y8j5tp56p6ke create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/9/p/9pt2pz5srn6gthdq019taegmadr7pfwxv24e5qajd5cbe create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/9/p/9pyzvf0969q6r8nwnmr7yhwyegx8m8vffgnbjh8darjcj create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/9/r/9rccrh1twhkb756ves35p72ydzvds4v9a58ckxv4ccjhy create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/9/r/9rwcjbaa7kxnvt61ysr5aa9a1htsgsc5akc4w41k3y6ht create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/9/s/9saeytr2fzvg7ep4x96tzbq20tv1b9mfv6q6q9bbrakrr create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/9/s/9sry4weyyekrmefx9xx4jcsfnf1dj0vy1e5vp1zsyyzny create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/9/t/9t1jhwp88r17fndf8a6mw93spgxbzcv3bgyvnhv70ca5g create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/9/t/9tv8bc75k3bzqhjeqpwkyv29kzt7p8bazzx3ty7artsk8 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/9/v/9vhw4120f7nv6dvvhmsyb1khrxmwbqkgs22k838edwedj create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/9/w/9wbbvcxcy4p2925nytewgyq3wh3tjbn6mp8d0cap1n7jw create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/9/w/9wdtbsr7qeshkz69p2kgshe0xsa45ntjw6ypx6a3142by create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/9/x/9x2e52cvp0d2ebjhvbb9gfcyjss12qtv2kr0cdjkcang4 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/9/x/9xn2apb41f7ev2vzwfh5002wp8jq1t9hgycz9b913znx2 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/9/x/9xnfg7md78205facm1zrbcwazzg87wrxkg6nhhyc7242p create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/9/x/9xrbx5dcssbk9fgs3250tt06s57txd90ddpdevz64y7kt create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/9/y/9yywvhjaq25kg1r9489qdcrbdp051mxnyexd4r98zcxw8 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/a/2/a2cd3q0q8nty2kamzc30yd3yrsca6cv5p1qwzb5jkkh2m create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/a/4/a44jjd8tqccyt151zdc50zv32f9v7e9x39qmxrf50pswy create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/a/4/a4r64smhx7gpf3pf0ztfjdp4cqr4fjdcgdwjzgdzxnbf2 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/a/6/a61dsw5z4095vy5a9nrzbbmctqj1s60t8a4nx4z5masbw create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/a/6/a6sv323xdcfz69fkq3hdepy0014265agv95f8v9685n56 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/a/7/a78f2j2cp06v3pxabfw8a6qfz1gzxpvnzm06d4rt8qtmw create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/a/7/a7k16nf5284aa24r0spdgrph37mpb4d7ws7ez4qwqzt3t create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/a/9/a9ygmyzxnswhedm95ey6jmpa2egzn9jceyd74zjj8ep8t create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/a/d/adqgep5rgg42hdagzzcaqcwkq8dnq5xywq41p8yzhjhxr create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/a/f/afg3jgfgacqbyp0e0hw5w3bw7ne6fqnsbymaw99qvym9a create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/a/g/ag3rh55v0g8j79j5bb7267tzqgdjrqzhpbvde5qp98sxt create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/a/j/aj2dkkddzftkjddb7ztsdbjz6dmpjjn6x3y295gq39n98 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/a/n/anhr0t9ghya12j6htrh77pjte2jeq23ftqyw4b2qjej52 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/a/p/ap712egc4y89qq05pep6z4f2y6mxw2xrc5yw0q9b25nd4 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/a/q/aqpdhzv05hc0sxkg8vgjmvymaqw3f6q26mwmty8vygs48 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/a/r/arfgc8zcqrbh0f5yzpjrqts7ganxrw1b6bpcvm9d3g1wm create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/a/t/atmpzxjkayxkqzjsefhdxc96qbm94etg252p41fx214pt create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/a/v/avvknp9mtyb13s8xzb43p22w4g6hasm6s9xn96z93hhh0 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/a/x/ax3adfvkr5t4afqhfb6tbtgg4heqcv34z860ftazx8f1e create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/a/x/ax87204a9gvnrwhsfs6gw3wd9dgp53k6wcj5v7t5vt39g create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/b/5/b5xkhvdfpwcw9t9dhmdggn32aycqwaftsqjc93ph6f9mr create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/b/7/b72dm8vrs55qdq3mbmh9s1rr4x1qyhfb9hz91w0fg6am6 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/b/7/b792ade597qyvp0q44d1pnbj9yrxc2aegmjtzxw2g6cwr create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/b/9/b922m26p9tjmqpcgjmm15a87y3q6ge5vr12vfd5cegjza create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/b/9/b9zjk64ptxd1rqjsh0g7yg9bwc3vr6jay0rqjnb2cgq3g create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/b/c/bcnnahc097sf7wfdz84yntp9h8xhb1cyntp5mtfbh0bcr create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/b/c/bcvpdd7znb1h88w52fyf4ss16j6g5x6y1xdrbch3f2498 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/b/f/bfdm55ahtbz3bstsb7wa1zc5cwe8j0kf1ac6kn2b3c9e0 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/b/g/bgnptbbyxjbtzx5r683vg13k1gv4c428jbfk27m58vwca create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/b/h/bh5pj4v790fdb76zgnp1h74skr5nv7mcty01stnyj86dc create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/b/h/bhjjkqvqr89xn9f94n91n25rqzs58s7gkq12c4qb19n82 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/b/h/bhyhcrg11qye6fygr8ve1jrv75z6vxzgwpc10ykare34p create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/b/j/bjf5jcbq1bh9rmwq2arjsd3nbkfj9cabtkjc6rytb707y create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/b/j/bjw8y3t1ypcbv2r80jsymm5yqtkbx41kqb7yax0dag4rp create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/b/k/bk7tg4v8q8489zy60xa9kjwfn9dtnf6f5841aw9mbz84j create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/b/m/bmckydcqpd8k65kkc18gahhkr8ckmc2tfs4fsc3rx54mw create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/b/m/bmwh1asmc9q9qptsthtjgw247pyeasgpcy17gx32ka5kc create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/b/n/bn21bfa4kh47929dmgnee4d1f05hgv6js583hxnzacqat create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/b/p/bp3te430cppsj8f0n1tr69zgme8zm3zdctwkts89tqkgj create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/b/q/bqtd1xq1cbjw4bv28vrdaptenxn31d03qfycsxd94dztw create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/b/s/bsbp7ykhgax3jbh9pm9v832kxgfpc8yj2wdqnj0dnbshc create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/b/s/bsvnbfz83cjp98eh9qrz4e7baqbcfr8w07c9mr8h8h76r create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/b/t/btpd8pnyd810hns19dkvp0pm6mbctydr6nxh2h5bj28f2 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/b/v/bv4dcsdt837x02eq88kajab91nq0ctkmc5fja6s14x49g create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/b/y/by0vf31j6shg87ajmc19gh3f36tc4vr6b7v81n73f21cg create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/b/y/by404cjxa879cfr3mkp8j5v99ewqdh1ka3bqptc3r7248 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/c/0/c00d75zm9sz89vs3pnwfs94sn7rxdxg25tyqgx5jbdbp0 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/c/1/c1xfgsxp69zhyth69xdj5wvrsj2v898bb12r1xvcpg390 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/c/3/c3t4rfdgtkkcy6efj3fyz1excf7pww2x15239qk3etjpy create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/c/4/c4szvzad8fpgnp25t9ea96c4nbkmts8gxsdkq301t1sbr rename implementation/fuzz/{inputs-genvm-modules-normalize-whitespace/708713afdb215a7fdd1b3246ec662a3d992648dc262ca1d5360be94b6d75313b => inputs-genvm-modules-complete-json/c/4/c4wkbgpf2hrga2ja9gan85nve6vs8ndygv9atc4gvx9zr} (50%) create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/c/5/c5gqzg4fntq5azg6h5f1n3k89v7c3zkd79a68gvqpa9jy create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/c/b/cb2jgg28j8qmpnrxvgwqwhy9rsa3qzrnqmzj9ppdafjag create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/c/c/ccn5t75s30qs8f06japgnm7sc9vq1m1qwrqs634edemx6 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/c/d/cdbjcc0akvqmxres0ak235a7j1g5x8af42gvewk0w0ary create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/c/d/cdswa6f4rx04dgwghpam5zkxp9c4yvnry1a9dm6708hcr create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/c/e/ce6q5vbgw31nc3nvdrve6xhx519nntpfp466y772cd562 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/c/f/cf6wctsh5htv0fhkw46ab49ws2dqrng17h3panefejg3c create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/c/f/cfm826zbpsn5r4wex6spd3043c4y5gcnbvyxmfpgj97w2 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/c/h/ch7v9q3a817bjvgqk19qhsrmre8hr82mhrjc2zq2m19gr create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/c/k/ck2mhak8fj2023hsbnd500aknqbzpvx1gghaxdfbmx9mr create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/c/n/cnnd3zk59t022ennk6sne1h1m5jzkv16e7tcqaaeh4ngc create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/c/q/cqg62jc16td8pgmwkmpb887sccg89r05ph3n0k2emj4jw create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/c/q/cqvndvgd652rw5bfgenmkatrxbypkxm658ykfnq29h4g0 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/c/s/csk3f07z6bfh43433v3t4sfrg22ybgsyfy42fqm6x1v3r create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/c/t/ctj41vh54xd59bpg48hmrjgz890q69jmejh756a0q8nnt create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/c/t/ctqthskpwwnmrm8g6khj3j8hxs5yfvhxgf086pq30jhtr create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/c/w/cw1cfp9bdvcqkqg5tm08ggrc5bda7zmqz8xq3k3d6ysam create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/c/w/cw1v9r7a7eysaxd0fpm6d58tbdg981d7z1jdjvcy9xmj6 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/c/w/cwa33vsd7h36xx5pzk5wzyk4hnn9ha7y87eh90vszdtwc create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/c/w/cwmpyftwdby1bddzawz9ee6bavbfagjwem3n3yvsa5936 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/c/z/czsfsvp41g06qv09d387sz4kknemgat7pv72apmrvav5r create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/d/1/d13vbtddm9r6x9jq3wdgpckfe4xzxgp19bh2x9ncjqwnc create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/d/1/d1q9ctz9pyrpym136n3yra4fhxqvzjks22hjm9y2errxy create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/d/2/d27grm16kjp4rfzwbzk3p7sda804ykfma4wmb53ar4998 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/d/4/d46ttf0zq9pvmgs3qjpdegznawqw1ndzbtbvs40the3h6 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/d/6/d6dqgr1jgbkmebh826wq0heyv6071h8j84qw4xmjdcrap create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/d/7/d7hsbgnjd6mtrdw1nsn4p31s2arbkw7gy8manf8vp4tz4 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/d/8/d82hxbrtkmkz7zt2vg3zq107ez9g017egpmcac6w50vbm create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/d/9/d91jv8nn4e18xh44q9kswsggmy91vy11ykx85ttpc6a7g create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/d/9/d93mqvmwkazrrm0gzftmzw2604x4pkpner8a4g52rgnhc create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/d/c/dctpvb2zt2hrkg19mkfqc9h9sk8yz3rxj95z6y2n6tbee create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/d/e/de70hek7tx48svhkcmgtr4jp6cp0nmzsgzqdp0qkrbjz0 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/d/e/dewbqrqshc4a4e544gy7bra0cc7ngwb8nk98zwexz4ek2 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/d/h/dhk6b8jk1e8d8gg6n97s15b4h6y9q8n0r5yp0r9wsmzgc create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/d/k/dk9ye8hftb4hf441w1rsnrk9zhps2tfwfm6xdz6v8497a create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/d/k/dkba0gfw61g6810w4p3njyk0d0wh0xf498g53ntcj4rjt create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/d/k/dkmhenq6tnsbsncefwrtcxcj2zc3h3jkdmcb7f95skgny create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/d/m/dm4nvqbr2pm9vaagx1nkz146334pgeaqhwkvdsw3pv9wy create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/d/n/dnw4ywfh2ctkm2tm83xfhj4nv0a6hph0220eznt31rf9t create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/d/p/dpafw3gwg4st7g6jsby4tqa9b6b8qz8yfsy3wxcqa8m9m create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/d/q/dqbjhhgnmdp394wyrsrdhhaarc93anpyxdb722st357r0 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/d/v/dv5tqa29astpjxqb9qb9nqy7a9p2yp7jff7ewbyqj4jjt create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/d/v/dv8djb7j6ykqpqp5xkznbzx0fx4j0fv9eeb86t21ngqc8 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/d/v/dvm11rn82hs4z3c99f5kvs8e08skq9frctwpbtgj6vxzw create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/d/v/dvp6az1bt52tr38crbmgm9ffvsaqm9me6x8w8z62qca6w create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/d/v/dvpzes6wrn2rt75rrgmvsx54ep3zb3e2psa7fvyghhd28 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/d/z/dzr40rh8bnnba51n7mg5cgxjhd6m83qazmq7w4yp7b13r create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/e/0/e02vsbkhabpazvdv8bwezwexe2e88n4dn8ve59wy3dkfp create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/e/0/e0c7785fpwp0z6r9he4bbkpk3xqjndx2yanmy2d4qdvcp create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/e/1/e1epvtd5v0g86azye9vbwtdfeq00p1sseg096m3j9cs80 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/e/1/e1mmg4qtd0ezmtb4bf6beemmaqh9h76hkfx8th8y94mey create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/e/2/e20857bs4stbmpedvjgn6avmv6weje9x8gk386v9jrspt create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/e/2/e2qxf7j5hfttj5g4d8e2hzb90pmtfb16brhwm4v6gx11p create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/e/4/e42jccxftrntwr3se574eq9e7ykcpcapyewr1rsa5kf6g create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/e/5/e59046tvy72ss2g7vv16ppqtgy8nrq349ajkt5esk7j60 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/e/5/e5pd5j0jb6rj91rnf8vdfv5mj7frgnh6b3q6ty1nff452 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/e/6/e6h4svg8jmemyt4177w80xen3bs5fstgc8jbkb6ttcrqg create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/e/6/e6tzw9ba8amkm74msqq0012txz5aqq9q9qe99f8csqr1c create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/e/7/e7cx24jfe7r8jg011g5j6z871zjjqc0hvtcsf8bksf9rp create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/e/8/e8erj3me5g573cnrzgk30d8350x3e8tdtkqn6banswm2y create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/e/8/e8g8n6g0e3b9b31ptzy2zdmf8q7xp9txsqhqhkhj78hp8 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/e/9/e9s4tg672179s8epq9vjc1t8xc7p426bjcz5arcz9p9py create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/e/b/eb127pt6r50fwsjxvtcvtgrqqyjskvzraqnmbz5c4nhs8 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/e/b/ebppt36hmn1raq16m83ywr2v1hfksbc677j05jz3a5cta create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/e/b/ebt503723z2emxs30rr989mgg7jcw5597bgjzdtbsn43y create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/e/b/ebv0ny12mz2f64ah46mqt4h1244vn25gtjdh4rcp40yxr create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/e/c/ec6n23yd7cbe5zk7fasp1c8h7h6r9pqx84kf59kkq9t6j create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/e/c/ecadpx0fdp3t2b9f0dq8tkyjz7bb17wp7qek6c5j2wbpy create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/e/c/ecbfma1cype1bcfa17gp8y8mzvsqb7r09f6trh09sv72y create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/e/d/edj2jxcnb53f4y97p9a8rgypqezdfap98c3wgneg7k3fg create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/e/d/edpwnq7kzhv187n0eefn36fxjqqkh811hpe5mschmt682 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/e/e/eegsza8ysntt59aawddf8nrjfz11yr5avv3mfzjpa4cct create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/e/f/efqfmbd7qgdmfcmx9gnpypcabepgxw28cyk07c90xqzkg create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/e/f/efthb1cez1ecc2v4ft3g9br7hgm9pn19295v93vj7xd98 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/e/g/egvnv6b036px26c1pfxdyetr2ke0t5k19q9x0697h0620 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/e/k/ekw1vwn92g566pqhj4psd63yh1t2mmrty1xvfba4v2dmg create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/e/k/ekw8e6jejmqxgs5feynsczrd46hhyhh4w8kr21xx78fs2 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/e/q/eqw32e17fbw05ezyr972xx9kp3yv30jgk5fk31yhwhhye create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/e/r/er4venxzh25x3nhg28jjsvkz3r9vdwrmn3mp51n9hjscc create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/e/r/er8fpyb5ted5r0ydph71yvwk92w9ad8jc7cwdey5d1wt8 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/e/r/erbzxzy6c0se61kh6fw955s9q7c74r6dqrn29pq7j7nva create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/e/t/etng0z2n7emt92qnkhvn9e40mre0cm7fj6cavcg3fa8pt create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/e/x/ex8ebm6z9e8603wab4pna1mf1cdec3wrp32jem3aw64zy create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/e/y/ey6frz98yn558h2wzg6a1q9m9mna6h73tfa0bazs7sn8j create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/f/0/f0kp1rtsrfr2e8b3kkp860q87jekvsmh09s5yh461kebm create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/f/0/f0vkjt3y2enqdjd79k21h1c0knams8se5fehqmjz3ezmj create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/f/0/f0w1wp4f4g2j3eev35nwd693ztg9rywxbf2xz2sw97efc create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/f/2/f2977sr15mekm241hw6307vdxw6jfpvtzcv96e0kpb30t create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/f/2/f2mw7jkye43zq3m8tcjfavsyxb0ydpfvhecr88cct7qqy create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/f/5/f5mxash0dv9x92eban9n4bja4a9f02re6s0cfdvgxggsg create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/f/7/f722yfy9gpzwpkyd441q5bxa9d6q0znw8cn5c38yakb8j create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/f/8/f894kk1aszm9zrt7r2rk33emns5jh7mt8v9ma5atbwd3t create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/f/8/f8f838tn0zrencezbxme6vs2k4ehm2m25vpvaakhz7ger create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/f/9/f9qj2956qabnb1858sgp1x5fqec9f81j796pe84cd0fnr create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/f/b/fbq5975x2xbt7q49nejc75g2gkgq1sazyfz2g4s7dbza0 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/f/d/fdarbet0j38wtdpacqvcedcp5nskf95r02gek39rmg8qw create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/f/d/fdxw7s5sstzbwkzhcmrd55mrgw5ydy3kwys6cpx1znspy create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/f/e/fe908rz29x6xjyhzyt3cdtz1btr6tgatnyhnc07a2d5vg create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/f/f/ffabam1yxkm107he6rdbwpthe7npazfph718gv3802qy8 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/f/f/ffzbr34j2epdjf3hm6gs0bn44dkv64q71s8vxrwn4ghfy create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/f/k/fkeqwc9ng2a7y42dae5ax8bv7zvms6s4jqb3ms9q1062a create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/f/m/fmw01nxpr25fjkpb44v57kfxy22jbhre9jk9xmz903st6 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/f/r/frr2tx60pfkvq45nsmz9b71pp579pdqd5g9njtv0bnw90 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/f/t/ft635tsas864bn4tkwtkbxf623k66fnffb27x07czte6t create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/f/v/fvgd11vja9g02rct4rgk4sbyz2pq8g6ce8sv4hc7wwbtr create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/f/v/fvn42ng8jwjq002dmsatnmn21as2fd1g5h4srjywym9g2 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/f/v/fvqnwfm2teefsk6cqhaahqvmaeppjcdzgfyph7y64cs06 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/f/x/fxv0dxa6py99hk9vmne6g5w317jh1spjt90vf5pkq904c create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/g/1/g1fsd04pp0ek735a5d3fzbxnxhpvbafw9hvh99kvbwz8p create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/g/3/g3htj4zjgtrxfvw4hmnbkzdp6bdh1j0x9rrj0804xz2g6 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/g/3/g3m2wz64dne91ntf7e90dhbf9srktb5p29aq26jzwk0bg create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/g/4/g4eq0tjf2yp04axpjztgdrg2q1m1rvzsvp351nc80dpam create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/g/5/g5pjkd2759bpn5f461dkws3h2j15yy7keerbyspd857b0 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/g/5/g5v21bx76yg9g3vd6kmxcnkh2dg6q0pctj5bhngfannba create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/g/6/g6t74w7s5gapgnshxyn15401mxhzm5hh3fve4cdmjwncr create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/g/7/g79z13a134phq13pmsjfmm4a8jzb5n2nbzjaq0bvj78ja create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/g/7/g7bwdm2hqypr8xfsgwq6cgekw1wqsrzpww5nwscttq46p create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/g/8/g8w3at38nnnj5pbjqbfs84vekef7cz8w0hppz1egw5dwg create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/g/9/g9qa3ry1jg09wd47m2xwpvgae90wnxtr2rvwf8e146qg6 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/g/9/g9qtzypr7b7x9c00zgpnw90pch381j378ttszp1jfb658 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/g/b/gb62dbqk4z17473cs581byf9df8452h86tz7r7vjja3er create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/g/b/gb6q6gtnpe7dtegw8qsney5km59ma0yn4690y7qg2hq04 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/g/d/gdawmvp49h1dt7259q310ypjasyxkq3fxs1svg1rr3c18 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/g/d/gdrt3ewgwe2zgrax02f99a2mz17c7mzej0zkhmc5sxz60 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/g/e/gejprghszwxv6ppp3mrranea2ptkgwenp6h2gpyn03rsm create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/g/m/gm4zy0ph9dzdwvdf02pvmmnb3z71fxqdzmk5tfcd95016 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/g/m/gmbdhbg2n4jtfvgpzkdrvag402km42qm3cvykssn1f7c2 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/g/m/gmgzrf6rdq7jav2mc6v518jtcs63dr389zfqj1awnmx82 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/g/m/gmyj4521be0c23ktwdyqs75esqbby40y7tc4ee40mg33e create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/g/n/gn161457rhjg8q1gtgwrxwmhv7fe4hd36n16x15e3zjmm create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/g/q/gqnz8wygwa45hanm7s90tvaw890psc9tg15712zb3p7j4 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/g/s/gs2391srmekajtdzkg6xvyr4rd6ndvp5279vwc0xn2bcp create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/g/s/gsxfwzkf15t5ab0xfzq7kwvzf9c3ej1vk3sdz9jpnykm8 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/g/t/gtnxy4531pmxv9847jz7perpg0cpz4wnkdkxgtydg3v60 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/g/v/gvh7p1xkxz9brkee1ea2ywawfgxqn5x8t9svnqt9244ea create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/g/w/gw0baa8yz1ty6dywpg4xjeawgmehkk287m40ygkx6f6zp create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/g/x/gx98f4rk68hcfbeqdx4pk7bzjz07vg6e7a3cxzx3v9wft create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/g/x/gxwg8stb3a070a7d4pg61a7rveqv0ccr5bhfpn7a4rstg create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/g/y/gyqgq9ge6v81dz4ah04p7s2wmgpxxvzqwytpzbmwz7b3a create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/g/z/gzj084c49th6y4ryysvh0027yvt77cfmxxn79qcyyhzgg create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/h/1/h1ybayrg290j0f6myh3tt1p63zzjx1mf29k0rwewqqchw create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/h/2/h213vzjvnvfx83c7abfycjh14x57afa1ceyznkx4tg0k8 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/h/2/h2wcbzae9g1dae8mbrgtcph3phzvn6xs0p4hgwcxnzeea create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/h/4/h4ep3m70qztxm8dd5ftste6jege5zgy5t6j021833j7ej create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/h/4/h4jhyy4pf9dk9sm960mm91qpnw8draq2q4mwvdngp56d2 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/h/5/h5wxz58hzw7pmvdgyt09dyarfk10n5g1rfxf5nep1531r create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/h/6/h62x0c4w7p7jsrz1h1ezka109ryjqetjtm9n6bth6ea4e create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/h/8/h8dk8bqhfqc4qy61zpzberwejg3b5evkeq3dma3js2t0e create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/h/8/h8eegm2adakfzwk3dh4hwkepwt67gpkyfw5x6ywrrs9jg create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/h/8/h8s754wqsxw871nt24v192x2eck1jz9w9s8bk89m7657m create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/h/a/ha1k3z20wf9ahpv6ttewhna2m0251hx8tvtsk13pdazb8 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/h/a/haagrvyam74hae6x8wsw3an148qa9caw8cjx2jwxpyznr create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/h/c/hc7wg34atrg7afqky1ajnxprcdzbbdqfg09fb92m063r2 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/h/f/hfh1j3r352wvahzbgnh8t449j3pm9kb4d1ax0ezky2b7j create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/h/g/hgw6n5jgrsxe3aydj10cvq35ma4wsg7qczhz1r64d7hn0 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/h/h/hhfkz92b2h54dsc8s1vzbhfzenxw9r7hx740z41fexja0 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/h/h/hhkddf8e6nbae84a2vgwbs0dcm847zgw3vctbsjd6j2ag create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/h/h/hhxkfkn00xvw37f3t551t5tcemzfqe9jegt9wgftas6c2 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/h/j/hjrr8ycqfat197v4k1hxtkt0zcyrvhgx8tmzx326zrd0r create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/h/n/hn2gy0dcn4srykfn7x9vabhne57j2x7y8dg4mhrjdw7wj create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/h/n/hn9qfpe5y3st6etwykvvb01v697hvpsbzgpq98e44xvpy create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/h/n/hnamj82p5zpjpyzzvxe1ztgte2spnpdqet0gm82frhmda create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/h/q/hq75ncgkjyx1j6h3h5vw9x9m3nw6f6h5rnkv1e1qx55qp create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/h/q/hqmnjzsw0nb75195g8emmhk4tjw68n44xwvgkz4w8063p create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/h/r/hrbxm7ybsmr740a6hbpeaavxczg1yq6eza16jvst2ppg0 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/h/s/hsznzbxvyc2zgevg0344280fmx3ahkk1hnjzr1715g3j6 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/h/t/ht4v0zqe12ekgy8jzscfbwprhaj8nwpz70p11k1zwsmzg create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/h/v/hvcgvt6d2g6613m3w203h288dzd2k6d92wn4kfvzbd7je create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/h/y/hyc888ywyvy5fr83bdyj7dxjeq62mc10s4xshxsqhw7wp create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/h/z/hz604cp3v0gt2pjkkzmqd84nx5qfv1d90x6qtre0n9qcg create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/j/0/j00hadzgnrkbdch4ja8rjvc6yc6fdk5canytgzgr7qhs2 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/j/1/j1wvajs26edvpejqkc27syy0dqxttg0epvf1dbzfvce18 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/j/4/j4nzmgpapfbw9q5gd1r99hzchwqmnc130bv3e102xeekw create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/j/4/j4zgfdkbdfjrbhyctrkkdwzyy6tv0xx7xdjd3syvq9pte create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/j/5/j5a2m63pssmgd46emw3s4mrk88agm1pr3ykwkpxx5nhna create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/j/7/j7gc0kzcgk2v3hq0ehgh1fcs2272ryyan83wm7p3gxa78 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/j/8/j83hfvekz2y5g1jhg1ykyemphdhfyarhmwg7smgvy28w8 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/j/8/j8rb6d6578z22xtepbzrc67xhp4cr6c5azxex2p3m841y create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/j/8/j8ytdmjfsm33pdkx1vneefqterd9ajy4w4caztg49m49r create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/j/b/jb8e27vb4h8m64gn1qepw3kgrzm6ygdtgrr03nb685w3y create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/j/b/jbqnh283dae76dtmhfkvq7nmd6qt8rva7y2hbz51fmdzy create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/j/d/jdats9sxdjzpxdfkb3kape5azt6yde0se4f664h6tefke create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/j/d/jdqkeac33py9haf2ervq2ggmcwhen40aj4sc291x22xv6 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/j/d/jdshqt7jp74v9e149zrv089cb812zcdqz9nsrvzyn54p4 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/j/e/je6jbergqyrxdm309dnn4x1t085j601kvppjycgfam982 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/j/k/jk1wdf80q75qrbb8zbzgycmahpcbtyjzgm502764ee8tp create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/j/p/jpqfx60td314ksyxv666k9awxntamx4zcnbxbyw8pvsrc create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/j/r/jr60zpc4dd7hc1cac6jydf05dyf59kv9g86pyygsmq76m create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/j/s/jsgx8wb9rbc02j98tz450a1xkst7j3v185atgmtsbcvfm create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/j/t/jtd53xadg9jt8grsvsh9a8bkceye8zr1rxx4wn1qh5gwp create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/j/v/jvm7y97hwf2hespz771jydy0mxe1gbgqqj86q67z3bxw8 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/j/w/jw14ta37vnr37zdj46hcvkxg3tzdbvv9hrkcmj98rhhy2 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/j/w/jwtrd5e0gr5evfwqz6n983xmpr1zqcpg53ap1gftvkx3p create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/j/y/jy2dwkz0s5qa16xke0n9d4zkaefmp349kbwyjpd6nj1ry create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/j/y/jy8tv9rzs0ewfa9c71aqc4t7qym7mjyywwxb0cjkm39tc create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/k/0/k0adjkygzy5vt8e1yhtb9r0dhg19hxzxcdtp5whmpe6a0 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/k/0/k0s9ekbqz6ts2av7eb8n3yjv4wytdaarys3ed9gvnz5b4 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/k/0/k0wkm3rtc76y4amxbq6a3bvj28wt28t9127hf7cverh92 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/k/1/k1rjtsqeqb0xm894bjt7vy73prmyhyhzpe9hg1nk349n4 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/k/1/k1wz7cgm8ra1sarnn2w689q42sed8xb9ve78mvj85knry create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/k/4/k4838jmv5xazrbrknbzv2w01bzfh74j3s7pw5wntdgzcg create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/k/6/k6nka9cctyna6x83rethzyb8d7vy6915xmv1xa90snb2c create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/k/8/k83gwrajz3mzbq960z8vb6sq1mf72n39dbja6a93mfqsy create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/k/a/karq2vdertx5s0530v4cc7hf5932h40dhsz5wbkh6ac38 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/k/b/kbmxbt4vz15rxtvy75a2c7zrcqq80ywr682e36936shdy create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/k/d/kd1dfh3hfn0b7cn92xd8fyrx6jx96e1sp3pwdq64wvpc8 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/k/d/kdbgas0n2fctkfdx4m1bevmrk333drnreerk4scwtmb0t create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/k/d/kddc8w3t0sa6bgt4hqfmk0s702fhw3aj2s23vrf113vhj create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/k/d/kdkd1aeh2c5te4wzhqg19vgrt8xay96svegd8190h53fa create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/k/g/kgaj8p56a7r8jrt40152sd7fcc4c90j1y2890rqat47kw create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/k/g/kgmgxgmb7s36z7t5w0c13v2kz5m7bf4qw7bgcys380f08 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/k/j/kj25phm9g9r65k34nf2bnbvktjcxarpnshyhss1hgzrcm create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/k/j/kjsqsn694mxs46wv9pxk5wr0v7w32h7yq1am1qxnrq3a8 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/k/k/kk2z2tx173gmjdhbh0tkeg62s8n7kcsv9gktyt864mhnm create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/k/m/km351mdw2h6myk3rettmc5nhmye9h8k67d8rd5s747qx2 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/k/n/kn0szcfjdy98emzg60xkmccxk3qdcsk6ads62zsf4vyqm create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/k/p/kpeapxmj9qkwvszvqsrktmkcrkvss8wq7dfm9p462p064 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/k/p/kpv7nrbmhdc6556cy33hd5d3fqcw518m68q4gcbsmzpma create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/k/q/kqkz5qnj4nkj1esk3m2v3tw4hzgcvd4aq2zcv0a0vv90a create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/k/r/kr3jxhewcm04e4px2m2znvdxjbr686pmewxrstvaxz9he create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/k/r/krh4hj84agz7txk2hsapr4r7gbpxac5ttbdefme657ysg create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/k/s/ksqdnet3e2y8ztax53q7hddaej4xqjgaz5kveh5eeyxbw create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/k/t/kt8wttecepmpdjfnc0ht0x0gwb6a1jq03s3yj444kqjcy create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/k/t/ktafsc8tmnxj0cxxmr3jes8gz76p54fs8wzfqh7pz9274 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/k/y/ky0p59y4jd3yq8j6gz8zz668capv25c5ya0s58qydxcst create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/k/y/ky4mbkfs19k1fmkfhx1ej4jqns8h1v8dyj58eg6j33dmt create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/k/y/kyst24ensy38wm2t0at68frg52g4g812hm20ws9ne1mde create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/k/y/kyx60vyjpbs117mf2gz69h6enprvzm0hcwt3z641kf5xe create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/k/z/kz3m9qrp2a5axfa9jy5h8ewtxanwygafwhx4r5fwbztce create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/m/0/m0wmr0cmdm54pr7gsbxyskvymrvkdyh003phfftngp9dj create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/m/1/m1qhqarrsg7mnv6d49q2ksgb6q2afqe8z16ct7dkxy9aw create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/m/3/m32t8fg331gpkvmxp9mkgyxyk8r1ndbqtfxxh7904mkda create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/m/3/m3ahpq4mgferty6rhpdp79xvte4cfjmzwtf53kmsxaz4g create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/m/3/m3mzkfk06j8qbanxvsdd0ctez8e8js8qq86w1n8s85kzj create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/m/4/m493qx6671gfk42fjcpzapxgjv932b4najhcdcvb87vcy create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/m/5/m5972ey1t36ymxs03rc2yz80xavs5dfjfc8gy1zgsm2yc create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/m/6/m68has31kcvzdmf3rmzmc54tja0yk4ebs21kt9v1htm16 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/m/6/m6abvdrjj88h71m40pamp9x2ty7p1v0h5afjcscnhfjvm create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/m/7/m7dpad9g37xfp5826mapya0j9fjfasbja8fsz721k7hse create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/m/7/m7k6a08a240xtex9khqvm8zkj6v4ane2yqvajq13a64q8 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/m/9/m9ypnkqd0fd7b89g1r8zw2sn6t1ffhc34chtkb4zt4g6p create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/m/a/mabk4cpr62tte8fpee0h93ej82ebadtdmzev02ankgxxc create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/m/b/mb6qw3wvss3wv90ty3689ayg9bh3hm1yj1v81d7mmkmrm create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/m/b/mbn6cytj7h2etyzm73q23gzwb3r3zzn46a26147c3ngtt create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/m/c/mc2g13cjhwvkmnjkzhap1wv65s0d6mrsfnpvgqjd4d43t create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/m/c/mcvp61x7qvhm54q7venkq2j0rxvcj8pkz54tktsmz696m create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/m/d/md3nvsew9f6adkessshmv0jp07cs9t1ammp05wfapy95w create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/m/d/mdzbn1tkad4dcz38gynvv3b9vjjxa753a6smp4w0gq8fw create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/m/f/mfe555xm1czdmsz55emnt639nhve81rb305236qf5cvw2 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/m/f/mfp4jnm2pgnprzney40fatb7pp46t42kb5fk5bwagzksy create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/m/f/mftybwf9g3rm5g31dmygq0ksw3hdxwpmr2msqhz46581g create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/m/g/mg5f0w63svmgw0k720qpmarnn3b7x1jfd6sxzfjrv0qsj create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/m/h/mh89vhjanjap5bqrg5n0y4awn9ey3b9hv04y0505ycf8m create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/m/h/mhkkk8x1hczb24rycg0wk93k0n71vrkd129shee2kb770 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/m/j/mj61caxe8khbx3femfbaeeqyf8rtjxxnvh7peyzvr6nr8 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/m/q/mq2w64nvaq0ek0d3n4trpsnkrx13bh73eexhqy3qz1n50 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/m/s/msjqvqw6nxfk7ccsbfqbphcbtjm3chag6qdy7t667dvxw create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/m/w/mw17vyjk6cj3yh3k8f0rdzt65b7j6tp7vzhht4e8g3ed6 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/m/w/mw3bwkxt1sgswfqnxb2ksghfs930jn8wt94e4ftsrh84g create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/m/w/mw95h82x0xmdasfnqp69gzrv0kag7fyytg72g7wsqp1qe create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/m/w/mwvwy5wqq8a78r9krk8bdrkc6dgy4rfwcjjjm0d6n04zm create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/m/x/mxys9rn0bks5a8stzkxj5z2e617qr56dn9r6y4amjf6aw create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/n/1/n1dtcpmdyg4mv2echk5hmx1gw435cy74yzqe63rv3w2xe create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/n/1/n1r5h883zt39y2k6qb2s6t6aj88pdr0f3vfs1fw18y5gr create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/n/1/n1v7ztryqh0pqzj368wwbxqkvm7qbvs33n66hxx7nrqwm create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/n/2/n2hmwrdvrc4nsc74704jejejj7axdre87g6zpggvrbd3r create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/n/2/n2j89f9aqmhsv9bnyeqzn91bpdt7b98hfef1h55f3svva create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/n/3/n3g9vf59rm5hmm5f7jxhc5st3mtygp36p0x9wwj2qtnbw create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/n/5/n5159m6kqnn4x3frndv6zm7acsqtt2abaknc1zk5px8ce create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/n/7/n7dy1dsabxynmvt3x0c6gehwykk90p57sm0dw4yv30wn2 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/n/8/n83y3d9a13g5dmxd7kahqes3eqy60fcv6v0xd2dhd783j create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/n/9/n9330495hyfzbhes4m5b24jh013b6wkyrb1nv59takz1e create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/n/9/n9afwk9mv069s48jzh27mv334kwn483b343g05psgg1h0 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/n/g/ng5dswqsnt0nz180w1njgh84rsevdd04qvzx737tqp4ny create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/n/g/ng7y0bw3s81dq313y936cngh5v3vbxst6c4s91v444jr2 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/n/g/ngk6cm42qnjr54s0gwv2gehm9t0v2kq8p3fb84fdbq6wm create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/n/j/njm94atgazn4sqp0y06bfer99f5n9d0z47251b0t1nbrc create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/n/j/njx918nmttxgteyaykpskyqswf27byd6dv7v3c6mr9ce8 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/n/m/nmysrgc94jph3xbps9f1zmwdyb5z68rtbqmhx2j43rv54 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/n/s/nsvc3ayxsqq26r2y7ehk3t0x139cmem9fyjmx7q07yr70 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/n/t/ntftmawhwhs2ha6c76dxh02gswf2h4vh6vham7m3sfdct create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/n/t/ntg935cet85vqftahbkvazp0hy57ee73y1gccm9hf8gqc create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/n/t/ntq3r5tp52qea2p58k13t20fp1y8z4y6sctkjf8bhmyec create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/n/v/nvav67fs6raebm8yy7tnxdrfn4nstrdcc7xtwj7ec4864 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/n/x/nx9ndd6wvxxe9pztv02yrhqsk0qhkk03hhsb6z4xqpqg6 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/n/x/nxhz15vspeftz6hvkdstfgdnmv7q9bh5v5tpasqshkx74 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/n/y/nysnaj2py5tjqqj394c5ndb89ycet34x94fw8dygjd4ew create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/n/z/nzf01kbbdv8hwahc86r3tjncbce3gynjtrz5t02p99bqy create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/p/0/p0p3c0xzs6tr5c3y2meepqx344ra5nnwzj5k4p3w197yw create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/p/1/p1kszrpddaqfpf849d2mmygc3pa2w2j7canbcmvbvns8y create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/p/2/p287mydv35s2xwtnrdnyc3pd56awqzh67157x8y6mpw9e create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/p/2/p2fx30snm4efr2k1x8mqpej6xdv2hpwc13vv2njapmt3c create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/p/3/p32zpavsm0j6fx9mbn21b8126993m8mc8fkpmheapshac create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/p/3/p368t8wtgrt740a0aeyfjjts65pyaw6w3sjpazhpt2h94 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/p/5/p5b4pv72mn0h9ekhe92x8zhn6r03n7j6fh0w6vz1spe9g create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/p/5/p5wwkgjqjyja1vak6tv7yxyaygm73g4psv9pexeqr9v1c create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/p/8/p868kv3f7ap1y7frhkfzjcxs3p4ja3za0e46vj4e245g6 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/p/a/patxxt64rh1qcbhctjpsbnhx1s8h1kt28aejhvs4nt71a create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/p/b/pb7f6wcyfjdk2c23myaqx4sqbef6wsy8j3dvz3m449fzr create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/p/b/pbejrk8cbm11vcgfv0k9vbc3v6mhntmfevse94ddtx87j create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/p/e/pek78ck9a0kyxemegbagc2rjfedsha7z80gt6fztyht2w create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/p/f/pfp74myh8qrgwatp0tvksqb3r6rze4am7zcerdfpbt95g create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/p/f/pftwz4qs3hsbf428b1mkem5f51drw08afybzybxz4kz24 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/p/g/pg8h16tn6nzr6s54fqys8twzwvt4nf3r1jnpjz1abaj6y create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/p/h/phrk1hwdpd06mahek2rzzrfqsmhng70qv6ttv0zfkva50 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/p/j/pj481h7q6wx5z6s7b4mrma2mj26awddz79ve8mttxh7fg create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/p/j/pj75key1yebs99qhj5qr9paqvy0ck0w2y7ph9e63n4xyc create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/p/m/pm1rsjzm5ffxrb7tjz9g2ttmbsg79bd399fgh3qffaxr0 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/p/r/prdm0t56pfry6xsf9mvc4w1rkrzy7crwff5srqf7zz2a6 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/p/s/psq0xew56vpdtmecqyd965zxgzke9tmb0tgj5nsbr6hke create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/p/v/pvk1zkyxmscw8gfnprjyjz3s4mq07nb1ndg0bf4ptr5x6 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/p/w/pwps76p2mngwb2g0s36t87nfw9771gnkbh88x64v3b9za create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/p/w/pwxm2140kkdd3y52kv6da4r8r7vk6bhxk2nhhj2wrrx90 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/p/z/pzdkt9mh86k8rq8ksknbzsvptc3274n2n1zjgswg1ehqm create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/q/0/q0nf8cpayztamyr7evqjfy6wga3zngxhk4kvx0dza4mv8 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/q/0/q0prpmnjbk9d5ymty9jcby1093s57dkm118qcs250k05y create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/q/2/q2m3epw20qqgp7shh42j9kbbeq79znh82r03bmv9jq832 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/q/3/q3p690xw3m4yzy85b9vb4t4cq42158d96rfd923m8aa2p create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/q/4/q45my1w9wtefsgpzdk6nvsz98z23k9wmz63w1ej6xe766 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/q/5/q5df95x81dj1gn9tpbj8w8r0ht7c3w60hskcn6mgjtrme create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/q/6/q6gebjxz3vwfj4pd0dc1kerm6gq0egmy1mxr4g82zjg64 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/q/9/q9a59mn46kmhq246znnmh6wzajqcmngexwn2qhbepg02a create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/q/9/q9d9n5fkfwsfdery7z1e7gsh5a9qdqmt0vfmkf185rp7p create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/q/9/q9e1emsm4enja8d8gdxb0173t52zah1pm9rbbatghtg6a create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/q/b/qb1dg1yxdy9fbywxyn689e3n8s17jqvs8vj3dm4c09f6p create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/q/c/qcq8fxxg0886x5118vcn9hn9re95kjbm64m1mnv1v2yv8 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/q/d/qd5dzxmff201w13fv07e00kgx7y6xzk80dmf6h0n6e8he create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/q/d/qdarjc9gm33ymfwkpsw3pdm4bm7e38hyakp6m5rd3hdf6 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/q/d/qdjjdjb5ztvry0ky10ryyx6j3a0v84f4jcfwa8v3swn3j create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/q/e/qebc2ds26vf511s9ev9e0ymjbmw01an7s43nypap1yd2j create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/q/g/qg7xyqbzsmnv7z2r12gtre42jtnn0evnz8sghsbzfbvty create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/q/g/qgqa29n6vmqy1cmsdgn259sa2gear6ansyytb7dsnj380 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/q/g/qgw5gj9bemrbyaptz86xrsvkgrz43e5fd766v8sptck92 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/q/m/qmsqzbe79qkxy5zv5daw6m1x7kh2f68gnxz4k8rxr7bet create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/q/m/qmtws2y6pk25cgmf8fg74p3z1e8tw9g7h72m2x5am92tc create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/q/n/qn0t88k7rhyt3jyq9174j4k059espkf5nnnvt1wn254ea create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/q/y/qyecheaygp5n2htj123hvsgfyfjdpmwb46v7yv8k9kbsp create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/q/z/qz9v320wcgc0ydng7frvmm58x4az6r1w8j5knk9c8ejma create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/r/0/r005wtb7jsahx35v0fgnd0d5cp3gnas1fv24jsbyt7fet create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/r/1/r1b4z4m42s02q4ab3gezsvp1ytyfhhmxag9sdxdrsh3cr create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/r/1/r1cdd04qajrp85sczjm8s2y26c7xyns3mwtypkx1n19y8 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/r/2/r254rs232s782j37a38v7f3cbgmxr8gw2450eq26f9732 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/r/2/r29f3etsn2vgy2342p8ezq2bbbn5mgwqx5mb67eyre9hg create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/r/3/r32mdfagf8w7x7tfzqsgb5k4abcfmk3p1kmggc8ya37dg create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/r/3/r3y803bhd7qf17s9aq8q5rxrh6384f5q48h7e3pbmn94y create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/r/7/r76wr3jct03x1hcdfyr9zc5de8kjzhqdftatb9p0txjy2 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/r/9/r9gm6axcx8mpe1qp1whakh1y0t0wgfahsvgwth5p7mwy4 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/r/a/ranvb980qjve6cak095mg75awgrd5p14s3bqnccs3d9vp create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/r/a/rar68phs41hzs9gfadgbhm8jp91n9zbyb73cbmvhc8mhy create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/r/d/rd9b48n0xygtvkx3nckbc9p4ky0kphkxj1b0zzpfa9bg4 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/r/f/rfcj6z0sr8yz063zzaxameqvq8kntp05bh1cp3tnefr06 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/r/g/rg8pj4jk38xv6fhpf1bbn2398bkvcgmkqq1crd55ja9ke create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/r/g/rg8q6t50jp1c9gst4ek2j8r245e2hw5a7gwvvpgnfhf1t create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/r/h/rhtmpfv58qes02wt4ktjxrwveypfwns9hne6hvk9mrd44 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/r/j/rj71xj66vxs4qczheqy8ne3rtf5jsa0e84d787cfeh8xe create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/r/k/rks8yw6t9nqwhn9ygqqjfn42egqbmz9tvxwjj4wtvvp7j create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/r/k/rkwm6vptf0pxp94z7dseek6j01pcsaxa669xmnj0gpy68 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/r/n/rngntzfd0n1t66793erxeqbgbp8etqkyfsg0m56gb19xe create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/r/n/rnvm0dbv2tezpv8wtgzehd84x35bbqecq4r3pp1ss5s7y create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/r/p/rphqjmqw0ersskqb60kag18j2xj99gcz1caszeapaxv82 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/r/p/rpjqfegw5ybpp8qakdfc3k8gwf3jwx1e9jpm8nvyjaa2g create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/r/q/rq57j344ypcyv8zrwr98g3g5n8h3pr19p8mcmmg6n1avc create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/r/v/rvfj1zs8cr9sxh03r64qr0y9rkmatt2qs2772k49fk6qt create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/r/v/rvh6nrpbnsfjd33qy67epbwhnvf8rjr75dhcz9w85t8ta create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/r/v/rvnx9h9xjstjgzmmj18r8g515kwvre8wrb9zr8btxhpyj create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/r/w/rw6hk88m8twgs79ndsypq12yb3ab7ae4n2t3z45gy2ey8 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/r/x/rx7phk58zbbyr4dcgzymj9fvdwpyazrc9f0qkpgb17p5w create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/r/y/ry06bs5fkzz4rmvkf6p1sv8mhyafa5bgaar290ae37kxc create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/r/z/rzes82sf0h9mheve4rtr7et2yahhaapjhjfajk33b8kbc create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/r/z/rzktdgqz67sfj5ykcwv0fr8e3ssaesj6zxnfbb2pgjczw create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/r/z/rzvcecva3rt0z8852gz1f9tgq7a7eebcfbs5dvhbp3hr4 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/s/0/s0z3smqnjfb2way9n0n9praqmjxjz6j87a2t6n04gzxha create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/s/3/s3as9nevz7ys78r8cyz6jhd7z2e50c8ve5npey393prya create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/s/5/s5dpj46txknxehtjbtm2mf8xe42m810vdfw9ngf5e2p6j create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/s/7/s7wz172q8pprzf70wxcd76nvqz816abpercv1m6d0tsje create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/s/8/s84jvn9nfee3kzs0w03f45814khxz0m70hns9mf3kbhrt create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/s/a/sab21yexdccy2rhr3w32q832kk62dbwj6rmj7wnnfsyj0 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/s/a/sah5nna4pvs9azyhqsq3rqz5dxsr2m9n1a557zm3ejx1a create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/s/a/saswj19zc6xv7kmsb087yrcrhdzb4hhvf3w65pxkmbkh8 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/s/b/sbepevrk0tzwspssqan4ycb1pk0scekrjak5y8t6jm9e6 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/s/b/sbj1y9pevygw3shngyczkcfcxbmjv4fmf0tqzcgy9w5wy create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/s/c/sch1bx4mpgstvnv1fns60d9hrzswpt6s8fgmvr6pjtzd6 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/s/e/sepyp22kyv11cm1njcbsxr0d0p9cnyaqjn8cpheqapv4c create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/s/e/sewpv417w2tcwf9p9n6sj8c5aa14eryvjt55q6m61byf0 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/s/h/shbzkkv7z5sq8ejty3d85npfsypfnrtktm61ce75wm8gm create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/s/k/sk04ga1sjg4xj9mvqfbxq5a8g0rczv1p5hmpjj7m25fw6 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/s/k/skw1n9487pkkys6qjqnwh3dxwnnfwwd3yrgg6cb49jjdc create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/s/m/smed9gvse0wb83r7aqgkstv6gdcck4h94tej49q39tp7m create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/s/m/smj6k6yx46bcymgfhcmn1hbhej3km6yfr3c9wvh5xe63t create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/s/n/sn8ydke73da5rnxnmpdn1sywrw0e9qw1z39rm8vw1875p create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/s/p/spd83sdtasgb2mw9aaaqtd2ag8bd8r8sg02e78e5nrhqr create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/s/p/spj3bsfmd8z424mnm6vnmqxxk2578bkzztxtxbw7td7a2 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/s/s/ssqjmb7kcpmjdfbwtqxt8mmfqwacc1ag8yprymdb8snep create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/s/s/sssc9q6nt579crjpv174n51wpjn43bn909zaxnaayb4sj create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/s/s/ssw4ytb1yy66twvrgjyfvwhfnkxk2kx2b0tdx3pcbsxae create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/s/t/sty8pqkhmpd4zqgwev6da6th185z93cx5xdwgvjmwjtr8 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/s/w/swxbs2ndsbvwv69rf6yrma4478m5tbq1rxaew320spk8c create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/s/x/sxgx73x7d4skxrhkga3978tv1ftdjwa13gf35gxpb4mz6 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/s/y/syemw0n5g8krbh5mfk84gqvbstaw9hxayxjqf9q0zwsra create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/s/y/symmye7nza29sez2a9029kn3h0g0f51yvm5ertx0cppyr create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/s/z/szkkg31h97m32mqp6qrs2p8f41cvr6y7f6qfkr7bcpzke create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/t/0/t0nc5jyr3pe6v9bs5qeaf4khz36sn00g713mypkjvb7me create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/t/3/t3gehabpnf1n5b1znxw3tssraag8cb1z8ff0zscdtsgwt create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/t/4/t4c0kr0xyk4tgzr9xmzgk85mw4abc3f7qw7hjrgyjhv9r create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/t/7/t7mp7d81dhz89yhk5yk0pxn4jxj751g9yh1kvqwtpd5ww create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/t/b/tb20b0s3mn5gg6e6jjxn7fs8khj3jh9de89q75yjp2tt2 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/t/b/tbh8ppt850729b51y9nq4wkbkeczhyr1qbxdyekh7ggcr create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/t/b/tbzd4c43m17kehn6f4gbvjv81512b7ma1ffaq1xm5e4vt create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/t/c/tccnrkkqzmrd17eh91gmtanm2hqs9jva7xed2a54cayd0 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/t/c/tcqdv403zxegm5pgexxxknh3xzd09f9s1zw488xfzab4r create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/t/e/tewt7a2mr57vzp3k7ydqxm7jdqg2xykadr6zbjkq1qys4 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/t/f/tfjrjfrctqnbnhkwyyjgkcfx3e93q9g42xfx6prq8mqg0 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/t/g/tghvavtjyjphjx3r37e97cpexy5871p899xet1x7m3ht4 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/t/h/thzkrf36h9dg6rwsac79b4k61gw1vqv6kw8n84ambexj4 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/t/j/tjz9ftrgnt21xexwhtaznm62y4d0n39sgcey08s4pqn32 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/t/k/tkqszdz6rkktpz6712rj3jfh5x4by23nm5tyrkgabbxqr create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/t/m/tm1sxtx5gy8fg6qqxy4fp89bd6j5p43r7k2rd53s0546m create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/t/q/tq0hzsh2sy3ddcsg40ew1mwzk7t3ndxnx76b7rk8pcbza create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/t/q/tqqm05gz4tfbkzy9d0x40p1cj1mpvv18yx53kpwk0hajm create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/t/q/tqsgy5jc4ey7xqhd0pqgv9peh363c6rtzpqgzxr2j370j create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/t/s/tsr9tph355x87s5cafd7358xfetzmna2jaje9tn63hnze create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/t/v/tvg7yr8tr3r56kpmyh06mgde04kmepy0y78fxwkqbvya2 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/t/v/tvmb6ayrwvh5vxfdq3xv5m2qvx77r48zezsjedz6mmae2 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/t/w/tw670yp56sq147e0zedx9kpbkqgceykbz4v2y6ga8wvjw create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/t/x/txnccemyszf7rt08ae038h1y4j3eangvrjm1f0qgfzw18 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/t/y/ty27wk4wrch8ce8vdw81g5azp09v462g422pxzsxn13jp create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/t/y/tyegkzzk97gb87zw8n0jxt3e1tm0af7ga1s127eg8y6g2 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/v/0/v01hpm7n0mr8w9zvfjj77ja3dds7d3t4eykavvjyb9m18 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/v/0/v0mpmhw2nvq116a2mq66w5ac0t0n0rh9n803ws906ezt6 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/v/0/v0n4ehya912q5endeyjvppc7xk4a3p4pywpme4e8cm0f2 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/v/1/v1bbhk3wqc0516fwh3znyrpb8ccx98nb67ytedg78vz4w create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/v/2/v226w6c43jv18cfeb1bnxkjc1dv6yvb16ghhpksdtdqam create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/v/2/v2ydwb8kqxsbsft3xqrjr6ph3ppj99yhenczadg1b6ate create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/v/3/v3fny0ttw0wsmmhf0tt3qwz2w8n455h6ty27qjq0pwqxe create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/v/3/v3rf8grtvs24scm7eht2f38dwpeffbmnxptb81yszkbh4 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/v/4/v4mx2wxpjc3ypdq1rfnjhg8jxg94pnfyzdsmkqq10vgz8 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/v/5/v5j0gwb56jzrre8y93et4mcv9x71s3takyq3bkf8kwagg create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/v/5/v5sxkhpw15t9fyev7w2frdce2kzxtrbmpvrpm77cvz15e create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/v/6/v6qzwvrh6qzg0q78njxekj5xg587hfb3x88dytjc91tqy create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/v/b/vbtrzd9d9nszedymt6vd4b1nfyv5g3ne00k2cqf1d4zxe create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/v/c/vc5ww77ga0fk0cdsxtckn180ensm0ns09dyavgvwky8r4 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/v/c/vcxkywx2ywv64776g163kf3cefzwz3694bxebr7svcyw8 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/v/f/vffm0hhkv5f63jcf4j05zb3n78vd0qp63fex6qeq4xkp2 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/v/f/vfy3tt0qcnf44gpzmrtys7c6m0payywz0ebwd84xq28ky create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/v/g/vgwa3yyr5zkyk7dg3xpn2rzrn42dgytz80he4m10560xy create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/v/h/vh8rwxqf32axt398a08qrrvz9h90qzhjq4x86evknssbc create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/v/m/vm4kkahbvyvb8sffh3rwmf7eq5bydnv3vx5gtap05a9z8 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/v/q/vqn8sf0fpgcf5nth6d4xkhjytxynyqpr8kp2z0nmxc0e6 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/v/q/vqpgxh7cnypmzrfphqxsy7qtxbqtsfnq1ep7ndksgww2e create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/v/s/vsdj5q9vxs4g30xzwvsbz7hg6nb9dn00tpdc6qk9tyhg2 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/v/s/vsyxzq69qcqef54brey6fwpv7dh8fqmmgmbzq7a6n4j60 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/v/t/vt2g24hdjy5vwww4x7y5p44azbjrvzjn1717jas35cteg create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/v/w/vwhwypz73cj0fkgaachvc33598qjb92j8ctd4j4ng3ndr create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/v/w/vwtnpjteafpchkcd19tsd63z67d4m1fxbrwy46skjamq0 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/v/y/vy8fz3gbm7k0mxgbew3vw5n5bmq6jhakqmttp3r5dbgrm create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/w/0/w0225bd73w9dazbw6jb2pgh84cpw34z4aga4jzn7b82ap create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/w/0/w07t44ytv2x4cagakbp8vyced6dxaffy35h6t2fsa3nk4 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/w/1/w1229a8rvrkwbhyktew1xb3frek1atfrxzw33kq8ssgh2 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/w/2/w2abwa2c3bmhgjnc1qmt5xrhnr96aarma0tk1n2dz8sft create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/w/3/w3nar75v78qkx3d928h0he832sqbkctbmcrk5fe9ws8ba create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/w/8/w8j0vfcsy4ex7923x16y4wrhh0h70kenfhy215j09jf28 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/w/8/w8rwmgvasgfdha3k13fyf7h2m6mg00kd77waj719nhbfe create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/w/a/wa556dm9x1mzfj5g2th5wmys862hry0ryn5rde5xg3be8 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/w/a/waf2pz4cpn6fxk22zay0d1g8k7nzbhffyf573ajvg3te6 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/w/a/wan3zs63jsevc9x72va1apz2t0ynsktxdxryvsf8bpt10 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/w/b/wbn6hd1d29xjz910rz9haaxdj81atz1gpmk6a8910w5jj create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/w/c/wc7166zk28n709g67hkqn9e4vb38t749q18srgb226yhy create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/w/d/wd1dqfbtbahx5wngqwwzjrs59v7p10gfcndybp0d96j6j create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/w/e/wee7ssenrycdbatem6dtavfcxnyqs4qmaca3cnaxk1bya create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/w/f/wfr8wqbx83fgq50mww2ect3a8j08189dy7v8663mrregr create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/w/g/wgbw3wrjhp2ywmqdg8cbgedrjy9apfx30bvm0c4h4w9gm create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/w/g/wggdm8g9sepcbgchsj5gjfhqnn15r4xgxzv225hd94dqa create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/w/g/wgjt61ess9kww0xvgsq35ej5new78w63pzkhfc252tchg create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/w/j/wj34mctxj9r2m4f7mp8cdmg1s776c0feyxn27h206yms6 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/w/j/wjj5qa0n4pp10ejqmt8qrzy6wvwkja36my7rhtc046q14 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/w/j/wjk3ttywnza76t2k3z4qvrbmyg2b9ptvr7ennmym1n59g create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/w/k/wkgbj1ak3n7ercyw3x0f3nf884dja04kcbg7wey0kfvxc create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/w/k/wkktvvynb5h7hgm8py35yf9ta0r6psqww75yg99ypm26g create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/w/k/wkygkggtxf22b7w6d38faw092jf348v67vp78dc7esj1r create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/w/n/wn5e036b3prr4germa9hy89g82j908yd9tn2pg7vds984 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/w/p/wp50v8nk3akyvwhn06hwzz1b1ssz04n83wwzc1ex5p0tm create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/w/p/wp8ksqp4rzj5czsyq8ad6dmr0zbw8mwqpzpyt5zgvs638 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/w/p/wphfn9cmx58aw16edmv25m9p93zvvp2wx3y9fd0e9ktey create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/w/q/wq79wfbcss4wzg30xfctg28sang4r84taka353p2j7v3w create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/w/r/wrs5t71dcjjsxgs535bvcjgn24hegspkjfzsdbajznj4c create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/w/s/wsfags1x4xer51704f72d0ymmk40kabm1eeezg84rf8r6 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/w/v/wv6st7rt4c9n8pqq0t14s83m0akm8cx594yv0fnc5zmg2 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/w/w/wwb5adm1pqs5928hf33j2sedc6j7try738exxnh6hypc0 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/w/x/wx7w4p18yyc3a01apa5qz0w31n0sbgbwj1jxvs3dwh8zc create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/w/x/wxd3t34dgg13219xkkbakdj9wgrshqsg2vqmj82t9hxm8 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/w/y/wyjsr0jjm2qa355th3w231761y1s3cwy2bqzd7eam3wbp create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/x/0/x0b9bjk8bhnjw5zvg8bmm9tbvfxjk0vp4ejy7yqn51cat create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/x/2/x2y2wxzhvb64j5v9y1wsvw25wskfh6r4r41dhcjx9vf84 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/x/3/x31bt75p0d02masjc7bb36dqz3w478hrp838z7vgnmkfc create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/x/3/x3xn1mgbwk5sm1eksq5rdw36x3qs1s410zsd7fyz8tb7e create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/x/7/x7z9y3rfa7r2p2as8e6cb9nd4mcze4ez61hrdc59bj774 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/x/9/x9vhhbw4nq8hp3bnekcxcgkxvmz213h30h866bw592m50 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/x/b/xbzbem843j8vaevc0551zcb60f8br7ezhwvyzfh8g9wdw create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/x/c/xc68061ezanp6hv7db2605w2r8v3r092yfcs6bnj4fxqy create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/x/c/xcz9d9fqq2p5tbxy7bg0kkye1zhj9dg932bfny8h7j412 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/x/d/xd1wspeerh2t52fm38ngbcv1a0vy24k6j2hykbtf6g7vw create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/x/f/xf7nec3t7xfrezaacqkt64v40xmbapa90ra07h5kqfj12 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/x/g/xg1nqjd3axjyy7vswtf683s63g7x4qbtynp1xjtpawrx0 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/x/g/xgzc2sv4nwbamhn2p6es0n4t1r292078dwe5w9mwr1j8t create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/x/h/xhmre3dg37xp3988dzxnkeaezejnbejvjt93b72g20c5j create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/x/k/xkbr14gk3a7rr65r5tk7hfat39pevmz8why3gn0tvh57c create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/x/k/xkmwqaa2w39myhpa309a3b2hdr202f63jfbr343nzhmza create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/x/m/xmq5ewyq17ta0sadjn489c4dpkgn5bnrmw6be2tmw0dw8 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/x/p/xp8sqt9zk6h6bt0tq2wf62xb1pck71qvn6r8d0g70mmja create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/x/p/xpvwyb9ad0rfk3yjwtfjha05etx8te3peqbwddheh8fwc create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/x/q/xq3tjskn8hg3ydkemexpfa75fvm46rasg29y7zjkd7za4 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/x/q/xqw45r3n70nw3wj6d638e448qf6zbrd0mt87j9698pyj0 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/x/t/xt8jyccjejyze9n80tsqf7xsjpq5nzt1c21g8rmsa1khr create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/x/v/xv4ne8e51c1hch84yxk993218ykvmfp3gq4c66r4x22hw create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/x/w/xwhde5sh1dmy64dvf4jjs30x34d291a6v6r2d03nqnpvj create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/y/1/y1xw7nq6z3pdayphm1vyc5rq1xsd9fadnp8m3y6ykep1y create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/y/3/y3qj9bnaksy94s17nvg4gef25kr6w1p125bq2ypdwv7ac create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/y/5/y51y1mf9z97wqwnxyrhj1qrt0tydh3cgttqwhpdxyk6by create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/y/5/y584nfccdc4nxyk791qsyfeysmt4w73kr0t1xbffvyv76 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/y/7/y730k36mhsd91yaj4gkennz0hng2992z3wc1d2b7xh1b6 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/y/7/y79exx9baft7b1mvwk8w7s45arqjmgph7gepxhyjyzfyp create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/y/7/y7jjsfaczgb0t5ef708k00m489v4b8nrkhz4y6nnxcf40 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/y/7/y7sz6nb3rz8wx7y76cymb95zt6z691adqvfnpv80yvsy2 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/y/8/y8gk3gmkbgwb9fe497h121av0m23cnd6zk1rr3fyp7hjr create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/y/a/ya3d6rzthp5zxm8jg5f6nh6yqs81fefzxdy874fmh6kg8 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/y/b/ybt964nbb5yb9ps6jtk72grm9y99dv2q7xc1nbk41h5fj create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/y/e/yex06rh07tncp10v3ck21ftg8esb1ez461dsrafy4b1cy create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/y/h/yh3g712vt6x1b3zhcj1s1p8jxfen93gtd42fxtjktwykp create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/y/m/ymr2htwvdcpehh55smk7n56fcmz7mmqcpfc2k83q51qkj create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/y/p/yp25xrkzh6cbr4afqcawsaredtfg9c5ape7seebsaax4m create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/y/p/ypj8a1df23mmj32g64hykewsqzng41a5zkj006089msq8 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/y/q/yqmec8gra9kkb9kp03kfsq32en9qf0xnq8e3628mk4msa create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/y/r/yr76r68778ggn6d8n6npcppzwrkfhzk84jxv5a4ty4rgr create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/y/s/yswexssab5tdnfx7511f6tk9p893yhkvfe0kw4scj03dw create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/y/t/yts85j4ftnhe1nvktt9j6dmch9hj2bhwcqzpjxvws2jrg create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/y/v/yvkhkdhwqsd9t5eq7ks3wf602jf3bwt8eczx2vybh6zc0 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/y/w/ywmhyj7wq34fhptmx4amwpefx3nqsww494c8wyrz37eva create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/y/x/yxfmh1pze8kkvezr7abqjr3tnhpqbbxp48kd2qwv0wk54 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/z/0/z0fk59m6rrgjabzhxmr32c09fh750e6k03zfa20k2vrxa create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/z/3/z34jsxa265sxqyb2cwx1thqtexxcar9egnjvnpcvkcrvj create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/z/3/z39kydgt59x46cdhf6vjhy39pe44ge3rvkhvb1nkwb2zr create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/z/3/z3eqckb2scmz4kjf7qt1yztrsfky9qraatwe03x51h7k6 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/z/3/z3yvw1ngyxqxp5wn2km4cd12kbm9cye0pz29wcymqgwgp create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/z/4/z472bjs2xxz4h1frvp4kewrt6hzem2ksne0k68c6w2478 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/z/5/z5f8bcx0bhxt4bc16ybcgjxgakyhz6081ej47wwb6yr6g create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/z/7/z70ryqvfm418005sc8r4bfm6v9ys4bbrmf0x5nnvsn1m0 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/z/8/z821q1k0181fd425hst88zjpc27fcjs81z16vp3f89rm6 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/z/9/z9ap3tct6s8eqh3eck88e5aw9sqj3ga4yzznm2m2j0604 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/z/a/zarvjgwmv5q3vwyd58gkw4gc501v7zkwjx00pkmgkft22 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/z/e/zer1ffmaw8crmjm9de5g8zqtarpbwr23epvv20t5bs8x8 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/z/e/zewv083f1apdzf1kg8vbg2jjfnvxgz9z00xa1h4mf65sa create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/z/f/zfg0xsh5caxhzqb9z8nwpjzsvr0q8g0fjkz3y4y43kdjg create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/z/g/zgqxey2qy8j4gqm0rd91k8fbx1p23t4nwmc7gabhafhsm create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/z/n/zn265mn3xv8v3v6zetbedfg88vjcvffa19ebk98pc5t6e create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/z/n/zntc9yjnew3s8krzbcd9m8b567mmq3d92b7e6henf2nhj create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/z/p/zp2gxjckb10wqsjcp91bmtqq9chq2qq5gpdyym87v6q1j create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/z/p/zpm8204mejxydjsawbrhn7vje3xjgaar0cpk7zy64ws0r create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/z/q/zqwvef7cvxgb2tybnypx352v6tdbpmr49vx29z97z5jtr create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/z/r/zrj953c90pegf5s0v2ehgb286hxrsgzh77cv6mx6zrfa2 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/z/t/zt8jegx974khs3jmwa42r8h98wypp5gsv1ss4s8je0166 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/z/t/ztkqhjcy7hbng5ndn9f36zaxab33220jxj2d0q7rgt72j create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/z/v/zvcfa3ehdtsn1djntnwmv4mn2jm17t3t659k2b3mn1ryj create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/z/w/zwby1chwm64dgy0t105pr72a9se3x4e75xennhgf7fct2 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/z/x/zx51d2g9vv3w2s6eh0fgy6esppgtmhary97nfcjqgtfza create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/z/y/zy2w7wxymampccdp97g9rzv76qwxk7vj05z145zenb364 create mode 100644 implementation/fuzz/inputs-genvm-modules-complete-json/z/z/zzn0apphqhr3g24550pxepzcyb5j2gp8px27fdjj1dt0r rename implementation/fuzz/inputs-genvm-modules-normalize-whitespace/{db91d0d947f0f760465149f837c4bc02bf02c988c7eb7824c7e9496a52884908 => 0/e/0ep4zdf4ndsymnwtkeh8j0tztts0y186jq1bcw3wch8pe} (100%) create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/0/p/0pez5en5ybbdkbh9axzqq7c4wgs8rt6mctebe7zf9kzy2 delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/03fc8c99262f53a7d60c876faf4876a11a92c2e045c0ad0fcc738399f61fc240 delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/06e3c1413644ebd520e20235d3dfa3145613e296cb645884ffe0b885b63bd99b delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/0a150ae9c5d282876a4e896149d6c486e771088809bf02e59aa61ca3d1c3e2e1 delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/0abd764f1b303911ca744e429fd729ffc1624caf063177229727e526f3dcbf1c create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/1/4/141ka9q4c2mgw2zqpxgjpmtbrpcx3q4ywn28z3hh4ss9r create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/1/s/1s28fveg5getzbynkzmb3t77fh8g40qsq7xp2ebfm6ye6 rename implementation/fuzz/inputs-genvm-modules-normalize-whitespace/{3caac4d461b724c627cbc28a9e96641f09c2130fb7e1fd0d9433bd63b2c12a20 => 1/t/1tk837vrcb0qn02pw3an43bzssvwsrc1e4t60fq0dknhc} (94%) delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/108efca4997c179e75c9c443def521c0bf14f754fa13984e8f6f4b25e3fe19fe delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/13a6f1b73ba94d5c0ec7097fac90f1bababca6ec0095c6d7dc6fdb4680112804 delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/18847d26806d9ac4981dd642d6aca8346cd197d647ed119cee4a6e5e95785183 rename implementation/fuzz/inputs-genvm-modules-normalize-whitespace/{85870cb385a6431e482529b899cc64ac4e3a3f99424bae342e19e1ef8cdc0fb5 => 2/n/2nnyph5gamd1qcg99hg1yc9c37h5ej81ny8m647z61vj0} (100%) delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/2afdb94524a31c4607ade0e66a33a33b94a6e6fc11da8e0ac8dc684550637684 delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/2b0698696f26299c279bef4360e307c4f4b41dcf7a8fe4db7b82dde9103e1351 delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/2b4985bd1778fbb5f2aec4265651102ad3f0d53bbada124914d102b92f2be068 delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/2f2af5cba25cf00d90e48678f7092ae6217152bc42205385d0b5996721c6034b rename implementation/fuzz/inputs-genvm-modules-normalize-whitespace/{a77a0b19ea78584e0ea34252aa4faab2839be31b03d9f27d7cd86f503153c038 => 3/x/3xfr5w22w0a3d2c5tddyy9qwdjg55hj6ve1xmfnpct7dj} (84%) delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/3462c1d0fca3164ff490caa99736fa4eae1ca1a646bf880d8b5894c955033450 delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/34e791781a9d100ab2a6d7dbef5b5eb38c4c38d887618ff5e94a8d117d4973df create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/4/2/42xap88t5te2mew2v7b9ekgkdy86ddj7kyg122q62qyrg create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/4/3/4372gqsr02f3mbqpv5sb1arw5zcrvjptra0e18973v30m create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/4/7/4730f4bwrk39hybvx8szvyvqyck9ad0aejq1zz34tgcby create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/4/b/4bnwwcdggzxf2s57jye94rmc46zrnd8551hd6gaaf0t0r create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/4/h/4htd2gzh8zfzp34tyr6eh4jq6nyqev0nbgvtpswcybxrg create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/4/r/4ryz1gbe2d7tkmrb5jab1m391vjpytdx7rgx8bne9ams6 delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/45eae75b464a765dc836ffdd8596874c5d0b7fa33c9685fb77ac22738dbe9edf delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/4888b4e22f1132f4617b3470c99715c2701e115f3023b1e7893d335bf807d4c9 delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/4a6cf03c7706d26879d5a738f98eed6e39312ebf0b61fa1f053b5cdb3c8869a0 delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/4b531e98e5057fd2eb3c96eb7973a5c90df1234758b5ee02c97755227a084e3f create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/5/0/50ae4phj2re52zkjrh1vj0vgh66t73e2870bhrrvey074 create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/5/f/5f0c7bjwbwg1ycpwcfp5dpmmjhvm7jk2smf2eevwcsvjy delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/50ae20ac4f0dd0a0d779ee8b2017b11348cfd8bd0f591c2abc2bbef7b6e95134 delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/53a4e4cb235b274fdb21397a1141b4d6b3ee23fdde1151f799df7c3d14c86ca1 delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/55443a6927436f5f0fde21ffd5ae2c195565e2e058ec518421272eb952dbd429 delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/59d573431e4724864b40110bb3247f783fb8fc33584238baeb1519ab70b426a1 delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/5c2c0103f79c00f4ee500842085ffcc424ebab149f1c85bdb6991de0b4ab86f6 delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/5ede5b56920816525591723816c93c0d9c830264b24ca58c386075072b388372 delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/5f77b630b55a04c392f2caf506e2660aa0352a11c83af4f9e58b01169ecb8ac9 create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/6/n/6nzf697cve2978kqd9jby90x9ekg4m8m5nysp1jngy8jc delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/61c8efd9f69870226747b046f077b5751015cea9dc045849d65509918973b62d delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/620942b003b1bae02a40d38e7b03f3cc4cc9e948192cfd5e7d0cb77b31d069ef create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/7/4/745nee7h4x050p29a485e3t21m37hsa5y7a1wn9amb3wc create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/7/c/7ccqs9fgddtc4r3s7axbxz1hr61n8rtqp6nsqc0cd5me6 create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/7/f/7fd6ksepb7j5k4q8t8e3p230agfvr357se20qa952dn2r delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/70546d5fca6e8da3f4320b2bb5a2789bedcac8a5bd30820c7aac957d3c6e1684 delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/7923fd9b7984beaa52f2b5e97ecbf0812db63788b4eb8411bcddc4be7267e428 delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/7b2c4f7c87a84c1e2c26e69abf2fe09932db7ef15b4c939c079cf8fb0d275cbc delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/823ded55b18db8759c77a9c684b6d3af75c26871dd228082e3f3171ebda3b5a4 delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/85ccfc35f2828100617311f9ebb138147e03034d9acbe149c2c1e93fba076dad delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/891736f80c0acb4550ebf3a53fbde2113736faefe5e5107d3d7da299369df1b0 delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/8fde056b9da569485292d132338d9af447d0b4e17345b30d0874b8ecccc88535 create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/9/1/91de5j750xjrn3kyfs8n3vgzt7389p5z7rnpxdw20mfrt create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/9/4/94g4fe2jqv1ktz8h03heqvtybz5zekjr3kzgmg73jyzzr rename implementation/fuzz/inputs-genvm-modules-normalize-whitespace/{931f79f6a9b51bb5822885e25a66a326e2beae862e1f8493d137870f9526abb0 => 9/7/974px1w6qq7hdrmawg8d8m32fccy4ft8jk32h57863qgc} (93%) delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/9cf2c3c978eb136c0bc952d366b028830ebe2391400b3d6658ac26f5dba36868 rename implementation/fuzz/inputs-genvm-modules-normalize-whitespace/{8d56d269122a8bbee7db1f44f0d2bc3a834cb31b8d2538dedc5c0b6ff9a740a0 => a/6/a68wtsxj7hgjqf2j6yrqfqcbqa85px7mms469p8j7n2bw} (60%) create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/a/g/agnftywfhe3yc82jp0f4jad4wad9awev9gbq4px62c0gc create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/a/w/awkr5j8mde696580twqnhkvrkjg1b8b44pf9vvyhbyc7m create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/a/w/awm30caerhxjrabr6nxxp215bmmnzp576ds3n0fnd5qej delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/abfcf8aa5a090baf9be4f429ac9bea30c17dff4b027e19ec303d143d82cc3eb6 create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/b/r/brgcy15k37c4rk914aktt5fxp4z1wdv0sjzp0va05a3s0 delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/b84925528f02d3ff5848684e0c537a889e6d301c9de358ab1959ec04e80af57f rename implementation/fuzz/inputs-genvm-modules-normalize-whitespace/{bd2ee8d25a19d78820a021ee98d8dc8de927f858011075464cb3ad4288b7ea0c => c/c/ccn0xhgpacrfh1aq74fkjnkd6qha0rp1z9zr22byttbgm} (100%) create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/c/s/csx0dgbx4j5a96gazak55b9ekgvw4f0cgk2zbmtjjjkve delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/c3bcfe060468c318a0bfea760a116fd4b1e13a3905ae812b4c4434484297ca53 delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/c48855b1773d9593d2bfbcd50023edf88c026e39d5a407c1b7b9155704add33d delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/c7df8fdd6edee85308057d4f35c608d9f2d2c8e9320b136bd312cf028400df4c delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/cd40d68396aa759f586d90a3b4887a2672716b4ef446659f26d1c030acedd63d delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/ceb8a0e16a10fb5086e705c9cc73174b55c7a826833b26ab01a78cc5304725ea delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/dbfa4b3440b784d342db858d835171514a94c3b3fee05e4cf9caae20e0d28df0 create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/e/4/e4692ckej8yvykkywjmdmz3y8k7062my5rk753watxfj0 create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/e/7/e7yvwya7erng8g6r1xee9we5cjd7ys97ejb0yc1crfpzr delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/e6237eb52e4a8750685e1fb8741b2c0cbac5e873876430bc8f6e41b5ad0209c8 delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/e72d0a509f5966a8861516dd92358725c1c5603183da9733bb90a125f85ab9db delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/e8b6d77f122eb966b7a7cbb6a225ef83020029066c04ad009ec529c273de0fa8 create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/f/7/f717g71e7k6bcs5q565ze8z0f8xna7kpmt8j8kcp7t4rg create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/f/j/fj28aaza5he7xhhe0pff01yxcft748ycsfxjkz1p1pzw2 rename implementation/fuzz/inputs-genvm-modules-normalize-whitespace/{5df9f5badb887ecd9ffbe572515a25f71356275d18023fb295c80fe8caba5c82 => f/r/fr6sp1b7ghxs3gbj8e6je6nc4bwqra5kpkd4exvremh62} (100%) create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/f/v/fvq371s3kcysrjp95yrvxpccg93687d7c8c7ypv3xadvg create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/f/w/fwqe7ds6xt6f0mvqxw2gvqmwnsgb15fas67ge0wb0mbe0 delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/f35d3bec11953f5914c9ca460ef3d4a1d5d7121d0ff0d8d6cdbc374ae9946c80 delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/f71d8c3bc20355985702ac0223e06edd28d33945777415f169219fc560a6025b delete mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/f7e3e6880c7e27d1a97f3a04f448e90e33f72f377cf4fcc582fa70c9fbeec75a create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/g/e/gecqrc9ajgg1n1dtv2p2v1qjxjgrh535tb5hwqkngyz00 create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/h/0/h013nnr4swa87d0ztyyvvrvqb28vkagw6m9xs5vb9grsm create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/h/8/h8rtrgawv1nqn6hg52g9t04zpspzmdrakrktftpejfg4y rename implementation/fuzz/inputs-genvm-modules-normalize-whitespace/{f9024ec809d2d43073ab044fa702ae963fdba5c841b38a76999eb5ccead5518c => h/x/hxj83xa25drwh4m46gdt9naav2gzf3yg450m3gtd5xhz4} (100%) create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/j/g/jgjsyahmzmj5v16fek5xs8aa88g7nbe8n3mwbkpy4j1ce create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/j/j/jj3w58ycvjfwcfsv728ky0ym67zwq5ghv97tr3gvs709w create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/j/j/jjnrnbpw8tdkz7x7d88ksjpvma82a1eesd5kq3ma7cbma rename implementation/fuzz/inputs-genvm-modules-normalize-whitespace/{0a53edc743484691ae84defc2db71ac98893a8c4afc437239bbf602cadafcf01 => j/m/jmg6zts2m7cnj5v0cwbp2tzeez12bya52tf8ff84vcx2a} (89%) rename implementation/fuzz/inputs-genvm-modules-normalize-whitespace/{07d0413d5429f6ec807e88ee6428490ca8c4e7486981a814b99a1dcd1460be16 => j/r/jrm84mt72pk0x462yncar39r2ztrbqa2hz12d6akabk92} (69%) create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/j/s/jshb3dwrfzdq3w7zdrmsf5e5ed4djp29a0a2khgaff4ky rename implementation/fuzz/inputs-genvm-modules-normalize-whitespace/{24047b08dbc5f81e0d744bee88f975cd94332c54c3887d0305a1b768ed23b1ab => k/j/kjzgqk5d7x3mfk4eh2r3erw2bcjvbb2a4tx5bmyg5h7gm} (67%) rename implementation/fuzz/inputs-genvm-modules-normalize-whitespace/{709af572b2d689c93a58cc6beacccd0e41429338d17234b2a1591ffbf28e0aaf => k/z/kz1hkv3gm079cr4xbqx5r88g7n89246m27dcckhwdrdn4} (65%) rename implementation/fuzz/inputs-genvm-modules-normalize-whitespace/{b39d82c804f3d1d5d8b219aab4aa30e36f7d4e00e1657d89a4dd2cefc2d47c58 => m/k/mkvxxgdq2vw8r2zch1k1m1j8w8s48dtst8d6ckyj2hzg6} (76%) create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/m/z/mzfpjfw6n0r4khj55ptpnx9cayycmwj6rb0hc1q6m8rb4 rename implementation/fuzz/inputs-genvm-modules-normalize-whitespace/{add963d8792a3271636e0d20bb97a73de0ea4220567f4fba49a271018360f15f => n/e/neab4np5n178b82cn37w6wwwew2s3hbz2xgtawd732qm4} (75%) create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/n/r/nr89fnaja49pea92v7ynqbrmtv9fmeqp91kpvb5cxkqj6 create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/p/1/p1zpp6xkygnwpdwe92026bywfvmxgwjmms4xp1p031gne create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/p/8/p8awpn2bs92fdxdgba32kgfqzy20eqysnv60tc2bdtw00 create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/p/e/pe5n6cfvewh2w64bd0za74k56dm5w120w067p8e7s8zqe rename implementation/fuzz/inputs-genvm-modules-normalize-whitespace/{167391bcddba8328537beeeb9c29c8ae7c1f4a23113ce87e30aa365fdec40971 => p/x/pxbseqw4f9cwxkxm8qjv999ydvxydehy9xfrw5x0pynjg} (84%) create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/q/d/qda1v0f123z425e6e676d6jxpx5depn9zy8mrncp32ky2 create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/q/d/qdq0byn41gzhq0wqwfcs77e4kcmftqh4r11237dqpg8j4 create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/q/j/qjrs2n5jd5yenchgmwk4zny08by0d7qtqq6paysg185h6 create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/q/w/qwk9094vv8hx6rmr13g2ncxp0wjj2pbvd73t02498vdq8 create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/r/5/r5jvp155d7b5sa8zbgqzc2x6176bgs69qz9g0hx99r79e create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/r/7/r7bm9nh1znjv0nh4qcy272s9brgsz64pz4dq52sx0ajec create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/r/m/rm7j76kpffjnykfqxcgx0925xen98vn3t09atfgxapbmp create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/r/r/rrg0pbmdjev80ps4d0js812gzqa86q9jpa7rj5dn29rrp create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/r/z/rzpzfk0mrj3zpzt3637hkx9zr4578xrbznyq6e2a7w91g create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/s/j/sjnxj30614zv737b6tjpmtnce0992v52bzadh2rhxzjf0 create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/s/s/ssx9arzd57h63sh653erphygbrm9xnq5t4e4yhs2yq5aj create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/s/w/swzsf6kzdrj1gay4k3mpkhwen689j0z2191zpt6qw1bm0 create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/s/y/sybedhhm3tshmxs05mtw2vsqenrw8wv4cvw7wae800jk0 create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/w/9/w9frp6nrpx9teh9vx8q52bwe8pz3gmzv2320qv3mttqtw create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/w/a/wa80m1ba2zgfd9y9bqvh13wfm83kqbqpsjwhv4mjmqebj create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/x/4/x4gh60rdd803axv8pw9kh27xe1enmx1nhfhmjrags7jy8 create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/y/b/yb68cax2dvjwqrnpxdq5srdezdcfwjfsfx6bsfcf9sk78 create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/y/q/yqzk58tjef594ycch1z76qwvtyms5e4rvj2989her6rey create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/z/9/z941c9vqggbmfxm8shz7vrn24bvpvajg0wpdeym43vbym create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/z/g/zg4trx14yq823m5wamqwvdrz61sh6vkhc4y3fps0m02w6 create mode 100644 implementation/fuzz/inputs-genvm-modules-normalize-whitespace/z/x/zx6bt1mp5mztw0q4nzhjezfq249br5779q1v51s6hk85e create mode 100644 implementation/tests/manager_permits.rs create mode 100644 support/tools/genvm-tool/flake.lock create mode 100644 support/tools/genvm-tool/flake.nix create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/0/a/0a77zeewq0aqkcykz2bb1xwhdr99xh4w8sajwhz3k7t4c create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/1/j/1j7zdpm0keqes0emct1z75fvxqefmg86rabh7f58b9j1t create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/1/m/1m3m9gb15nqrbsf0s0yxnwkxw8bvjchzt3g3292nfh89r create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/1/q/1qak1b713h3b43wtnpxr3k4szgy779kkpt5fmmgk3k7r8 create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/1/r/1rm6sfd3ce16mneqnza9a6k44q1q0b630bjyjsr0bay2t create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/2/8/2889c6vjcr10k5mct810px4hwtvkjjmfap9g4v7vqw49m create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/2/z/2z4a5kfma2ck2ptfr4vgf7x93035nc52ysw31vcmwz8kw create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/2/z/2ztshe2mqj9vscyc77m0xhsh0mv1syb797gssvxk2qgrg create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/3/x/3xjn8db9k106dpsxg3p4safeddkwr8vyb46fzb42ycz6m create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/4/e/4ebgh2n7349b26mbas03fnk883hbd8bfxkk3ta9gfzem8 create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/4/y/4ysd4fq9jcf95wvg4fpk73h0ftkyg9dy285arxvgqft2j create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/5/n/5n4jjf4c6xgejs3ntw83jbpakfa8y26hrka9d9ykpkt7y create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/5/n/5nmfkg45vmckx2qj1gpm5tbke0cqedmvsgky5pa5fdhdj create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/5/q/5qh9w8qmh1wz818jmnwnrd7aqq6f7y39hx1fqn6gret3a create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/6/d/6dge9wd637rp52z6bazc1ph9jbacfknb3mpkcwff8y8be create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/7/a/7a5y9xan3xzj1vam3946gqbepe41p9471t8f5gw0gfn1a create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/7/d/7dbdpj58bhnvv9zcqaxg1gyf6ty707k25avczqr13n414 create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/7/m/7mne39ergtbmmtvn8dwf5rts4c4k5aaac1t6rf7rypn1y create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/8/h/8h406nwh64gfav2jxv96hbmhrb1t03yqf5e61s7jf7tfm create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/8/v/8vfbamrjk4e8cm6nc8qmp2jx8dkqqhfs13wkb1hb5rq90 create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/9/v/9vv23ykykwfjgdmnsfxxtrqzfjx5r0b30frarwecm3rd8 create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/9/w/9wmhqg28z4ergsgh57jkqfyx2p66pdby1r1zs5wth779p create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/a/0/a03g00cnvfa0gs77n0y1ww0tdn8908qnja4bxw9a7gpmw create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/a/4/a4r7021ggw1rmd6qz9p3spjn266k328eh5vr8pg0pm2gm create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/a/7/a77dmacngtkjb9y93610yv1zh6a9rtd1s6dt3vjz00zcg create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/a/y/ayj9117xzpdr731k488aawap9fa25kbg0yez0yxncp8r6 create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/a/z/azat8a0r7d706fa7tnee39qz1rbdc92ekgbxtgejyntrc create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/b/h/bh4wrvwgtw8a0xw2836gdfej6qxws1f66ej3hxtqf480m create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/c/a/ca97ypke6xjfbx2n9jpex04wy2rd0nh950vrteqpx6v4c create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/d/1/d1ftb59xsy5bkg0vacd8wrqrhm8c5s10zd3xcpc7edtn4 create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/d/j/djj76h73qn7c9yh8r6xfzehdtvsbn0hkxvz5gz1w8cw6r create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/d/s/ds11mdgj72234c89gzj6k0cvw7vpb5v2n3c9327rmch3a create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/d/t/dtjpxmbt1f6axs8fh7kz3mb4xhstcm3erhpx9nnsmb5c8 create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/d/t/dtt19bz3ejma3sk5xyk9bhnz9yna4w7qwc5xvqt12n1nr create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/e/5/e56xem18j5vnd0db4gmheccjjrzp6114yjzv0rwpvkarr create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/e/k/ek756vzwq4xcp9g7ncsqmc1n0bpa2391kkjwa70anx5km create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/e/w/ewzaq332qw2ef9kjhgknt5jgzqvq9959t18a9netpe7n2 create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/f/2/f2977sr15mekm241hw6307vdxw6jfpvtzcv96e0kpb30t create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/f/r/frwjgmhyf89bdtnk33sfphqeqh99a7bgnjm672d7491sp create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/f/s/fs19mwm00yyna4dgd0vfdd2xsz9tta24qqnb3a5j2fpb4 create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/f/y/fypk3np5dvfnrfhm0mp84qmkpbweyfyqtenpb9tybvv4r create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/g/k/gk362a9qzh27m009584enyxjpcgrydm3gpvh2pfgragvp create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/g/s/gs7tthhttxra0bw9pr03wswx1zca0tasa30ban582d1ry create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/h/0/h0rj8gfaf7y5xn1bd2f55dxj285rfmrv8ja0zypgzdjc2 create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/h/2/h2vx2w3me566fak1gsq86nytqw7x2egsgqntxh4m811rp create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/h/8/h8s06mw10v4v4hs3dd98gfqnpszmh7s2bfp3bgstpx1sc create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/h/k/hke62nwahseh6j9eb81qsbggf2k585jf9t01jf5s49cwt create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/j/g/jgvxrb5teh8rxxejpttpqzwhthnpctfrnxma8fzf07zqt create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/j/q/jq7d47bpcygzb55b6jsjwh3xb7nq5p5swwg52dv0je7qt create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/j/t/jt773qwy1mj50y89pkcasgkkm326mykrzx9yvx94ts4kw create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/j/t/jtccnqnb3bh8m1dtcd9wh20qxbyhggqhk559d4xafecb8 create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/k/3/k3n2b2f1cvyd4nnep751pzhdh0fn48ga066nn3d2jpgs4 create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/k/j/kjy0wbyzj59320tt8pbm99abg80w2v0spd0v1g1q2jxaa create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/k/p/kpwwz2fcjyav4ksb0rcj53w0ansjt4gb9yh1ynhapkz0j create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/k/q/kq8w2s5d2rq6ncbzmryxtzcc62jz09j2vzb1cs24t8mcg create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/k/x/kxhe3xvpb0s9d1f7ffrnkwrg2h61dqqkgrn5sap51xkqm create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/n/v/nv0apj6jkdsnskpkhbnpxhq75zq69msynk5mab9mgv54w create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/p/s/ps065bgt8wk77gp5pacydweetf153h40chbhpb5advvj8 create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/q/1/q1ggb0ky8p285276zx7gnbyfd8ygy3t6r7vrv2x9y7rjt create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/q/q/qqkmwgxfve8zjkwyag90ha2k573f1sfja9mcxttfmqkw0 create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/q/r/qr29xg001ksft36c10dn085qrsfwqv5q0414rnrpwtzgm create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/r/2/r284nd9vqrwfx5hb6ve95gbakraz1e9xmqse6m6vyme7e create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/r/5/r5zhh7c9j6r51vggk6ab360krd6x252xxgh3tdfp6rksa create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/r/e/rekdzpff11mgb0mtkjc166frrkre6smknejpqcpt7f22p create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/r/t/rt4ykf5cbnv0n2r107km5yh0qqmx2vsjjp1qwybpjgy5t rename implementation/fuzz/inputs-genvm-modules-normalize-whitespace/0c8b1f460e1ccc4f95050847930e5b35a31848eeee570692c202ed78f3fa739e => support/tools/genvm-tool/fuzz/inputs/scheduling/s/k/sk5exxw1y9681n735pv748t072w0qz7s9x23907cdjj68 (50%) create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/t/1/t1hj52m3a64ws7xxvx7qps9xsnhehfay1k6j88tr3ypke create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/t/r/trx1968q1gq2y6dbw2nbykrrendnqp1stprbxvd7mt558 create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/t/t/ttdsymeqpec56fp0mrr72150b33memf6nex59dhnbkqfg create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/t/z/tzhred7xrw335kmfkv4px3sppvkjx8kff9ykcv0nfr1gg create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/t/z/tzm8wzrej02f2qhyj478hsx7jzj55ge236qr4hz2zajdt create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/v/9/v9bn5x7sxq15qm7yckys35008w0b0d4mp59gcp0th6hta create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/v/q/vq8mbh2tkh0ce4nq474f49zrg0egdva3bxyhs2m0jhv7m create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/w/9/w9b8068b46gp8ajd9xzv810kws9j84q5eb2rj5pgmn9t2 create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/w/9/w9exq2ds6v5amcy0sv3q7s2c8thhpb4tafjsmawb7f132 create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/w/b/wbjmbw1dr6vxhxvmavvsa3zq0h4mm209am9qteesgyekg create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/w/h/whza2rpj7esftmqkevjy248v1p0a13rawsz3cz6jg172a create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/w/j/wj4fkpxc9em4mhhhymes72frtqxkjckgjbvwn5cdnzy4e create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/x/2/x25dmm6a6rbny7cwfsbxz7p2ccdjg3r42cfxqjaqj0vea create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/x/3/x36pbrm7ks6yytqpe4eye2n8zbhh15n3fnkng3p5m9a9w create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/x/f/xfnb5rmw43mb4w66amzywjvy5hrdy2zj8s9wy15g2vfsc create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/x/h/xhebxpwxedbn894hgq2x212xe7nc1ky9brgnqpt9emw6y create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/x/x/xxthw0f8kdytdk9ww8p3pb17seayx23p73ye02px68sra create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/x/y/xyechygnfyvjc98mxcqa33pxqdtny51d3avkgy5rqrmbc create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/y/1/y1x97drzaeq8r3a3tsb44s1xhd9rq5w34jceqcf0cnzk2 create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/y/3/y3sn7080jpjd06ttd23q11f1gvf7e8x2vksq2987e74x6 create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/y/b/yb13jgagqpa7t8wm0p1cm563cqzs3t7bnma3b1c15ks70 create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/y/e/yep2zs7xpq5936mtf5cachrha72j0j1512g1c8hbgvmnm create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/y/m/ym3qhn9hdmjz2f11dxm70nzrwmb4akc4rh536xmx8mkvj create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/z/1/z1tpx2ffwwzkhmysrgp72696z3nwm1b9c0w0rjqs21ge0 create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/z/a/za6qeb27pkvczyyz2b75c59yaa5c9sghmgrekcr80ea28 create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/z/e/zef9y9cs4pzfg1bwv73k271wp240w8gnpgdbs1532n3mp create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/z/j/zj9tvrqmg8a20yn6zjg040fznngbfzqc6wd8hxqjza3de create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/z/m/zm2bj14d4d6jx6t9dwevjz9qjy9hvkm6yq0zsvhq0traa create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/z/t/ztgvvbrx1tcewc1h1932qkz4qm9yjdbs7g3072ackqg06 create mode 100644 support/tools/genvm-tool/fuzz/inputs/scheduling/z/t/ztkrh5101zazqjygn9mnf19aa05ybkq9hyh1w1as4pbjc create mode 100755 support/tools/genvm-tool/fuzz/src/scheduling.py create mode 100644 support/tools/genvm-tool/genvm_tool/cmd_fuzz_corpus.py create mode 100644 support/tools/genvm-tool/genvm_tool/io.py create mode 100644 support/tools/genvm-tool/genvm_tool/misc.py create mode 100644 support/tools/genvm-tool/genvm_tool/tests/exec/process.py create mode 100644 support/tools/genvm-tool/genvm_tool/tests/tags.py create mode 100644 support/tools/genvm-tool/unit_tests/test_codegen_detail_suffix.py create mode 100644 support/tools/genvm-tool/unit_tests/test_fuzz_corpus.py create mode 100644 support/tools/genvm-tool/unit_tests/test_fuzz_corpus_layout.py create mode 100644 support/tools/genvm-tool/unit_tests/test_process_teardown.py create mode 100644 support/tools/genvm-tool/unit_tests/test_service_semaphores.py create mode 100644 support/tools/genvm-tool/unit_tests/test_service_startup.py create mode 100644 support/tools/genvm-tool/unit_tests/test_watchdog_ack.py create mode 100644 tests/presets/python-fuzz.txt create mode 100644 tests/runner/genvm_tool_plugins/afl.py create mode 100644 tests/runner/genvm_tool_plugins/runners.py create mode 100644 tests/runner/gvm_extra/case_inputs.py create mode 100644 tests/system/cross-major-observability/assets/v0.2/caller.py create mode 100644 tests/system/cross-major-observability/assets/v0.2/chain.py create mode 100644 tests/system/cross-major-observability/assets/v0.3/chain.py create mode 100644 tests/system/cross-major-observability/assets/v0.3/helper.py create mode 100644 tests/system/cross-major-observability/assets/v0.3/observer.py create mode 100644 tests/system/cross-major-observability/assets/v0.3/self_recursion.py create mode 100644 tests/system/cross-major-observability/assets/v0.3/trap.py create mode 100644 tests/system/cross-major-observability/assets/v0.3/user_error.py create mode 100644 tests/system/cross-major/assets/v0.2/blob_caller.py create mode 100644 tests/system/cross-major/assets/v0.2/busy.py create mode 100644 tests/system/cross-major/assets/v0.2/callee.py create mode 100644 tests/system/cross-major/assets/v0.2/caller.py create mode 100644 tests/system/cross-major/assets/v0.2/chain.py create mode 100644 tests/system/cross-major/assets/v0.2/loop.py create mode 100644 tests/system/cross-major/assets/v0.2/signer_caller.py create mode 100644 tests/system/cross-major/assets/v0.2/user_error.py create mode 100644 tests/system/cross-major/assets/v0.3/blob.py create mode 100644 tests/system/cross-major/assets/v0.3/callee.py create mode 100644 tests/system/cross-major/assets/v0.3/caller.py create mode 100644 tests/system/cross-major/assets/v0.3/chain.py create mode 100644 tests/system/cross-major/assets/v0.3/loop.py create mode 100644 tests/system/cross-major/assets/v0.3/signer.py create mode 100644 tests/system/cross-major/assets/v0.3/state_caller.py create mode 100644 tests/system/cross-major/assets/v0.3/structured_error.py create mode 100644 tests/system/cross-major/assets/v0.3/user_error.py create mode 100644 tests/system/cross-major/cases/routing.jsonnet create mode 100644 tests/system/cross-major/cases/routing.null-mismatch-caller_null-mismatch-callee_null-mismatch.stdout create mode 100644 tests/system/cross-major/cases/routing.null-same-caller_null-same-callee_null-same-major.stdout create mode 100644 tests/system/cross-major/cases/routing.plain-error-caller_plain-error-callee_plain-error.stdout create mode 100644 tests/system/cross-major/cases/routing.plain-error-reverse-caller_plain-error-reverse-callee_plain-error-reverse.stdout create mode 100644 tests/system/cross-major/cases/routing.signer-caller_signer-callee_signer.stdout create mode 100644 tests/system/cross-major/cases/routing.structured-error-caller_structured-error-callee_structured-error.stdout create mode 100644 tests/system/cross-major/cases/routing.v02-to-v03-caller_v02-to-v03-callee_v02-to-v03.stdout create mode 100644 tests/system/cross-major/cases/routing.v03-to-v02-caller_v03-to-v02-callee_v03-to-v02.stdout create mode 100644 tests/tags.json create mode 100644 webdriver/src/prj/src/duration.test.ts create mode 100644 webdriver/src/prj/src/semaphore.test.ts create mode 100644 webdriver/src/prj/src/semaphore.ts create mode 100644 webdriver/src/prj/ts-node-register.mjs diff --git a/.claude/settings.local.json b/.claude/settings.local.json new file mode 100644 index 00000000..d61c7860 --- /dev/null +++ b/.claude/settings.local.json @@ -0,0 +1,3 @@ +{ + "outputStyle": "Brief Style" +} diff --git a/.claude/skills/agentic-fuzzing/SKILL.md b/.claude/skills/agentic-fuzzing/SKILL.md index 4c03387a..d7b00ca0 100644 --- a/.claude/skills/agentic-fuzzing/SKILL.md +++ b/.claude/skills/agentic-fuzzing/SKILL.md @@ -119,8 +119,8 @@ leader-vs-validator comparison does not catch a load failure either: it is identical in all three modes, so the hashes agree and the case still passes β€” a probe whose contract never loaded reports `βœ“` in 50ms. Before believing a pass, open `genvm.log.gz` and check the run reached the contract, and decode the `hash` -artifact β€” `base64 -d < hash` on a failed load reads `invalid_contract -malformed_runner`. +artifact β€” `base64 -d < hash` on a failed load reads `invalid_contract runner +malformed`. ## Reviewing a PR for a failing test diff --git a/.claude/skills/commit-style/SKILL.md b/.claude/skills/commit-style/SKILL.md index 6857ba4c..793c27c3 100644 --- a/.claude/skills/commit-style/SKILL.md +++ b/.claude/skills/commit-style/SKILL.md @@ -140,3 +140,6 @@ submodule's linter reformat leak an unrelated file into a commit. Each repo's per-repo pre-commit hook (git-hooks.nix, in its flake) runs on the repo you commit in. Do not `--no-verify` (even for gitlink bumps β€” let the hooks run). Full workflow (order, pushing, `--force-with-lease` after a rebase): `/submodules`. + +Never include text like "bump the v0.3 gitlink". State the underlying change, +not name the file updated diff --git a/.claude/skills/rust-test-style/SKILL.md b/.claude/skills/rust-test-style/SKILL.md index 8eaa1851..4b6feb9d 100644 --- a/.claude/skills/rust-test-style/SKILL.md +++ b/.claude/skills/rust-test-style/SKILL.md @@ -1,6 +1,6 @@ --- name: rust-test-style -description: GenVM Rust test conventions and style. Use when writing, adding, or modifying Rust tests β€” inline `#[cfg(test)] mod tests`, integration tests under a crate's `tests/`, helpers, assertions, and how `genvm-tool test` discovers them. +description: Always read before writing, adding, or modifying Rust tests β€” where to put and how to run them. --- # Writing Rust tests in GenVM diff --git a/.claude/skills/spec/SKILL.md b/.claude/skills/spec/SKILL.md index ad9f83bb..d5eed14e 100644 --- a/.claude/skills/spec/SKILL.md +++ b/.claude/skills/spec/SKILL.md @@ -30,11 +30,13 @@ generated pages stay the single source of truth: - Constants: `:ref:`gvm-def-consts-value-<group>-<name>`` / `:ref:`gvm-def-const-<name>`` from `spec/appendix/constants.rst` - (generated from `executor/codegen/data/public-abi.json`) or + (generated from `executor/codegen/data/public-abi.json`), + `internal-constants.rst` (from `internal-constants.json`) or `constants-pending.rst` (from `public-abi-pending.json`). **Never edit these .rst by hand** β€” edit the JSON and regenerate - ([genvm-tool.md](../../../docs/contributing/howto/genvm-tool.md)). New - not-yet-stabilized constants go to the pending JSON. + ([genvm-tool.md](../../../docs/contributing/howto/genvm-tool.md)). Constants a + contract cannot read go to the internal JSON, not-yet-stabilized ones to the + pending JSON. - Error outcomes: `:ref:`gvm-def-str-trie-value-vm-error-...`` β€” every "traps with" / "rejected with" must link the exact vm_error entry. - Glossary terms: `:term:`sub-VM`` etc. on first use in a section. diff --git a/.claude/skills/test/SKILL.md b/.claude/skills/test/SKILL.md index a7e23014..eb1206ad 100644 --- a/.claude/skills/test/SKILL.md +++ b/.claude/skills/test/SKILL.md @@ -5,6 +5,17 @@ description: Runs tests for the GenVM project. Use after making code changes to See docs/contributing/howto/testing/README.md. +Invoke the tool as: + +``` +nix run '.?submodules=1#genvm-tool' -- test run --filter-tag '!fuzz' ... +``` + +A bare `genvm-tool` from `PATH` is a nix store copy pinned at dev-shell build +time; it goes stale against the working tree and fails with import errors such +as `ModuleNotFoundError: No module named 'genvm_tool.tests.exec.process'`. +`build/genvm_tool.sh` has the same problem β€” it bakes the store path in. + After fixing a failure, rerun with `--filter-continue <file>` (path printed in the failure summary) before any full rerun β€” see the "Fix–rerun loop" section there. diff --git a/.genvm-monorepo-root b/.genvm-monorepo-root index 08b44276..e11b5104 100644 --- a/.genvm-monorepo-root +++ b/.genvm-monorepo-root @@ -8,6 +8,7 @@ "v0.2" ], "artifacts_dir": "build/test-artifacts", + "tags_registry": "tests/tags.json", "extra_python_paths": [ "tests/runner" ] diff --git a/.genvm-tool.py b/.genvm-tool.py index 35b4bf13..7b4ca8ed 100644 --- a/.genvm-tool.py +++ b/.genvm-tool.py @@ -1,4 +1,5 @@ -"""Manager-root genvm-tool project config. +""" +Manager-root genvm-tool project config. Loaded once by genvm-tool (``common.load_project``) before any subcommand runs; subcommands ask it for what they need: @@ -20,12 +21,14 @@ def tests(ctx): - """Umbrella test suite (was the top-level .ya-test.py). + """ + Umbrella test suite (was the top-level .ya-test.py). ``ctx`` is the runner's configuration ``Context``. Collectors close over the imports below (resolved lazily when collection runs). """ import json + import platform import sys from pathlib import Path @@ -56,32 +59,21 @@ def tests(ctx): + '\n' ) - ctx.run_parser.add_argument( - '--fuzz-timeout', - type=int, - default=30, - help='Timeout for each fuzzing run in seconds', - ) - - ctx.run_parser.add_argument( - '--fuzz-update-corpus', - default=False, - action='store_true', - help='Whether to update the fuzzing corpus', - ) - import genvm_tool_plugins ctx.shared.logger.trace( 'import path', path=sys.path, plugins_path=genvm_tool_plugins.__path__ ) from genvm_tool_plugins import ( + afl, cargo, genvm, integration, pytest, ) + afl.register(ctx) + def collect_rust(ctx: genvm_tool.tests.stage.collection.Context): for t in filter(lambda x: x.name == 'Cargo.toml', ctx.shared.git_files): ctx.shared.logger.debug('discovered Cargo.toml', path=t) @@ -101,54 +93,47 @@ def collect_rust(ctx: genvm_tool.tests.stage.collection.Context): name = f'{name.parent}/{name.stem}' cargo.cargo_fuzz( ctx, - genvm_tool.tests.test.Description( - name, - console_pool=True, - ), + genvm_tool.tests.test.Description(name), rust_root_dir=rust_root_dir, name=fuzz_file.stem, ) def collect_pytest(ctx: genvm_tool.tests.stage.collection.Context): - p = ctx.shared.root_dir.joinpath( - 'executors', 'v0.3.x', 'runners', 'genlayer-py-std' - ) - pytest.pytest( - ctx, - genvm_tool.tests.test.Description( - str(p.relative_to(ctx.shared.root_dir)), - console_pool=True, - tags=frozenset({'unit'}), - ), - project_root_dir=p, - ) - - ci = ctx.shared.root_dir.joinpath('support', 'ci') - pytest.pytest( - ctx, - genvm_tool.tests.test.Description( - str(ci.relative_to(ctx.shared.root_dir)), - console_pool=True, - tags=frozenset({'unit'}), - ), - project_root_dir=ci, - ) - - fuzz_files = list(p.glob('fuzz/src/*.py')) - fuzz_files.sort() - for fuzz_file in fuzz_files: - name = fuzz_file.relative_to(ctx.shared.root_dir) - name = f'{name.parent}/{name.stem}' - continue # for now let's disable it - pytest.py_fuzz( + dirs = [ + ctx.shared.root_dir.joinpath('executors', 'v0.3.x', 'runners', 'genlayer-py-std'), + ctx.shared.root_dir.joinpath('support', 'tools', 'genvm-tool'), + ctx.shared.root_dir.joinpath('support', 'ci'), + ] + for p in dirs: + pytest.pytest( ctx, genvm_tool.tests.test.Description( - name, + str(p.relative_to(ctx.shared.root_dir)), + console_pool=True, + tags=frozenset({'unit'}), ), project_root_dir=p, - name=fuzz_file.stem, ) + # A project's flake carries python-afl and AFL++ on x86_64-linux only, + # so elsewhere the case could not start at all + if (platform.system(), platform.machine()) != ('Linux', 'x86_64'): + continue + + fuzz_files = list(p.glob('fuzz/src/*.py')) + fuzz_files.sort() + for fuzz_file in fuzz_files: + name = fuzz_file.relative_to(ctx.shared.root_dir) + name = f'{name.parent}/{name.stem}' + pytest.py_fuzz( + ctx, + genvm_tool.tests.test.Description( + name, + ), + project_root_dir=p, + name=fuzz_file.stem, + ) + ctx.add_collector(collect_rust) ctx.add_collector(collect_pytest) @@ -196,6 +181,7 @@ def collect_integration(ctx: genvm_tool.tests.stage.collection.Context): ci = ctx.shared.ci manager_port = genvm.get_manager_port(ctx.configuration) + manager_semaphore = ctx.new_semaphore('manager-listener') if no_manager: manager_impl = genvm.ExternalManagerService( @@ -225,6 +211,7 @@ def collect_integration(ctx: genvm_tool.tests.stage.collection.Context): manager_service = ctx.new_service( name='manager', manager=manager_impl, + semaphores=[manager_semaphore], ) manager_service.meta = { 'port': manager_port, @@ -250,11 +237,47 @@ def collect_integration(ctx: genvm_tool.tests.stage.collection.Context): webdriver_service=webdriver_service, ci=ci, ) + integration.integration_test_directory( + ctx, + cases_dir=ctx.shared.root_dir / 'tests' / 'system' / 'cross-major' / 'cases', + reroute_to=build_info['executor_versions']['v0.3'], + save_hashes=False, + manager_service=manager_service, + modules_service=modules_service, + webdriver_service=webdriver_service, + ci=ci, + ) ctx.collect_dir('tests/system/permits', manager_service=manager_service) - ctx.collect_dir('tests/system/manager-socket') - ctx.collect_dir('tests/system/cross-major') - ctx.collect_dir('tests/system/cross-major-observability') + limited_manager_service = None + if not no_manager: + limited_manager_service = ctx.new_service( + name='manager-cross-major-small-message-cap', + manager=genvm.ManagerService( + bin_path=build_dir / 'out' / 'bin' / 'genvm-modules', + log_path=tests_output_root / 'cross-major-small-message-cap.log', + env=ctx.configuration, + ci=ci, + config={'max_message_bytes': 1024 * 1024}, + ), + semaphores=[manager_semaphore], + ) + ctx.collect_dir( + 'tests/system/manager-socket', + manager_semaphore=manager_semaphore, + build_dir=build_dir, + ci=ci, + ) + ctx.collect_dir( + 'tests/system/cross-major', + manager_service=manager_service, + limited_manager_service=limited_manager_service, + managed_manager=not no_manager, + ) + ctx.collect_dir( + 'tests/system/cross-major-observability', + manager_service=manager_service, + ) ctx.add_collector(collect_integration) diff --git a/.github/workflows/incl_release_build_test_cell_build.yaml b/.github/workflows/incl_release_build_test_cell_build.yaml index c39ed002..d79e09bb 100644 --- a/.github/workflows/incl_release_build_test_cell_build.yaml +++ b/.github/workflows/incl_release_build_test_cell_build.yaml @@ -77,6 +77,7 @@ jobs: with: token: ${{ secrets.NIX_CACHE_PUSH_TOKEN }} include: genvm|genlayer|(^lief) + exclude: genvm-cpython-objs - name: Upload artifacts uses: actions/upload-artifact@v4 diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..462fc4a4 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,54 @@ +# GenVM manager + +Before changing this repository, read the matching page in `docs/contributing/`: +a task has a how-to, a design question has an explanation + +<!-- below is generated with `genvm-tool docs` --> + +## Tutorial +- [first-contribution.md](docs/contributing/tutorial/first-contribution.md) β€” patch an executor, branch, commit, push, open the PR + +## Howto +- [genvm-tool.md](docs/contributing/howto/genvm-tool.md) β€” the umbrella tool: man page, test and git subcommands, codegen +- [pr.md](docs/contributing/howto/pr.md) β€” branch model, PR action panel, `ci-safe` / `run-full-tests`, merge gates +- [setup.md](docs/contributing/howto/setup.md) β€” first-time clone: submodules, vendored trees, nix/direnv dev shell + +## Howto/Building +- [build.md](docs/contributing/howto/building/build.md) β€” debug build: configure + ninja, targets, outputs, cargo quirks +- [docs.md](docs/contributing/howto/building/docs.md) β€” building and publishing the website, spec vs impl-spec, ADRs +- [runners.md](docs/contributing/howto/building/runners.md) β€” where runners come from: build on Linux, download elsewhere + +## Howto/Committing +- [git-third-party.md](docs/contributing/howto/committing/git-third-party.md) β€” how vendored trees (wasmtime, …) are pinned and patched +- [runners.md](docs/contributing/howto/committing/runners.md) β€” clearing runner dev-mode and refreshing hashes before a commit +- [submodules.md](docs/contributing/howto/committing/submodules.md) β€” repo topology, gitlink bumps, pre-commit hooks, push order + +## Howto/Docs +- [style.md](docs/contributing/howto/docs/style.md) β€” prose conventions for guides, specs, ADRs and commit bodies + +## Howto/Extending +- [add-host-function.md](docs/contributing/howto/extending/add-host-function.md) β€” new executor↔host protocol method +- [add-llm-provider.md](docs/contributing/howto/extending/add-llm-provider.md) β€” new LLM backend in the manager +- [add-wasi-function.md](docs/contributing/howto/extending/add-wasi-function.md) β€” new gl_call method or raw WASI function +- [modify-runner.md](docs/contributing/howto/extending/modify-runner.md) β€” runner dev-mode and hash refresh +- [modify-wasmtime.md](docs/contributing/howto/extending/modify-wasmtime.md) β€” patching vendored wasmtime, trap plumbing +- [write-a-script.md](docs/contributing/howto/extending/write-a-script.md) β€” conventions for helper scripts and pre-commit hooks + +## Howto/Releasing +- [release-build.md](docs/contributing/howto/releasing/release-build.md) β€” nix packages, platforms, release assets +- [versioning.md](docs/contributing/howto/releasing/versioning.md) β€” release trains, `.genvm-monorepo-root`, version tools + +## Howto/Testing +- [README.md](docs/contributing/howto/testing/README.md) β€” `genvm-tool test`: filters, presets, continue files +- [fuzzing.md](docs/contributing/howto/testing/fuzzing.md) β€” AFL fuzz targets, host sysctl prep +- [integration.md](docs/contributing/howto/testing/integration.md) β€” jsonnet cases, tags, golden `.stdout`/`.hash` files, services +- [python.md](docs/contributing/howto/testing/python.md) β€” Python tests, direct pytest for genlayer-py-std +- [rust.md](docs/contributing/howto/testing/rust.md) β€” Rust tests: where they go, how to run them, coverage + +## Explanation +- [docs-layout.md](docs/contributing/explanation/docs-layout.md) β€” the 4 kinds of page and where each belongs +- [executor-lines.md](docs/contributing/explanation/executor-lines.md) β€” why several executor lines ship side by side, and what it costs +- [fuzz.md](docs/contributing/explanation/fuzz.md) β€” why fuzz targets get fake entropy and no CmpLog +- [merge-model.md](docs/contributing/explanation/merge-model.md) β€” why merging is a maintainer panel rather than a merge queue +- [shared-submodule-cache.md](docs/contributing/explanation/shared-submodule-cache.md) β€” why submodules are worktrees of one cache repo, not clones +- [vendored-trees.md](docs/contributing/explanation/vendored-trees.md) β€” why third-party sources are patch series, not forks diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index b62f2919..00000000 --- a/CLAUDE.md +++ /dev/null @@ -1,53 +0,0 @@ -# GenVM manager - -Before changing this repository, read the matching page in `docs/contributing/`: -a task has a how-to, a design question has an explanation - -<!-- below is generated with `genvm-tool docs` --> - -## Tutorial -- [first-contribution.md](docs/contributing/tutorial/first-contribution.md) β€” patch an executor, branch, commit, push, open the PR - -## Howto -- [genvm-tool.md](docs/contributing/howto/genvm-tool.md) β€” the umbrella tool: man page, test and git subcommands, codegen -- [pr.md](docs/contributing/howto/pr.md) β€” branch model, PR action panel, `ci-safe` / `run-full-tests`, merge gates -- [setup.md](docs/contributing/howto/setup.md) β€” first-time clone: submodules, vendored trees, nix/direnv dev shell - -## Howto/Building -- [build.md](docs/contributing/howto/building/build.md) β€” debug build: configure + ninja, targets, outputs, cargo quirks -- [docs.md](docs/contributing/howto/building/docs.md) β€” building and publishing the website, spec vs impl-spec, ADRs -- [runners.md](docs/contributing/howto/building/runners.md) β€” where runners come from: build on Linux, download elsewhere - -## Howto/Committing -- [git-third-party.md](docs/contributing/howto/committing/git-third-party.md) β€” how vendored trees (wasmtime, …) are pinned and patched -- [runners.md](docs/contributing/howto/committing/runners.md) β€” clearing runner dev-mode and refreshing hashes before a commit -- [submodules.md](docs/contributing/howto/committing/submodules.md) β€” repo topology, gitlink bumps, pre-commit hooks, push order - -## Howto/Docs -- [style.md](docs/contributing/howto/docs/style.md) β€” prose conventions for guides, specs, ADRs and commit bodies - -## Howto/Extending -- [add-host-function.md](docs/contributing/howto/extending/add-host-function.md) β€” new executor↔host protocol method -- [add-llm-provider.md](docs/contributing/howto/extending/add-llm-provider.md) β€” new LLM backend in the manager -- [add-wasi-function.md](docs/contributing/howto/extending/add-wasi-function.md) β€” new gl_call method or raw WASI function -- [modify-runner.md](docs/contributing/howto/extending/modify-runner.md) β€” runner dev-mode and hash refresh -- [modify-wasmtime.md](docs/contributing/howto/extending/modify-wasmtime.md) β€” patching vendored wasmtime, trap plumbing -- [write-a-script.md](docs/contributing/howto/extending/write-a-script.md) β€” conventions for helper scripts and pre-commit hooks - -## Howto/Releasing -- [release-build.md](docs/contributing/howto/releasing/release-build.md) β€” nix packages, platforms, release assets -- [versioning.md](docs/contributing/howto/releasing/versioning.md) β€” release trains, `.genvm-monorepo-root`, version tools - -## Howto/Testing -- [README.md](docs/contributing/howto/testing/README.md) β€” `genvm-tool test`: filters, presets, continue files -- [fuzzing.md](docs/contributing/howto/testing/fuzzing.md) β€” AFL fuzz targets, host sysctl prep -- [integration.md](docs/contributing/howto/testing/integration.md) β€” jsonnet cases, tags, golden `.stdout`/`.hash` files, services -- [python.md](docs/contributing/howto/testing/python.md) β€” Python tests, direct pytest for genlayer-py-std -- [rust.md](docs/contributing/howto/testing/rust.md) β€” Rust tests: where they go, how to run them, coverage - -## Explanation -- [docs-layout.md](docs/contributing/explanation/docs-layout.md) β€” the 4 kinds of page and where each belongs -- [executor-lines.md](docs/contributing/explanation/executor-lines.md) β€” why several executor lines ship side by side, and what it costs -- [merge-model.md](docs/contributing/explanation/merge-model.md) β€” why merging is a maintainer panel rather than a merge queue -- [shared-submodule-cache.md](docs/contributing/explanation/shared-submodule-cache.md) β€” why submodules are worktrees of one cache repo, not clones -- [vendored-trees.md](docs/contributing/explanation/vendored-trees.md) β€” why third-party sources are patch series, not forks diff --git a/CLAUDE.md b/CLAUDE.md new file mode 120000 index 00000000..47dc3e3d --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/SECURITY.md b/SECURITY.md index eefc9fc1..d60a8d1b 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -6,8 +6,8 @@ identical and trustworthy across all validators. Please treat security issues ac ## Reporting a vulnerability **Do not open a public issue.** Report privately via GitHub's -[private vulnerability reporting](https://github.com/genlayerlabs/genvm/security/advisories/new), -or email kira@yeager.ai +[private vulnerability reporting](https://github.com/genlayerlabs/genvm-manager/security/advisories/new), +or email kira@genlayerlabs.com Include a description, affected component/version, and a reproduction (a contract, calldata, or test case) where possible. We aim to acknowledge within a few business days. @@ -37,3 +37,12 @@ In scope: the executor, runners/SDK, modules (LLM/web/manager), the install/mani the CI/release supply chain. Out of scope: issues requiring a pre-compromised host or operator machine, and non-default deployments that expose the loopback-only manager to untrusted networks (though we still want to know). + +## Trust boundary + +The following relationships are trusted. Hardening them is welcome, but a report that assumes +one side is hostile is not treated as a vulnerability: + +- host and GenVM +- executor and manager +- the local disk and loopback in general diff --git a/crates/fuzzing/Cargo.lock b/crates/fuzzing/Cargo.lock new file mode 100644 index 00000000..9a86e14f --- /dev/null +++ b/crates/fuzzing/Cargo.lock @@ -0,0 +1,177 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "cobs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" +dependencies = [ + "thiserror", +] + +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + +[[package]] +name = "genvm-fuzzing" +version = "0.1.0" +dependencies = [ + "mutatis", + "postcard", + "serde", +] + +[[package]] +name = "mutatis" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "468ca2a8bc8087a0b2c11d21e7dca51d92cc2662a6993d6b56bac2701795f202" +dependencies = [ + "mutatis-derive", + "rand", +] + +[[package]] +name = "mutatis-derive" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5708f6e65277d4a89733aff227b43d0989c5ead03999b6f1f33a8ec7a9978361" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "postcard" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" +dependencies = [ + "cobs", + "embedded-io 0.4.0", + "embedded-io 0.6.1", + "serde", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" diff --git a/crates/fuzzing/Cargo.toml b/crates/fuzzing/Cargo.toml new file mode 100644 index 00000000..7d5398d6 --- /dev/null +++ b/crates/fuzzing/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "genvm-fuzzing" +version = "0.1.0" +edition = "2021" +publish = false + +[dependencies] +# `default-features = false` keeps heapless, spin and lock_api out of the tree +postcard = { version = "1.1.3", default-features = false, features = [ + "use-std", +] } +serde = { version = "1.0.219", features = ["derive"] } + +[dependencies.mutatis] +default-features = false +features = ["derive", "std"] +version = "0.5.3" diff --git a/crates/fuzzing/preload/Cargo.lock b/crates/fuzzing/preload/Cargo.lock new file mode 100644 index 00000000..bb34fecc --- /dev/null +++ b/crates/fuzzing/preload/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "genvm-fuzz-preload" +version = "0.1.0" diff --git a/crates/fuzzing/preload/Cargo.toml b/crates/fuzzing/preload/Cargo.toml new file mode 100644 index 00000000..38d05633 --- /dev/null +++ b/crates/fuzzing/preload/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "genvm-fuzz-preload" +version = "0.1.0" +edition = "2021" +publish = false + +[lib] +crate-type = ["cdylib"] + +[dependencies] diff --git a/crates/fuzzing/preload/src/lib.rs b/crates/fuzzing/preload/src/lib.rs new file mode 100644 index 00000000..56d24168 --- /dev/null +++ b/crates/fuzzing/preload/src/lib.rs @@ -0,0 +1,78 @@ +//! `LD_PRELOAD` shim that makes the OS randomness syscalls deterministic. +//! +//! Loaded into AFL targets only, via `AFL_PRELOAD`. Without it every process +//! seeds its hashers from real entropy, so a saved crash need not replay and +//! coverage differs between runs. + +use core::ffi::{c_int, c_uint, c_void}; +use std::cell::Cell; + +thread_local! { + /// Number of `u64` words handed out by this thread. + static WORDS: Cell<u64> = const { Cell::new(0) }; +} + +fn next_word() -> u64 { + // splitmix64 over the call counter + let mut z = WORDS.with(|words| { + let current = words.get(); + words.set(current.wrapping_add(1)); + current.wrapping_add(0x9e37_79b9_7f4a_7c15) + }); + z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + z ^ (z >> 31) +} + +/// # Safety +/// `buffer` must be writable for `length` bytes, or `length` must be zero. +unsafe fn fill(buffer: *mut c_void, length: usize) { + if length == 0 { + return; + } + assert!(!buffer.is_null(), "genvm-fuzz-preload: null buffer"); + + let mut written = 0; + while written < length { + let word = next_word().to_le_bytes(); + let chunk = word.len().min(length - written); + std::ptr::copy_nonoverlapping(word.as_ptr(), (buffer as *mut u8).add(written), chunk); + written += chunk; + } +} + +/// # Safety +/// See [`fill`]. Matches the libc signature, flags are ignored: every flag +/// combination is satisfiable when the entropy is fake. +#[no_mangle] +pub unsafe extern "C" fn getrandom(buffer: *mut c_void, length: usize, _flags: c_uint) -> isize { + fill(buffer, length); + length as isize +} + +/// # Safety +/// See [`fill`]. Unlike libc this accepts any `length`; the 256 byte cap exists +/// to bound how much real entropy one call may drain. +#[no_mangle] +pub unsafe extern "C" fn getentropy(buffer: *mut c_void, length: usize) -> c_int { + fill(buffer, length); + 0 +} + +#[cfg(test)] +mod tests { + use super::next_word; + + #[test] + fn threads_have_the_same_sequence() { + let threads: Vec<_> = (0..4) + .map(|_| std::thread::spawn(|| [next_word(), next_word()])) + .collect(); + let sequences: Vec<_> = threads + .into_iter() + .map(|thread| thread.join().unwrap()) + .collect(); + + assert!(sequences.windows(2).all(|pair| pair[0] == pair[1])); + } +} diff --git a/crates/fuzzing/src/depth_limit.rs b/crates/fuzzing/src/depth_limit.rs new file mode 100644 index 00000000..afc0d6ab --- /dev/null +++ b/crates/fuzzing/src/depth_limit.rs @@ -0,0 +1,408 @@ +use core::fmt; +use serde::de::{self, DeserializeSeed, Visitor}; + +pub(crate) struct Deserializer<D> { + inner: D, + remaining: usize, +} + +impl<D> Deserializer<D> { + pub(crate) fn new(inner: D, remaining: usize) -> Self { + Self { inner, remaining } + } + + fn descend<'de, V>(self, visitor: V) -> Result<(D, LimitVisitor<V>), D::Error> + where + D: de::Deserializer<'de>, + { + let remaining = self + .remaining + .checked_sub(1) + .ok_or_else(|| de::Error::custom("deserialization depth limit exceeded"))?; + Ok(( + self.inner, + LimitVisitor { + inner: visitor, + remaining, + }, + )) + } +} + +macro_rules! delegate { + ($($method:ident $(($($arg:ident: $ty:ty),*))?);+ $(;)?) => { + $( + fn $method<V>(self, $($($arg: $ty,)*)? visitor: V) -> Result<V::Value, Self::Error> + where + V: Visitor<'de>, + { + self.inner.$method($($($arg,)*)? visitor) + } + )+ + }; +} + +impl<'de, D> de::Deserializer<'de> for Deserializer<D> +where + D: de::Deserializer<'de>, +{ + type Error = D::Error; + + delegate! { + deserialize_any(); + deserialize_bool(); + deserialize_i8(); + deserialize_i16(); + deserialize_i32(); + deserialize_i64(); + deserialize_i128(); + deserialize_u8(); + deserialize_u16(); + deserialize_u32(); + deserialize_u64(); + deserialize_u128(); + deserialize_f32(); + deserialize_f64(); + deserialize_char(); + deserialize_str(); + deserialize_string(); + deserialize_bytes(); + deserialize_byte_buf(); + deserialize_unit(); + deserialize_unit_struct(name: &'static str); + deserialize_identifier(); + deserialize_ignored_any(); + } + + fn deserialize_option<V>(self, visitor: V) -> Result<V::Value, Self::Error> + where + V: Visitor<'de>, + { + let (inner, visitor) = self.descend(visitor)?; + inner.deserialize_option(visitor) + } + + fn deserialize_newtype_struct<V>( + self, + name: &'static str, + visitor: V, + ) -> Result<V::Value, Self::Error> + where + V: Visitor<'de>, + { + let (inner, visitor) = self.descend(visitor)?; + inner.deserialize_newtype_struct(name, visitor) + } + + fn deserialize_seq<V>(self, visitor: V) -> Result<V::Value, Self::Error> + where + V: Visitor<'de>, + { + let (inner, visitor) = self.descend(visitor)?; + inner.deserialize_seq(visitor) + } + + fn deserialize_tuple<V>(self, len: usize, visitor: V) -> Result<V::Value, Self::Error> + where + V: Visitor<'de>, + { + let (inner, visitor) = self.descend(visitor)?; + inner.deserialize_tuple(len, visitor) + } + + fn deserialize_tuple_struct<V>( + self, + name: &'static str, + len: usize, + visitor: V, + ) -> Result<V::Value, Self::Error> + where + V: Visitor<'de>, + { + let (inner, visitor) = self.descend(visitor)?; + inner.deserialize_tuple_struct(name, len, visitor) + } + + fn deserialize_map<V>(self, visitor: V) -> Result<V::Value, Self::Error> + where + V: Visitor<'de>, + { + let (inner, visitor) = self.descend(visitor)?; + inner.deserialize_map(visitor) + } + + fn deserialize_struct<V>( + self, + name: &'static str, + fields: &'static [&'static str], + visitor: V, + ) -> Result<V::Value, Self::Error> + where + V: Visitor<'de>, + { + let (inner, visitor) = self.descend(visitor)?; + inner.deserialize_struct(name, fields, visitor) + } + + fn deserialize_enum<V>( + self, + name: &'static str, + variants: &'static [&'static str], + visitor: V, + ) -> Result<V::Value, Self::Error> + where + V: Visitor<'de>, + { + let (inner, visitor) = self.descend(visitor)?; + inner.deserialize_enum(name, variants, visitor) + } + + fn is_human_readable(&self) -> bool { + self.inner.is_human_readable() + } +} + +struct LimitVisitor<V> { + inner: V, + remaining: usize, +} + +impl<'de, V> Visitor<'de> for LimitVisitor<V> +where + V: Visitor<'de>, +{ + type Value = V::Value; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + self.inner.expecting(formatter) + } + + fn visit_none<E>(self) -> Result<Self::Value, E> + where + E: de::Error, + { + self.inner.visit_none() + } + + fn visit_some<D>(self, deserializer: D) -> Result<Self::Value, D::Error> + where + D: de::Deserializer<'de>, + { + self.inner + .visit_some(Deserializer::new(deserializer, self.remaining)) + } + + fn visit_newtype_struct<D>(self, deserializer: D) -> Result<Self::Value, D::Error> + where + D: de::Deserializer<'de>, + { + self.inner + .visit_newtype_struct(Deserializer::new(deserializer, self.remaining)) + } + + fn visit_seq<A>(self, seq: A) -> Result<Self::Value, A::Error> + where + A: de::SeqAccess<'de>, + { + self.inner.visit_seq(LimitSeqAccess { + inner: seq, + remaining: self.remaining, + }) + } + + fn visit_map<A>(self, map: A) -> Result<Self::Value, A::Error> + where + A: de::MapAccess<'de>, + { + self.inner.visit_map(LimitMapAccess { + inner: map, + remaining: self.remaining, + }) + } + + fn visit_enum<A>(self, data: A) -> Result<Self::Value, A::Error> + where + A: de::EnumAccess<'de>, + { + self.inner.visit_enum(LimitEnumAccess { + inner: data, + remaining: self.remaining, + }) + } +} + +struct LimitSeed<S> { + inner: S, + remaining: usize, +} + +impl<'de, S> DeserializeSeed<'de> for LimitSeed<S> +where + S: DeserializeSeed<'de>, +{ + type Value = S::Value; + + fn deserialize<D>(self, deserializer: D) -> Result<Self::Value, D::Error> + where + D: de::Deserializer<'de>, + { + self.inner + .deserialize(Deserializer::new(deserializer, self.remaining)) + } +} + +struct LimitSeqAccess<A> { + inner: A, + remaining: usize, +} + +impl<'de, A> de::SeqAccess<'de> for LimitSeqAccess<A> +where + A: de::SeqAccess<'de>, +{ + type Error = A::Error; + + fn next_element_seed<T>(&mut self, seed: T) -> Result<Option<T::Value>, Self::Error> + where + T: DeserializeSeed<'de>, + { + self.inner.next_element_seed(LimitSeed { + inner: seed, + remaining: self.remaining, + }) + } + + fn size_hint(&self) -> Option<usize> { + self.inner.size_hint() + } +} + +struct LimitMapAccess<A> { + inner: A, + remaining: usize, +} + +impl<'de, A> de::MapAccess<'de> for LimitMapAccess<A> +where + A: de::MapAccess<'de>, +{ + type Error = A::Error; + + fn next_key_seed<K>(&mut self, seed: K) -> Result<Option<K::Value>, Self::Error> + where + K: DeserializeSeed<'de>, + { + self.inner.next_key_seed(LimitSeed { + inner: seed, + remaining: self.remaining, + }) + } + + fn next_value_seed<V>(&mut self, seed: V) -> Result<V::Value, Self::Error> + where + V: DeserializeSeed<'de>, + { + self.inner.next_value_seed(LimitSeed { + inner: seed, + remaining: self.remaining, + }) + } + + fn size_hint(&self) -> Option<usize> { + self.inner.size_hint() + } +} + +struct LimitEnumAccess<A> { + inner: A, + remaining: usize, +} + +impl<'de, A> de::EnumAccess<'de> for LimitEnumAccess<A> +where + A: de::EnumAccess<'de>, +{ + type Error = A::Error; + type Variant = LimitVariantAccess<A::Variant>; + + fn variant_seed<V>(self, seed: V) -> Result<(V::Value, Self::Variant), Self::Error> + where + V: DeserializeSeed<'de>, + { + let (value, variant) = self.inner.variant_seed(LimitSeed { + inner: seed, + remaining: self.remaining, + })?; + Ok(( + value, + LimitVariantAccess { + inner: variant, + remaining: self.remaining, + }, + )) + } +} + +struct LimitVariantAccess<A> { + inner: A, + remaining: usize, +} + +impl<'de, A> de::VariantAccess<'de> for LimitVariantAccess<A> +where + A: de::VariantAccess<'de>, +{ + type Error = A::Error; + + fn unit_variant(self) -> Result<(), Self::Error> { + self.inner.unit_variant() + } + + fn newtype_variant_seed<T>(self, seed: T) -> Result<T::Value, Self::Error> + where + T: DeserializeSeed<'de>, + { + self.inner.newtype_variant_seed(LimitSeed { + inner: seed, + remaining: self.remaining, + }) + } + + fn tuple_variant<V>(self, len: usize, visitor: V) -> Result<V::Value, Self::Error> + where + V: Visitor<'de>, + { + let remaining = self + .remaining + .checked_sub(1) + .ok_or_else(|| de::Error::custom("deserialization depth limit exceeded"))?; + self.inner.tuple_variant( + len, + LimitVisitor { + inner: visitor, + remaining, + }, + ) + } + + fn struct_variant<V>( + self, + fields: &'static [&'static str], + visitor: V, + ) -> Result<V::Value, Self::Error> + where + V: Visitor<'de>, + { + let remaining = self + .remaining + .checked_sub(1) + .ok_or_else(|| de::Error::custom("deserialization depth limit exceeded"))?; + self.inner.struct_variant( + fields, + LimitVisitor { + inner: visitor, + remaining, + }, + ) + } +} diff --git a/crates/fuzzing/src/lib.rs b/crates/fuzzing/src/lib.rs new file mode 100644 index 00000000..6727a013 --- /dev/null +++ b/crates/fuzzing/src/lib.rs @@ -0,0 +1,177 @@ +pub use mutatis; +pub use postcard; +pub use serde; + +mod depth_limit; + +/// Corpus files above this are refused rather than decoded. +pub const MAX_CORPUS_BYTES: usize = 8 * 1024; + +/// Maximum number of nested containers in a corpus value. +pub const MAX_CORPUS_DEPTH: usize = 128; + +/// Decodes a corpus file into the value a target consumes. A file AFL produced +/// by other means than the custom mutator will not decode, hence the `Option`. +pub fn decode<T: serde::de::DeserializeOwned>(bytes: &[u8]) -> Option<T> { + if bytes.len() > MAX_CORPUS_BYTES { + return None; + } + let mut deserializer = postcard::Deserializer::from_bytes(bytes); + serde::Deserialize::deserialize(depth_limit::Deserializer::new( + &mut deserializer, + MAX_CORPUS_DEPTH, + )) + .ok() +} + +/// Encodes a value back into the bytes AFL stores. Only fails when `T`'s +/// `Serialize` does, which for a fuzz input is a bug rather than an input error. +pub fn encode<T: serde::Serialize>(value: &T) -> Vec<u8> { + postcard::to_allocvec(value).expect("fuzz input must serialize") +} + +/// Defines the `afl_custom_*` entry points AFL++ dlopens, mutating `$ty` +/// structurally instead of editing its serialized bytes. +/// +/// Expand it in a crate with `crate-type = ["cdylib"]`, next to the `#[path]` +/// module that also defines the target's input type: +/// +/// ```ignore +/// #[path = "../../shared/leader-result-input.rs"] +/// mod input; +/// +/// genvm_fuzzing::mutator!(input::Input); +/// ``` +#[macro_export] +macro_rules! mutator { + ($ty:ty) => { + struct MutatorState { + session: $crate::mutatis::Session, + buf: ::std::vec::Vec<u8>, + calls: u64, + } + + /// How many mutations one call stacks. + /// + /// A session applies a single mutation, which on its own explores about + /// as far from a seed as one bit flip does. AFL's own havoc stage owes + /// much of its reach to stacking, and a structural mutator that does + /// not stack loses to it. + const MAX_STACKED: u64 = 8; + + #[no_mangle] + pub extern "C" fn afl_custom_init( + _afl: *mut ::core::ffi::c_void, + seed: u32, + ) -> *mut ::core::ffi::c_void { + let state = ::std::boxed::Box::new(MutatorState { + session: $crate::mutatis::Session::new().seed(seed as u64), + buf: ::std::vec::Vec::new(), + calls: 0, + }); + ::std::boxed::Box::into_raw(state) as *mut ::core::ffi::c_void + } + + /// # Safety + /// `data` is what [`afl_custom_init`] returned. + #[no_mangle] + pub unsafe extern "C" fn afl_custom_deinit(data: *mut ::core::ffi::c_void) { + drop(::std::boxed::Box::from_raw(data as *mut MutatorState)); + } + + /// # Safety + /// `buf` is readable for `buf_size` bytes and `out_buf` is writable, as + /// the AFL++ custom mutator API specifies. The returned buffer stays + /// alive until the next call, which is when AFL is done with it. + #[no_mangle] + pub unsafe extern "C" fn afl_custom_fuzz( + data: *mut ::core::ffi::c_void, + buf: *mut u8, + buf_size: usize, + out_buf: *mut *const u8, + _add_buf: *mut u8, + _add_buf_size: usize, + max_size: usize, + ) -> usize { + use $crate::mutatis::Mutate as _; + + let state = &mut *(data as *mut MutatorState); + let input = ::core::slice::from_raw_parts(buf, buf_size); + + // A seed AFL minimized, spliced or havoc'd is no longer a valid + // encoding; starting over from the default beats emitting garbage. + let mut value: $ty = $crate::decode(input).unwrap_or_default(); + + // splitmix64 over the call counter, so the count varies without a + // second source of randomness to seed and replay + state.calls = state.calls.wrapping_add(1); + let mut z = state.calls.wrapping_mul(0x9e37_79b9_7f4a_7c15); + z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + for _ in 0..=(z >> 32) % MAX_STACKED { + let _ = state.session.mutate(&mut value); + } + + state.buf = $crate::encode(&value); + + // Truncating would corrupt the encoding, so an oversized mutation + // is dropped in favour of the input it came from. + let max_size = max_size.min($crate::MAX_CORPUS_BYTES); + if state.buf.len() > max_size { + state.buf.clear(); + state + .buf + .extend_from_slice(&input[..input.len().min(max_size)]); + } + + *out_buf = state.buf.as_ptr(); + state.buf.len() + } + }; +} + +#[cfg(test)] +mod tests { + use super::{decode, encode, MAX_CORPUS_DEPTH}; + use serde::{Deserialize, Serialize}; + use std::collections::BTreeMap; + + #[derive(Debug, Deserialize, PartialEq, Serialize)] + struct Mixed { + values: Vec<Option<u16>>, + named: BTreeMap<String, bool>, + } + + #[derive(Debug, Deserialize, PartialEq, Serialize)] + enum Nested { + More(Box<Nested>), + End, + } + + fn nested(depth: usize) -> Nested { + (0..depth).fold(Nested::End, |inner, _| Nested::More(Box::new(inner))) + } + + #[test] + fn decode_preserves_normal_values() { + let value = Mixed { + values: vec![Some(42), None], + named: BTreeMap::from([("answer".to_owned(), true)]), + }; + + assert_eq!(decode(&encode(&value)), Some(value)); + } + + #[test] + fn decode_accepts_bounded_nesting() { + let value = nested(MAX_CORPUS_DEPTH - 1); + + assert_eq!(decode(&encode(&value)), Some(value)); + } + + #[test] + fn decode_rejects_excessive_nesting() { + let bytes = encode(&nested(MAX_CORPUS_DEPTH)); + + assert_eq!(decode::<Nested>(&bytes), None); + } +} diff --git a/crates/modules-interfaces/Cargo.lock b/crates/modules-interfaces/Cargo.lock index 433aeb5d..d218bf78 100644 --- a/crates/modules-interfaces/Cargo.lock +++ b/crates/modules-interfaces/Cargo.lock @@ -41,12 +41,6 @@ version = "1.0.97" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f" -[[package]] -name = "arbitrary" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" - [[package]] name = "arrayvec" version = "0.7.6" @@ -281,7 +275,6 @@ name = "genvm_modules_interfaces" version = "0.1.0" dependencies = [ "anyhow", - "arbitrary", "bytes", "chrono", "genlayer_calldata", diff --git a/crates/modules-interfaces/Cargo.toml b/crates/modules-interfaces/Cargo.toml index 63a37027..9d5122e5 100644 --- a/crates/modules-interfaces/Cargo.toml +++ b/crates/modules-interfaces/Cargo.toml @@ -4,12 +4,10 @@ version = "0.1.0" edition = "2021" [features] -arbitrary = ["dep:arbitrary"] default = [] [dependencies] anyhow = "1.0.97" -arbitrary = { version = "1.0", optional = true } bytes = { version = "1.11.0", features = ["serde"] } chrono = "0.4.45" genlayer_calldata = { path = "../../executors/v0.3.x/executor/crates/calldata" } diff --git a/crates/modules-interfaces/codegen/data/host-fns.json b/crates/modules-interfaces/codegen/data/host-fns.json index 245044ac..e9d9e0e5 100644 --- a/crates/modules-interfaces/codegen/data/host-fns.json +++ b/crates/modules-interfaces/codegen/data/host-fns.json @@ -15,6 +15,18 @@ "run_nested": 8 } }, + { + "type": "enum", + "repr": "u8", + "name": "result_code", + "values": { + "return": 0, + "user_error": 1, + "vm_error": 2, + "internal_error": 3, + "fatal_vm_error": 4 + } + }, { "type": "enum", "repr": "u8", @@ -25,14 +37,6 @@ "forbidden": 2 } }, - { - "type": "str_trie", - "name": "vm_error_detail", - "values": [ - "internal", - "external" - ] - }, { "type": "const", "name": "current_major", diff --git a/crates/modules-interfaces/src/abi_stub.rs b/crates/modules-interfaces/src/abi_stub.rs index e829c327..c0dd0892 100644 --- a/crates/modules-interfaces/src/abi_stub.rs +++ b/crates/modules-interfaces/src/abi_stub.rs @@ -11,7 +11,6 @@ use genlayer_calldata::Address; Eq, Debug, )] -#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] pub enum On { #[serde(rename = "finalized")] #[calldata(rename = "finalized")] @@ -94,38 +93,3 @@ fn default_datetime() -> chrono::DateTime<chrono::Utc> { .unwrap() .to_utc() } - -#[cfg(feature = "arbitrary")] -impl<'a> arbitrary::Arbitrary<'a> for MessageData { - fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result<Self> { - use arbitrary::Arbitrary; - - let ts = u32::arbitrary(u)?; - let Some(datetime) = chrono::DateTime::<chrono::Utc>::from_timestamp_secs(ts as i64) else { - return Err(arbitrary::Error::NotEnoughData); - }; - - let chain_id_bytes: [u8; 32] = Arbitrary::arbitrary(u)?; - let chain_id = primitive_types::U256::from_big_endian(&chain_id_bytes); - - let value_bytes: [u8; 32] = Arbitrary::arbitrary(u)?; - let value = primitive_types::U256::from_big_endian(&value_bytes); - - Ok(Self { - contract_address: Arbitrary::arbitrary(u)?, - sender_address: Arbitrary::arbitrary(u)?, - origin_address: Arbitrary::arbitrary(u)?, - signer_address: Arbitrary::arbitrary(u)?, - chain_id: num_bigint::BigInt::from_bytes_be( - num_bigint::Sign::Plus, - &chain_id.to_big_endian(), - ), - value: num_bigint::BigInt::from_bytes_be( - num_bigint::Sign::Plus, - &value.to_big_endian(), - ), - is_init: bool::arbitrary(u)?, - datetime, - }) - } -} diff --git a/crates/modules-interfaces/src/domain.rs b/crates/modules-interfaces/src/domain.rs index aec09051..a083b58c 100644 --- a/crates/modules-interfaces/src/domain.rs +++ b/crates/modules-interfaces/src/domain.rs @@ -152,6 +152,7 @@ pub enum ResultCode { UserError = 1, VmError = 2, InternalError = 3, + FatalVmError = 4, } #[derive(Debug, Clone, serde::Serialize)] @@ -347,9 +348,12 @@ where enc.push_str(match kind { ResultCode::Return => "Return", ResultCode::UserError => "UserError", - // An internal error is never handed to a caller as a result, so it - // shares the `VMError` label rather than inventing a fourth one. - ResultCode::VmError | ResultCode::InternalError => "VMError", + // Neither an internal error nor fatality is part of the outcome + // itself: the first is never handed to a caller as a result, and the + // second only says the caller may not catch it. The value folded is + // the same `vm_error` either way, so a timeout hashes alike whether + // or not the line that served it can raise fatality. + ResultCode::VmError | ResultCode::InternalError | ResultCode::FatalVmError => "VMError", })?; enc.push_map_k("result")?; diff --git a/crates/modules-interfaces/src/domain/fees/mod.rs b/crates/modules-interfaces/src/domain/fees/mod.rs index 0119dcc9..c0efedde 100644 --- a/crates/modules-interfaces/src/domain/fees/mod.rs +++ b/crates/modules-interfaces/src/domain/fees/mod.rs @@ -14,13 +14,9 @@ use crate::On; genlayer_calldata::Encode, genlayer_calldata::Decode, )] -#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] pub struct InternalMessageParams { - #[cfg_attr(feature = "arbitrary", arbitrary(with = crate::abi::arb::arb_u256))] pub leader_timeunits_allocation: U256, - #[cfg_attr(feature = "arbitrary", arbitrary(with = crate::abi::arb::arb_u256))] pub validator_timeunits_allocation: U256, - #[cfg_attr(feature = "arbitrary", arbitrary(with = crate::abi::arb::arb_u256))] pub execution_budget_per_round: U256, /// Per-round rotation allocations; `rotations[0]` is the initial round, the /// rest are appeal rounds. Must be non-empty. @@ -31,21 +27,17 @@ pub struct InternalMessageParams { /// so we derive `appeal_rounds = rotations.len() - 1` instead. The ABI /// encoder re-inserts it so the encoded bytes (and their `keccak256` fee-param /// pin) match the chain's field layout. - #[cfg_attr(feature = "arbitrary", arbitrary(with = crate::abi::arb::arb_vec_u256))] pub rotations: Vec<U256>, /// Per-time-unit GEN price cap locked at activation (consensus CON-549, /// v0.6-dev). The chain charges at this cap as the funding multiplier and /// cancels the tx if the global price exceeds it; `MessagePayments` requires /// it to be non-zero for internal messages. - #[cfg_attr(feature = "arbitrary", arbitrary(with = crate::abi::arb::arb_u256))] pub max_price_gen_per_time_unit: U256, /// Max gas price applied to the storage-fee component (v0.6-dev). Revert /// guard in `_calculateRoundFees`; must be non-zero for internal messages. - #[cfg_attr(feature = "arbitrary", arbitrary(with = crate::abi::arb::arb_u256))] pub storage_fee_max_gas_price: U256, /// Max gas price applied to the receipt-fee component (v0.6-dev). Revert /// guard in `_calculateRoundFees`; must be non-zero for internal messages. - #[cfg_attr(feature = "arbitrary", arbitrary(with = crate::abi::arb::arb_u256))] pub receipt_fee_max_gas_price: U256, } @@ -110,6 +102,7 @@ impl MessageAllocationNode { abi::encode(roots) } + #[allow(clippy::if_same_then_else)] pub fn matches_internal( &self, on: On, @@ -130,6 +123,7 @@ impl MessageAllocationNode { } } + #[allow(clippy::if_same_then_else)] pub fn matches_external( &self, recipient: genlayer_calldata::Address, diff --git a/crates/modules-interfaces/src/host_fns.rs b/crates/modules-interfaces/src/host_fns.rs index d3cfcbe1..ae6be202 100644 --- a/crates/modules-interfaces/src/host_fns.rs +++ b/crates/modules-interfaces/src/host_fns.rs @@ -1,11 +1,9 @@ // This file is auto-generated. Do not edit! -#![allow(dead_code, clippy::redundant_static_lifetimes)] +#![allow(dead_code, clippy::all)] use serde::{Deserialize, Serialize}; -use std::borrow::Cow; - #[derive( Debug, PartialEq, @@ -88,6 +86,61 @@ impl TryFrom<u8> for Methods { ::genlayer_calldata::Decode, )] #[repr(u8)] +pub enum ResultCode { + Return = 0, + UserError = 1, + VmError = 2, + InternalError = 3, + FatalVmError = 4, +} + +impl ResultCode { + pub const SIZE: usize = 5; + pub fn value(self) -> u8 { + match self { + ResultCode::Return => 0, + ResultCode::UserError => 1, + ResultCode::VmError => 2, + ResultCode::InternalError => 3, + ResultCode::FatalVmError => 4, + } + } + pub fn str_snake_case(self) -> &'static str { + match self { + ResultCode::Return => "return", + ResultCode::UserError => "user_error", + ResultCode::VmError => "vm_error", + ResultCode::InternalError => "internal_error", + ResultCode::FatalVmError => "fatal_vm_error", + } + } +} + +impl TryFrom<u8> for ResultCode { + type Error = (); + + fn try_from(value: u8) -> Result<Self, ()> { + match value { + 0 => Ok(ResultCode::Return), + 1 => Ok(ResultCode::UserError), + 2 => Ok(ResultCode::VmError), + 3 => Ok(ResultCode::InternalError), + 4 => Ok(ResultCode::FatalVmError), + _ => Err(()), + } + } +} +#[derive( + Debug, + PartialEq, + Clone, + Copy, + Serialize, + Deserialize, + ::genlayer_calldata::Encode, + ::genlayer_calldata::Decode, +)] +#[repr(u8)] pub enum Errors { Ok = 0, EvmReverted = 1, @@ -124,34 +177,6 @@ impl TryFrom<u8> for Errors { } } } -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] -pub struct VmErrorDetail(pub Cow<'static, str>); - -impl From<VmErrorDetail> for String { - fn from(val: VmErrorDetail) -> String { - val.0.into() - } -} -#[rustfmt::skip] -impl VmErrorDetail { - pub const fn internal() -> Self { Self(Cow::Borrowed("internal")) } - pub const fn external() -> Self { Self(Cow::Borrowed("external")) } -} - -#[rustfmt::skip] -impl VmErrorDetail { - /// Whether `s` is a well-formed `vm_error_detail` path. - pub fn is_valid_(s: &str) -> bool { - if matches!(s, - "internal" | - "external" - ) { - return true; - } - false - } -} - pub const CURRENT_MAJOR: u8 = 0; pub const CURRENT_MAJOR_STR: &'static str = "v0.0.0"; diff --git a/crates/modules-interfaces/src/manager_api.rs b/crates/modules-interfaces/src/manager_api.rs index 8b46009a..aa87d711 100644 --- a/crates/modules-interfaces/src/manager_api.rs +++ b/crates/modules-interfaces/src/manager_api.rs @@ -1,6 +1,6 @@ // This file is auto-generated. Do not edit! -#![allow(dead_code, clippy::redundant_static_lifetimes)] +#![allow(dead_code, clippy::all)] use serde::{Deserialize, Serialize}; diff --git a/crates/modules-interfaces/src/nested.rs b/crates/modules-interfaces/src/nested.rs index 57021cf8..94577089 100644 --- a/crates/modules-interfaces/src/nested.rs +++ b/crates/modules-interfaces/src/nested.rs @@ -249,9 +249,11 @@ mod tests { assert_ne!(hash(ResultCode::Return), hash(ResultCode::UserError)); assert_ne!(hash(ResultCode::Return), hash(ResultCode::VmError)); - // An internal error is never a caller-visible outcome, so it shares the - // `VMError` label rather than adding a kind a peer line might not know. + // Neither an internal error nor fatality changes the outcome value, so + // both fold as a plain `VMError` rather than adding a kind a peer line + // might not know. assert_eq!(hash(ResultCode::VmError), hash(ResultCode::InternalError)); + assert_eq!(hash(ResultCode::VmError), hash(ResultCode::FatalVmError)); } #[test] diff --git a/docs/adr/012. runner memory accounting.md b/docs/adr/012. runner memory accounting.md index 0571d2aa..db235999 100644 --- a/docs/adr/012. runner memory accounting.md +++ b/docs/adr/012. runner memory accounting.md @@ -221,4 +221,4 @@ surface comes later in a runner release. point. - Spec: gl_call payloads, the permissions note, the runners page (load action, loaded set, custom lifetime), the RAM-limiting page (charge rule), - and `memory_limiter_consts.runner_load_cost` in the public ABI constants. + and `memory_limiter_consts.runner_load_cost` in the internal constants. diff --git a/docs/adr/013. pre-validating untrusted decoded inputs.md b/docs/adr/013. pre-validating untrusted decoded inputs.md index 229d36ef..1317889b 100644 --- a/docs/adr/013. pre-validating untrusted decoded inputs.md +++ b/docs/adr/013. pre-validating untrusted decoded inputs.md @@ -44,27 +44,28 @@ and handle its failure correctly. One total function, `parse_leader_result`, a pure function of the proposed bytes so every honest node derives the same result: -1. **Absent or empty** β†’ `absent_leader_nondet_output`. (Empty must equal +1. **Absent or empty** β†’ `leader_fault nondet_output absent`. (Empty must equal absent: `Supervisor::push_nondet_result` pads gaps with empty `Bytes`.) 2. Result code not in `Return`(0)/`UserError`(1)/`VmError`(2) β†’ - `leader_output malformed`. This includes `InternalError`(3): the enum admits - it but the wire format does not. + `leader_fault nondet_output malformed`. This includes `InternalError`(3): + the enum admits it but the wire format does not. 3. **`Return`/`UserError`**: the payload must satisfy `calldata::decode` - (canonical, depth ≀ 128, no trailing bytes) else `leader_output malformed`. - Accepted bytes are preserved **exactly** β€” validation never re-encodes, or - validators would hash a value other than the one proposed. + (canonical, depth ≀ 128, no trailing bytes) else + `leader_fault nondet_output malformed`. Accepted bytes are preserved + **exactly** β€” validation never re-encodes, or validators would hash a value + other than the one proposed. 4. **`VmError`**: the payload must be UTF-8, carry **no ` # ` detail** (Rule 1b: an honest leader never publishes one), then, *in order*: 1. If the code is in the **derived-outcome namespace** (equals or - space-extends `absent_leader_nondet_output` or `leader_output`), it is - replaced by `leader_output uses_this_error <h>`. Checked *before* validity - so proposing the literal `leader_output malformed` cannot echo verbatim. + space-extends `leader_fault nondet_output`), it is replaced by + `leader_fault nondet_output uses_this_error <h>`. Checked *before* + validity so proposing the literal `leader_fault nondet_output malformed` + cannot echo verbatim. 2. Otherwise the code must be a valid `vm_error` trie path, including the canonical spelling of any parameter (`exit_code +7`/`007` are rejected β€” same `i32`, different bytes, and codes reach the hash). The check is **generated** from the trie data, so it cannot drift. Fail β†’ - `leader_output malformed`. Pending-ABI codes (below) are not trie paths, - hence not proposable. + `leader_fault nondet_output malformed`. The parsed-or-substituted result then flows through the **standard consensus comparison** β€” handed to the contract's validator function like any proposal. @@ -75,17 +76,19 @@ directly is removed. Fees (`consume_nondet_output`) are charged on the accepted/substituted result's **own** length β€” rejected bytes never enter the result or the hash. -**Derived-outcome namespace.** Validators derive `absent_leader_nondet_output`, -`leader_output malformed`, `leader_output uses_this_error <h>` and -`leader_output extra <h>` themselves. A proposal must never be byte-equal to -what a validator derives from rejecting it, or "leader proposed X" and -"proposal rejected as X" become indistinguishable (namespace hygiene, not +**Derived-outcome namespace.** Validators derive +`leader_fault nondet_output absent`, `leader_fault nondet_output malformed`, +`leader_fault nondet_output uses_this_error <h>` and +`leader_fault nondet_output extra <h>` themselves. A proposal must never be +byte-equal to what a validator derives from rejecting it, or "leader proposed X" +and "proposal rejected as X" become indistinguishable (namespace hygiene, not anti-spoofing β€” a dishonest leader controls its own hash regardless). So an -in-namespace proposal maps to `leader_output uses_this_error <h>`, `<h>` = first -6 chars of `gvm32(sha3_256(proposed code))`. A self-replacing fixed point -(~2³⁰ brute force) maps to the sentinel `…uses_this_error fix_point`, which -can't collide with an `<h>` (9 chars vs always 6). `<h>` collisions are -harmless β€” the value carries no meaning. +in-namespace proposal maps to +`leader_fault nondet_output uses_this_error <h>`, `<h>` = first 6 chars of +`gvm32(sha3_256(proposed code))`. A self-replacing fixed point (~2³⁰ brute +force) maps to the sentinel `…uses_this_error fix_point`, which can't collide +with an `<h>` (9 chars vs always 6). `<h>` collisions are harmless β€” the value +carries no meaning. ### Rule 1.5 β€” surplus leader results (`Validator` and `Sync`) @@ -94,14 +97,15 @@ run reaching a block with no result is Rule 1.1; the complement β€” a leader publishing **more** results than the run consumed β€” was silently tolerated, hiding a leader whose execution diverged into extra blocks. Now: if the leader supplied more results than blocks run, the run's result is replaced by -`leader_output extra <h>`, `<h>` = first 6 chars of `gvm32(sha3_256(b))` where -`b` is the **full `[result_code][data]` buffer** the run would otherwise have -returned (`RunOk::as_bytes()`) β€” fingerprinting the discarded result rather -than losing it. This is **not** a non-deterministic disagreement (it's a -property of the counts, not a re-executed block). No fix-point escape is needed: -a `leader_output` code is never accepted verbatim (Rule 1.4), so this can't be -byte-equal to a proposal. The outcome depends only on the two counts, so *when* -the check runs (after the block count is final) is unspecified. +`leader_fault nondet_output extra <h>`, `<h>` = first 6 chars of +`gvm32(sha3_256(b))` where `b` is the **full `[result_code][data]` buffer** the +run would otherwise have returned (`RunOk::as_bytes()`) β€” fingerprinting the +discarded result rather than losing it. This is **not** a non-deterministic +disagreement (it's a property of the counts, not a re-executed block). No +fix-point escape is needed: a `leader_fault nondet_output` code is never +accepted verbatim (Rule 1.4), so this can't be byte-equal to a proposal. The +outcome depends only on the two counts, so *when* the check runs (after the +block count is final) is unspecified. ### Rule 1b β€” publishing the leader's own result (`Leader`) @@ -160,15 +164,12 @@ Rule 1b governs `Leader`. ### Where the constants live -`malformed_entry`, `leader_output malformed`, `leader_output uses_this_error -{str}` and `leader_output extra {str}` go in the **pending** ABI data -(`executor/codegen/data/public-abi-pending.json`), the -[ADR-012](<012. runner memory accounting.md>) mechanism for public-ABI -constants that must not regenerate the runner-hashed `public_abi.py`. Generated -into `common` only β†’ hash-neutral. Cost: the `vm_error` taxonomy temporarily -lives in two shapes; fold the pending codes into the trie at the next -hash-moving release train. Until then the generated validity check rejects them, -which is what makes them unproposable under Rule 1.4. +`malformed_entry`, `leader_fault nondet_output malformed`, +`leader_fault nondet_output uses_this_error {str}` and +`leader_fault nondet_output extra {str}` live in the public ABI trie. The +generated validity check rejects the leader-fault nondet-output subtree for +leader proposals because it is executor-derived, which is what makes those +codes unproposable under Rule 1.4. ## Consequences @@ -182,14 +183,12 @@ leader results collapse into one total parse feeding one comparison path. **Negative.** Consensus-visible behaviour change (results/calls accepted before are now rejected; nondet `VMError` loses its ` # detail`); ships with a version -bump ([ADR-010](<010. versions.md>)), runner hashes unmoved thanks to the -pending mechanism. Rust-SDK contracts relying on `MainCalldata::parse`'s lenient -defaulting (non-Map payload, wrong-typed `args`) now get `malformed_entry`. A -`""` key on a deployment is now an error (was silently ignored on init) β€” eight -integration fixtures sent one and their goldens move to `malformed_entry`. The -`vm_error` taxonomy is split until the next hash-moving train. Validating -`Return` costs one decode pass over host-bounded leader bytes on the validator -path. +bump ([ADR-010](<010. versions.md>)). Rust-SDK contracts relying on +`MainCalldata::parse`'s lenient defaulting (non-Map payload, wrong-typed `args`) +now get `malformed_entry`. A `""` key on a deployment is now an error (was +silently ignored on init) β€” eight integration fixtures sent one and their +goldens move to `malformed_entry`. Validating `Return` costs one decode pass +over host-bounded leader bytes on the validator path. **Neutral.** A leader still has an arbitrary-*content* channel via a *well-formed* `Return`/`UserError` (inherent β€” the rules bound shape, not @@ -250,8 +249,7 @@ calldata, extra keys or observable nondet details. - Spec: proposed-result rules and derived namespace in `subvm-result-encoding`; `03-vm/04-determinism-mode-switching.rst` covers all three modes; Rule 2 on the calling-convention page with its failure surfaces in `03-vm/01-startup.rst` - and under `CallContract`. The rst backend gained `--rst-anchor-ns` so the - pending appendix does not redefine the real `gvm-def-str-trie-vm-error` label. + and under `CallContract`. ### Deferred @@ -260,5 +258,3 @@ calldata, extra keys or observable nondet details. tree and would move hashes. Rule 1 makes the guard defensive, not load-bearing. - **`message.rs` `CallKey::UNNAMED` fallback** on a decode failure β€” should be an error. -- **Folding the pending codes into the `vm_error` trie** at the next hash-moving - train. diff --git a/docs/adr/015. cross-major contract calls.md b/docs/adr/015. cross-major contract calls.md index eff91c2e..08c75fc1 100644 --- a/docs/adr/015. cross-major contract calls.md +++ b/docs/adr/015. cross-major contract calls.md @@ -141,7 +141,7 @@ computed by the caller and carried on the wire. The table is complete against | `method_hosts` | Constructed by the manager: host 0 by default, `consume_result` and `run_nested` on host 1. | | `host_hello_data` | Copied from the outer request; entry 1 is cleared for the new manager socketpair. | | Effectful and nondeterministic allocations | Cleared: `leader_nondet_results` and `record_actions` are empty because a deterministic child produces neither; `bucket_totals`, `message_fee_allocation` and `initial_time_units_allocation` are empty or zero because the child imports its remaining deterministic budget instead of being granted a fresh one. | -| `remaining_recursion` | Parent budget minus one. Carried as a remainder, not a spent count, so the bound stays the one the chain's root minted even when the lines disagree on `VM_RECURSION` -- but the receiving executor clamps it to its own limit, since a remainder is not an authority to exceed one. Also set, for a chain root, by the `initial_recursion` debug override. | +| `remaining_recursion` | Parent budget minus one. Carried as a remainder, not a spent count, so the bound stays the one the chain's root minted even when the lines disagree on `VM_RECURSION` -- but the receiving executor clamps it to its own limit, since a remainder is not an authority to exceed one. The manager clamps it again, to `CROSS_MAJOR_RECURSION` (6), so a delegated chain is bounded far more tightly than an in-process one: a hop costs a whole executor process, and the manager is the only party that sees the chain as a whole. The clamp also bounds the subtree below a delegated call, which is intended -- a chain that has already crossed a boundary does not get to recurse deeply in-process either. Also set, for a chain root, by the `initial_recursion` debug override. | | `nested.memory_limit` | The derived limiter's remaining memory, made an explicit process-spawn limit. | | `nested.stack` | Caller stack with the caller contract appended. | | `nested.permissions` | `CallContract`-derived permissions. Unknown envelope bits decode as denied. | diff --git a/docs/contributing/explanation/README.md b/docs/contributing/explanation/README.md index 918220f7..a7ab0ac3 100644 --- a/docs/contributing/explanation/README.md +++ b/docs/contributing/explanation/README.md @@ -5,6 +5,7 @@ lives in the spec, and these pages only motivate it and link out - [docs-layout.md](docs-layout.md) β€” the 4 kinds of page and where each belongs - [executor-lines.md](executor-lines.md) β€” why several executor lines ship side by side, and what it costs +- [fuzz.md](fuzz.md) β€” why fuzz targets get fake entropy and no CmpLog - [merge-model.md](merge-model.md) β€” why merging is a maintainer panel rather than a merge queue - [shared-submodule-cache.md](shared-submodule-cache.md) β€” why submodules are worktrees of one cache repo, not clones - [vendored-trees.md](vendored-trees.md) β€” why third-party sources are patch series, not forks diff --git a/docs/contributing/explanation/fuzz.md b/docs/contributing/explanation/fuzz.md new file mode 100644 index 00000000..d9e7015f --- /dev/null +++ b/docs/contributing/explanation/fuzz.md @@ -0,0 +1,39 @@ +# Why Fuzzing Is Set Up This Way + +## Fake Entropy + +A fuzz target is only useful if a saved crash reproduces. Real entropy breaks +that: `std` seeds tokio's scheduler and every `HashMap` from `getrandom`, so two +runs over one input take different task orders and different bucket layouts, and +an input AFL saved as a crash need not crash again + +So fuzz cases β€” and no other test β€” run with `crates/fuzzing/preload` in +`AFL_PRELOAD`, a shim whose `getrandom`/`getentropy` hand out a fixed sequence. +AFL turns that into an `LD_PRELOAD` for the target alone, which is why it is a +preload rather than a patch: the seeding happens in `std`, below any code this +repository owns, and interposing the symbol is the only place to stand + +`glibc` seeds its allocator from inside the loader, before a preloaded symbol can +win, so that call stays random β€” it perturbs heap layout only, and AFL already +disables ASLR. The mechanism also assumes a dynamically linked target: a static +or `musl` build would ignore the preload and say nothing, so the entropy would +come back silently rather than as a failure + +## No CmpLog + +CmpLog logs both operands of every comparison, letting AFL write a magic value +straight into an input instead of guessing it. It is the standard answer for a +parser gated on tags and headers, and it does nothing for us + +Our targets are `arbitrary`-driven: input bytes are *interpreted* to construct a +value, not compared against constants, so there is almost nothing for CmpLog to +log. Measured over 60s per side, `genvm-common-encode` found 1791 edges either +way, and `genvm-common-decode` 1782 against 1785 β€” noise + +The cost is not noise. CmpLog is a second instrumentation, and AFL wants it as a +separate binary so the fuzzing loop stays fast, which means building the whole +dependency graph twice. For one crate that came to 1.2G + +Worth revisiting for a target that parses raw bytes rather than driving +`arbitrary` β€” `runners-parse` reads real zip data and was never measured, since +building its second graph pulls in wasmtime diff --git a/docs/contributing/howto/README.md b/docs/contributing/howto/README.md index 7988ef81..7a5349b8 100644 --- a/docs/contributing/howto/README.md +++ b/docs/contributing/howto/README.md @@ -17,7 +17,7 @@ - [rust.md](testing/rust.md) β€” Rust tests: where they go, how to run them, coverage - [python.md](testing/python.md) β€” Python tests, direct pytest for genlayer-py-std - [integration.md](testing/integration.md) β€” jsonnet cases, tags, golden `.stdout`/`.hash` files, services -- [fuzzing.md](testing/fuzzing.md) β€” AFL fuzz targets, host sysctl prep +- [fuzzing.md](testing/fuzzing.md) β€” AFL fuzz targets, seeding a corpus from a run, host sysctl prep ## committing/ - [submodules.md](committing/submodules.md) β€” repo topology, gitlink bumps, pre-commit hooks, push order diff --git a/docs/contributing/howto/genvm-tool.md b/docs/contributing/howto/genvm-tool.md index 3fbd80d6..8df682b6 100644 --- a/docs/contributing/howto/genvm-tool.md +++ b/docs/contributing/howto/genvm-tool.md @@ -13,23 +13,38 @@ has no wrapper | `codegen` | render a data JSON into rust/python/rst/go | | `build-manifest` | write the manager's `data/manifest.yaml` | | `docs` | render the `docs/contributing/` indexes, the source of the list in `CLAUDE.md` | +| `fuzz-corpus` | seed the fuzz corpora from a finished run ([testing/fuzzing.md](testing/fuzzing.md)) | | `git` | `ls`, `list-repo`, `create-branches`, `check-for-push` across the manager and every executor submodule ([submodules.md](committing/submodules.md)) | ## Codegen `ninja -C build codegen` regenerates from data JSON; never edit an output by -hand. Host protocol data is shared, public ABI data comes from the primary line +hand. Host protocol data is shared, per-line data lives in +`executors/<line>.x/executor/codegen/data/` | Data | Generated | |---|---| | `crates/modules-interfaces/codegen/data/host-fns.json` | `…/src/host_fns.rs`, `tests/runner/origin/host_fns.py` | | `crates/modules-interfaces/codegen/data/manager-api.json` | `…/src/manager_api.rs`, `tests/runner/origin/manager_api.py`, `docs/website/src/impl-spec/appendix/manager-socket-consts.rst` | -| `executors/<primary>.x/executor/codegen/data/public-abi.json` | `tests/runner/origin/public_abi.py`, `docs/website/src/spec/appendix/constants.rst` | + +Per line, with the manager-global outputs taken from the primary line only: + +| Data | Per-line output | From the primary line only | +|---|---|---| +| `public-abi.json` | `executor/crates/sdk-rs/src/abi/consts.rs`, `runners/genlayer-py-std/src/genlayer/vm/public_abi.py` | `tests/runner/origin/public_abi.py`, `docs/website/src/spec/appendix/constants.rst` | +| `public-abi-pending.json` | `executor/crates/common/src/public_abi_pending.rs` | `docs/website/src/spec/appendix/constants-pending.rst` | +| `internal-constants.json` | `executor/crates/common/src/internal_constants.rs` | `docs/website/src/spec/appendix/internal-constants.rst` | ## Suite Layout The suite is `tests(ctx)` in the root `.genvm-tool.py`, which registers collectors. Plugins in `tests/runner/genvm_tool_plugins/` are plain importable modules, with no runner-specific registry. Per-suite definitions live next to -the tests in `tests/system/<name>/test.py`, pulled in with `ctx.collect_dir`. -Each executor line carries its own `.genvm-tool.py` +the tests in `tests/system/<name>/test.py`, pulled in with `ctx.collect_dir`, or +reuse `integration_test_directory` for contract-execution Jsonnet. Each +executor line carries its own `.genvm-tool.py` + +Services that bind the same listener declare the same `ctx.new_semaphore` +value. The scheduler finishes a service and its dependent cases before starting +the next service holding that semaphore; `genvm-tool test show plan` displays +the resulting lifetime boundaries diff --git a/docs/contributing/howto/testing/README.md b/docs/contributing/howto/testing/README.md index 0e5886f3..b8429cb0 100644 --- a/docs/contributing/howto/testing/README.md +++ b/docs/contributing/howto/testing/README.md @@ -20,7 +20,8 @@ subcommand Presets in `tests/presets/`: `release.txt` is `integration & stable & !bench`, `rust.txt` is `(rust | integration) & !bench & !fuzz`, `rust-fuzz.txt` is -`rust & fuzz`, `python.txt` is `python`. `genvm-tool test show tags` lists every +`rust & fuzz`, `python.txt` is `python & !needs-fuzz`, `python-fuzz.txt` is +`python & needs-fuzz`. `genvm-tool test show tags` lists every tag, the integration ones are in [integration.md](integration.md) ## Fix–Rerun Loop @@ -37,6 +38,8 @@ genvm-tool test run --filter-continue <name-or-path> Rust from every tracked `Cargo.toml` and its `fuzz/*.rs`, python from `executors/v0.3.x/runners/genlayer-py-std`, integration from each line's -`tests/integration/`, system from `tests/system/<name>/test.py`. Collection is -wired in the root `.genvm-tool.py`, runner configuration in +`tests/integration/`, and system from `tests/system/`. System suites use the +same Jsonnet collector when they execute contracts; protocol-level assertions +use `test.py`, which must register 1 `genvm-tool` case per assertion. Collection +is wired in the root `.genvm-tool.py`, runner configuration in `.genvm-monorepo-root` ([genvm-tool.md](../genvm-tool.md)) diff --git a/docs/contributing/howto/testing/fuzzing.md b/docs/contributing/howto/testing/fuzzing.md index 5901182f..621697e6 100644 --- a/docs/contributing/howto/testing/fuzzing.md +++ b/docs/contributing/howto/testing/fuzzing.md @@ -1,17 +1,165 @@ # Fuzzing -Rust fuzz targets run under AFL (`cargo-afl`); a target is any `fuzz/*.rs` next -to a tracked `Cargo.toml`: +Rust and Python fuzz targets run under AFL. A Rust target is any `fuzz/*.rs` +next to a tracked `Cargo.toml`; a Python target is any `fuzz/src/*.py` in a +collected Python project: ```bash -genvm-tool test run --filter-tag "$(cat tests/presets/rust-fuzz.txt)" # rust & fuzz +genvm-tool test run --filter-tag "$(cat tests/presets/rust-fuzz.txt)" # rust & fuzz +genvm-tool test run --filter-tag "$(cat tests/presets/python-fuzz.txt)" # python & needs-fuzz ``` -Python fuzz targets exist, but their collection is currently disabled in -`.genvm-tool.py`. Some integration cases carry the `fuzz` tag too +A corpus is committed next to its target: `fuzz/inputs-<target>` for Rust, +`fuzz/inputs/<target>` for Python. Beside it, `<corpus>-curated` holds seeds +written by hand β€” see [Curated Seeds](#curated-seeds) -Options: `--fuzz-timeout N` (seconds per run, default 30) and -`--fuzz-update-corpus` (write interesting inputs back) +A generated entry is named `gvm32(sha3_224(input))` and stored under the first +two characters of that name, `<corpus>/<n0>/<n1>/<name>`, so a corpus of +thousands of entries stays a tree of small directories. Reading takes an entry +wherever it sits, so entries an older run left directly in the corpus dir are +still used; a corpus update rewrites the whole dir and shards them + +`needs-fuzz` is the tag to select on: it means the case drives AFL and wants the +host prepared as below, whereas the plain `fuzz` tag some integration cases carry +only says the case is randomized. The CI cell that runs the AFL cases is `other` + +Python targets are collected on x86_64 Linux only β€” the pinned python-afl and +AFL++ are packaged for that platform alone + +The shared AFL test plugin owns the process fleet for both languages. Options: +`--fuzz-timeout D` (per run, a duration such as `30s` or `5m`, default `30s`), +`--fuzz-concurrent N` (targets side by side, default: cores minus two), +`--fuzz-jobs N` (processes per target, default: the cores `--fuzz-concurrent` +leaves over, so one), and `--fuzz-update-corpus` (write interesting inputs back) + +A fleet is N instances sharing one output dir; they only exchange corpora every +`AFL_SYNC_TIME` (20 minutes, halved for the `-M` instance), so below that it is N +independent searches β€” still worth it, since every instance's crashes are +collected. That is why the default spends cores across targets rather than within +one: a second instance of a target re-searches what the first one covers, while +another target covers code neither reaches + +AFL's live UI needs the terminal to itself, so it is only shown when a single +target runs at a time (`--fuzz-concurrent 1`). Otherwise every instance writes to +`<output dir>/<instance>.log` and the case reports the closing `afl-whatsup` +summary + +`--fuzz-concurrent` sizes the fleet; what physically bounds overlap is the case +pool, `--max-concurrent` (default: the core count). A fuzz case holds one permit +of it per fuzzer process it starts + +A crash is copied into the target's generated corpus (`fuzz/inputs-<target>`, +named as a corpus entry) as soon as it is reported, so every later run replays +it β€” the next run wipes the crash dir it was saved in. Commit it along with the +fix; put it in `<corpus>-curated` instead if a corpus update must never drop it + +Findings of both languages live under `build/test-artifacts/fuzz/<project>/`. +Reruns resume from that dir (`AFL_AUTORESUME`) and therefore ignore `-i`: remove +it to pick up a corpus that changed. Corpus updates `cmin` the fleet's queues +and replace the committed corpus with what survives, laid out as above β€” so the +first update of a corpus moves every file it keeps + +## Curated Seeds + +`cmin` judges an input by the edges it reaches, so a seed written to pin a shape +down β€” the one input covering a format's corner β€” is dropped as soon as some +havoc'd blob happens to cover the same edges. Put such an input in +`<corpus>-curated` instead: `fuzz/inputs-<target>-curated` for Rust, +`fuzz/inputs/<target>-curated` for Python + +That dir is read-only to the tooling. Every run stages it together with the +generated corpus into the `-seeds` dir it hands to `-i`, and neither +`--fuzz-update-corpus` nor `fuzz-corpus` ever writes or deletes anything in it. +Name the files for humans β€” `deeply-nested-object`, not a content hash; the +staging renames them on the way in, and a corpus update drops the copy `cmin` +kept of a curated seed rather than committing it twice. A `README.txt` there is +not a seed, so the dir can say what its inputs are for + +A curated seed still has to buy something: `fuzz-corpus` counts its coverage as +already-covered, so a harvested blob reaching nothing beyond it is not added + +## Seeding a Corpus From a Test Run + +Random bytes are almost never a zip or a calldata frame, so a target with a +committed corpus of a dozen entries spends its budget rediscovering the reject +path. The integration suite produces real calldata, contract code and leader +results on every run; `fuzz-corpus` harvests them from the artifacts dir into +`fuzz/inputs-<target>`, named by content hash: + +```bash +genvm-tool test run --filter-tag integration # produce the artifacts +genvm-tool fuzz-corpus --dry-run # what it would add, and why not +genvm-tool fuzz-corpus +``` + +Each candidate is kept only if `afl-showmap` says it reaches an edge the corpus +does not already cover, so the committed corpus grows by seeds that buy +something. It also drops what a corpus should not carry: a blob above +`--max-size`, or one embedding a path of the machine that produced it. The check +needs the targets built; `--no-verify` skips it + +A harvest only adds; run the targets once with `--fuzz-update-corpus` afterwards +to `cmin` the union back down, and check the repo delta before committing + +Nothing is harvested for a `-structured` target or its op-sequence kin: they take +a serialized value rather than a format, which a run does not produce. Seed those +by driving their mutator, as below + +## Structured Targets + +A target whose input is a value rather than a byte format comes in two halves. +`<name>-raw` takes the bytes verbatim and is what covers the parser. Next to it, +`<name>-structured` takes a [`postcard`] encoding of the value, and a mutator +crate at `fuzz/mutators/<name>/` mutates that value structurally β€” AFL loads it +through `AFL_CUSTOM_MUTATOR_LIBRARY`, which the test plugin sets whenever such a +crate exists + +The input type lives in its own file under `fuzz/shared/`, pulled into both the +target and the mutator crate with `#[path = ...] mod`. Keep that file free of +anything the executor owns: the mutator crate is a small `cdylib` and must not +have to link `genvm` to build + +AFL's own mutations are deliberately left on. Handing every mutation to the +custom mutator (`AFL_CUSTOM_MUTATOR_ONLY`) measures worse β€” a havoc'd buffer that +fails to decode costs one cheap early return, while the ones that do decode reach +values the structural mutator never proposes. `GENVM_FUZZ_MUTATOR_ONLY=1` turns +it on to compare + +### Seeding One + +A structural mutator only mutates a value it already has, so a `-structured` +target needs a non-empty corpus before it explores anything. Generate one by +driving the built mutator library, which makes every seed valid by construction: + +```python +import ctypes, pathlib +from genvm_tool.misc import fuzz_input_name, fuzz_input_path + +lib = ctypes.CDLL('<target-dir>/debug/libfuzz_mutator_<name>.so') +lib.afl_custom_init.restype = ctypes.c_void_p +lib.afl_custom_init.argtypes = [ctypes.c_void_p, ctypes.c_uint] +lib.afl_custom_fuzz.restype = ctypes.c_size_t +lib.afl_custom_fuzz.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_size_t, + ctypes.POINTER(ctypes.c_char_p), ctypes.c_char_p, ctypes.c_size_t, ctypes.c_size_t] + +state, current = lib.afl_custom_init(None, 0), b'\x00' # the default value +for _ in range(60): + out = ctypes.c_char_p() + n = lib.afl_custom_fuzz(state, current, len(current), ctypes.byref(out), None, 0, 4096) + current = ctypes.string_at(out, n) + path = fuzz_input_path(pathlib.Path('fuzz/inputs-<name>'), fuzz_input_name(current)) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(current) +``` + +Two things to check in the result. Every variant of the input type should appear +across the seeds β€” a mutator that cannot construct one leaves that variant's code +unreachable, and seeds generated from it will not cover it either. And the type's +own `serde` impls are not always a corpus codec: `calldata::Value` describes +itself to the format (a null is a unit, a number is an `i64` when it fits), which +`postcard` cannot read back, so it has a separate `Corpus` representation + +[`postcard`]: https://docs.rs/postcard ## Host Preparation diff --git a/docs/contributing/howto/testing/integration.md b/docs/contributing/howto/testing/integration.md index 7593f763..4c8c3b65 100644 --- a/docs/contributing/howto/testing/integration.md +++ b/docs/contributing/howto/testing/integration.md @@ -21,6 +21,14 @@ stays in-process, which is what all other cases do. A routed callee must pin its runner by hash: the nested executor runs with debug mode disabled, where `:test` does not resolve. See `misc/routed_call` on v0.3.x +A multi-line system case may also set `reroute_to` on an individual step, for +example while deploying a fixture with the executor line that owns its public +ABI. Use `${executorV02}` or `${executorV03}` rather than pinning a release +version; the collector unfolds these from `build/info.json` + +Set `expected_executor_route_requests` to assert the exact hook calls. Each +entry has `contract_address`, numeric `state_mode`, and `advisory_major` + A step's declared public-ABI `major` is read from the contract's root slot, so the manager routes it the way a real host would. A step may set `major` explicitly when its subject is what the executor does with a major the manager @@ -37,8 +45,8 @@ would otherwise refuse to route β€” see `runner/major_mismatch` Stability is the second argument of `util.features(paths, stability)` in the case's jsonnet `tags`; legacy v0.2.x uses a `stable/` top-level directory -instead. `--filter-tag` tokenizes alphabetic words only, so a hyphenated tag β€” -`semi-stable`, any `feature-*` β€” is unmatchable there; use `--filter-name` +instead. `--filter-tag` accepts alphanumeric tag names containing `-` and `_`; +matching remains exact ## Running a Subset @@ -76,8 +84,11 @@ exposed. A top-level `debug_mode` in the jsonnet lowers that to `safe`, `safe-unbounded` or raises it to `unsafe-tracing`; `disabled` is rejected, because `reroute_to` β€” what points a case at its own line's executor β€” is honored only from `safe` up. Below `unsafe` a contract must name its runner by -hash instead of `:test`, read from -`build/out/executor/<version>/data/latest.json` +hash instead of `:test`. Write `@RUNNER_LATEST_<line>_<runner>@` for that hash, +for example `py-genlayer:@RUNNER_LATEST_v0.3_py-genlayer@`: the harness replaces +it with the uid in `build/out/executor/<version>/data/latest.json` before the +code reaches an executor, so a runner change needs no test edit. A literal uid +still works and goes stale ## Services @@ -85,3 +96,6 @@ The manager, the modules and the webdriver start automatically; `--no-manager` and `--no-webdriver` reuse externally running ones (a manual webdriver is `bash webdriver/build-and-run.sh`). After WASM files change, `./build/out/executor/<version>/bin/genvm precompile` saves test time + +`--no-manager` omits cases that require a manager with a suite-owned config or +restart lifecycle diff --git a/docs/contributing/howto/testing/rust.md b/docs/contributing/howto/testing/rust.md index 0d5aa57f..5a26cfe3 100644 --- a/docs/contributing/howto/testing/rust.md +++ b/docs/contributing/howto/testing/rust.md @@ -11,6 +11,27 @@ shared `crates/` and executor crates are all picked up 4. Tests for `abc.rs` go into `abc_test.rs`, referenced with `#[path = "abc_test.rs"] mod tests;` +## Extra Tags + +Files in `tests/`, `examples/` and `fuzz/` can add tags to their own case with a +comment anywhere in the file: + +```rust +// genvm-tool-test-tags: feature-web-request-body, needs-web +``` + +They add to `rust` plus `unit`, `example`, or `fuzz` and `needs-fuzz`, depending +on the case kind + +Library and binary cases declare tags in the crate's `.ya-test-config.json`: + +```json +{"tags": {"lib": ["feature-schema-primitive"], "bin/genvm": ["smoke"]}} +``` + +Feature tags expand to their declared dash-prefix ancestors. The complete +vocabulary is in `tests/tags.json`; unknown tags fail collection + ## How To Run ```bash diff --git a/docs/schemas/default-config.json b/docs/schemas/default-config.json index 3f56aa01..692e39fe 100644 --- a/docs/schemas/default-config.json +++ b/docs/schemas/default-config.json @@ -175,10 +175,38 @@ "description": "path to manifest file" }, "permits": { - "type": ["integer", "null"], + "type": ["object", "null"], "default": null, - "minimum": 1, - "description": "concurrency permits; if null will be auto-discovered; full VM takes 2, sync VM takes 1" + "description": "concurrency budget, one permit is one gigabyte of RAM; a total below the most expensive run is raised to it", + "additionalProperties": false, + "properties": { + "total": { + "type": ["integer", "null"], + "default": null, + "minimum": 1, + "description": "total permits; if null, free memory in gibibytes times `per_gib`" + }, + "per_gib": { + "oneOf": [ + { "type": "integer", "minimum": 1 }, + { "type": "string", "pattern": "^\\s*[0-9]+\\s*(/\\s*[0-9]+\\s*)?$" } + ], + "default": 1, + "description": "permits handed out per gigabyte of free memory, an integer or an exact ratio like `1/4`; raise it to overcommit, lower it to undercommit" + }, + "sync": { + "type": "integer", + "default": 4, + "minimum": 1, + "description": "permits a sync run costs" + }, + "nondet": { + "type": "integer", + "default": 8, + "minimum": 1, + "description": "permits a nondet run costs, it also bounds the minimum of `total`" + } + } } }, "required": ["manifest_path"] diff --git a/docs/schemas/runners.json b/docs/schemas/runners.json index 779fe9ed..22a543f0 100644 --- a/docs/schemas/runners.json +++ b/docs/schemas/runners.json @@ -13,12 +13,12 @@ }, "runner-id": { "type": "string", - "description": "Identifier of a runner to load", + "description": "Identifier of a runner to load. Hashes are matched in their canonical GVM32 form; the executor's decoder is additionally lenient about case, the `i`/`l`/`o` aliases and ignored `-`, but ids are not written that way", "anyOf": [ { - "pattern": "^[a-zA-Z0-9_-]+:[a-zA-Z0-9_=-]+$", + "pattern": "^[a-zA-Z0-9_-]+:[0-9abcdefghjkmnpqrstvwxyz]{52}$", "not": { "pattern": "^(contract|chain|custom):" }, - "description": "`name:hash` β€” packaged runner by human readable name and archive hash (in debug mode hash may be `test`/`latest`). `contract`/`chain`/`custom` are reserved and cannot be used as a name" + "description": "`name:hash` β€” packaged runner by human readable name and archive hash, the latter 52 GVM32 characters (in dev mode the magic `test0000...` hash, which is itself well-formed GVM32). `contract`/`chain`/`custom` are reserved and cannot be used as a name" }, { "const": "contract", @@ -29,21 +29,22 @@ "description": "`chain:<address>[:<a|f>[:<slot>]]` β€” code blob read from a `0x` 20 byte address at a GVM32-encoded 32 byte slot, `a`=accepted / `f`=finalized. `<a|f>` and `<slot>` are optional: `<a|f>` defaults to `a` and `<slot>` to the target contract's root code slot" }, { - "pattern": "^custom:[a-zA-Z0-9_=-]+$", - "description": "`custom:<hash>` β€” runner registered at runtime" + "pattern": "^custom:[0-9abcdefghjkmnpqrstvwxyz]{52}$", + "description": "`custom:<hash>` β€” runner registered at runtime, addressed by 52 GVM32 characters" } ] }, + "NestedInitAction": { + "description": "A rule nested inside another one. `$schema` is only accepted on the top level object.", + "allOf": [ + { "$ref": "#/definitions/InitAction" }, + { "not": { "required": ["$schema"] } } + ] + }, "InitAction": { "description": "A single rule for loading.", "type": "object", "allOf": [ - { - "type": "object", - "properties": { - "$schema": { "type": "string" } - } - }, { "oneOf": [ { @@ -95,9 +96,12 @@ } ], "properties": { + "$schema": { "type": "string" }, "AddEnv": { "description": "Adds environment variable", "type": "object", + "additionalProperties": false, + "required": ["name", "val"], "properties": { "name": { "type": "string", @@ -112,6 +116,8 @@ "MapFile": { "description": "Makes file available at specified path in genvm", "type": "object", + "additionalProperties": false, + "required": ["file", "to"], "properties": { "file": { "type": "string", @@ -146,26 +152,30 @@ "description": "Execute sequence", "type": "array", "items": { - "$ref": "#/definitions/InitAction" + "$ref": "#/definitions/NestedInitAction" } }, "When": { "description": "Conditionally execute action", "type": "object", + "additionalProperties": false, + "required": ["cond", "action"], "properties": { "cond": { "$ref": "#/definitions/WasmMode" }, - "action": { "$ref": "#/definitions/InitAction" } + "action": { "$ref": "#/definitions/NestedInitAction" } } }, "With": { "description": "Makes given runner current without executing it's action. It means that MapFile will fork with files from this archive. Useful for creating lock files", "type": "object", + "additionalProperties": false, + "required": ["runner", "action"], "properties": { "runner": { "description": "runner id to make current", "$ref": "#/definitions/runner-id" }, - "action": { "$ref": "#/definitions/InitAction" } + "action": { "$ref": "#/definitions/NestedInitAction" } } } }, diff --git a/docs/website/generate.py b/docs/website/generate.py index 13d5d6e0..b1bab11e 100755 --- a/docs/website/generate.py +++ b/docs/website/generate.py @@ -23,7 +23,8 @@ def _newest_line() -> str: - """The newest active executor line, e.g. `v0.3`. + """ + The newest active executor line, e.g. `v0.3`. Decided by version order, NOT by position: `active-versions` is hand-ordered and `genvm-tool check-versions bump` APPENDS the new line, so after the next @@ -318,7 +319,8 @@ def version_to_anchor(ver): def build_runner_version_list(rid): - """Build list of (hash, version_range_str) for previous versions of a runner. + """ + Build list of (hash, version_range_str) for previous versions of a runner. Groups consecutive versions sharing the same hash into ranges. Skips the current (latest) hash. diff --git a/docs/website/merge_txts.py b/docs/website/merge_txts.py index a805bc4d..19170776 100644 --- a/docs/website/merge_txts.py +++ b/docs/website/merge_txts.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 -"""Concatenate every ``*.txt`` under a directory into one bundle. +""" +Concatenate every ``*.txt`` under a directory into one bundle. Used to fold a sphinx ``-b text`` build of one section into a single ``_static/ai/<section>.txt`` file (the "docs for LLMs" bundles). Each entry is diff --git a/docs/website/src/impl-spec/01-core-architecture/01-components.rst b/docs/website/src/impl-spec/01-core-architecture/01-components.rst index d802182e..b8cddb1d 100644 --- a/docs/website/src/impl-spec/01-core-architecture/01-components.rst +++ b/docs/website/src/impl-spec/01-core-architecture/01-components.rst @@ -116,7 +116,7 @@ software floating point implementation for deterministic mode - Curated standard library for deterministic execution - Support for some necessary libraries (NumPy, PIL) -GenVM requires some built-in runners to be accessible by contracts. They are identified by hashes of their ``tar`` contents +GenVM requires some built-in runners to be accessible by contracts. They are identified by hashes of their ``zip`` contents :term:`Host` Interface ~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/website/src/impl-spec/02-vm/03-consensus.rst b/docs/website/src/impl-spec/02-vm/03-consensus.rst index e60f75cd..b6cf04b6 100644 --- a/docs/website/src/impl-spec/02-vm/03-consensus.rst +++ b/docs/website/src/impl-spec/02-vm/03-consensus.rst @@ -23,10 +23,10 @@ results: leader's result at ``call_no`` from this vector and feeds it back to the contract as the second argument of the non-det block. If the leader produced fewer entries than the validator demands, the run aborts with - ``VmError::absent_leader_nondet_output``. + ``VmError::leader_fault().nondet_output().absent()``. ``call_no`` is a monotonically increasing counter incremented per ``RunNondet`` -invocation. The hard cap is ``public_abi::top_limits::NONDET_BLOCKS`` (``4096``); +invocation. The hard cap is ``internal_constants::top_limits::NONDET_BLOCKS`` (``4096``); exceeding it produces ``VmError::oom().ram().limit()``. The counter is what binds a leader's i-th non-det result to the validator's i-th non-det check β€” the ordering of ``RunNondet`` calls in deterministic code MUST match between leader diff --git a/docs/website/src/impl-spec/appendix/manager-api.yaml b/docs/website/src/impl-spec/appendix/manager-api.yaml index 54a52ef2..e2a25f08 100644 --- a/docs/website/src/impl-spec/appendix/manager-api.yaml +++ b/docs/website/src/impl-spec/appendix/manager-api.yaml @@ -57,6 +57,12 @@ paths: max: type: integer description: Maximum number of permits + per_sync_run: + type: integer + description: Permits a sync run costs + per_nondet_run: + type: integer + description: Permits a nondet run costs executions: type: object description: Status of running executions @@ -98,6 +104,8 @@ paths: result: type: string example: module_started + '400': + $ref: '#/components/responses/ErrorResponse' '500': $ref: '#/components/responses/ErrorResponse' @@ -132,6 +140,8 @@ paths: result: type: string enum: [module_stopped, module_not_running] + '400': + $ref: '#/components/responses/ErrorResponse' '500': $ref: '#/components/responses/ErrorResponse' @@ -175,6 +185,8 @@ paths: result: type: string example: module_restarted + '400': + $ref: '#/components/responses/ErrorResponse' '500': $ref: '#/components/responses/ErrorResponse' @@ -278,6 +290,8 @@ paths: type: string format: byte nullable: true + '400': + $ref: '#/components/responses/ErrorResponse' '500': $ref: '#/components/responses/ErrorResponse' delete: @@ -317,6 +331,8 @@ paths: example: timeout during shutdown genvm_id: type: integer + '400': + $ref: '#/components/responses/ErrorResponse' /genvm/{genvm_id}/artifact: get: @@ -373,6 +389,8 @@ paths: data_base64: type: string format: byte + '400': + $ref: '#/components/responses/ErrorResponse' '500': $ref: '#/components/responses/ErrorResponse' @@ -408,6 +426,8 @@ paths: specified_major: type: integer description: Detected major version + '400': + $ref: '#/components/responses/ErrorResponse' '500': $ref: '#/components/responses/ErrorResponse' @@ -444,6 +464,8 @@ paths: example: log_level_set level: type: string + '400': + $ref: '#/components/responses/ErrorResponse' '500': $ref: '#/components/responses/ErrorResponse' @@ -484,7 +506,10 @@ paths: description: Number of available permits post: summary: Set permits - description: Set the number of execution permits. + description: >- + Set the number of execution permits, one permit is one gigabyte of RAM. + A value below the cost of the most expensive run is rejected, and the + response carries the unchanged maximum. operationId: setPermits requestBody: required: true @@ -513,6 +538,8 @@ paths: example: permits_set permits: type: integer + '400': + $ref: '#/components/responses/ErrorResponse' '500': $ref: '#/components/responses/ErrorResponse' @@ -615,6 +642,8 @@ paths: response: type: string nullable: true + '400': + $ref: '#/components/responses/ErrorResponse' '500': $ref: '#/components/responses/ErrorResponse' @@ -642,6 +671,8 @@ paths: type: string nullable: true description: Human-readable error description, or null if unknown + '400': + $ref: '#/components/responses/ErrorResponse' '500': $ref: '#/components/responses/ErrorResponse' diff --git a/docs/website/src/spec/02-execution-environment/01-wasm.rst b/docs/website/src/spec/02-execution-environment/01-wasm.rst index 17839325..0d47f3a4 100644 --- a/docs/website/src/spec/02-execution-environment/01-wasm.rst +++ b/docs/website/src/spec/02-execution-environment/01-wasm.rst @@ -16,15 +16,31 @@ Enabled WASM Features and Proposals :ref:`gvm-def-det-mode` Additional Limitations ---------------------------------------------- -Only following ``f32.*`` and ``f64.*`` operations are allowed: +An operation on a floating point value is allowed *iff* it moves or copies bits +without interpreting them as numbers. The complete allowed set is: - ``f32.store``, ``f64.store`` - ``f32.load``, ``f64.load`` - ``f32.const``, ``f64.const`` - ``f32.reinterpret_i32``, ``f64.reinterpret_i64`` - ``i32.reinterpret_f32``, ``i64.reinterpret_f64`` - -Any other floating point operation is considered non-deterministic and is not allowed in :ref:`gvm-def-det-mode`. +- ``f32x4.splat``, ``f64x2.splat`` +- ``f32x4.extract_lane``, ``f64x2.extract_lane`` +- ``f32x4.replace_lane``, ``f64x2.replace_lane`` + +Every other ``f32.*``, ``f64.*``, ``f32x4.*`` and ``f64x2.*`` operation is +considered non-deterministic β€” this covers arithmetic, ``min``/``max`` +(including the relaxed and pseudo-min/max forms), ``sqrt``, rounding, +``abs``/``neg``/``copysign``, comparisons, fused multiply-add, and every +conversion between a float and an integer or between float widths. Reaching one +of them in :ref:`gvm-def-det-mode` traps with +:ref:`gvm-def-str-trie-value-vm-error-wasm-trap-nondet-instruction`; a module +merely containing them is not rejected. + +Operations on ``v128`` that do not name a float type (``v128.load``, +``v128.store``, ``v128.const``, ``i8x16.shuffle``, ``v128.bitselect``, the +integer lane operations, …) are not floating point operations and are +unrestricted. :ref:`gvm-def-non-det-mode` does not have these limitations, allowing all floating point operations. diff --git a/docs/website/src/spec/02-execution-environment/02-wasip1.rst b/docs/website/src/spec/02-execution-environment/02-wasip1.rst index beeabe00..51e2bdf4 100644 --- a/docs/website/src/spec/02-execution-environment/02-wasip1.rst +++ b/docs/website/src/spec/02-execution-environment/02-wasip1.rst @@ -128,6 +128,7 @@ Fail with ``Notsup`` error code: - ``proc_raise`` - ``sched_yield`` - ``fd_pwrite`` +- ``fd_renumber`` Functions @@ -155,18 +156,88 @@ caller that wants a graceful fallback must handle that error explicitly. #. ``proc_exit(0)`` is equivalent to :ref:`gvm-def-return` of ``null`` value. #. ``proc_exit(x)`` where :math:`x \neq 0` results in :ref:`gvm-def-vm-error` +.. _gvm-def-vfs-path-resolution: + +Path Resolution +~~~~~~~~~~~~~~~ + +``path_*`` functions take a directory :term:`FD` and a path. Resolution is +purely lexical and identical in both modes: + +#. The directory descriptor's own path and the supplied path are concatenated + and split on ``/``. Empty and ``.`` components are dropped, a ``..`` + component pops the preceding one, and a ``..`` that would escape the root is + dropped. There are no symlinks, so ``dirflags`` never changes the result. +#. The resulting absolute path must lie within a preopened subtree, otherwise + the call fails with ``Notcapable``. +#. The path is then walked through the :ref:`gvm-def-vfs`: a missing component + fails with ``Noent``, and a component that resolves to a file while more + components remain fails with ``Badf``. + +A descriptor that is not a directory fails with ``Badf`` before any of this. + +File Metadata +~~~~~~~~~~~~~ + +The :ref:`gvm-def-vfs` stores no metadata, so every ``Filestat`` reports +``dev``, ``ino``, ``atim``, ``mtim`` and ``ctim`` as ``0``; ``size`` is the +file's length in octets and ``0`` for a directory. Timestamps are **not** +derived from the transaction timestamp β€” a file has no modification time at +all. + ``path_open`` Function ~~~~~~~~~~~~~~~~~~~~~~ +#. The path is resolved as in `Path Resolution`_. +#. ``oflags``, ``fs_rights_base``, ``fs_rights_inheriting`` and ``fdflags`` are + ignored. The :ref:`gvm-def-vfs` is read-only, so ``creat``, ``excl`` and + ``trunc`` never take effect β€” opening a non-existent path fails with + ``Noent`` rather than creating it β€” and ``directory`` does not restrict the + result. +#. On success a fresh descriptor is allocated (see + :ref:`gvm-def-fd-allocation`) that refers to the resolved file or directory. + A file descriptor starts at offset ``0``. +#. The descriptor is released again if the call fails, so a failed open leaks + neither a descriptor nor its :ref:`gvm-def-ram-consumption`. + ``path_filestat_get`` Function ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Resolves the path as in `Path Resolution`_ and returns the ``Filestat`` +described in `File Metadata`_ with ``filetype`` ``regular_file`` or +``directory`` and ``nlink`` ``0``. + ``fd_readdir`` Function ~~~~~~~~~~~~~~~~~~~~~~~ +A descriptor that is not a directory fails with ``Badf``. + +The entry sequence of a directory is fixed and does not depend on how the +:ref:`gvm-def-vfs` was populated: + +#. ``.`` β€” filetype ``directory`` +#. ``..`` β€” filetype ``directory``, and refers to the root itself in the root + directory +#. the directory's children in ascending order of their names, compared as raw + octet sequences; a child's filetype is ``directory`` or ``regular_file`` + +``d_ino`` is ``0`` for every entry, ``d_namlen`` is the name's length in +octets, and ``d_next`` is the cookie of the following entry β€” cookies number +the sequence above from ``1``. The ``cookie`` argument skips that many leading +entries; a cookie past the end yields no entries. + +Entries are written as a ``Dirent`` immediately followed by the name, until the +buffer is exhausted. A truncated entry is not backed out: the last ``Dirent`` +or name may be cut mid-way, and the call then returns ``buf_len`` to signal +that the caller must retry with a larger buffer. Otherwise it returns the +number of octets written. + ``fd_tell`` Function ~~~~~~~~~~~~~~~~~~~~ +Returns the descriptor's current offset. ``stdout``/``stderr`` fail with +``Spipe``, a directory with ``Notsup``. + ``fd_datasync`` Function ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -180,12 +251,22 @@ Does nothing and always returns success. ``fd_seek`` Function ~~~~~~~~~~~~~~~~~~~~ -``fd_renumber`` Function -~~~~~~~~~~~~~~~~~~~~~~~~ +Moves a file descriptor's offset and returns the new value. + +#. ``set`` and ``cur`` clamp the result into :math:`[0, \texttt{size}]` instead + of failing: a negative target becomes ``0`` and a target beyond the end + becomes the file's size. +#. ``end`` is not supported and fails with ``Notsup``. +#. ``stdout``/``stderr`` fail with ``Spipe``, a directory with ``Notsup``. ``fd_prestat_dir_name`` Function ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Writes the preopened directory's name β€” the last component of its path, or ``/`` +for the root. ``fd_prestat_get`` returns the ``Prestat`` whose ``pr_name_len`` +determines the name's length. A buffer shorter than that name fails with +``Overflow``; any non-directory descriptor fails with ``Badf``. + ``fd_prestat_get`` Function ~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -198,21 +279,59 @@ Does nothing and always returns success. ``fd_write`` Function ~~~~~~~~~~~~~~~~~~~~~ -``fd_pread`` Function -~~~~~~~~~~~~~~~~~~~~~ +Only ``stdout`` and ``stderr`` are writable; a file or directory descriptor +fails with ``Rofs``. The written octets are diagnostic output: they are not part +of the :ref:`gvm-def-vm-result` and are not covered by the +:ref:`gvm-def-execution-hash`. + +The call reports every supplied octet as written and never fails on the +underlying stream, so a contract cannot observe whether the :term:`Host` +accepted, buffered or discarded the output. ``fd_read`` Function ~~~~~~~~~~~~~~~~~~~~ +Reads a file descriptor's contents into the ``iovec``\ s in order, starting at +the descriptor's offset and advancing it by the number of octets read. Reading +at or past the end of the file yields ``0`` octets rather than an error. +``stdout``/``stderr`` fail with ``Acces``, a directory with ``Isdir``. + +``fd_pread`` Function +~~~~~~~~~~~~~~~~~~~~~ + +As ``fd_read``, except that it starts at the explicit ``offset`` and leaves the +descriptor's own offset unchanged. + ``fd_filestat_get`` Function ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Returns the ``Filestat`` described in `File Metadata`_ with ``nlink`` ``1`` and +``filetype``: + +- ``regular_file`` for a file descriptor, whose ``size`` is the file's length +- ``directory`` for a directory descriptor +- ``character_device`` for ``stdout``/``stderr`` + ``fd_fdstat_get`` Function ~~~~~~~~~~~~~~~~~~~~~~~~~~ +``fs_flags`` is always empty β€” no descriptor is appending, non-blocking or +synchronous β€” and ``fs_rights_inheriting`` always equals ``fs_rights_base``. +The rights are fixed per descriptor kind and never include a mutating right: + +- a file descriptor: ``regular_file`` with the read, seek, tell, advise, sync, + datasync, readdir, readlink, ``path_open`` and filestat-get rights +- a directory descriptor: ``directory`` with the read, ``path_open``, readdir, + readlink and filestat-get rights +- ``stdout``/``stderr``: ``unknown`` filetype with the write right only + ``fd_close`` Function ~~~~~~~~~~~~~~~~~~~~~ +Deallocates the descriptor (see :ref:`gvm-def-fd-allocation`) and releases the +:ref:`gvm-def-ram-consumption` charged for its contents, if any. An unknown +descriptor fails with ``Badf``. + ``fd_advise`` Function ~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/02-gl_call.rst b/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/02-gl_call.rst index 8aafb847..026faa5d 100644 --- a/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/02-gl_call.rst +++ b/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/02-gl_call.rst @@ -296,7 +296,9 @@ meta-properties, including the *param* ``runner`` and *param* The caller receives the sandbox result (:ref:`gvm-def-subvm-result-encoding`) and may handle both :ref:`gvm-def-vm-error` and :ref:`gvm-def-user-error`; -storage writes performed by the sandbox are not reverted on error. +storage writes performed by the sandbox are not reverted on error. If the +sandbox terminates with :ref:`gvm-def-fatal-vm-error`, the caller terminates +with the same fatal VM error instead. .. _gvm-def-gl-call-register-runner: @@ -573,6 +575,21 @@ Payload Causes VM to exit with ``ContractReturn``. Encodes return value using :ref:`gvm-def-calldata-encoding` format. +.. _gvm-def-gl-call-observable-discretion: + +Implementation Discretion +------------------------- + +Where a message below says an implementation may ignore it, that discretion +covers only the side effect the message asks for β€” writing a log line, +recording a timing. The **value returned to the guest** is never at an +implementation's discretion: it is fixed by this specification for the mode the +call is made in, and such a call MUST NOT fail. A message whose result differs +between two conformant implementations would make +:ref:`gvm-def-det-mode` execution diverge across validators; widening or +narrowing what a message returns therefore requires a new +:ref:`GenVM version <gvm-def-contract-version>`. + ``Trace.Message`` Message ------------------------- @@ -595,13 +612,14 @@ Payload .. note:: - Implementations may choose to ignore this message and return an error. + Whether anything is logged is implementation-defined (see + :ref:`gvm-def-gl-call-observable-discretion`); the call returns no value and + always succeeds. Requirements ~~~~~~~~~~~~ #. GenVM version 0.1.10 or higher -#. :term:`GenVM` implementation is allowed ignore this message .. _tracing-runtime-microsec: @@ -622,13 +640,14 @@ Payload .. note:: - Implementations may choose to ignore this message and return an error. + The returned value is not at an implementation's discretion β€” in + :ref:`gvm-def-det-mode` it is exactly ``0``, and the call never fails (see + :ref:`gvm-def-gl-call-observable-discretion`). Requirements ~~~~~~~~~~~~ #. GenVM version 0.1.10 or higher -#. :term:`GenVM` implementation is allowed ignore this message ``Yield`` Message ----------------- @@ -646,13 +665,14 @@ Payload .. note:: - Implementations may choose to ignore this message. + Whether the implementation actually yields anything is implementation-defined + (see :ref:`gvm-def-gl-call-observable-discretion`); the call returns no value + and always succeeds. Requirements ~~~~~~~~~~~~ #. GenVM version 0.3.0 or higher -#. :term:`GenVM` implementation is allowed ignore this message .. _get-timestamp: diff --git a/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/03-schemas.rst b/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/03-schemas.rst index 96074cae..2aeff786 100644 --- a/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/03-schemas.rst +++ b/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/03-schemas.rst @@ -43,22 +43,19 @@ stage; it is replaced by a derived :ref:`gvm-def-vm-error` and handed to the comparison as if it had been proposed. #. The buffer is non-empty. An absent or empty proposal yields - :ref:`gvm-def-str-trie-value-vm-error-absent-leader-nondet-output`. + :ref:`gvm-def-str-trie-value-vm-error-leader-fault-nondet-output-absent`. #. ``result_code`` is one of :ref:`gvm-def-enum-value-result-code-return`, :ref:`gvm-def-enum-value-result-code-user-error` or - :ref:`gvm-def-enum-value-result-code-vm-error`. Any other byte β€” including - :ref:`gvm-def-enum-value-result-code-internal-error`, which the enumeration - admits but the wire format does not β€” yields - :ref:`gvm-def-pending-str-trie-value-vm-error-leader-output-malformed`. + :ref:`gvm-def-enum-value-result-code-vm-error`; any other byte yields + :ref:`gvm-def-str-trie-value-vm-error-leader-fault-nondet-output-malformed`. #. For **return** and **user_error**, ``data`` is valid :ref:`gvm-def-calldata-encoding` with no trailing bytes; otherwise - :ref:`gvm-def-pending-str-trie-value-vm-error-leader-output-malformed`. + :ref:`gvm-def-str-trie-value-vm-error-leader-fault-nondet-output-malformed`. #. For **vm_error**, ``data`` is UTF-8, carries no ``" # "`` detail, and names a :ref:`gvm-def-str-trie-vm-error` path, including the canonical spelling of - any parameter. Codes outside the trie β€” among them every - :doc:`pending constant <../../appendix/constants-pending>` β€” yield - :ref:`gvm-def-pending-str-trie-value-vm-error-leader-output-malformed`. + any parameter. Codes outside the trie yield + :ref:`gvm-def-str-trie-value-vm-error-leader-fault-nondet-output-malformed`. An accepted result is preserved **byte for byte**: validation never re-encodes, so every node hashes the value that was proposed. @@ -71,14 +68,14 @@ Derived-Outcome Namespace Following :ref:`gvm-def-vm-error` codes are derived by the consuming executor rather than proposed: -#. :ref:`gvm-def-str-trie-value-vm-error-absent-leader-nondet-output` -#. :ref:`gvm-def-pending-str-trie-value-vm-error-leader-output-malformed` -#. :ref:`gvm-def-pending-str-trie-value-vm-error-leader-output-uses-this-error` -#. :ref:`gvm-def-pending-str-trie-value-vm-error-leader-output-extra` +#. :ref:`gvm-def-str-trie-value-vm-error-leader-fault-nondet-output-absent` +#. :ref:`gvm-def-str-trie-value-vm-error-leader-fault-nondet-output-malformed` +#. :ref:`gvm-def-str-trie-value-vm-error-leader-fault-nondet-output-uses-this-error` +#. :ref:`gvm-def-str-trie-value-vm-error-leader-fault-nondet-output-extra` -A proposed code that equals, or extends at a space boundary, either -``absent_leader_nondet_output`` or ``leader_output`` is replaced by -:ref:`gvm-def-pending-str-trie-value-vm-error-leader-output-uses-this-error` +A proposed code that equals, or extends at a space boundary, +``leader_fault nondet_output`` is replaced by +:ref:`gvm-def-str-trie-value-vm-error-leader-fault-nondet-output-uses-this-error` whose parameter is the first 6 characters of the :doc:`../../04-contract-interface/06-gvm32` encoding of ``sha3_256`` of the proposed code. A proposal that is its own replacement maps to the parameter diff --git a/docs/website/src/spec/02-execution-environment/04-runners.rst b/docs/website/src/spec/02-execution-environment/04-runners.rst index 6b1cd2d7..b45f1edf 100644 --- a/docs/website/src/spec/02-execution-environment/04-runners.rst +++ b/docs/website/src/spec/02-execution-environment/04-runners.rst @@ -29,26 +29,55 @@ forms (see the ``runner-id`` definition in the runner.json schema): ``contract``, ``chain`` and ``custom`` are reserved prefixes and cannot be used as human-readable ids. +.. _gvm-def-chain-runner-state: + +``chain:`` State Visibility +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A ``chain:`` id is code, but its resolution is an ordinary +:ref:`gvm-def-det-mode` storage read and is subject to the same rules: + +#. The read resolves against the state the executing transaction was fixed to + by consensus, not against whatever a node's chain tip happens to be. Every + validator of the transaction therefore reads the same octets, whichever + :term:`Host` serves them. +#. ``f`` (finalized) reads the state at the last finalized block; ``a`` + (accepted, the default) reads the accepted state β€” the same view a + read-only :ref:`gvm-def-gl-call-call-contract` child observes by default + (see :ref:`contract-execution-flow`). Neither view includes the executing + transaction's own uncommitted writes, so a contract cannot deploy code and + load it as a runner in the same transaction. +#. The choice between the two is part of the id string and thus part of the + runner graph, never a node-local default. + +A resolution that finds no code, or code that is not a valid :term:`runner`, +fails like any other malformed runner rather than falling back to another view. + Hash Format ~~~~~~~~~~~ -``hash`` is the SHA3 256-bit hash of the runner's contents, encoded with +``hash`` is a 256-bit hash of the runner's contents, encoded with :doc:`../04-contract-interface/06-gvm32` (GVM32, a lowercase Crockford Base32). This keeps it free of filesystem-illegal characters and case-insensitive. +The algorithm differs per id form: + +- ``<human-readable-id>:<hash>`` β€” SHA-256, matching the hash the runner is + packaged and distributed under. +- ``custom:<hash>`` β€” SHA3-256 of the registered blob. + Runner Layout ------------- -For any of the layouts a file list is constructed. Each file in this name has: - -- a name +For any of the layouts a file list is constructed. Each entry name: - #. it must not start with ``/`` - #. path separator must be ``/`` - #. it must not contain ``.`` or ``..`` path components +- Must not be empty or start with ``/`` +- Must use ``/`` as the path separator; backslashes are forbidden +- Must not contain empty, ``.`` or ``..`` path components +- Must not end with ``/`` unless the entry is a directory -- contents (raw bytes slice) +Each file also has contents as a raw byte slice 1. ZIP Archive ~~~~~~~~~~~~~~ @@ -57,7 +86,17 @@ Used if runner bytes represent a ZIP archive - If successful, extracts the archive contents and processes it as a structured :term:`runner` package - This format supports complex :term:`runners <Runner>` with multiple files, dependencies, and configuration -- Only allowed compression is ``stored`` (no compression) +- Only allowed compression is ``stored`` (no compression), declared by both the central directory and the entry's local header +- A stored entry's compressed and uncompressed sizes must be equal +- A stored file's contents must match its declared CRC-32 +- The local header must agree with the central directory on the entry name, and, + unless the entry has a data descriptor (general-purpose flag bit 3), on the + CRC-32 and the sizes. The name is compared as raw bytes and must be valid + UTF-8, so an entry no two readers would decode alike is rejected rather than + silently resolved +- Encrypted entries (general-purpose flag bit 0) are rejected +- An entry ending with ``/`` is a directory; its size and CRC-32 must be 0, and it is omitted from the file list +- If several entries share a name, the file list holds the last of them 2. Raw WASM ~~~~~~~~~~~ @@ -128,11 +167,30 @@ Schema is available in :doc:`../appendix/runner-schema`\. It must be a valid JSON object with described below structure +Each action object accepts exactly the fields shown below. The top-level object may also contain a string ``$schema`` +annotation + +``Seq``, ``When`` and ``With`` nest actions. Nesting deeper than +:ref:`gvm-def-consts-value-runner-limits-init-action-depth`, or a NUL octet in +any action string, is rejected with +:ref:`gvm-def-str-trie-value-vm-error-invalid-contract-runner-malformed`\. + AddEnv ~~~~~~ Adds an environment variable to the GenVM environment with variable interpolation support using ``${}`` syntax. +The name is at most :ref:`gvm-def-consts-value-runner-limits-env-name-len` +octets and consists of ASCII characters other than ``=``, whitespace and control +characters. The value after interpolation is at most +:ref:`gvm-def-consts-value-runner-limits-env-value-len` octets. Violating either +is rejected with +:ref:`gvm-def-str-trie-value-vm-error-invalid-contract-runner-malformed`\. + +The interpolated value's length in octets is charged as +:ref:`gvm-def-ram-consumption`; exceeding the budget results in +:ref:`gvm-def-str-trie-value-vm-error-out-of-memory`\. + Example ^^^^^^^ @@ -298,7 +356,8 @@ Startup ------- Runner actions are executed left-recursively, until :ref:`gvm-def-start-wasm` is reached. -If it was not reached, it will result in a :ref:`gvm-def-vm-error` with ``error_inval`` code. +If it was not reached, it will result in a :ref:`gvm-def-vm-error` with +``invalid_contract runner malformed`` code. Loading a :term:`runner` goes through a single **load action**, defined per :term:`sub-VM`. Each :term:`sub-VM` owns a **loaded-runner set**: the runner diff --git a/docs/website/src/spec/03-vm/01-startup.rst b/docs/website/src/spec/03-vm/01-startup.rst index 97c50836..b580d192 100644 --- a/docs/website/src/spec/03-vm/01-startup.rst +++ b/docs/website/src/spec/03-vm/01-startup.rst @@ -40,7 +40,7 @@ The entry payload is host-supplied and is checked against :ref:`gvm-def-contract-call-conv` before permissions are read and before any runner loads. A payload that violates the convention produces the :ref:`gvm-def-vm-error` -:ref:`gvm-def-pending-str-trie-value-vm-error-malformed-entry` as the execution +:ref:`gvm-def-str-trie-value-vm-error-malformed-entry` as the execution result, like any other failure at this point. Messages emitted by :ref:`gvm-def-post-message` and ``DeployContract`` are @@ -106,8 +106,12 @@ with :ref:`gvm-def-str-trie-value-vm-error-out-of-nondet-blocks`. Creating the :term:`sub-VM` charges :ref:`gvm-def-consts-value-memory-limiter-consts-vm-spawn-cost` octets of -:ref:`gvm-def-ram-consumption` to the new :term:`sub-VM`; the charge is -released when it finishes. +:ref:`gvm-def-ram-consumption` to the new :term:`sub-VM`, plus +:ref:`gvm-def-consts-value-memory-limiter-consts-storage-page-inherited` octets +for every storage region it inherits already written from its caller +(:ref:`gvm-def-gl-call-sandbox` and :ref:`gvm-def-gl-call-run-nondet` inherit +their caller's; the initial entry and :ref:`gvm-def-gl-call-call-contract` +inherit none). Both charges are released when it finishes. After these checks, startup applies the `Custom-Runner Grants`_ for the new :term:`sub-VM`, then performs the runner load action for the entry runner and @@ -202,7 +206,8 @@ inherited permission, never add one. :ref:`gvm-def-gl-call-run-nondet` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The child executes in :ref:`gvm-def-non-det-mode`. +The child executes in :ref:`gvm-def-non-det-mode`, on its own RAM budget seeded +with what the caller had remaining (see :doc:`03-ram-limiting`). - :ref:`gvm_vm_field_permissions` are all false, including :ref:`gvm-perm-deterministic`. diff --git a/docs/website/src/spec/03-vm/03-ram-limiting.rst b/docs/website/src/spec/03-vm/03-ram-limiting.rst index a6c34c3c..2c7db4d2 100644 --- a/docs/website/src/spec/03-vm/03-ram-limiting.rst +++ b/docs/website/src/spec/03-vm/03-ram-limiting.rst @@ -1,9 +1,13 @@ Resource Limiting ================= -:ref:`gvm-def-det-mode` and :ref:`gvm-def-non-det-mode` have two separate RAM budgets. -Each budget starts at 4294967295 octets (4 GiB). -All :term:`sub-VM` instances within the same mode share the same budget. +:ref:`gvm-def-det-mode` and :ref:`gvm-def-non-det-mode` have separate RAM +budgets: what one consumes is never charged to the other. +The deterministic budget starts at 4294967295 octets (4 GiB). +Every :ref:`gvm-def-gl-call-run-nondet` gets its own non-deterministic budget, +starting at what its caller had remaining at the moment of the call, so a +nondet block never gets more RAM than its caller had left. +All :term:`sub-VM` instances within one budget share it. .. _gvm-def-ram-consumption: @@ -42,10 +46,17 @@ The following operations consume RAM: and ``RegisterRunner`` ``gl_call``\ s, and receiving a custom-runner grant at sub-VM creation (see :doc:`../02-execution-environment/04-runners` and :ref:`gvm-meta-property-custom-runners`) +- **Storage writes**: writing to a 32-octet aligned region of a + :term:`Storage Slot` costs + :ref:`gvm-def-consts-value-memory-limiter-consts-new-storage-page` octets the + first time that region is written. Regions the :term:`sub-VM` inherited + already written from its caller, and repeated writes to a region, cost nothing - **Sub-VM creation**: each new :term:`sub-VM` costs - :ref:`gvm-def-consts-value-memory-limiter-consts-vm-spawn-cost` octets, - charged to the new :term:`sub-VM` at creation (see :doc:`01-startup`) and - released when it finishes + :ref:`gvm-def-consts-value-memory-limiter-consts-vm-spawn-cost` octets, plus + :ref:`gvm-def-consts-value-memory-limiter-consts-storage-page-inherited` + octets for every 32-octet region already written in the storage it inherits. + Both are charged to the new :term:`sub-VM` at creation (see :doc:`01-startup`) + and released when it finishes The runner load cost (:ref:`gvm-def-consts-value-memory-limiter-consts-runner-load-cost`) is a fixed per-load overhead @@ -59,6 +70,12 @@ This applies to runner charges as well: memory consumed by loading or registering a runner is released when the registering :term:`sub-VM` finishes, like any other charge. There are no permanent charges. +Storage write charges are the exception: they follow the storage they paid for. +When a :ref:`gvm-def-gl-call-sandbox` child returns and its caller takes over the +child's storage, the charge for the regions the child wrote first is transferred +to the caller rather than released, so a caller cannot use repeated +:ref:`gvm-def-gl-call-sandbox` calls to accumulate storage it never pays for. + Other Limits ------------ diff --git a/docs/website/src/spec/03-vm/05-result.rst b/docs/website/src/spec/03-vm/05-result.rst index 7cd20124..572d30ac 100644 --- a/docs/website/src/spec/03-vm/05-result.rst +++ b/docs/website/src/spec/03-vm/05-result.rst @@ -21,6 +21,16 @@ exceeding resource limits. It uses predefined string error codes. +.. _gvm-def-fatal-vm-error: + +.. rubric:: Fatal VM Error + +Every VM error is either **fatal** or not; both draw their code from the same +set. A non-fatal error of a :term:`sub-VM` is returned to its caller as +:ref:`gvm-def-subvm-result-encoding`. A fatal one is not catchable: the caller +terminates with the same VM error, and propagation continues until the topmost +VM boundary + .. _gvm-def-vm-error-code: VM Error Code Format @@ -37,9 +47,25 @@ VM Error Code Format separated from the public code by ``#`` surrounded by a single space on each side. -The detail MUST be a deterministic function of execution (the full string is -covered by the :ref:`gvm-def-execution-hash`), but its content carries no -compatibility promise β€” see :ref:`gvm-def-vm-error-compat`. +The full string, detail included, is covered by the +:ref:`gvm-def-execution-hash`, so it must be reproducible octet for octet by +every implementation running the same execution. A detail MUST therefore be +composed only of: + +#. octets the execution itself produced or consumed β€” a guest-supplied string, + a decoded field of the calldata or of a runner description; +#. values fixed by the WASM module or by this specification β€” a function or + memory index, a limit, a constant from :doc:`../appendix/constants`; +#. literal text chosen by the code that raises the error. + +Anything the host or the runtime environment supplies MUST NOT appear, even +indirectly. In particular: filesystem paths, operating-system error strings or +numbers, host addresses and pointer values, elapsed times, locale-dependent or +platform-width-dependent formatting, thread or process identifiers, and +implementation build or version strings. + +Beyond that constraint the detail's content carries no compatibility promise β€” +see :ref:`gvm-def-vm-error-compat`. Consumers MUST compare and match VM error codes only by the public code (the part before the first `` # ``). A consumer matching a known code ``P`` MUST @@ -52,16 +78,24 @@ with further space-separated components. Represents a user-produced error in utf-8 format. +.. rubric:: Effects of a Non-Returning Run + +Only a :ref:`gvm-def-return` carries effects. A topmost run that ends in +:ref:`gvm-def-user-error` or :ref:`gvm-def-vm-error` reports no +``storage_changes`` and no ``emissions``, whatever it wrote or emitted before +failing, and its :ref:`gvm-def-execution-hash` covers those empty fields. + .. _gvm-def-internal-error: InternalError ------------- -It is a special :ref:`gvm-def-vm-result` that represents an internal error in the VM, -such as: :term:`Host` communication failures or :term:`Module` unavailability. +Not a :ref:`gvm-def-vm-result` but the absence of one: the executor could not +run the contract to a verdict at all, because of a :term:`Host` communication +failure or :term:`Module` unavailability. -Internal errors are not visible by the contracts. Most likely :term:`Host` will -vote *timeout* if encounters such an error +Internal errors are not visible by the contracts and are reported only to the +:term:`Host`, which will most likely vote *timeout* if it encounters one Non-Deterministic Block Result Encoding --------------------------------------- @@ -70,6 +104,9 @@ Non-Deterministic Block Result Encoding - :ref:`gvm-def-user-error`\: utf-8 string - :ref:`gvm-def-vm-error`\: utf-8 string +These three are the only codes a leader-proposed non-deterministic block result +may carry; validators treat every other byte as a malformed leader result + Contract Result Encoding ------------------------ @@ -128,7 +165,9 @@ with the following keys (in this order): #. ``backtrace`` #. ``data`` β€” the contract result value +#. ``data_fees_consumed`` #. ``data_fees_remaining`` +#. ``emissions`` β€” emitted messages and events, in emission order #. ``kind`` β€” the :ref:`gvm-def-vm-result` result code #. ``storage_changes`` #. ``subvm_hashes`` β€” see :ref:`gvm-def-subvm-hash` @@ -137,6 +176,11 @@ with the following keys (in this order): Two runs that agree on the deterministic result produce the same execution hash, so consensus can compare a single 32-byte value instead of the full result. +``emissions`` covers the whole content of every emitted message and event, not +just its metered cost: two emissions can carry different calldata or different +event topics for the same fee, so a fee-only commitment would let nodes agree on +the hash while committing divergent side effects + .. _gvm-def-subvm-hash: Sub-VM Result Hash @@ -149,8 +193,11 @@ into a rolling SHA3-256 accumulator. The finalized accumulator is the For each deterministic sub-VM, its *small hash* is folded into the accumulator. The small hash is a SHA3-256 digest over a :ref:`gvm-def-calldata-encoding` encoded map: -#. ``kind`` β€” the exact string ``"Return"``, ``"UserError"``, or ``"VMError"`` -#. ``result`` β€” for ``Return``/``UserError`` the result value in :ref:`gvm-def-calldata-encoding`; for ``VMError`` the error code string +#. ``kind`` β€” the exact string ``"Return"``, ``"UserError"`` or ``"VMError"``. + Fatality is not an outcome of its own, only a statement about who may catch + it, so a fatal result hashes as ``"VMError"`` +#. ``result`` β€” for ``Return``/``UserError`` the result value in + :ref:`gvm-def-calldata-encoding`; for ``VMError`` the error code string #. ``subvm_hashes`` β€” that sub-VM's own finalized accumulator, making the hash recursive over the whole deterministic call tree #. ``wasm_store_hashes`` @@ -171,7 +218,7 @@ A consumer in :ref:`gvm-def-sync-mode` or :ref:`gvm-def-validator-mode` consumes one leader-proposed result per non-deterministic block it runs. If the leader supplied **more** results than the run consumed, the surplus blocks were never reached: the run's result is replaced by -:ref:`gvm-def-pending-str-trie-value-vm-error-leader-output-extra`, whose +:ref:`gvm-def-str-trie-value-vm-error-leader-fault-nondet-output-extra`, whose parameter is the first 6 characters of the :ref:`gvm-def-gvm32` encoding of the ``sha3_256`` digest of the complete ``[result_code][data]`` :ref:`sub-VM result buffer <gvm-def-subvm-result-encoding>` the run would diff --git a/docs/website/src/spec/04-contract-interface/02-abi.rst b/docs/website/src/spec/04-contract-interface/02-abi.rst index 6fa33522..83c06ef3 100644 --- a/docs/website/src/spec/04-contract-interface/02-abi.rst +++ b/docs/website/src/spec/04-contract-interface/02-abi.rst @@ -7,6 +7,12 @@ contracts interact with each other. The ABI provides a standardized way to encode method calls, handle parameters, and manage contract schemas while supporting both deterministic and non-deterministic operations. +The public ABI includes the stable values listed in +:doc:`../appendix/constants`. Values listed in +:doc:`../appendix/internal-constants` bound what a run may do, but they are not +part of that surface: a contract cannot read them, and nothing in the ABI is +required to keep them stable + .. _gvm-def-contract-call-conv: Method Calling Convention @@ -45,7 +51,7 @@ set. Element types inside ``args``/``kwargs`` are matched against the method signature by the runner, not here. Violations are reported as -:ref:`gvm-def-pending-str-trie-value-vm-error-malformed-entry`; where that +:ref:`gvm-def-str-trie-value-vm-error-malformed-entry`; where that surfaces depends on the entry point, see :ref:`gvm-vm-startup-entry-validation` and :ref:`gvm-def-gl-call-call-contract`. diff --git a/docs/website/src/spec/appendix/constants-pending.rst b/docs/website/src/spec/appendix/constants-pending.rst index d70b1b90..0b076926 100644 --- a/docs/website/src/spec/appendix/constants-pending.rst +++ b/docs/website/src/spec/appendix/constants-pending.rst @@ -1,33 +1,2 @@ Constants ========= - -.. _gvm-def-pending-str-trie-vm-error: - -vm_error --------- - -Type: str_trie - -.. _gvm-def-pending-str-trie-value-vm-error-malformed-entry: - -``malformed_entry`` -~~~~~~~~~~~~~~~~~~~ - -.. _gvm-def-pending-str-trie-value-vm-error-leader-output-malformed: - -``leader_output malformed`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. _gvm-def-pending-str-trie-value-vm-error-leader-output-uses-this-error: - -``leader_output uses_this_error`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Param: str - -.. _gvm-def-pending-str-trie-value-vm-error-leader-output-extra: - -``leader_output extra`` -~~~~~~~~~~~~~~~~~~~~~~~ - -Param: str diff --git a/docs/website/src/spec/appendix/constants.rst b/docs/website/src/spec/appendix/constants.rst index aa114299..d744f684 100644 --- a/docs/website/src/spec/appendix/constants.rst +++ b/docs/website/src/spec/appendix/constants.rst @@ -29,13 +29,6 @@ vm_error Value: ``2`` -.. _gvm-def-enum-value-result-code-internal-error: - -internal_error -~~~~~~~~~~~~~~ - -Value: ``3`` - .. _gvm-def-enum-storage-type: storage_type @@ -106,48 +99,6 @@ can_use_balance_for_message_fees Value: ``1`` -.. _gvm-def-consts-memory-limiter-consts: - -memory_limiter_consts ---------------------- - -Type: u32 - -.. _gvm-def-consts-value-memory-limiter-consts-table-entry: - -table_entry -~~~~~~~~~~~ - -Value: ``64`` - -.. _gvm-def-consts-value-memory-limiter-consts-file-mapping: - -file_mapping -~~~~~~~~~~~~ - -Value: ``256`` - -.. _gvm-def-consts-value-memory-limiter-consts-fd-allocation: - -fd_allocation -~~~~~~~~~~~~~ - -Value: ``96`` - -.. _gvm-def-consts-value-memory-limiter-consts-runner-load-cost: - -runner_load_cost -~~~~~~~~~~~~~~~~ - -Value: ``4096`` - -.. _gvm-def-consts-value-memory-limiter-consts-vm-spawn-cost: - -vm_spawn_cost -~~~~~~~~~~~~~ - -Value: ``134217728`` - .. _gvm-def-consts-root-offsets: root_offsets @@ -204,76 +155,6 @@ permissions Value: ``37`` -.. _gvm-def-consts-top-limits: - -top_limits ----------- - -Type: u32 - -.. _gvm-def-consts-value-top-limits-nondet-blocks: - -nondet_blocks -~~~~~~~~~~~~~ - -Value: ``4096`` - -.. _gvm-def-consts-value-top-limits-locked-slots: - -locked_slots -~~~~~~~~~~~~ - -Value: ``256`` - -.. _gvm-def-consts-value-top-limits-upgraders: - -upgraders -~~~~~~~~~ - -Value: ``32`` - -.. _gvm-def-consts-value-top-limits-vm-recursion: - -vm_recursion -~~~~~~~~~~~~ - -Value: ``512`` - -.. _gvm-def-consts-value-top-limits-web-request-min-space: - -web_request_min_space -~~~~~~~~~~~~~~~~~~~~~ - -Value: ``65536`` - -.. _gvm-def-consts-value-top-limits-web-render-min-space: - -web_render_min_space -~~~~~~~~~~~~~~~~~~~~ - -Value: ``134217728`` - -.. _gvm-def-consts-value-top-limits-max-fds: - -max_fds -~~~~~~~ - -Value: ``1024`` - -.. _gvm-def-consts-value-top-limits-wasm-call-depth: - -wasm_call_depth -~~~~~~~~~~~~~~~ - -Value: ``1024`` - -.. _gvm-def-consts-value-top-limits-wasm-stack-value-slots: - -wasm_stack_value_slots -~~~~~~~~~~~~~~~~~~~~~~ - -Value: ``65535`` - .. _gvm-def-enum-special-method: special_method @@ -307,10 +188,34 @@ Type: str_trie ``timeout`` ~~~~~~~~~~~ -.. _gvm-def-str-trie-value-vm-error-absent-leader-nondet-output: +.. _gvm-def-str-trie-value-vm-error-malformed-entry: + +``malformed_entry`` +~~~~~~~~~~~~~~~~~~~ + +.. _gvm-def-str-trie-value-vm-error-leader-fault-nondet-output-absent: -``absent_leader_nondet_output`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``leader_fault nondet_output absent`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. _gvm-def-str-trie-value-vm-error-leader-fault-nondet-output-malformed: + +``leader_fault nondet_output malformed`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. _gvm-def-str-trie-value-vm-error-leader-fault-nondet-output-uses-this-error: + +``leader_fault nondet_output uses_this_error`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Param: str + +.. _gvm-def-str-trie-value-vm-error-leader-fault-nondet-output-extra: + +``leader_fault nondet_output extra`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Param: str .. _gvm-def-str-trie-value-vm-error-exit-code: @@ -484,10 +389,10 @@ Param: i32 ``fee too_many_rounds`` ~~~~~~~~~~~~~~~~~~~~~~~ -.. _gvm-def-str-trie-value-vm-error-host-forbidden: +.. _gvm-def-str-trie-value-vm-error-forbidden: -``host_forbidden`` -~~~~~~~~~~~~~~~~~~ +``forbidden`` +~~~~~~~~~~~~~ .. _gvm-def-str-trie-value-vm-error-evm-reverted: @@ -499,21 +404,21 @@ Param: i32 ``invalid_contract`` ~~~~~~~~~~~~~~~~~~~~ -.. _gvm-def-str-trie-value-vm-error-invalid-contract-absent-runner-comment: +.. _gvm-def-str-trie-value-vm-error-invalid-contract-runner-absent: + +``invalid_contract runner absent`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. _gvm-def-str-trie-value-vm-error-invalid-contract-runner-malformed: -``invalid_contract absent_runner_comment`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``invalid_contract runner malformed`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. _gvm-def-str-trie-value-vm-error-invalid-contract-not-utf8-text: ``invalid_contract not_utf8_text`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. _gvm-def-str-trie-value-vm-error-invalid-contract-malformed-runner: - -``invalid_contract malformed_runner`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - .. _gvm-def-str-trie-value-vm-error-invalid-contract-major-mismatch: ``invalid_contract major_mismatch`` @@ -536,6 +441,16 @@ This error remains terminal for top-level and runner loads. During :ref:`gvm-def ``invalid_contract wasm entrypoint`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. _gvm-def-str-trie-detail-vm-error-internal: + +``# internal`` +~~~~~~~~~~~~~~ + +.. _gvm-def-str-trie-detail-vm-error-external: + +``# external`` +~~~~~~~~~~~~~~ + .. _gvm-def-const-event-max-topics: event_max_topics diff --git a/docs/website/src/spec/appendix/index.rst b/docs/website/src/spec/appendix/index.rst index 9cda9e86..613e7734 100644 --- a/docs/website/src/spec/appendix/index.rst +++ b/docs/website/src/spec/appendix/index.rst @@ -6,4 +6,5 @@ Appendix constants Pending Constants <constants-pending> + Internal Constants <internal-constants> runner-schema diff --git a/docs/website/src/spec/appendix/internal-constants.rst b/docs/website/src/spec/appendix/internal-constants.rst new file mode 100644 index 00000000..44b72f37 --- /dev/null +++ b/docs/website/src/spec/appendix/internal-constants.rst @@ -0,0 +1,156 @@ +Constants +========= + +.. _gvm-def-consts-memory-limiter-consts: + +memory_limiter_consts +--------------------- + +Type: u32 + +.. _gvm-def-consts-value-memory-limiter-consts-table-entry: + +table_entry +~~~~~~~~~~~ + +Value: ``64`` + +.. _gvm-def-consts-value-memory-limiter-consts-file-mapping: + +file_mapping +~~~~~~~~~~~~ + +Value: ``256`` + +.. _gvm-def-consts-value-memory-limiter-consts-fd-allocation: + +fd_allocation +~~~~~~~~~~~~~ + +Value: ``96`` + +.. _gvm-def-consts-value-memory-limiter-consts-runner-load-cost: + +runner_load_cost +~~~~~~~~~~~~~~~~ + +Value: ``4096`` + +.. _gvm-def-consts-value-memory-limiter-consts-vm-spawn-cost: + +vm_spawn_cost +~~~~~~~~~~~~~ + +Value: ``134217728`` + +.. _gvm-def-consts-value-memory-limiter-consts-new-storage-page: + +new_storage_page +~~~~~~~~~~~~~~~~ + +Value: ``256`` + +.. _gvm-def-consts-value-memory-limiter-consts-storage-page-inherited: + +storage_page_inherited +~~~~~~~~~~~~~~~~~~~~~~ + +Value: ``128`` + +.. _gvm-def-consts-top-limits: + +top_limits +---------- + +Type: u32 + +.. _gvm-def-consts-value-top-limits-nondet-blocks: + +nondet_blocks +~~~~~~~~~~~~~ + +Value: ``4096`` + +.. _gvm-def-consts-value-top-limits-locked-slots: + +locked_slots +~~~~~~~~~~~~ + +Value: ``256`` + +.. _gvm-def-consts-value-top-limits-upgraders: + +upgraders +~~~~~~~~~ + +Value: ``32`` + +.. _gvm-def-consts-value-top-limits-vm-recursion: + +vm_recursion +~~~~~~~~~~~~ + +Value: ``512`` + +.. _gvm-def-consts-value-top-limits-web-request-min-space: + +web_request_min_space +~~~~~~~~~~~~~~~~~~~~~ + +Value: ``65536`` + +.. _gvm-def-consts-value-top-limits-web-render-min-space: + +web_render_min_space +~~~~~~~~~~~~~~~~~~~~ + +Value: ``134217728`` + +.. _gvm-def-consts-value-top-limits-max-fds: + +max_fds +~~~~~~~ + +Value: ``1024`` + +.. _gvm-def-consts-value-top-limits-wasm-call-depth: + +wasm_call_depth +~~~~~~~~~~~~~~~ + +Value: ``1024`` + +.. _gvm-def-consts-value-top-limits-wasm-stack-value-slots: + +wasm_stack_value_slots +~~~~~~~~~~~~~~~~~~~~~~ + +Value: ``65535`` + +.. _gvm-def-consts-runner-limits: + +runner_limits +------------- + +Type: u32 + +.. _gvm-def-consts-value-runner-limits-env-name-len: + +env_name_len +~~~~~~~~~~~~ + +Value: ``128`` + +.. _gvm-def-consts-value-runner-limits-env-value-len: + +env_value_len +~~~~~~~~~~~~~ + +Value: ``1048576`` + +.. _gvm-def-consts-value-runner-limits-init-action-depth: + +init_action_depth +~~~~~~~~~~~~~~~~~ + +Value: ``128`` diff --git a/docs/website/src/spec/changelog.rst b/docs/website/src/spec/changelog.rst new file mode 100644 index 00000000..2751adcb --- /dev/null +++ b/docs/website/src/spec/changelog.rst @@ -0,0 +1,54 @@ +Changelog +========= + +User-observable changes to the specification, grouped by executor line (see +:doc:`01-core-architecture/03-versioning`). Internal refactors are omitted; +see the repository history for those + +.. _gvm-changelog-v0-3: + +v0.3 +---- + +Breaking +~~~~~~~~ + +#. ``runner.json`` rejects unknown top-level and nested fields; a runner that + relied on an extra field being silently ignored fails to load. The single + top-level ``$schema`` string annotation is still accepted. See + :doc:`appendix/runner-schema` and the action definitions in + :doc:`02-execution-environment/04-runners` +#. ZIP-packaged :term:`runners <Runner>` are accepted under narrower rules; an + archive that previously loaded despite violating one of them now fails. + See the "ZIP Archive" layout in :doc:`02-execution-environment/04-runners`: + + #. Compression must read ``stored`` in both the central directory and the + entry's local header + #. A stored entry's compressed and uncompressed sizes must be equal + #. Every entry's CRC-32 is validated against its declared value + #. A directory entry must carry no contents; it is skipped rather than + added to the file list + #. Entry names are validated at parse time: no empty name, no leading + ``/``, no backslash, no empty, ``.`` or ``..`` path component, no + trailing ``/`` on a file + #. Entries sharing a name resolve to the last of them + +Changed +~~~~~~~ + +#. A missing or malformed runner archive/comment header is now reported as + :ref:`gvm-def-str-trie-value-vm-error-invalid-contract-runner-absent` or + :ref:`gvm-def-str-trie-value-vm-error-invalid-contract-runner-malformed` + respectively, instead of the former ``invalid_contract absent_runner_comment`` + and ``invalid_contract malformed_runner`` codes. A malformed runner archive + also now reports the precise + :ref:`gvm-def-str-trie-value-vm-error-invalid-contract-runner-malformed` + code where it previously surfaced a bare ``invalid_contract`` +#. A :ref:`gvm-def-gl-call-run-nondet` block starts on a budget seeded with its + caller's remaining RAM instead of a fresh 4 GiB one, so it can no longer use + more RAM than the deterministic caller had left. See + :doc:`03-vm/03-ram-limiting` +#. A resource error raised while a module is being instantiated keeps its own + code β€” e.g. exhausting the memory budget there reports + :ref:`gvm-def-str-trie-value-vm-error-out-of-memory-wasm-memory` β€” instead of + being reported as a bare ``invalid_contract`` diff --git a/docs/website/src/spec/index.rst b/docs/website/src/spec/index.rst index f0cfc7b3..5f1b2029 100644 --- a/docs/website/src/spec/index.rst +++ b/docs/website/src/spec/index.rst @@ -13,6 +13,7 @@ The specification has been split into multiple sections for better organization :caption: Contents: glossary + changelog 01-core-architecture/index 02-execution-environment/index 03-vm/index diff --git a/executors/v0.2.x b/executors/v0.2.x index 5b23854c..9da582c9 160000 --- a/executors/v0.2.x +++ b/executors/v0.2.x @@ -1 +1 @@ -Subproject commit 5b23854c49cc978995b5e1d4ac50c64f1c576736 +Subproject commit 9da582c9eb7b8f801cc00a4fd8a20f5dedd33da3 diff --git a/executors/v0.3.x b/executors/v0.3.x index 8a3a80cc..a54b15b7 160000 --- a/executors/v0.3.x +++ b/executors/v0.3.x @@ -1 +1 @@ -Subproject commit 8a3a80cc93c3433c9d34690171d8fbd2e2540738 +Subproject commit a54b15b7f3d4d81e362b80e6d2a678f89ae52b37 diff --git a/flake.nix b/flake.nix index fce2b678..dee0485b 100644 --- a/flake.nix +++ b/flake.nix @@ -150,7 +150,6 @@ mermaid-cli ]; packages-py-test = with pkgs; [ - # aflplusplus # currently we don't run fuzzing on CI python312 poetry # pytest + plugins so `poetry run -- pytest` has the @@ -736,7 +735,10 @@ ''; }; devShells.mock-tests = pkgs.mkShell { - packages = packages-0 ++ packages-rust ++ packages-debug-test; + # `pre-commit` itself, not `enabledPackages`: the push-refs unit test + # drives a `language: system` hook, so it needs no hook tooling. + packages = + packages-0 ++ packages-rust ++ packages-debug-test ++ [ pre-commit-check.config.package ]; shellHook = shell-hook-base; }; devShells.full = pkgs.mkShell { diff --git a/implementation/.ya-test-config.json b/implementation/.ya-test-config.json index e3659a87..bc011a52 100644 --- a/implementation/.ya-test-config.json +++ b/implementation/.ya-test-config.json @@ -14,5 +14,12 @@ "ATOMAKEY", "LSQLITE3_SRC", "LUA_INCLUDE" - ] + ], + "tags": { + "lib": [ + "feature-manager-socket", + "feature-version-routing", + "feature-web-request-signed" + ] + } } diff --git a/implementation/Cargo.lock b/implementation/Cargo.lock index 37e629a6..ee5803e6 100644 --- a/implementation/Cargo.lock +++ b/implementation/Cargo.lock @@ -123,9 +123,6 @@ name = "arbitrary" version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" -dependencies = [ - "derive_arbitrary", -] [[package]] name = "archery" @@ -721,17 +718,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "derive_arbitrary" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "digest" version = "0.10.7" @@ -1019,7 +1005,6 @@ dependencies = [ name = "genlayer_calldata" version = "0.0.2" dependencies = [ - "arbitrary", "bytes", "chrono", "genlayer_calldata_derive", @@ -1043,7 +1028,6 @@ dependencies = [ name = "genlayer_sdk" version = "0.0.2" dependencies = [ - "arbitrary", "bytes", "chrono", "genlayer_calldata", @@ -1060,7 +1044,6 @@ name = "genvm-common" version = "0.3.0" dependencies = [ "anyhow", - "arbitrary", "archery", "bytes", "chrono", diff --git a/implementation/default.nix b/implementation/default.nix index 0694077a..ee480581 100644 --- a/implementation/default.nix +++ b/implementation/default.nix @@ -17,9 +17,21 @@ let lib = pkgs.lib; + monorepo = builtins.fromJSON (builtins.readFile (root-src + "/.genvm-monorepo-root")); + release-src = get-root-subtree ( + [ + ".genvm-monorepo-root" + "install" + "libs/unhardcoded-engine/llm_policy" + "libs/unhardcoded-engine/llm_policy.lua" + "support/manifest-base.yaml" + ] + ++ builtins.map (line: "executors/${line}.x/manifest.json") monorepo.active-versions + ); + # The shipped LLM dispatch script requires the `llm_policy` package, which # lives in the unhardcoded-engine submodule instead of under install/. - llm-policy-src = root-src + "/libs/unhardcoded-engine"; + llm-policy-src = release-src + "/libs/unhardcoded-engine"; make-for-target = target: @@ -38,6 +50,8 @@ let "crates/modules-interfaces" "crates/calldata" "crates/calldata-derive" + # a dev-dependency of the executor crates below, but cargo still needs its manifest + "crates/fuzzing" # executor crates come from the v0.3.x submodule mount "executors/v0.3.x/executor/crates" ]; @@ -57,7 +71,7 @@ let srcs = [ exe - ../install + (release-src + "/install") ] ++ compiled-libs.${target}; @@ -103,7 +117,7 @@ let # Assemble data/manifest.yaml from the active executor submodules # (executor-version + available-after) and the static base fields. - genvm-tool -C ${root-src} build-manifest --output "$out/data/manifest.yaml" + genvm-tool -C ${release-src} build-manifest --output "$out/data/manifest.yaml" patch-rpath --codesign --search-dir "$out/lib" --rpath '$ORIGIN/../lib' "$out/bin/genvm-modules" find "$out/lib" -type f -name '*.so' -not -name 'libc.so' | while read lib; do diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/1 b/implementation/fuzz/inputs-genvm-modules-complete-json-curated/1 similarity index 100% rename from implementation/fuzz/inputs-genvm-modules-complete-json/1 rename to implementation/fuzz/inputs-genvm-modules-complete-json-curated/1 diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json-curated/2 b/implementation/fuzz/inputs-genvm-modules-complete-json-curated/2 new file mode 100644 index 00000000..490b564b --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json-curated/2 @@ -0,0 +1,3 @@ +{ "fo": [ { "fo": [ ] +} ] +} \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/0/0/00d3ga150hdc2vahv7sek5vhev60wgxrqmchnxqg3tkgj b/implementation/fuzz/inputs-genvm-modules-complete-json/0/0/00d3ga150hdc2vahv7sek5vhev60wgxrqmchnxqg3tkgj new file mode 100644 index 00000000..ab36f9a2 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/0/0/00d3ga150hdc2vahv7sek5vhev60wgxrqmchnxqg3tkgj @@ -0,0 +1 @@ +"UUUMbcγ€» )rbbbe," diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/0/0/00esba677v1ndtj0816kwhjph61a9mtjqrhqp45j4pn8j b/implementation/fuzz/inputs-genvm-modules-complete-json/0/0/00esba677v1ndtj0816kwhjph61a9mtjqrhqp45j4pn8j new file mode 100644 index 00000000..97f03a8d --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/0/0/00esba677v1ndtj0816kwhjph61a9mtjqrhqp45j4pn8j @@ -0,0 +1,54 @@ +{ + + + + + + + + + + "foo": "bar", + + + + + + + "foo": "bar", + + "foo": "bar", + + + + + + + + "foo": "bar" + + + + + + + + + + + + + + +! + + + + + + + + + + + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/0/0/00m6tt8xf34nkv5f4jvcv7t5qnbvq8e6vspmf0we96ydt b/implementation/fuzz/inputs-genvm-modules-complete-json/0/0/00m6tt8xf34nkv5f4jvcv7t5qnbvq8e6vspmf0we96ydt new file mode 100644 index 00000000..454ce545 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/0/0/00m6tt8xf34nkv5f4jvcv7t5qnbvq8e6vspmf0we96ydt @@ -0,0 +1,3 @@ +{ + +} \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/0/1/0113bt3yk923gfp1a4cnxp8n9k0nqjsd6003qrspy8ng0 b/implementation/fuzz/inputs-genvm-modules-complete-json/0/1/0113bt3yk923gfp1a4cnxp8n9k0nqjsd6003qrspy8ng0 new file mode 100644 index 00000000..cf5c6cfd --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/0/1/0113bt3yk923gfp1a4cnxp8n9k0nqjsd6003qrspy8ng0 @@ -0,0 +1,4 @@ +{ + "foo": [ + [ + [[[[[[[[[[[[[[[[[[[[ [[[[[[[[ [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/0/3/03tgtrmf50h7edgb1jvdpp767qs4j6vjfp33ty9rr9w1t b/implementation/fuzz/inputs-genvm-modules-complete-json/0/3/03tgtrmf50h7edgb1jvdpp767qs4j6vjfp33ty9rr9w1t new file mode 100644 index 00000000..d98ebe76 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/0/3/03tgtrmf50h7edgb1jvdpp767qs4j6vjfp33ty9rr9w1t @@ -0,0 +1 @@ +0e555 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/0/4/045698ck5t9h0fn9dbv9x26q7gxekqbn9chzzbskfp53c b/implementation/fuzz/inputs-genvm-modules-complete-json/0/4/045698ck5t9h0fn9dbv9x26q7gxekqbn9chzzbskfp53c new file mode 100644 index 00000000..cfa49983 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/0/4/045698ck5t9h0fn9dbv9x26q7gxekqbn9chzzbskfp53c @@ -0,0 +1 @@ +12.1E111 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/0/5/05r1ay1bw7ngb8zmtkdfkyvrka6qbjs9jwv46mt3cbgde b/implementation/fuzz/inputs-genvm-modules-complete-json/0/5/05r1ay1bw7ngb8zmtkdfkyvrka6qbjs9jwv46mt3cbgde new file mode 100644 index 00000000..93a9ef1c --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/0/5/05r1ay1bw7ngb8zmtkdfkyvrka6qbjs9jwv46mt3cbgde @@ -0,0 +1 @@ + " \uDbbb\uDDbBu" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/0/5/05z7425x8bd6nvkk4kgq4f2499n09vpv1paeafe025c6c b/implementation/fuzz/inputs-genvm-modules-complete-json/0/5/05z7425x8bd6nvkk4kgq4f2499n09vpv1paeafe025c6c new file mode 100644 index 00000000..b914e297 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/0/5/05z7425x8bd6nvkk4kgq4f2499n09vpv1paeafe025c6c @@ -0,0 +1 @@ + [1111111111.1111111111E111, 1111111111.1111111111E111, 2,2, 11.111111111111E111, 2, 111111111111611111111] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/0/8/082658bdw9cssdkncjmxvzbhmff4n1j4pwhntw0n9mj2t b/implementation/fuzz/inputs-genvm-modules-complete-json/0/8/082658bdw9cssdkncjmxvzbhmff4n1j4pwhntw0n9mj2t new file mode 100644 index 00000000..941c5b4a --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/0/8/082658bdw9cssdkncjmxvzbhmff4n1j4pwhntw0n9mj2t @@ -0,0 +1,3 @@ +{ + "":{ "":{ +, \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/0/9/09j7nh5jet076pmv9f8pqe6swkjsqdqc9yxw67105wrnc b/implementation/fuzz/inputs-genvm-modules-complete-json/0/9/09j7nh5jet076pmv9f8pqe6swkjsqdqc9yxw67105wrnc new file mode 100644 index 00000000..1404c1e8 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/0/9/09j7nh5jet076pmv9f8pqe6swkjsqdqc9yxw67105wrnc @@ -0,0 +1 @@ +"\f \\ζ—₯112 " \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/0/a/0a34bxzvvbkk9zdf1wh22spsbdwbqeqerm0e43pv2ngct b/implementation/fuzz/inputs-genvm-modules-complete-json/0/a/0a34bxzvvbkk9zdf1wh22spsbdwbqeqerm0e43pv2ngct new file mode 100644 index 00000000..8a919d7d --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/0/a/0a34bxzvvbkk9zdf1wh22spsbdwbqeqerm0e43pv2ngct @@ -0,0 +1,2 @@ + { + " \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/0/a/0a52mzaj1hv1tb2txtrjrs0ejszt3fgn469604sqnqj6w b/implementation/fuzz/inputs-genvm-modules-complete-json/0/a/0a52mzaj1hv1tb2txtrjrs0ejszt3fgn469604sqnqj6w new file mode 100644 index 00000000..5f7e8093 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/0/a/0a52mzaj1hv1tb2txtrjrs0ejszt3fgn469604sqnqj6w @@ -0,0 +1 @@ +"\f\f\f\ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/0/a/0a7d5brepkxdgktr1f7xwgdg5swstym8vsx6fg3ncrmp4 b/implementation/fuzz/inputs-genvm-modules-complete-json/0/a/0a7d5brepkxdgktr1f7xwgdg5swstym8vsx6fg3ncrmp4 new file mode 100644 index 00000000..5e0da963 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/0/a/0a7d5brepkxdgktr1f7xwgdg5swstym8vsx6fg3ncrmp4 @@ -0,0 +1 @@ +22919112111111111111.111111111 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/0/a/0asfwrht8gf6bz9tf3ag2ah4sdtaexyeeaatfy86cfbse b/implementation/fuzz/inputs-genvm-modules-complete-json/0/a/0asfwrht8gf6bz9tf3ag2ah4sdtaexyeeaatfy86cfbse new file mode 100644 index 00000000..36e83cac --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/0/a/0asfwrht8gf6bz9tf3ag2ah4sdtaexyeeaatfy86cfbse @@ -0,0 +1,2 @@ + +"Ρ€\\uba d r" diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/0/a/0avcwmc836hmvcx4tcsgxjt88gfqgr6crja3s08qjb2em b/implementation/fuzz/inputs-genvm-modules-complete-json/0/a/0avcwmc836hmvcx4tcsgxjt88gfqgr6crja3s08qjb2em new file mode 100644 index 00000000..7341a61b --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/0/a/0avcwmc836hmvcx4tcsgxjt88gfqgr6crja3s08qjb2em @@ -0,0 +1,2 @@ +[ + , t \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/0/b/0b3zfjthc54aqca5f5q9776emq6rtmc4psze4xt987swm b/implementation/fuzz/inputs-genvm-modules-complete-json/0/b/0b3zfjthc54aqca5f5q9776emq6rtmc4psze4xt987swm new file mode 100644 index 00000000..000ffd24 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/0/b/0b3zfjthc54aqca5f5q9776emq6rtmc4psze4xt987swm @@ -0,0 +1 @@ +00 [ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/0/d/0dkbwhymbmwe8yfb2af3mdmd6sreg4w60t8km5ctnghr8 b/implementation/fuzz/inputs-genvm-modules-complete-json/0/d/0dkbwhymbmwe8yfb2af3mdmd6sreg4w60t8km5ctnghr8 new file mode 100644 index 00000000..97166e39 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/0/d/0dkbwhymbmwe8yfb2af3mdmd6sreg4w60t8km5ctnghr8 @@ -0,0 +1,16 @@ +{ + "foo": [ + "bar", + "baz\n\\z", + 123, + -123, + -1e10, + 1e20, false, + [1111111111111111111111111111, 2, 3] + ], + " bar": [ + "русскиС Π±ΡƒΠΊΠ²έ‹ ζ—₯本", + "mbbbbbbbbbbbbbbaz\t\uabbbbb", + "ΠΊΠΈΠ΅ Π±ΡƒΠΊΠ²Ρ‹ ζ—₯uabbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/0/h/0h92aedsc1y7xgr695dmyvyf4zm587ab3yd1acdhrvr9a b/implementation/fuzz/inputs-genvm-modules-complete-json/0/h/0h92aedsc1y7xgr695dmyvyf4zm587ab3yd1acdhrvr9a new file mode 100644 index 00000000..2ce3e2ea --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/0/h/0h92aedsc1y7xgr695dmyvyf4zm587ab3yd1acdhrvr9a @@ -0,0 +1,2 @@ + +"Ρ€bbba===========\ua11E |" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/0/h/0hfwbfz1w1wys0ckdwxg5y4tvhshzwmq46h9mh79qw1ae b/implementation/fuzz/inputs-genvm-modules-complete-json/0/h/0hfwbfz1w1wys0ckdwxg5y4tvhshzwmq46h9mh79qw1ae new file mode 100644 index 00000000..d5b545fe --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/0/h/0hfwbfz1w1wys0ckdwxg5y4tvhshzwmq46h9mh79qw1ae @@ -0,0 +1,4 @@ + + + + "bar -123, \\\\\\\\\\\\\\\\bbbbbebbΠΊbbbΠ±ΡƒΡ™\\本\\\\\\ΠΊbbbbubbbubbbcΠ΅ Π±ΡƒΡ„\\本\\Π±ΡƒΡ„cΠ΅ Π±ΡƒΡ„\\本\\Π±ΡƒΡ„\\本\\\\\\ΠΊΠ²Ρ‹ 旁ba z" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/0/j/0j3dp76k6cfjc5zkzec3cn0t5h0t58w96yyzeyhzsm6he b/implementation/fuzz/inputs-genvm-modules-complete-json/0/j/0j3dp76k6cfjc5zkzec3cn0t5h0t58w96yyzeyhzsm6he new file mode 100644 index 00000000..b182156f --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/0/j/0j3dp76k6cfjc5zkzec3cn0t5h0t58w96yyzeyhzsm6he @@ -0,0 +1,3 @@ +{ + "foo": [ + "ba n \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/0/j/0j89xg8cfb3xb62hhx33xhhd813c2qrfrwjjgj5ebsw7w b/implementation/fuzz/inputs-genvm-modules-complete-json/0/j/0j89xg8cfb3xb62hhx33xhhd813c2qrfrwjjgj5ebsw7w new file mode 100644 index 00000000..4ca669b3 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/0/j/0j89xg8cfb3xb62hhx33xhhd813c2qrfrwjjgj5ebsw7w @@ -0,0 +1,5 @@ +{ + "foo 222": [ + " " + ] +} \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/0/m/0m4rse79871djv54arat6a564d9wk6b32et8xfaatccqr b/implementation/fuzz/inputs-genvm-modules-complete-json/0/m/0m4rse79871djv54arat6a564d9wk6b32et8xfaatccqr new file mode 100644 index 00000000..4fee252a --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/0/m/0m4rse79871djv54arat6a564d9wk6b32et8xfaatccqr @@ -0,0 +1,19 @@ +{ + "f faloo": [ + "bar", + "baz\n\\z", + 123, + -123, + -1e10, + null, + true, + false, + [1, 2, 3] + ], + "bar": [ + "русскиС Π±ΡƒΠΊΨ²Ρ‹ ζ—₯b", + "bbbb" + ], + "bar": [ + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/9a70598e97b57c7a6c6d7d77cc64509b13c0caf72203e8cfbe0c6f2e3446c804 b/implementation/fuzz/inputs-genvm-modules-complete-json/0/m/0mddxqyra4jsre5cw5fmk3fg3y4gx74fc32dxwggnkqz6 similarity index 50% rename from implementation/fuzz/inputs-genvm-modules-normalize-whitespace/9a70598e97b57c7a6c6d7d77cc64509b13c0caf72203e8cfbe0c6f2e3446c804 rename to implementation/fuzz/inputs-genvm-modules-complete-json/0/m/0mddxqyra4jsre5cw5fmk3fg3y4gx74fc32dxwggnkqz6 index 3834720e..6d03ac6b 100644 --- a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/9a70598e97b57c7a6c6d7d77cc64509b13c0caf72203e8cfbe0c6f2e3446c804 +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/0/m/0mddxqyra4jsre5cw5fmk3fg3y4gx74fc32dxwggnkqz6 @@ -1,17 +1,16 @@ - + - - - - \ No newline at end of file +" + + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/0/n/0nz26d1454cv5768t5j9se7txfecpxxsm9x8s7r91qf1e b/implementation/fuzz/inputs-genvm-modules-complete-json/0/n/0nz26d1454cv5768t5j9se7txfecpxxsm9x8s7r91qf1e new file mode 100644 index 00000000..703e58f3 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/0/n/0nz26d1454cv5768t5j9se7txfecpxxsm9x8s7r91qf1e @@ -0,0 +1 @@ +22222222222222.22222222 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/0/p/0paa3j6967xv5d1a2074nyqtnbbtv76pvbpzgevzfgy54 b/implementation/fuzz/inputs-genvm-modules-complete-json/0/p/0paa3j6967xv5d1a2074nyqtnbbtv76pvbpzgevzfgy54 new file mode 100644 index 00000000..395a6a3e --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/0/p/0paa3j6967xv5d1a2074nyqtnbbtv76pvbpzgevzfgy54 @@ -0,0 +1 @@ + "σ—₯—σ—₯—bσ—₯—σ—₯—" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/0/p/0prjncdjcy75m9m6q0qk6ezw8zj1596rhj36e402ew754 b/implementation/fuzz/inputs-genvm-modules-complete-json/0/p/0prjncdjcy75m9m6q0qk6ezw8zj1596rhj36e402ew754 new file mode 100644 index 00000000..f61a6a1a --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/0/p/0prjncdjcy75m9m6q0qk6ezw8zj1596rhj36e402ew754 @@ -0,0 +1,2 @@ +[, + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/0/q/0qn2kcn21t453s1b2ngz23bwnbzkm23e5ks6x0v2z5td0 b/implementation/fuzz/inputs-genvm-modules-complete-json/0/q/0qn2kcn21t453s1b2ngz23bwnbzkm23e5ks6x0v2z5td0 new file mode 100644 index 00000000..2f4081f3 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/0/q/0qn2kcn21t453s1b2ngz23bwnbzkm23e5ks6x0v2z5td0 @@ -0,0 +1,7 @@ +{"ar + + + + + +", \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/0/r/0rtkxs54nn6ex33a27wg1ppmthvkknkp3mep8f5b98k50 b/implementation/fuzz/inputs-genvm-modules-complete-json/0/r/0rtkxs54nn6ex33a27wg1ppmthvkknkp3mep8f5b98k50 new file mode 100644 index 00000000..424faaef --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/0/r/0rtkxs54nn6ex33a27wg1ppmthvkknkp3mep8f5b98k50 @@ -0,0 +1,2 @@ + +"\u0000Ρ‹" diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/0/t/0tjjkt9cpff1d660acdtdzdpfv43mfctgbpy3he9beysw b/implementation/fuzz/inputs-genvm-modules-complete-json/0/t/0tjjkt9cpff1d660acdtdzdpfv43mfctgbpy3he9beysw new file mode 100644 index 00000000..17931f67 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/0/t/0tjjkt9cpff1d660acdtdzdpfv43mfctgbpy3he9beysw @@ -0,0 +1 @@ +"\fn" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/0/w/0wqka8ved58jyqtjbr88vpkg80h3sbf3pjyznxq7j1904 b/implementation/fuzz/inputs-genvm-modules-complete-json/0/w/0wqka8ved58jyqtjbr88vpkg80h3sbf3pjyznxq7j1904 new file mode 100644 index 00000000..322876ac --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/0/w/0wqka8ved58jyqtjbr88vpkg80h3sbf3pjyznxq7j1904 @@ -0,0 +1 @@ +2111111.1121111111111 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/0/y/0y24j1rns05r9jsbwe40h08etjq468myc7zyqnr06d3et b/implementation/fuzz/inputs-genvm-modules-complete-json/0/y/0y24j1rns05r9jsbwe40h08etjq468myc7zyqnr06d3et new file mode 100644 index 00000000..0eb41820 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/0/y/0y24j1rns05r9jsbwe40h08etjq468myc7zyqnr06d3et @@ -0,0 +1 @@ +0.11 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/0/y/0ynj4895jyafkpw0jnvzmxdnhkv2p5jz9p6kc483sgk50 b/implementation/fuzz/inputs-genvm-modules-complete-json/0/y/0ynj4895jyafkpw0jnvzmxdnhkv2p5jz9p6kc483sgk50 new file mode 100644 index 00000000..a8e99ef1 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/0/y/0ynj4895jyafkpw0jnvzmxdnhkv2p5jz9p6kc483sgk50 @@ -0,0 +1 @@ +" Ρ‚\u000000 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/1/2/12qz7rfp162bvpwtqdcsjfcte6gnaqhbn0brtbqmrfeyy b/implementation/fuzz/inputs-genvm-modules-complete-json/1/2/12qz7rfp162bvpwtqdcsjfcte6gnaqhbn0brtbqmrfeyy new file mode 100644 index 00000000..a36d2a76 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/1/2/12qz7rfp162bvpwtqdcsjfcte6gnaqhbn0brtbqmrfeyy @@ -0,0 +1,17 @@ + { + "":0, + "": { + "": + { + "":0, + "": { + "": + { +"":0, + "": { + " + + + + + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/1/2/12v1440bpgmpf20twrkek8mv0d8005q3q499gz8acnzmm b/implementation/fuzz/inputs-genvm-modules-complete-json/1/2/12v1440bpgmpf20twrkek8mv0d8005q3q499gz8acnzmm new file mode 100644 index 00000000..16593f0b --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/1/2/12v1440bpgmpf20twrkek8mv0d8005q3q499gz8acnzmm @@ -0,0 +1 @@ +-0 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/1/4/149cmt9ms89v020yrryvpa8jvmc20h657s4n8p4ypamy0 b/implementation/fuzz/inputs-genvm-modules-complete-json/1/4/149cmt9ms89v020yrryvpa8jvmc20h657s4n8p4ypamy0 new file mode 100644 index 00000000..692b8050 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/1/4/149cmt9ms89v020yrryvpa8jvmc20h657s4n8p4ypamy0 @@ -0,0 +1 @@ +fals \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/1/4/14qyfsbfxsjvhq4h4px6vne04dr3tedqcnk65kx1h0x5a b/implementation/fuzz/inputs-genvm-modules-complete-json/1/4/14qyfsbfxsjvhq4h4px6vne04dr3tedqcnk65kx1h0x5a new file mode 100644 index 00000000..147a3b66 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/1/4/14qyfsbfxsjvhq4h4px6vne04dr3tedqcnk65kx1h0x5a @@ -0,0 +1,2 @@ +{ " @fo" : [ 3 ] +} \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/1/5/152sp9r1b3r2ws1hk69wtkfwp9rrxedhcyss8bdcnvaha b/implementation/fuzz/inputs-genvm-modules-complete-json/1/5/152sp9r1b3r2ws1hk69wtkfwp9rrxedhcyss8bdcnvaha new file mode 100644 index 00000000..cc7ce787 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/1/5/152sp9r1b3r2ws1hk69wtkfwp9rrxedhcyss8bdcnvaha @@ -0,0 +1,17 @@ +{ + "foo": [ + "baz\n\\z", + 123, + -123, + -1e10, + null, + true, + false, + [1, 2, 3] + ], + "bar": [ + "русскиС Π±ΡƒΠΊΨ²Ρ‹ ζ—₯本az\t\uabbbbb", + "bbbb" + ] +} + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/1/5/15bp4vcf7xkp37ac1718tt0rxvgts1agh64p5aw1fhxbj b/implementation/fuzz/inputs-genvm-modules-complete-json/1/5/15bp4vcf7xkp37ac1718tt0rxvgts1agh64p5aw1fhxbj new file mode 100644 index 00000000..0fa37116 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/1/5/15bp4vcf7xkp37ac1718tt0rxvgts1agh64p5aw1fhxbj @@ -0,0 +1 @@ +2222222222222222222222222E2 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/1/5/15sf0t7k419d92gz7xhsav1gcmq07n07wydnnwsrgphym b/implementation/fuzz/inputs-genvm-modules-complete-json/1/5/15sf0t7k419d92gz7xhsav1gcmq07n07wydnnwsrgphym new file mode 100644 index 00000000..c4906337 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/1/5/15sf0t7k419d92gz7xhsav1gcmq07n07wydnnwsrgphym @@ -0,0 +1,20 @@ +{ + "foo": [ + "bar" + ], + "babbbbbbbbbbbbbaz\t\\r": [ + 123, + -123, + -1e10, + 1e20, + null, + true, + false, + [1, 2, 3] + ], + "bar": [ + "русскиС Π±ΡƒΠΊΠ²Ρ‹ ζ—₯本", + "mbbbbbbbbbbbbbbaz\t\uabbbbb", + "mbbbbbbbbbbbbbbaz\t\\uabbbbb" + ] +} \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/1/7/17hz2hbxc3cj4tw3tje2j8d5ty7kc6kfpsmfte6p2twm2 b/implementation/fuzz/inputs-genvm-modules-complete-json/1/7/17hz2hbxc3cj4tw3tje2j8d5ty7kc6kfpsmfte6p2twm2 new file mode 100644 index 00000000..512eabc9 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/1/7/17hz2hbxc3cj4tw3tje2j8d5ty7kc6kfpsmfte6p2twm2 @@ -0,0 +1 @@ + " \uDCb5" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/1/8/184jstbv6j02k45pq0cvc2v78est9zdsmbnrffr0r7nv2 b/implementation/fuzz/inputs-genvm-modules-complete-json/1/8/184jstbv6j02k45pq0cvc2v78est9zdsmbnrffr0r7nv2 new file mode 100644 index 00000000..6f12b734 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/1/8/184jstbv6j02k45pq0cvc2v78est9zdsmbnrffr0r7nv2 @@ -0,0 +1,2 @@ + +"Ρ€jξ—¨750 1\uDbbb\uDc11 ! e111@eC@1b11111111111111111;111111.11p" diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/1/9/19dp9v5s7zg0r44q1q10j4cw9xczdd69pvt8v340a6f8c b/implementation/fuzz/inputs-genvm-modules-complete-json/1/9/19dp9v5s7zg0r44q1q10j4cw9xczdd69pvt8v340a6f8c new file mode 100644 index 00000000..efcbec52 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/1/9/19dp9v5s7zg0r44q1q10j4cw9xczdd69pvt8v340a6f8c @@ -0,0 +1 @@ +299999999999999999992E+ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/1/a/1acaehj72a3q4n0svg1mw6sr73c755f4fs585a33prhzp b/implementation/fuzz/inputs-genvm-modules-complete-json/1/a/1acaehj72a3q4n0svg1mw6sr73c755f4fs585a33prhzp new file mode 100644 index 00000000..cd477695 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/1/a/1acaehj72a3q4n0svg1mw6sr73c755f4fs585a33prhzp @@ -0,0 +1,22 @@ +{ + "foo": [ + "bar", + "baz\n\\{", + 123, +23, + -1e10, + 1e20, + null, + true, + false, + [1, 2, 3] + ], + "": [ ], + "bar":[ ], + "": [ ], + "bar": [ + "русскиС Π±ΡƒΠΊΠ²Π‡ ζ—₯本", + "mbbbbbbbbbbbbbbaz\t\uabbbbb", + "mbbbbbbbbbbbbbbaz\t\\tabbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/1/a/1aj0wvbm2cwde34147rf5ha82ry271c5rc0agxjrc47x4 b/implementation/fuzz/inputs-genvm-modules-complete-json/1/a/1aj0wvbm2cwde34147rf5ha82ry271c5rc0agxjrc47x4 new file mode 100644 index 00000000..8671c168 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/1/a/1aj0wvbm2cwde34147rf5ha82ry271c5rc0agxjrc47x4 @@ -0,0 +1 @@ +2e-3333 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/1/a/1ajgxbt71nyw23zrd1qy323jtwynep5d2pdaav2tffxp8 b/implementation/fuzz/inputs-genvm-modules-complete-json/1/a/1ajgxbt71nyw23zrd1qy323jtwynep5d2pdaav2tffxp8 new file mode 100644 index 00000000..270841da --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/1/a/1ajgxbt71nyw23zrd1qy323jtwynep5d2pdaav2tffxp8 @@ -0,0 +1 @@ +"f\\.111111,\\\\ζ—₯1 ', " \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/1/b/1b6evrr1ecfjkt9k5kg0tk3xh25s33wx4ww96573naghw b/implementation/fuzz/inputs-genvm-modules-complete-json/1/b/1b6evrr1ecfjkt9k5kg0tk3xh25s33wx4ww96573naghw new file mode 100644 index 00000000..1ee1d282 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/1/b/1b6evrr1ecfjkt9k5kg0tk3xh25s33wx4ww96573naghw @@ -0,0 +1 @@ + "с\\ffffffffffabba\": ffffffffffabba\":fffabba\": [22222222222222222ffffffffffhbba\":f1222bbbb" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/1/c/1cph9wc8w0a8z0w2028vtcx5xs4dgyzf5w1s7jdppr3xy b/implementation/fuzz/inputs-genvm-modules-complete-json/1/c/1cph9wc8w0a8z0w2028vtcx5xs4dgyzf5w1s7jdppr3xy new file mode 100644 index 00000000..035ff3c1 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/1/c/1cph9wc8w0a8z0w2028vtcx5xs4dgyzf5w1s7jdppr3xy @@ -0,0 +1 @@ + [1,22, , 21, \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/1/f/1f72s7zs5t4wmrbvk0yg71wxveqe55ztfpg8jbsa3bqmg b/implementation/fuzz/inputs-genvm-modules-complete-json/1/f/1f72s7zs5t4wmrbvk0yg71wxveqe55ztfpg8jbsa3bqmg new file mode 100644 index 00000000..4672bf57 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/1/f/1f72s7zs5t4wmrbvk0yg71wxveqe55ztfpg8jbsa3bqmg @@ -0,0 +1,3 @@ + { + "":0, + } diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/1/f/1fghdv1x7bnfg9pavb755sf0sc40qmhsbcwqf5nrs7ae4 b/implementation/fuzz/inputs-genvm-modules-complete-json/1/f/1fghdv1x7bnfg9pavb755sf0sc40qmhsbcwqf5nrs7ae4 new file mode 100644 index 00000000..6909f8f5 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/1/f/1fghdv1x7bnfg9pavb755sf0sc40qmhsbcwqf5nrs7ae4 @@ -0,0 +1,15 @@ + [ + + + + + + + + + + + + + +[[[[[[[[[[[[[[[[[[[S \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/1/f/1fjcx4dk7z22dava03ptrv3ymfjycm4m4ffx7j0d98m4p b/implementation/fuzz/inputs-genvm-modules-complete-json/1/f/1fjcx4dk7z22dava03ptrv3ymfjycm4m4ffx7j0d98m4p new file mode 100644 index 00000000..1ad28cc7 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/1/f/1fjcx4dk7z22dava03ptrv3ymfjycm4m4ffx7j0d98m4p @@ -0,0 +1 @@ + "Yσ—₯—11 [," \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/1/j/1jj15a1rq85qvscnan6she76mdje40y78df623nha4fyj b/implementation/fuzz/inputs-genvm-modules-complete-json/1/j/1jj15a1rq85qvscnan6she76mdje40y78df623nha4fyj new file mode 100644 index 00000000..7430f02c --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/1/j/1jj15a1rq85qvscnan6she76mdje40y78df623nha4fyj @@ -0,0 +1,3 @@ +{ "o": [ { "o": [ +} ] +} \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/1/k/1k3r491j8kg26xvpnfe3a0yb3ve9wvy43c3n09dgv7ctw b/implementation/fuzz/inputs-genvm-modules-complete-json/1/k/1k3r491j8kg26xvpnfe3a0yb3ve9wvy43c3n09dgv7ctw new file mode 100644 index 00000000..9a13412d --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/1/k/1k3r491j8kg26xvpnfe3a0yb3ve9wvy43c3n09dgv7ctw @@ -0,0 +1 @@ +1011111111111111111111111111 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/1/k/1knejvhs0c6k96dxf9zspqgns0xwvzn3ntmnc3m7hfgbw b/implementation/fuzz/inputs-genvm-modules-complete-json/1/k/1knejvhs0c6k96dxf9zspqgns0xwvzn3ntmnc3m7hfgbw new file mode 100644 index 00000000..06249e40 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/1/k/1knejvhs0c6k96dxf9zspqgns0xwvzn3ntmnc3m7hfgbw @@ -0,0 +1 @@ +3e-922 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/1/m/1msmyw0naed8mgex7ss57qmqbfwapn427xst8jsqe3ngj b/implementation/fuzz/inputs-genvm-modules-complete-json/1/m/1msmyw0naed8mgex7ss57qmqbfwapn427xst8jsqe3ngj new file mode 100644 index 00000000..4df6ce0c --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/1/m/1msmyw0naed8mgex7ss57qmqbfwapn427xst8jsqe3ngj @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/1/r/1rmm9pj961n1pq2vsy22xpbxw0nhyq93px4r60hahzjma b/implementation/fuzz/inputs-genvm-modules-complete-json/1/r/1rmm9pj961n1pq2vsy22xpbxw0nhyq93px4r60hahzjma new file mode 100644 index 00000000..44620ac5 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/1/r/1rmm9pj961n1pq2vsy22xpbxw0nhyq93px4r60hahzjma @@ -0,0 +1,2 @@ +[ + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/1/s/1sf4137gha4gmqm24trscfjqc7jm4479w6qj1mex11fry b/implementation/fuzz/inputs-genvm-modules-complete-json/1/s/1sf4137gha4gmqm24trscfjqc7jm4479w6qj1mex11fry new file mode 100644 index 00000000..5adc2b29 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/1/s/1sf4137gha4gmqm24trscfjqc7jm4479w6qj1mex11fry @@ -0,0 +1 @@ +21111111111111111111111111.1 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/1/v/1v0k1nwfyggr3b0wc35azwddz7t89e408jvrrapd5edrw b/implementation/fuzz/inputs-genvm-modules-complete-json/1/v/1v0k1nwfyggr3b0wc35azwddz7t89e408jvrrapd5edrw new file mode 100644 index 00000000..8ffaa9d0 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/1/v/1v0k1nwfyggr3b0wc35azwddz7t89e408jvrrapd5edrw @@ -0,0 +1,12 @@ +{ + + + + + + + + + + "foo": "bar", +  \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/1/v/1vfxfq59pf73na9hbcz0b6vdm97rwqjegsmyfg1vqsm82 b/implementation/fuzz/inputs-genvm-modules-complete-json/1/v/1vfxfq59pf73na9hbcz0b6vdm97rwqjegsmyfg1vqsm82 new file mode 100644 index 00000000..4acc4aea --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/1/v/1vfxfq59pf73na9hbcz0b6vdm97rwqjegsmyfg1vqsm82 @@ -0,0 +1,17 @@ +{ + "foo": [ + "bar", + "baz\n\\zt\uabbbbb", + "mbbbbbbb", + 123, + -123, + -1e10, + 1e20, + false, + [1,3] + ], + "bar": [ + "русскиС z\n\\zt\uabbbbb", + "mbmbbbbbbbbbbbbbbaz\t\bbbbaz\t\\uabbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/1/x/1x74f8x8hn228h57pkdye6wysbhsrrhjw49z4gepbqzvj b/implementation/fuzz/inputs-genvm-modules-complete-json/1/x/1x74f8x8hn228h57pkdye6wysbhsrrhjw49z4gepbqzvj new file mode 100644 index 00000000..5b4c53e5 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/1/x/1x74f8x8hn228h57pkdye6wysbhsrrhjw49z4gepbqzvj @@ -0,0 +1 @@ + [[ [[1, ,,,,, \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/1/y/1ytqg1fmemq3efa31gttwkc9ks89mdx2dbzc5qhgj92s4 b/implementation/fuzz/inputs-genvm-modules-complete-json/1/y/1ytqg1fmemq3efa31gttwkc9ks89mdx2dbzc5qhgj92s4 new file mode 100644 index 00000000..b43d02d4 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/1/y/1ytqg1fmemq3efa31gttwkc9ks89mdx2dbzc5qhgj92s4 @@ -0,0 +1 @@ + [1111111111111111111111111, 2, 3] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/1/z/1zbxn03hxa23gxe1md3gbn3zv7cngzcerj1j675n9780a b/implementation/fuzz/inputs-genvm-modules-complete-json/1/z/1zbxn03hxa23gxe1md3gbn3zv7cngzcerj1j675n9780a new file mode 100644 index 00000000..3cc762b5 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/1/z/1zbxn03hxa23gxe1md3gbn3zv7cngzcerj1j675n9780a @@ -0,0 +1 @@ +"" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/2/0/20n3w7v3kk9wzmhsayk4at70qkrajx55jgnepk7z6hr0r b/implementation/fuzz/inputs-genvm-modules-complete-json/2/0/20n3w7v3kk9wzmhsayk4at70qkrajx55jgnepk7z6hr0r new file mode 100644 index 00000000..8eaee9c6 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/2/0/20n3w7v3kk9wzmhsayk4at70qkrajx55jgnepk7z6hr0r @@ -0,0 +1 @@ + [1111111111116111111161e-111111112, 311, 11E1] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/2/1/210xq1kcqh6mwx1jv8e4m8p45ep59sdmv28pmzy4pw1a4 b/implementation/fuzz/inputs-genvm-modules-complete-json/2/1/210xq1kcqh6mwx1jv8e4m8p45ep59sdmv28pmzy4pw1a4 new file mode 100644 index 00000000..bafe7a35 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/2/1/210xq1kcqh6mwx1jv8e4m8p45ep59sdmv28pmzy4pw1a4 @@ -0,0 +1,2 @@ + [ [{ + "emo": 1, 2, 3nul \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/2/1/21ya1p19ay28psnn5f4zhyd2t4kfvda8a4y3dpc14v5f4 b/implementation/fuzz/inputs-genvm-modules-complete-json/2/1/21ya1p19ay28psnn5f4zhyd2t4kfvda8a4y3dpc14v5f4 new file mode 100644 index 00000000..5e955612 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/2/1/21ya1p19ay28psnn5f4zhyd2t4kfvda8a4y3dpc14v5f4 @@ -0,0 +1,5 @@ +{ "o": [ { "o": [ { "o": [ { "o": [ +} ] +} +} ] +} \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/2/3/23vz35x95rw4mrq1a5fxzy55qjks164pc29znrc45cdtp b/implementation/fuzz/inputs-genvm-modules-complete-json/2/3/23vz35x95rw4mrq1a5fxzy55qjks164pc29znrc45cdtp new file mode 100644 index 00000000..8fa7afe5 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/2/3/23vz35x95rw4mrq1a5fxzy55qjks164pc29znrc45cdtp @@ -0,0 +1 @@ +11111111111111111111111e-522 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/2/5/25c2p7k1zb2gkvyfdaz725jw76qrm1wncjwqkqc4kk1m8 b/implementation/fuzz/inputs-genvm-modules-complete-json/2/5/25c2p7k1zb2gkvyfdaz725jw76qrm1wncjwqkqc4kk1m8 new file mode 100644 index 00000000..5d20f5fc --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/2/5/25c2p7k1zb2gkvyfdaz725jw76qrm1wncjwqkqc4kk1m8 @@ -0,0 +1,2 @@ + +"Ρ€\\ub` d [[1, B\bΓ΅ r" diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/2/6/26wa4awjzxj0n7cxbv03m54mfedmavr2f6m01vv3zvtte b/implementation/fuzz/inputs-genvm-modules-complete-json/2/6/26wa4awjzxj0n7cxbv03m54mfedmavr2f6m01vv3zvtte new file mode 100644 index 00000000..8d2f53d0 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/2/6/26wa4awjzxj0n7cxbv03m54mfedmavr2f6m01vv3zvtte @@ -0,0 +1 @@ + [[ [[ [[[[[[[[[[[[[[[[[[[1, \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/2/9/29tp9v5282spx6x5aagjzrpyypvmtpkq3r0rrfdg4m67c b/implementation/fuzz/inputs-genvm-modules-complete-json/2/9/29tp9v5282spx6x5aagjzrpyypvmtpkq3r0rrfdg4m67c new file mode 100644 index 00000000..476f449f --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/2/9/29tp9v5282spx6x5aagjzrpyypvmtpkq3r0rrfdg4m67c @@ -0,0 +1,3 @@ + + +"\r" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/2/b/2babs58b5a87gfxksxtm649ypqgmkacmejzry836t92k0 b/implementation/fuzz/inputs-genvm-modules-complete-json/2/b/2babs58b5a87gfxksxtm649ypqgmkacmejzry836t92k0 new file mode 100644 index 00000000..00a55f04 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/2/b/2babs58b5a87gfxksxtm649ypqgmkacmejzry836t92k0 @@ -0,0 +1 @@ +[[]] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/2/c/2c95ns6xkaakcy132nmxmq4x2zgqr90x0dmsnz627ed66 b/implementation/fuzz/inputs-genvm-modules-complete-json/2/c/2c95ns6xkaakcy132nmxmq4x2zgqr90x0dmsnz627ed66 new file mode 100644 index 00000000..c17c703a --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/2/c/2c95ns6xkaakcy132nmxmq4x2zgqr90x0dmsnz627ed66 @@ -0,0 +1,2 @@ +2E+2 + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/2/d/2d8qwhcyefn0frp5r1ca0x7c5fx7z73799v9jgh5kznrm b/implementation/fuzz/inputs-genvm-modules-complete-json/2/d/2d8qwhcyefn0frp5r1ca0x7c5fx7z73799v9jgh5kznrm new file mode 100644 index 00000000..d96f13d8 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/2/d/2d8qwhcyefn0frp5r1ca0x7c5fx7z73799v9jgh5kznrm @@ -0,0 +1 @@ + 1 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/2/e/2ej6zv22vbtxd1h5fyy6aqs1709dm2kcner8xgmdjw3gy b/implementation/fuzz/inputs-genvm-modules-complete-json/2/e/2ej6zv22vbtxd1h5fyy6aqs1709dm2kcner8xgmdjw3gy new file mode 100644 index 00000000..24752035 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/2/e/2ej6zv22vbtxd1h5fyy6aqs1709dm2kcner8xgmdjw3gy @@ -0,0 +1,10 @@ + { + "":0, + "" + + + "": + " +: { + + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/2/h/2hmtt4mqq71y2pvf4twdsffeyvc71wp3h2hvs5ahme1nr b/implementation/fuzz/inputs-genvm-modules-complete-json/2/h/2hmtt4mqq71y2pvf4twdsffeyvc71wp3h2hvs5ahme1nr new file mode 100644 index 00000000..d9779a6d --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/2/h/2hmtt4mqq71y2pvf4twdsffeyvc71wp3h2hvs5ahme1nr @@ -0,0 +1 @@ +"\ri" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/2/j/2jng17dez65x6wswjp25acskmrx7b5tcmq386s35axwnc b/implementation/fuzz/inputs-genvm-modules-complete-json/2/j/2jng17dez65x6wswjp25acskmrx7b5tcmq386s35axwnc new file mode 100644 index 00000000..3d72cb7a --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/2/j/2jng17dez65x6wswjp25acskmrx7b5tcmq386s35axwnc @@ -0,0 +1 @@ +{ "Ρ€\ \u \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/2/k/2kqfqq0x171fhs9f654mhmhzdam101sx78y1pg4xvzj2e b/implementation/fuzz/inputs-genvm-modules-complete-json/2/k/2kqfqq0x171fhs9f654mhmhzdam101sx78y1pg4xvzj2e new file mode 100644 index 00000000..8669055f --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/2/k/2kqfqq0x171fhs9f654mhmhzdam101sx78y1pg4xvzj2e @@ -0,0 +1 @@ + " \uDAbb" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/2/k/2ksxsdkrcj75gxj5bh515daqdr49gxas5gq4w3mne8fxe b/implementation/fuzz/inputs-genvm-modules-complete-json/2/k/2ksxsdkrcj75gxj5bh515daqdr49gxas5gq4w3mne8fxe new file mode 100644 index 00000000..5b06994e --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/2/k/2ksxsdkrcj75gxj5bh515daqdr49gxas5gq4w3mne8fxe @@ -0,0 +1,2 @@ + +"Ρ€\\uba5R [[1, B\bΓ΅ r" diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/2/n/2n3jwfv96hh6722m749ghsxbesbbkgqgajvx8czn183fa b/implementation/fuzz/inputs-genvm-modules-complete-json/2/n/2n3jwfv96hh6722m749ghsxbesbbkgqgajvx8czn183fa new file mode 100644 index 00000000..3f33343f --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/2/n/2n3jwfv96hh6722m749ghsxbesbbkgqgajvx8czn183fa @@ -0,0 +1 @@ + "ς€»ƒ сскڸʡ6Π±ΡƒΠΊbbcbbbbb:bbb A A Bxbz" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/2/n/2n4t3zk62tgnhnhvqgzvfw352g9yy662b1fn2087cqc18 b/implementation/fuzz/inputs-genvm-modules-complete-json/2/n/2n4t3zk62tgnhnhvqgzvfw352g9yy662b1fn2087cqc18 new file mode 100644 index 00000000..0598260e --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/2/n/2n4t3zk62tgnhnhvqgzvfw352g9yy662b1fn2087cqc18 @@ -0,0 +1 @@ + "Yσ—₯—111 Ρƒ\u01bb: Ρƒ\uabbb [," \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/2/p/2p0k398b7qp548psbch15wvnbqehqgkz4pf2gtj1tpb7m b/implementation/fuzz/inputs-genvm-modules-complete-json/2/p/2p0k398b7qp548psbch15wvnbqehqgkz4pf2gtj1tpb7m new file mode 100644 index 00000000..c43ac613 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/2/p/2p0k398b7qp548psbch15wvnbqehqgkz4pf2gtj1tpb7m @@ -0,0 +1 @@ +2222222222222222222222222 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/2/p/2pm3p5fxq3fpxt5g0j5tpje93fn9r6c4b4qz0zmfn1df2 b/implementation/fuzz/inputs-genvm-modules-complete-json/2/p/2pm3p5fxq3fpxt5g0j5tpje93fn9r6c4b4qz0zmfn1df2 new file mode 100644 index 00000000..20dcc940 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/2/p/2pm3p5fxq3fpxt5g0j5tpje93fn9r6c4b4qz0zmfn1df2 @@ -0,0 +1 @@ + [1, 2, 3] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/2/q/2q195hbtgbyx52kbb43mdbrmt6gzsc3693yd49g1j2jga b/implementation/fuzz/inputs-genvm-modules-complete-json/2/q/2q195hbtgbyx52kbb43mdbrmt6gzsc3693yd49g1j2jga new file mode 100644 index 00000000..d5ccd0ac --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/2/q/2q195hbtgbyx52kbb43mdbrmt6gzsc3693yd49g1j2jga @@ -0,0 +1 @@ + "Ρ€ B\b" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/2/r/2rmtbeedk3mnayqb6z0dj77583dh0arvgz4456d2yp2wy b/implementation/fuzz/inputs-genvm-modules-complete-json/2/r/2rmtbeedk3mnayqb6z0dj77583dh0arvgz4456d2yp2wy new file mode 100644 index 00000000..632808b1 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/2/r/2rmtbeedk3mnayqb6z0dj77583dh0arvgz4456d2yp2wy @@ -0,0 +1 @@ +"UUUTmbbbbb00\\\\\\\\\\\\\\u0bbb\u0bbbb\\\\\\\\\\\\u0bbb\u0bbbbbb," diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/2/s/2s4as110gqppmf984s9600v742ztpwrbf4gkgkrayw2q0 b/implementation/fuzz/inputs-genvm-modules-complete-json/2/s/2s4as110gqppmf984s9600v742ztpwrbf4gkgkrayw2q0 new file mode 100644 index 00000000..f4976c44 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/2/s/2s4as110gqppmf984s9600v742ztpwrbf4gkgkrayw2q0 @@ -0,0 +1,3 @@ +{ + "1" : [ {} ] +} \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/2/t/2t0ftjj6197wjb7dqch5ammz2f5esnh0zrw4fh1vzk35e b/implementation/fuzz/inputs-genvm-modules-complete-json/2/t/2t0ftjj6197wjb7dqch5ammz2f5esnh0zrw4fh1vzk35e new file mode 100644 index 00000000..f32a5804 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/2/t/2t0ftjj6197wjb7dqch5ammz2f5esnh0zrw4fh1vzk35e @@ -0,0 +1 @@ +true \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/2/w/2wn6p7whcvzg482hfct6tzax39zggc7bt3qqxj393awqr b/implementation/fuzz/inputs-genvm-modules-complete-json/2/w/2wn6p7whcvzg482hfct6tzax39zggc7bt3qqxj393awqr new file mode 100644 index 00000000..07815a83 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/2/w/2wn6p7whcvzg482hfct6tzax39zggc7bt3qqxj393awqr @@ -0,0 +1,2 @@ + [0, null, + 1, 2, 3] diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/2/x/2xa502ckstahndqb4bqhxdwq7hzsxbw4a3dh9008n5grr b/implementation/fuzz/inputs-genvm-modules-complete-json/2/x/2xa502ckstahndqb4bqhxdwq7hzsxbw4a3dh9008n5grr new file mode 100644 index 00000000..7cbf9175 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/2/x/2xa502ckstahndqb4bqhxdwq7hzsxbw4a3dh9008n5grr @@ -0,0 +1,2 @@ + +"Ρ€b r" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/2/y/2ykbambjhke293td0wnyhmqnprz3dr3kckw2t81vdeq32 b/implementation/fuzz/inputs-genvm-modules-complete-json/2/y/2ykbambjhke293td0wnyhmqnprz3dr3kckw2t81vdeq32 new file mode 100644 index 0000000000000000000000000000000000000000..18bc64aafc080e266dd5ab7a93ead1199c776ad1 GIT binary patch literal 129 wcmb>CQUC&_wETP}D}`wM0IU>Q2}r9BcAyZe!v&KAvVj0i1ZWt8f`Xy~0I*ga-T(jq literal 0 HcmV?d00001 diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/2/z/2z3p4rpamny067vhq88ke789xdnyt23zn2ywjq7sbxy6p b/implementation/fuzz/inputs-genvm-modules-complete-json/2/z/2z3p4rpamny067vhq88ke789xdnyt23zn2ywjq7sbxy6p new file mode 100644 index 00000000..831ce971 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/2/z/2z3p4rpamny067vhq88ke789xdnyt23zn2ywjq7sbxy6p @@ -0,0 +1 @@ +111111111111111101111115 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/2/z/2zazarnhgndp91dxsf8m775r96x1hb59bpaa1942fe22p b/implementation/fuzz/inputs-genvm-modules-complete-json/2/z/2zazarnhgndp91dxsf8m775r96x1hb59bpaa1942fe22p new file mode 100644 index 00000000..82ce1f87 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/2/z/2zazarnhgndp91dxsf8m775r96x1hb59bpaa1942fe22p @@ -0,0 +1 @@ + " , B\rσ—₯—000 00000 , B\rσ—₯—000 000000\u0000 00000 , B\rσ—₯—000 00000000\u00000" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/3/0/30d131f7jpbsdzp0t3f3ry9shwecm2mbn3sfxqy4vxqrt b/implementation/fuzz/inputs-genvm-modules-complete-json/3/0/30d131f7jpbsdzp0t3f3ry9shwecm2mbn3sfxqy4vxqrt new file mode 100644 index 00000000..e84a8d7c --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/3/0/30d131f7jpbsdzp0t3f3ry9shwecm2mbn3sfxqy4vxqrt @@ -0,0 +1 @@ + "bξ—¨750 " \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/3/0/30hhad75c83y5s8a6z9ftdbnnmatgktgsasr18av2cj36 b/implementation/fuzz/inputs-genvm-modules-complete-json/3/0/30hhad75c83y5s8a6z9ftdbnnmatgktgsasr18av2cj36 new file mode 100644 index 00000000..ecdbee94 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/3/0/30hhad75c83y5s8a6z9ftdbnnmatgktgsasr18av2cj36 @@ -0,0 +1,6 @@ + "Ρ€\\u ? 12" + + + + + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/3/2/325vkc6r7q84h3g63whr2hdc69chys6bafjqyr21vd556 b/implementation/fuzz/inputs-genvm-modules-complete-json/3/2/325vkc6r7q84h3g63whr2hdc69chys6bafjqyr21vd556 new file mode 100644 index 00000000..68163bd9 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/3/2/325vkc6r7q84h3g63whr2hdc69chys6bafjqyr21vd556 @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/3/2/32wc7343p2qsgg1vr2c8cfxf29rxjte6xaf1j3kqrthp2 b/implementation/fuzz/inputs-genvm-modules-complete-json/3/2/32wc7343p2qsgg1vr2c8cfxf29rxjte6xaf1j3kqrthp2 new file mode 100644 index 00000000..775c4a0f --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/3/2/32wc7343p2qsgg1vr2c8cfxf29rxjte6xaf1j3kqrthp2 @@ -0,0 +1,2 @@ +111111111111111111111e-777071111 + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/3/2/32zvz3ffjf0rsaayrf4drj4x6a87162a5dxa9x5gbyy40 b/implementation/fuzz/inputs-genvm-modules-complete-json/3/2/32zvz3ffjf0rsaayrf4drj4x6a87162a5dxa9x5gbyy40 new file mode 100644 index 00000000..297ee259 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/3/2/32zvz3ffjf0rsaayrf4drj4x6a87162a5dxa9x5gbyy40 @@ -0,0 +1,4 @@ +"f\\\\]\\\\@\\\\" + + + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/3/3/335qkpa6a6a55kg78v14wk3yyesena6v3304gw7mp792t b/implementation/fuzz/inputs-genvm-modules-complete-json/3/3/335qkpa6a6a55kg78v14wk3yyesena6v3304gw7mp792t new file mode 100644 index 00000000..4beeea30 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/3/3/335qkpa6a6a55kg78v14wk3yyesena6v3304gw7mp792t @@ -0,0 +1,7 @@ +1e-515 + + + + + + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/3/3/33ew3bbq8bsesg4jf189jf9ard8cnp9m4sxsxsb96q3fj b/implementation/fuzz/inputs-genvm-modules-complete-json/3/3/33ew3bbq8bsesg4jf189jf9ard8cnp9m4sxsxsb96q3fj new file mode 100644 index 00000000..9e071003 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/3/3/33ew3bbq8bsesg4jf189jf9ard8cnp9m4sxsxsb96q3fj @@ -0,0 +1,21 @@ +{ + "foo": [ + "bar", + "baz\n\\{", + 123, + -123, + -1e10, + 1e20, + null, + true, + false, + [1, 2, 3] + ], + "": [ ], + "mbb -barar": [ ], + "": [ ], + "bar": [ + "русскиС null,bbbbbbbbbbb", + "mbbbbbbbbbbbbbbaz\t\\uabbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/3/3/33qz17sfvejgr7dfczk6w4qbt77qwanv1307w4b8ja0ma b/implementation/fuzz/inputs-genvm-modules-complete-json/3/3/33qz17sfvejgr7dfczk6w4qbt77qwanv1307w4b8ja0ma new file mode 100644 index 00000000..05dce3c8 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/3/3/33qz17sfvejgr7dfczk6w4qbt77qwanv1307w4b8ja0ma @@ -0,0 +1 @@ +"\\ζ³›" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/3/4/34j4w84w1cnqb7cr41p52saatagrqsxk8wj2ehhbpc7fw b/implementation/fuzz/inputs-genvm-modules-complete-json/3/4/34j4w84w1cnqb7cr41p52saatagrqsxk8wj2ehhbpc7fw new file mode 100644 index 0000000000000000000000000000000000000000..e2800224b0651e9e28ebf1ac1074b9948df0c1f3 GIT binary patch literal 127 wcmb>CQUC&_wETP}D}`wM0IU>Q2}r9B8c>MU;erW**idP7VFd*S1qDR~0C0L8WB>pF literal 0 HcmV?d00001 diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/3/5/35237jnwf7qj4e3jy7dngv203sb8da6za3tmdtaz8m71y b/implementation/fuzz/inputs-genvm-modules-complete-json/3/5/35237jnwf7qj4e3jy7dngv203sb8da6za3tmdtaz8m71y new file mode 100644 index 00000000..5d2b9651 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/3/5/35237jnwf7qj4e3jy7dngv203sb8da6za3tmdtaz8m71y @@ -0,0 +1,5 @@ +{ + "foo" + + + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/3/5/35wd5p3vpd27tw2r15btrpq06gt1b708brb74z1z6k5f8 b/implementation/fuzz/inputs-genvm-modules-complete-json/3/5/35wd5p3vpd27tw2r15btrpq06gt1b708brb74z1z6k5f8 new file mode 100644 index 00000000..b1514891 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/3/5/35wd5p3vpd27tw2r15btrpq06gt1b708brb74z1z6k5f8 @@ -0,0 +1 @@ +"]" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/3/6/36qhwstm4ezxvnsem3n9rkbwe5971cq5r9eyvt51n1ca4 b/implementation/fuzz/inputs-genvm-modules-complete-json/3/6/36qhwstm4ezxvnsem3n9rkbwe5971cq5r9eyvt51n1ca4 new file mode 100644 index 00000000..39e6a263 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/3/6/36qhwstm4ezxvnsem3n9rkbwe5971cq5r9eyvt51n1ca4 @@ -0,0 +1,2 @@ + [[ + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/3/7/37223ehxrzsjx4h98n1t8tzxdeez2q54n4ghv7y87h99y b/implementation/fuzz/inputs-genvm-modules-complete-json/3/7/37223ehxrzsjx4h98n1t8tzxdeez2q54n4ghv7y87h99y new file mode 100644 index 00000000..d1ae23d0 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/3/7/37223ehxrzsjx4h98n1t8tzxdeez2q54n4ghv7y87h99y @@ -0,0 +1,3 @@ +[[[[[[[[[[[[[[[[[ + + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/3/7/37epan24avchbwj139h8gwrgrn168hpdn6wctp4w3kvnw b/implementation/fuzz/inputs-genvm-modules-complete-json/3/7/37epan24avchbwj139h8gwrgrn168hpdn6wctp4w3kvnw new file mode 100644 index 00000000..bcdb0d9e --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/3/7/37epan24avchbwj139h8gwrgrn168hpdn6wctp4w3kvnw @@ -0,0 +1,2 @@ + +"Ρ€bbbbQbb
 r" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/3/8/38t9f41n2ynf9n1r6yq9wv9h9zygjpz50ghzy7qzfxb4e b/implementation/fuzz/inputs-genvm-modules-complete-json/3/8/38t9f41n2ynf9n1r6yq9wv9h9zygjpz50ghzy7qzfxb4e new file mode 100644 index 00000000..0332bd79 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/3/8/38t9f41n2ynf9n1r6yq9wv9h9zygjpz50ghzy7qzfxb4e @@ -0,0 +1 @@ +"σ—₯Ίubb [1" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/3/9/39jhye51bpxdznqxjc3n7txbqtdxw8tk999evs6pe6pty b/implementation/fuzz/inputs-genvm-modules-complete-json/3/9/39jhye51bpxdznqxjc3n7txbqtdxw8tk999evs6pe6pty new file mode 100644 index 00000000..8622d77c --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/3/9/39jhye51bpxdznqxjc3n7txbqtdxw8tk999evs6pe6pty @@ -0,0 +1,13 @@ +{ + "foo": [ + "bar", + "baz\n\\z", 1e20, + null, + true, [1, 2, 3] + ], + "ba\\uabbbbbr": [ + "русскиСCΠ±ΡƒΠΊΠ²Ρ‹ ζ—₯本", + "mbbbbbbbbbbbbbbaz\t\uabbbbb", + "mbbbbbbbbbbbbbbaz\t\\uabbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/3/b/3b5snsywt1yg9608yk9dtkzcsw8j226873xmgrbtx5e2w b/implementation/fuzz/inputs-genvm-modules-complete-json/3/b/3b5snsywt1yg9608yk9dtkzcsw8j226873xmgrbtx5e2w new file mode 100644 index 00000000..365702a1 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/3/b/3b5snsywt1yg9608yk9dtkzcsw8j226873xmgrbtx5e2w @@ -0,0 +1,7 @@ + "Ρ€750 UwUoUσ—₯£σ—₯Ή2" + + + + + + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/3/b/3bc93yvqp1d1zes49sk9vwyf8enqcmkfectpyjg700p5r b/implementation/fuzz/inputs-genvm-modules-complete-json/3/b/3bc93yvqp1d1zes49sk9vwyf8enqcmkfectpyjg700p5r new file mode 100644 index 00000000..a5804c8d --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/3/b/3bc93yvqp1d1zes49sk9vwyf8enqcmkfectpyjg700p5r @@ -0,0 +1 @@ + "с  ! e111@eC@1bz" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/3/c/3cv357mm22rrbcdb29bv4883ajrnye3p05wh3p4e9vvde b/implementation/fuzz/inputs-genvm-modules-complete-json/3/c/3cv357mm22rrbcdb29bv4883ajrnye3p05wh3p4e9vvde new file mode 100644 index 00000000..883359d0 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/3/c/3cv357mm22rrbcdb29bv4883ajrnye3p05wh3p4e9vvde @@ -0,0 +1,29 @@ +222222222922222222222E+2 + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/3/d/3dygjt8z5jp6gss3nfv19ht36jhan7xc4x46dh9wxxb14 b/implementation/fuzz/inputs-genvm-modules-complete-json/3/d/3dygjt8z5jp6gss3nfv19ht36jhan7xc4x46dh9wxxb14 new file mode 100644 index 00000000..7006372f --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/3/d/3dygjt8z5jp6gss3nfv19ht36jhan7xc4x46dh9wxxb14 @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + " \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/3/e/3ejnm2369k6hgrghespqr1pe0qdxrwv8yqrwjpevdvk68 b/implementation/fuzz/inputs-genvm-modules-complete-json/3/e/3ejnm2369k6hgrghespqr1pe0qdxrwv8yqrwjpevdvk68 new file mode 100644 index 00000000..671d029f --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/3/e/3ejnm2369k6hgrghespqr1pe0qdxrwv8yqrwjpevdvk68 @@ -0,0 +1,16 @@ +{ + "foo": [ + "bar", + "baz\n\\{", + -1e10, + 1e20, + null, + true, + [1, 2, 3] + ], "barrrrrrrrrr": [ ], + "brrrr": [ ], + "bar": [ + "русскиС mbbbbbbbbb", + "mbbbbbbbbbbbbbbaz\t\\uabbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/3/g/3gtdrvgpzq462t4wsnd5k20xxfvpwckw3b1gg9077ey4t b/implementation/fuzz/inputs-genvm-modules-complete-json/3/g/3gtdrvgpzq462t4wsnd5k20xxfvpwckw3b1gg9077ey4t new file mode 100644 index 00000000..7ae10e20 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/3/g/3gtdrvgpzq462t4wsnd5k20xxfvpwckw3b1gg9077ey4t @@ -0,0 +1,2 @@ +{ + "foo": [[[[[[[[[[[[[[[[[[ "bar", [[[[[[ "bar", 333333333333333333333333333333.3 [ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/3/h/3htw69mppz8dkd8kkysh4gc00j3021v0zv38sy6nhq02p b/implementation/fuzz/inputs-genvm-modules-complete-json/3/h/3htw69mppz8dkd8kkysh4gc00j3021v0zv38sy6nhq02p new file mode 100644 index 00000000..cfb918ef --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/3/h/3htw69mppz8dkd8kkysh4gc00j3021v0zv38sy6nhq02p @@ -0,0 +1,3 @@ +"f\\t\\uabbf\\t\\uabbbb" + + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/3/j/3jep0893yw33esbfnsx3vtpn03jw2f2kav5261w8tk8fj b/implementation/fuzz/inputs-genvm-modules-complete-json/3/j/3jep0893yw33esbfnsx3vtpn03jw2f2kav5261w8tk8fj new file mode 100644 index 00000000..83df38e6 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/3/j/3jep0893yw33esbfnsx3vtpn03jw2f2kav5261w8tk8fj @@ -0,0 +1 @@ +2522222222222111111111e \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/3/k/3kbz0gcyv9ydzwdszy7q1c4ktka3rfn4d8pk9e4318etr b/implementation/fuzz/inputs-genvm-modules-complete-json/3/k/3kbz0gcyv9ydzwdszy7q1c4ktka3rfn4d8pk9e4318etr new file mode 100644 index 00000000..e36060f1 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/3/k/3kbz0gcyv9ydzwdszy7q1c4ktka3rfn4d8pk9e4318etr @@ -0,0 +1,10 @@ +[[[[[[[[ + + +3, [[[[[[[[[[[[[[[[[ + + +3, [[[[[[[[ + + +3, ]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]][[ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/3/k/3kv8nxj26hfm5ez2vrp38j59q0rpgvxffcvd8x0vme6n4 b/implementation/fuzz/inputs-genvm-modules-complete-json/3/k/3kv8nxj26hfm5ez2vrp38j59q0rpgvxffcvd8x0vme6n4 new file mode 100644 index 00000000..d1543335 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/3/k/3kv8nxj26hfm5ez2vrp38j59q0rpgvxffcvd8x0vme6n4 @@ -0,0 +1,4 @@ + [111, +true, + false, + 111] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/3/m/3mdxxf03629wvddwgvezn1r428tykhd7spzdd5hn2fcmm b/implementation/fuzz/inputs-genvm-modules-complete-json/3/m/3mdxxf03629wvddwgvezn1r428tykhd7spzdd5hn2fcmm new file mode 100644 index 00000000..303e55ee --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/3/m/3mdxxf03629wvddwgvezn1r428tykhd7spzdd5hn2fcmm @@ -0,0 +1,22 @@ +{ + "foo": [ + "bar", + "baz\n\\z", + 123, + -123, + -1e10, + 1e20, + null, + true, + false, + [1, 2, 3] + ], + "bar": [ + "русскиС Π±ΡƒΠΊΠ²Ρ‹ ζ—₯本", + "mbbbbbbbb}bbbbbaz\t\uabbbbb", + "mbbbbbbbbbbbbbbaz\t\\uabbbbb" + + ], + "bar": [ + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/3/q/3qbv6wqxrtgkh6rbsmf4n7ta1xyndvfaedd6rbk08peap b/implementation/fuzz/inputs-genvm-modules-complete-json/3/q/3qbv6wqxrtgkh6rbsmf4n7ta1xyndvfaedd6rbk08peap new file mode 100644 index 00000000..385b2caa --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/3/q/3qbv6wqxrtgkh6rbsmf4n7ta1xyndvfaedd6rbk08peap @@ -0,0 +1 @@ + "ς€»ƒ B Bxbz" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/3/q/3qtfgm5c9jtkw8ae8gwh01s6tkk18my2ag0y7epc1ckd2 b/implementation/fuzz/inputs-genvm-modules-complete-json/3/q/3qtfgm5c9jtkw8ae8gwh01s6tkk18my2ag0y7epc1ckd2 new file mode 100644 index 00000000..2974771b --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/3/q/3qtfgm5c9jtkw8ae8gwh01s6tkk18my2ag0y7epc1ckd2 @@ -0,0 +1 @@ +"bbbbbIbbbΪ²Ρ‹ ζ—₯1 qo" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/3/r/3r1fn4qcz970rxcapmbg7axet37yk0c30pden7zfnbrcp b/implementation/fuzz/inputs-genvm-modules-complete-json/3/r/3r1fn4qcz970rxcapmbg7axet37yk0c30pden7zfnbrcp new file mode 100644 index 00000000..7f922eb6 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/3/r/3r1fn4qcz970rxcapmbg7axet37yk0c30pden7zfnbrcp @@ -0,0 +1 @@ + "\bσ—₯¬" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/3/t/3tx2a7qvwfc8df5hzpjcyhkc65hvjwfk4b9v649x59n46 b/implementation/fuzz/inputs-genvm-modules-complete-json/3/t/3tx2a7qvwfc8df5hzpjcyhkc65hvjwfk4b9v649x59n46 new file mode 100644 index 00000000..ebc2c46d --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/3/t/3tx2a7qvwfc8df5hzpjcyhkc65hvjwfk4b9v649x59n46 @@ -0,0 +1 @@ + "\bσ—₯—Zσ—₯—1 " \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/3/w/3w7w2k51qnmw7z4j79g0dngx0x3xv81fqx11rmktjvd5w b/implementation/fuzz/inputs-genvm-modules-complete-json/3/w/3w7w2k51qnmw7z4j79g0dngx0x3xv81fqx11rmktjvd5w new file mode 100644 index 00000000..309d3e2a --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/3/w/3w7w2k51qnmw7z4j79g0dngx0x3xv81fqx11rmktjvd5w @@ -0,0 +1 @@ +nu \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/3/z/3zrna3xrtgxct2en27tz1pb3v7dvtt5qh2fynhyvmhwew b/implementation/fuzz/inputs-genvm-modules-complete-json/3/z/3zrna3xrtgxct2en27tz1pb3v7dvtt5qh2fynhyvmhwew new file mode 100644 index 00000000..371cab64 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/3/z/3zrna3xrtgxct2en27tz1pb3v7dvtt5qh2fynhyvmhwew @@ -0,0 +1,4 @@ +{ + "foo": [ + [[[[[[[[": [ +111111 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/4/0/4039y0m4gwtzm6ybeecvxq6d297w7h1h3zwfpqeqwejp6 b/implementation/fuzz/inputs-genvm-modules-complete-json/4/0/4039y0m4gwtzm6ybeecvxq6d297w7h1h3zwfpqeqwejp6 new file mode 100644 index 00000000..a3d2a4fd --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/4/0/4039y0m4gwtzm6ybeecvxq6d297w7h1h3zwfpqeqwejp6 @@ -0,0 +1 @@ +[[[[[,,,, \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/4/2/42ckpme1rg6chpv2jg81r2mp6a1pqeb7jnpxkrj1zvvz0 b/implementation/fuzz/inputs-genvm-modules-complete-json/4/2/42ckpme1rg6chpv2jg81r2mp6a1pqeb7jnpxkrj1zvvz0 new file mode 100644 index 00000000..c98d8a65 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/4/2/42ckpme1rg6chpv2jg81r2mp6a1pqeb7jnpxkrj1zvvz0 @@ -0,0 +1,18 @@ +{ + "foo": [ + "bar", + "baz\n\\{", + 123, + -123, + -1e10, + 1e20, + null, + true, + false,2222222222222222222.2 ], + "bar": [ ], + "bar": [ + "русскиС b UUUUUUUUUU\n\\zUΠ±ΡƒΠΊΠ²Π‡ ζ—₯本", + "mbbbbbbbbbbbbbbaz\t\uabbbbb", + "mbbbbbbbbbbbbbbaz\t\\uabbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/4/2/42txhvm8f01c4cdcaymqg3nmjsbc91sxz2xryj3nmng3a b/implementation/fuzz/inputs-genvm-modules-complete-json/4/2/42txhvm8f01c4cdcaymqg3nmjsbc91sxz2xryj3nmng3a new file mode 100644 index 00000000..ec882fff --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/4/2/42txhvm8f01c4cdcaymqg3nmjsbc91sxz2xryj3nmng3a @@ -0,0 +1 @@ + "bbbbaz\tс\\fffffffffffffffZffffffffffffffffffffffffffffffffffffffffffffaTba\/:fffffffffffffffffffabba\/:fffffffabba\/: [bb [b333" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/4/4/44nkn4y353ss52zenyq9e8gr7vcarz4y36dtff3nd9af8 b/implementation/fuzz/inputs-genvm-modules-complete-json/4/4/44nkn4y353ss52zenyq9e8gr7vcarz4y36dtff3nd9af8 new file mode 100644 index 00000000..f04a39a3 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/4/4/44nkn4y353ss52zenyq9e8gr7vcarz4y36dtff3nd9af8 @@ -0,0 +1,9 @@ +{ + "foo":{ + "fTo":{ "foL":{ + "foo":{ + "foo":{ + "foo":{ "foL":{ + "fo "fo{ + "foo":{ "foL":{ + " "fo \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/4/5/454pdwe2qc8f6ck1zfkd4czrtxzbgf92qywfftqgxjhr0 b/implementation/fuzz/inputs-genvm-modules-complete-json/4/5/454pdwe2qc8f6ck1zfkd4czrtxzbgf92qywfftqgxjhr0 new file mode 100644 index 00000000..8a2b2f0f --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/4/5/454pdwe2qc8f6ck1zfkd4czrtxzbgf92qywfftqgxjhr0 @@ -0,0 +1 @@ +true \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/4/6/46hzz84st57awtf74kqg6x1ph1nc1b550hxnn8ky5e46p b/implementation/fuzz/inputs-genvm-modules-complete-json/4/6/46hzz84st57awtf74kqg6x1ph1nc1b550hxnn8ky5e46p new file mode 100644 index 00000000..7efeed6c --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/4/6/46hzz84st57awtf74kqg6x1ph1nc1b550hxnn8ky5e46p @@ -0,0 +1,3 @@ +{ "o, + +} \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/4/6/46s556f8ke1g4dtjahxs94mzecqzw1n3twh8m9vtxdk5c b/implementation/fuzz/inputs-genvm-modules-complete-json/4/6/46s556f8ke1g4dtjahxs94mzecqzw1n3twh8m9vtxdk5c new file mode 100644 index 00000000..d8898001 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/4/6/46s556f8ke1g4dtjahxs94mzecqzw1n3twh8m9vtxdk5c @@ -0,0 +1 @@ + [[1, $ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/4/7/47a21mjw0h89fdx7t27e856cxrbv2ahsh3xybqjfqrq36 b/implementation/fuzz/inputs-genvm-modules-complete-json/4/7/47a21mjw0h89fdx7t27e856cxrbv2ahsh3xybqjfqrq36 new file mode 100644 index 00000000..a082b719 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/4/7/47a21mjw0h89fdx7t27e856cxrbv2ahsh3xybqjfqrq36 @@ -0,0 +1,2 @@ + +"Ρ€jᛨ -1e 6 1111p" diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/4/7/47aenj50bwcb52ts5me0mptej88cabpzew64b22ncw82m b/implementation/fuzz/inputs-genvm-modules-complete-json/4/7/47aenj50bwcb52ts5me0mptej88cabpzew64b22ncw82m new file mode 100644 index 00000000..0f9c7bb4 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/4/7/47aenj50bwcb52ts5me0mptej88cabpzew64b22ncw82m @@ -0,0 +1 @@ +"u`b\nH" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/4/7/47zdxfswt8ax6ccwdq71k2wzyr2pv2sd7hcpcb3yes5rm b/implementation/fuzz/inputs-genvm-modules-complete-json/4/7/47zdxfswt8ax6ccwdq71k2wzyr2pv2sd7hcpcb3yes5rm new file mode 100644 index 00000000..37490b57 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/4/7/47zdxfswt8ax6ccwdq71k2wzyr2pv2sd7hcpcb3yes5rm @@ -0,0 +1,2 @@ +"\r" + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/4/8/483rdf9z0bqhr2s4f0s7yw5gawhafpd26csq0xrn78ehe b/implementation/fuzz/inputs-genvm-modules-complete-json/4/8/483rdf9z0bqhr2s4f0s7yw5gawhafpd26csq0xrn78ehe new file mode 100644 index 00000000..696462c1 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/4/8/483rdf9z0bqhr2s4f0s7yw5gawhafpd26csq0xrn78ehe @@ -0,0 +1,2 @@ +55555555555555555555555552 + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/4/8/488m98q123vvxhqx0aagr7fptytjjm2sesvzstq7vnjd4 b/implementation/fuzz/inputs-genvm-modules-complete-json/4/8/488m98q123vvxhqx0aagr7fptytjjm2sesvzstq7vnjd4 new file mode 100644 index 00000000..ab205500 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/4/8/488m98q123vvxhqx0aagr7fptytjjm2sesvzstq7vnjd4 @@ -0,0 +1,2 @@ + +"Ρ„jᛨ7Ρ€jᛨ750 1\uDbbb\uDc12.50 1=uDbbb^uDc12.1111111111pΡ€jᛨ750 1\uDbbb\uDc12.11" diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/4/9/49076gsq2b7p3hqv2h0fssazfg0mbppj9hr8yxyhbykwg b/implementation/fuzz/inputs-genvm-modules-complete-json/4/9/49076gsq2b7p3hqv2h0fssazfg0mbppj9hr8yxyhbykwg new file mode 100644 index 00000000..f3629c7a --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/4/9/49076gsq2b7p3hqv2h0fssazfg0mbppj9hr8yxyhbykwg @@ -0,0 +1,8 @@ +{ + "foo": [ + "bar", + "bazdn\\z", + 123, + + null, + -123, "222 -1e [[[[[[[[ [[[[ v ]]\]]]]]]]]] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/4/9/49438ym1f018m8ctq0gbb06cgkamh1yb5yhztnt2sbgwm b/implementation/fuzz/inputs-genvm-modules-complete-json/4/9/49438ym1f018m8ctq0gbb06cgkamh1yb5yhztnt2sbgwm new file mode 100644 index 00000000..9ceb43ce --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/4/9/49438ym1f018m8ctq0gbb06cgkamh1yb5yhztnt2sbgwm @@ -0,0 +1,2 @@ + +"Ρ€\\uba5R [1,[ B\bΓ΅ r" diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/4/e/4e6m73vd7pqnte9f4dmahm7eddc8n3nb78hf0f9d14b9j b/implementation/fuzz/inputs-genvm-modules-complete-json/4/e/4e6m73vd7pqnte9f4dmahm7eddc8n3nb78hf0f9d14b9j new file mode 100644 index 00000000..79722980 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/4/e/4e6m73vd7pqnte9f4dmahm7eddc8n3nb78hf0f9d14b9j @@ -0,0 +1 @@ +" \uDbbb\uD \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/4/f/4f82mbvdzrxewmjfyk3g5xg07xn8j49tc92n845861rty b/implementation/fuzz/inputs-genvm-modules-complete-json/4/f/4f82mbvdzrxewmjfyk3g5xg07xn8j49tc92n845861rty new file mode 100644 index 00000000..2117d3df --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/4/f/4f82mbvdzrxewmjfyk3g5xg07xn8j49tc92n845861rty @@ -0,0 +1 @@ +1111555555555111111111111111e-522222 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/b82254c422c4a0de4fde9cf66488007a6f787d972f1ff42a8f0d3a07204fdaff b/implementation/fuzz/inputs-genvm-modules-complete-json/4/g/4g0fnmrht3baacq1spykfq06esnj3smp0rd5e7jp79eyc similarity index 50% rename from implementation/fuzz/inputs-genvm-modules-normalize-whitespace/b82254c422c4a0de4fde9cf66488007a6f787d972f1ff42a8f0d3a07204fdaff rename to implementation/fuzz/inputs-genvm-modules-complete-json/4/g/4g0fnmrht3baacq1spykfq06esnj3smp0rd5e7jp79eyc index e45ec267dfee16f0f6e2acde04fdb36ea6e01978..7149a86e786efbe357d3136d69e0982d78051b67 100644 GIT binary patch literal 35 McmY#V;vxVP03S^Or2qf` literal 48 icmZQzU|`@<;erE2MP4xAQBX($3n~D)JiI)DP!#|`e*(w= diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/4/h/4hn2107rfefkdjabs9rhdsw49zx8y2gx8p2c0qgt21926 b/implementation/fuzz/inputs-genvm-modules-complete-json/4/h/4hn2107rfefkdjabs9rhdsw49zx8y2gx8p2c0qgt21926 new file mode 100644 index 00000000..c93ce603 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/4/h/4hn2107rfefkdjabs9rhdsw49zx8y2gx8p2c0qgt21926 @@ -0,0 +1 @@ +"Ρ€ΠΊa [1," \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/4/p/4pr8k38eg2vs9bccsw8mnjcnx5te4y1g11rgme8h3gybg b/implementation/fuzz/inputs-genvm-modules-complete-json/4/p/4pr8k38eg2vs9bccsw8mnjcnx5te4y1g11rgme8h3gybg new file mode 100644 index 00000000..1b4bb28e --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/4/p/4pr8k38eg2vs9bccsw8mnjcnx5te4y1g11rgme8h3gybg @@ -0,0 +1,4 @@ +{ + "foo": [ + ], + "bar" a \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/4/p/4pyjs7hxbkmv5bb8nk2ey2d0wjx8k82f81c6768pc92ge b/implementation/fuzz/inputs-genvm-modules-complete-json/4/p/4pyjs7hxbkmv5bb8nk2ey2d0wjx8k82f81c6768pc92ge new file mode 100644 index 00000000..ab3e6631 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/4/p/4pyjs7hxbkmv5bb8nk2ey2d0wjx8k82f81c6768pc92ge @@ -0,0 +1 @@ + "bσ—Έ—50 UwIoUσ—­£" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/4/q/4qp2afa7b9e3hmb9et592k41sh0f15patxjf2v5y5ax28 b/implementation/fuzz/inputs-genvm-modules-complete-json/4/q/4qp2afa7b9e3hmb9et592k41sh0f15patxjf2v5y5ax28 new file mode 100644 index 00000000..666e9d2b --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/4/q/4qp2afa7b9e3hmb9et592k41sh0f15patxjf2v5y5ax28 @@ -0,0 +1 @@ + [1, , \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/4/q/4qqkbm0kf3s0n4adszk893w6w8wnyjvf1e76ye91gg6tw b/implementation/fuzz/inputs-genvm-modules-complete-json/4/q/4qqkbm0kf3s0n4adszk893w6w8wnyjvf1e76ye91gg6tw new file mode 100644 index 00000000..03e19810 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/4/q/4qqkbm0kf3s0n4adszk893w6w8wnyjvf1e76ye91gg6tw @@ -0,0 +1 @@ +[[[[[[[[[[[[[[[[[[[[[[[ [[[[[[[[[[[[[ [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[w[ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/4/r/4rd4fhxdd75yrzwqrt2e1e9wewtthq7qdsxqqdng0b060 b/implementation/fuzz/inputs-genvm-modules-complete-json/4/r/4rd4fhxdd75yrzwqrt2e1e9wewtthq7qdsxqqdng0b060 new file mode 100644 index 00000000..52d8cd2f --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/4/r/4rd4fhxdd75yrzwqrt2e1e9wewtthq7qdsxqqdng0b060 @@ -0,0 +1,2 @@ + + "\"\"n" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/4/r/4rshs07vxp9p01tfz7pvd0jraxfs0jee1nw0cwkehxx94 b/implementation/fuzz/inputs-genvm-modules-complete-json/4/r/4rshs07vxp9p01tfz7pvd0jraxfs0jee1nw0cwkehxx94 new file mode 100644 index 00000000..207fbfb3 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/4/r/4rshs07vxp9p01tfz7pvd0jraxfs0jee1nw0cwkehxx94 @@ -0,0 +1,2 @@ +22222223222222222222222222.22222222E+2 + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/4/w/4wey7y6s511635spzptgtg0zhbcsa537xqby99h0nv250 b/implementation/fuzz/inputs-genvm-modules-complete-json/4/w/4wey7y6s511635spzptgtg0zhbcsa537xqby99h0nv250 new file mode 100644 index 00000000..a7f9700d --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/4/w/4wey7y6s511635spzptgtg0zhbcsa537xqby99h0nv250 @@ -0,0 +1 @@ +222222333333333333333333332E+21 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/5/0/50sf3sj3zza00020bnq6r03axvz1vksraj1573jebfd6g b/implementation/fuzz/inputs-genvm-modules-complete-json/5/0/50sf3sj3zza00020bnq6r03axvz1vksraj1573jebfd6g new file mode 100644 index 00000000..083012d3 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/5/0/50sf3sj3zza00020bnq6r03axvz1vksraj1573jebfd6g @@ -0,0 +1,17 @@ +{ + "foo": [ + "bar", + "baz\n\\{", + -1e10, + 1e20, + null, + true, + [1, 2, 3] + ], + "barrrrrrrrrr": [ ], + "brrrr": [ ], + "bar": [ + "ՀусскиС mbbbbZZZZZZZZZZZZZZZZZZZZZZZZZZbbbbb", + "bbbw" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/5/1/511bbrs4vynrjpt7pmjj4897n9mfpbe50b7rwn7x9pgk2 b/implementation/fuzz/inputs-genvm-modules-complete-json/5/1/511bbrs4vynrjpt7pmjj4897n9mfpbe50b7rwn7x9pgk2 new file mode 100644 index 00000000..2a49e5ee --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/5/1/511bbrs4vynrjpt7pmjj4897n9mfpbe50b7rwn7x9pgk2 @@ -0,0 +1,2 @@ + + "\/'''3''''H'''''''" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/5/1/51dz1xs6nywx9v858m9bnd8ast8y973mrmt6s2kgtp9we b/implementation/fuzz/inputs-genvm-modules-complete-json/5/1/51dz1xs6nywx9v858m9bnd8ast8y973mrmt6s2kgtp9we new file mode 100644 index 00000000..e16ae2da --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/5/1/51dz1xs6nywx9v858m9bnd8ast8y973mrmt6s2kgtp9we @@ -0,0 +1 @@ +611e-5221111511 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/5/2/521ask5rah2rmhtk8w53w5dvfptr9anhetwce58yktk1e b/implementation/fuzz/inputs-genvm-modules-complete-json/5/2/521ask5rah2rmhtk8w53w5dvfptr9anhetwce58yktk1e new file mode 100644 index 00000000..b0f2f550 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/5/2/521ask5rah2rmhtk8w53w5dvfptr9anhetwce58yktk1e @@ -0,0 +1 @@ + [[1, 21, 2, "Ρ€ΠΊu[1\\\\Π±ΡƒΡ„\f\\\\\\\\\\\\\\с Aqbz\\\\\\\\\\\\\\\\\\\\\\\\\\\bbbΠ΅ Π±ΡƒΡ€bbbbIbbβ€» 4/[\/[rΡ„\\\\\\\\\\\\\\\\с Bqbz" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/5/3/53nstbc50pvj2vw2nave0q2ry661kggrh2xrzfbnabzeg b/implementation/fuzz/inputs-genvm-modules-complete-json/5/3/53nstbc50pvj2vw2nave0q2ry661kggrh2xrzfbnabzeg new file mode 100644 index 00000000..ec3b7ad8 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/5/3/53nstbc50pvj2vw2nave0q2ry661kggrh2xrzfbnabzeg @@ -0,0 +1,2 @@ +{ + "foo": [[[[[[[[[[[[[[[[[123, [[[[[[[[[[[[[[[[[123, [[[[[[[[[[[[[[[[[[[[[[123, "ba23, [[[[[[[[[[r ------[ 1911111111 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/5/5/5532hjx36v13jedse38grc2h9jcqcrq3panpry2b2cq5p b/implementation/fuzz/inputs-genvm-modules-complete-json/5/5/5532hjx36v13jedse38grc2h9jcqcrq3panpry2b2cq5p new file mode 100644 index 00000000..0b07b320 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/5/5/5532hjx36v13jedse38grc2h9jcqcrq3panpry2b2cq5p @@ -0,0 +1 @@ +0e5555555515 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/5/5/55pm4s9k7pjvttjg3vmhhwkvgtp5rt9rren3884mtychm b/implementation/fuzz/inputs-genvm-modules-complete-json/5/5/55pm4s9k7pjvttjg3vmhhwkvgtp5rt9rren3884mtychm new file mode 100644 index 00000000..7628e8ca --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/5/5/55pm4s9k7pjvttjg3vmhhwkvgtp5rt9rren3884mtychm @@ -0,0 +1 @@ + "Yσ—₯—[," \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/5/5/55y8fgk28sjsw9kc4ct5vaw0nscda5yjqb3wxn81bjdpj b/implementation/fuzz/inputs-genvm-modules-complete-json/5/5/55y8fgk28sjsw9kc4ct5vaw0nscda5yjqb3wxn81bjdpj new file mode 100644 index 00000000..9c3306ba --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/5/5/55y8fgk28sjsw9kc4ct5vaw0nscda5yjqb3wxn81bjdpj @@ -0,0 +1 @@ +33333333333331111111111e-11133331133333 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/5/7/57h0p1x6sm52v6sgcw3k06afpbfxmb9d3xhzjretpzxhy b/implementation/fuzz/inputs-genvm-modules-complete-json/5/7/57h0p1x6sm52v6sgcw3k06afpbfxmb9d3xhzjretpzxhy new file mode 100644 index 00000000..2339d8b0 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/5/7/57h0p1x6sm52v6sgcw3k06afpbfxmb9d3xhzjretpzxhy @@ -0,0 +1 @@ + ",ζ—€bb `B\bσ—₯—" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/5/7/57q7gt19ynvbezapg0t7ercqeqbyetjdb4h3f7xy7s28y b/implementation/fuzz/inputs-genvm-modules-complete-json/5/7/57q7gt19ynvbezapg0t7ercqeqbyetjdb4h3f7xy7s28y new file mode 100644 index 00000000..f10725b2 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/5/7/57q7gt19ynvbezapg0t7ercqeqbyetjdb4h3f7xy7s28y @@ -0,0 +1 @@ + [1.11111111111111111111111111111111111111111111111111] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/5/7/57x0e8msvtg6fh2rgj738kh0zrmsnr65xhp8sknvdaf0g b/implementation/fuzz/inputs-genvm-modules-complete-json/5/7/57x0e8msvtg6fh2rgj738kh0zrmsnr65xhp8sknvdaf0g new file mode 100644 index 00000000..d8a5a36d --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/5/7/57x0e8msvtg6fh2rgj738kh0zrmsnr65xhp8sknvdaf0g @@ -0,0 +1,20 @@ +{ + "foo": [ + "bar" + ], + "babbbbbbbbbbb\bσ—₯—[8bbaz\t\\r": [ + 123, + -123, + -1e10, + 1e20, + null, + true, + false, + [1, 2, 3] + ], + "bar": [ + "русскиС Π±ΡƒΠΊΠ²Ρ‹ ζ—₯本", + "mbbbbbbbbbbbbbbaz\t\uabbbbb", + "mbbbbbbbbbbbbbbaz\t\\uabbbbb" + ] +} \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/5/8/58s6jqed3mz3m6eaqe529dqf6qrbr9pzx13mer6gwwnvy b/implementation/fuzz/inputs-genvm-modules-complete-json/5/8/58s6jqed3mz3m6eaqe529dqf6qrbr9pzx13mer6gwwnvy new file mode 100644 index 00000000..20fc751c --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/5/8/58s6jqed3mz3m6eaqe529dqf6qrbr9pzx13mer6gwwnvy @@ -0,0 +1 @@ +[ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/5/a/5a982thpv82fgbs86s3dtzsz5wnxt0yqz4dvhhwz3r1ye b/implementation/fuzz/inputs-genvm-modules-complete-json/5/a/5a982thpv82fgbs86s3dtzsz5wnxt0yqz4dvhhwz3r1ye new file mode 100644 index 00000000..ee5d6290 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/5/a/5a982thpv82fgbs86s3dtzsz5wnxt0yqz4dvhhwz3r1ye @@ -0,0 +1 @@ +"σ—₯Ίubb" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/5/b/5ba7bygx5f8z6r1s2a2519a6c625ytcfwmz4q9nt641qr b/implementation/fuzz/inputs-genvm-modules-complete-json/5/b/5ba7bygx5f8z6r1s2a2519a6c625ytcfwmz4q9nt641qr new file mode 100644 index 00000000..665f3214 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/5/b/5ba7bygx5f8z6r1s2a2519a6c625ytcfwmz4q9nt641qr @@ -0,0 +1,3 @@ +222229222222222222222222222 + + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/5/c/5ckz0981gspjax26nkh652wc7tcb54w7fy5csjms0d66e b/implementation/fuzz/inputs-genvm-modules-complete-json/5/c/5ckz0981gspjax26nkh652wc7tcb54w7fy5csjms0d66e new file mode 100644 index 00000000..d9d40020 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/5/c/5ckz0981gspjax26nkh652wc7tcb54w7fy5csjms0d66e @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/5/h/5h2g0chm9b3g0gtyv3vwa5125zj6h9wpewa1d9ttasjcc b/implementation/fuzz/inputs-genvm-modules-complete-json/5/h/5h2g0chm9b3g0gtyv3vwa5125zj6h9wpewa1d9ttasjcc new file mode 100644 index 00000000..f7d38d12 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/5/h/5h2g0chm9b3g0gtyv3vwa5125zj6h9wpewa1d9ttasjcc @@ -0,0 +1,3 @@ +{ + "foo": + true, [[Π΅ Π₯本 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/5/h/5h3sncn3nmnesyx8kpmqmvfkvkj44xzebssp7xnczq6hr b/implementation/fuzz/inputs-genvm-modules-complete-json/5/h/5h3sncn3nmnesyx8kpmqmvfkvkj44xzebssp7xnczq6hr new file mode 100644 index 00000000..f06f2512 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/5/h/5h3sncn3nmnesyx8kpmqmvfkvkj44xzebssp7xnczq6hr @@ -0,0 +1 @@ +6611e-5221111511 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/5/j/5j5yksckw2vp4xrca0xy5xnsa8sy9ytjw2wt1zhhndx36 b/implementation/fuzz/inputs-genvm-modules-complete-json/5/j/5j5yksckw2vp4xrca0xy5xnsa8sy9ytjw2wt1zhhndx36 new file mode 100644 index 0000000000000000000000000000000000000000..e17ee0bfba3707ccd41e91f3ab85c2e42eaf9c03 GIT binary patch literal 32 Xcmb>CQUC%aB`bw$5DS$dz@PvCTEzub literal 0 HcmV?d00001 diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/5/k/5ksca34p8jvyyv4r4t9aq8fqz0wyr6c5w6a5j1cns3wtj b/implementation/fuzz/inputs-genvm-modules-complete-json/5/k/5ksca34p8jvyyv4r4t9aq8fqz0wyr6c5w6a5j1cns3wtj new file mode 100644 index 00000000..06695d27 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/5/k/5ksca34p8jvyyv4r4t9aq8fqz0wyr6c5w6a5j1cns3wtj @@ -0,0 +1,5 @@ +{ + "foo": 1e20, + "\\uabb@bb" + + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/5/m/5m13t0nyge0w12tkmwt21zv8gphpckwv79ky7syc8fz04 b/implementation/fuzz/inputs-genvm-modules-complete-json/5/m/5m13t0nyge0w12tkmwt21zv8gphpckwv79ky7syc8fz04 new file mode 100644 index 00000000..2eaa299e --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/5/m/5m13t0nyge0w12tkmwt21zv8gphpckwv79ky7syc8fz04 @@ -0,0 +1 @@ +191111.111111111111111E-1111 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/5/m/5meq1wankjerhv937773xend4eakwq3xfyaescec2yy54 b/implementation/fuzz/inputs-genvm-modules-complete-json/5/m/5meq1wankjerhv937773xend4eakwq3xfyaescec2yy54 new file mode 100644 index 00000000..9536b4fc --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/5/m/5meq1wankjerhv937773xend4eakwq3xfyaescec2yy54 @@ -0,0 +1 @@ +"]no" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/5/n/5nxmsy3nbk3n9x2eezrgbjj21j80xj62n9c8svwt2weva b/implementation/fuzz/inputs-genvm-modules-complete-json/5/n/5nxmsy3nbk3n9x2eezrgbjj21j80xj62n9c8svwt2weva new file mode 100644 index 00000000..6c2406e8 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/5/n/5nxmsy3nbk3n9x2eezrgbjj21j80xj62n9c8svwt2weva @@ -0,0 +1 @@ +222222222222222222222222212.2 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/5/p/5pzkgwrwycj09hfrjtmyfhpgmnerkapbh6xhp55gskj9w b/implementation/fuzz/inputs-genvm-modules-complete-json/5/p/5pzkgwrwycj09hfrjtmyfhpgmnerkapbh6xhp55gskj9w new file mode 100644 index 00000000..961ee90c --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/5/p/5pzkgwrwycj09hfrjtmyfhpgmnerkapbh6xhp55gskj9w @@ -0,0 +1 @@ +2222222222.222222222222 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/5/q/5q2611whwhv6gmbtrtwm09b0cbpnjr0mcnrvfas0ky1ma b/implementation/fuzz/inputs-genvm-modules-complete-json/5/q/5q2611whwhv6gmbtrtwm09b0cbpnjr0mcnrvfas0ky1ma new file mode 100644 index 00000000..6250e8d0 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/5/q/5q2611whwhv6gmbtrtwm09b0cbpnjr0mcnrvfas0ky1ma @@ -0,0 +1 @@ +"bQzbQz\t\\\\" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/5/r/5rhhmvfczvhj7ba6jaxjqhx7d44drdy8nv5ypw3przvme b/implementation/fuzz/inputs-genvm-modules-complete-json/5/r/5rhhmvfczvhj7ba6jaxjqhx7d44drdy8nv5ypw3przvme new file mode 100644 index 00000000..35fb2f40 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/5/r/5rhhmvfczvhj7ba6jaxjqhx7d44drdy8nv5ypw3przvme @@ -0,0 +1,2 @@ + +"\u0400Ρ‹\u00000Ρ€\\wba r" diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/5/s/5sjmmjcchvtx07q0s65jd41npa39zadh6pdfavdebrk3a b/implementation/fuzz/inputs-genvm-modules-complete-json/5/s/5sjmmjcchvtx07q0s65jd41npa39zadh6pdfavdebrk3a new file mode 100644 index 00000000..b08a37ef --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/5/s/5sjmmjcchvtx07q0s65jd41npa39zadh6pdfavdebrk3a @@ -0,0 +1 @@ +411.111666666666666666661E11 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/b99e1959cf0de1171606c1971933f6b68cfc618b1937fc3117af71e4877ea0f1 b/implementation/fuzz/inputs-genvm-modules-complete-json/5/s/5st59vnef87wq5077p90tfrqwq4zwa7seh3erqrztx5xt similarity index 76% rename from implementation/fuzz/inputs-genvm-modules-normalize-whitespace/b99e1959cf0de1171606c1971933f6b68cfc618b1937fc3117af71e4877ea0f1 rename to implementation/fuzz/inputs-genvm-modules-complete-json/5/s/5st59vnef87wq5077p90tfrqwq4zwa7seh3erqrztx5xt index 492619e57d29cb0927cf9acbc7ce06651e2d130f..d121d3fbcef0e0a5153dc087c2a80df3afdcbcc0 100644 GIT binary patch literal 42 Vcmb>CQs7chP~t)Z3TS*F7XU=w0)_wp literal 47 Wcmd<$;^JapU{C;3FdzkEV4(plg8|<F diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/5/t/5tm51hcwchb3mvj75w58m2grm3dm3n82nvv3c0e6mwv0m b/implementation/fuzz/inputs-genvm-modules-complete-json/5/t/5tm51hcwchb3mvj75w58m2grm3dm3n82nvv3c0e6mwv0m new file mode 100644 index 00000000..3db31d9f --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/5/t/5tm51hcwchb3mvj75w58m2grm3dm3n82nvv3c0e6mwv0m @@ -0,0 +1 @@ +229191121611111111111 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/5/t/5tmrg5k6rz6vg0cmfwbrnmbzybvxptypmp4ghactbwkzm b/implementation/fuzz/inputs-genvm-modules-complete-json/5/t/5tmrg5k6rz6vg0cmfwbrnmbzybvxptypmp4ghactbwkzm new file mode 100644 index 00000000..f1defc9e --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/5/t/5tmrg5k6rz6vg0cmfwbrnmbzybvxptypmp4ghactbwkzm @@ -0,0 +1,4 @@ +"\bbbbbbbbbbbbbbbbΠ΅ Π±f (Ρ€bbbabbbbb" + + + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/5/t/5trdrhkddcaqa06fhrjy51knjj279hx2a87rtc9rwgs7c b/implementation/fuzz/inputs-genvm-modules-complete-json/5/t/5trdrhkddcaqa06fhrjy51knjj279hx2a87rtc9rwgs7c new file mode 100644 index 00000000..bff308d6 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/5/t/5trdrhkddcaqa06fhrjy51knjj279hx2a87rtc9rwgs7c @@ -0,0 +1 @@ + [81111.111111111111111] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/5/w/5wqvwve9j7bc3f8x0jb6z5qtff5tqedkwkad3zr0g7y5t b/implementation/fuzz/inputs-genvm-modules-complete-json/5/w/5wqvwve9j7bc3f8x0jb6z5qtff5tqedkwkad3zr0g7y5t new file mode 100644 index 00000000..b61c64e3 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/5/w/5wqvwve9j7bc3f8x0jb6z5qtff5tqedkwkad3zr0g7y5t @@ -0,0 +1 @@ +-0 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/5/y/5y8k3vxza1nxqkpp32q8r5027bbbv4qbfy37ad1knt6w4 b/implementation/fuzz/inputs-genvm-modules-complete-json/5/y/5y8k3vxza1nxqkpp32q8r5027bbbv4qbfy37ad1knt6w4 new file mode 100644 index 00000000..3d4b64b9 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/5/y/5y8k3vxza1nxqkpp32q8r5027bbbv4qbfy37ad1knt6w4 @@ -0,0 +1,2 @@ +2.222E-222222222223 + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/5/y/5yxxpcxncfjgm9e5bff4v1wbh7pny1gd26hqybg84sabg b/implementation/fuzz/inputs-genvm-modules-complete-json/5/y/5yxxpcxncfjgm9e5bff4v1wbh7pny1gd26hqybg84sabg new file mode 100644 index 00000000..335e1808 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/5/y/5yxxpcxncfjgm9e5bff4v1wbh7pny1gd26hqybg84sabg @@ -0,0 +1,6 @@ +{ + "foo": [ + "bar", + [1, [ + "bar", + [1, 2, 23, "222 -1e nu \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/5/z/5z2n879zp21139g3x4ra4m07sh9s2gn1fwwnn1m1sn1br b/implementation/fuzz/inputs-genvm-modules-complete-json/5/z/5z2n879zp21139g3x4ra4m07sh9s2gn1fwwnn1m1sn1br new file mode 100644 index 00000000..bd11bdc3 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/5/z/5z2n879zp21139g3x4ra4m07sh9s2gn1fwwnn1m1sn1br @@ -0,0 +1 @@ +[1, ,  \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/5/z/5zmgmc6sv8n8tzcz1s8j5cpzq2n7a26ermg60hyz4t1h4 b/implementation/fuzz/inputs-genvm-modules-complete-json/5/z/5zmgmc6sv8n8tzcz1s8j5cpzq2n7a26ermg60hyz4t1h4 new file mode 100644 index 00000000..26e418b5 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/5/z/5zmgmc6sv8n8tzcz1s8j5cpzq2n7a26ermg60hyz4t1h4 @@ -0,0 +1 @@ + "II\"\"\"" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/6/0/601srznj4t6bk5w9x4pcr0zwxmh6avsnt7gpn0fm7ak3c b/implementation/fuzz/inputs-genvm-modules-complete-json/6/0/601srznj4t6bk5w9x4pcr0zwxmh6avsnt7gpn0fm7ak3c new file mode 100644 index 00000000..38fc72a7 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/6/0/601srznj4t6bk5w9x4pcr0zwxmh6avsnt7gpn0fm7ak3c @@ -0,0 +1,2 @@ + +"Ρ€\\σ—΄ˆ" diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/6/0/606g3wx92025zvx1dnn197sr2sxja0yvrdy1p97xdxthw b/implementation/fuzz/inputs-genvm-modules-complete-json/6/0/606g3wx92025zvx1dnn197sr2sxja0yvrdy1p97xdxthw new file mode 100644 index 00000000..56a6051c --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/6/0/606g3wx92025zvx1dnn197sr2sxja0yvrdy1p97xdxthw @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/6/1/61mta1s9373nq89nvskpew6nssr5b9gvk6dyv1r0jrcz0 b/implementation/fuzz/inputs-genvm-modules-complete-json/6/1/61mta1s9373nq89nvskpew6nssr5b9gvk6dyv1r0jrcz0 new file mode 100644 index 00000000..e812e42f --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/6/1/61mta1s9373nq89nvskpew6nssr5b9gvk6dyv1r0jrcz0 @@ -0,0 +1,2 @@ +{ + "f o \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/6/2/620n716s2sn0567k9wwcq1vs380t3ap6m3526mkcgdhtt b/implementation/fuzz/inputs-genvm-modules-complete-json/6/2/620n716s2sn0567k9wwcq1vs380t3ap6m3526mkcgdhtt new file mode 100644 index 00000000..aa7b6468 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/6/2/620n716s2sn0567k9wwcq1vs380t3ap6m3526mkcgdhtt @@ -0,0 +1 @@ +{ " @fo": [ 3 ],} \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/6/2/62148mftv5xr6nej0mnpbg4eygjz8mqgfgnt0qsve1pt4 b/implementation/fuzz/inputs-genvm-modules-complete-json/6/2/62148mftv5xr6nej0mnpbg4eygjz8mqgfgnt0qsve1pt4 new file mode 100644 index 00000000..0d3d0e85 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/6/2/62148mftv5xr6nej0mnpbg4eygjz8mqgfgnt0qsve1pt4 @@ -0,0 +1,16 @@ +{ + "foo": [ + "b [1,ar", + 123, + -123, + -1e10, + 1e20, + [1, 2, 3] + ], + "bar": [ + "", + "mbbbbbѐусскиС Π±ΛƒΠΊΠ²Ρ‹ ζ—₯本", + "mbbbbbbbbbbbbbbaz\t\uabbbbb", + "mbbbbbbbbbbbbbbaz\t\\uabbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/6/2/62kak2fjfc2q7p4515s3gt8c42n443gqq58qt9hwcnzs0 b/implementation/fuzz/inputs-genvm-modules-complete-json/6/2/62kak2fjfc2q7p4515s3gt8c42n443gqq58qt9hwcnzs0 new file mode 100644 index 00000000..e3df2b2e --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/6/2/62kak2fjfc2q7p4515s3gt8c42n443gqq58qt9hwcnzs0 @@ -0,0 +1 @@ + " t]u
 \b" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/6/4/64p3kq1c31cejcj9qtn0rz5k7pzevbjyah0t1jsj84cbt b/implementation/fuzz/inputs-genvm-modules-complete-json/6/4/64p3kq1c31cejcj9qtn0rz5k7pzevbjyah0t1jsj84cbt new file mode 100644 index 00000000..379c1ca5 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/6/4/64p3kq1c31cejcj9qtn0rz5k7pzevbjyah0t1jsj84cbt @@ -0,0 +1 @@ +222222222222222222222 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/6/5/65jh0f2sd99x7d18esapxv0x0gvqmzzz8r3fw4sg1gvw0 b/implementation/fuzz/inputs-genvm-modules-complete-json/6/5/65jh0f2sd99x7d18esapxv0x0gvqmzzz8r3fw4sg1gvw0 new file mode 100644 index 00000000..5d2d990e --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/6/5/65jh0f2sd99x7d18esapxv0x0gvqmzzz8r3fw4sg1gvw0 @@ -0,0 +1 @@ + "Ρ€bbbbaz\t\uab111111 bbbусbbbbbbbbbaz\t}}}}}}}}}}}}}}}}}}}}}}}bbbaz\t}}}}}}}}}}}}}}}}}}}}}}}}}uabBqbz" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/6/8/68ka1qexyqvggrsm11r04zrg8bt4ygm1c5jr683y1xvk6 b/implementation/fuzz/inputs-genvm-modules-complete-json/6/8/68ka1qexyqvggrsm11r04zrg8bt4ygm1c5jr683y1xvk6 new file mode 100644 index 00000000..ff656e0b --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/6/8/68ka1qexyqvggrsm11r04zrg8bt4ygm1c5jr683y1xvk6 @@ -0,0 +1 @@ +"UUUIbcγ€» rfbbb," diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/6/9/698cfc2vbwxfdptecap1mxfgd5xeprwj203yq0fj9smhp b/implementation/fuzz/inputs-genvm-modules-complete-json/6/9/698cfc2vbwxfdptecap1mxfgd5xeprwj203yq0fj9smhp new file mode 100644 index 00000000..ef073cc4 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/6/9/698cfc2vbwxfdptecap1mxfgd5xeprwj203yq0fj9smhp @@ -0,0 +1 @@ +n \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/6/c/6c3qj9g5qak95p6mw2p3p7814bgwesxeceqwd97f83464 b/implementation/fuzz/inputs-genvm-modules-complete-json/6/c/6c3qj9g5qak95p6mw2p3p7814bgwesxeceqwd97f83464 new file mode 100644 index 00000000..eeec8edc --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/6/c/6c3qj9g5qak95p6mw2p3p7814bgwesxeceqwd97f83464 @@ -0,0 +1,21 @@ +{ + "foo": [ + "bar", + "baz\n\\z", + 123, + -123, + -8e10, + 1e20, + null, + true, + false, + [1, 2, 3] + +], + "babbbbbbaσ—₯ƒ\uabbbr" + : [ + "русскиС Π±ΡƒΠΊΠ²Ρ‹ ζ—₯本\uabbbbb", + "mbbbbbbaz\t", + "mbbΠ²Ρ‹ ζ—₯本\uabbbbbbaσ—₯ƒ\uabbbbb" + ] +} \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/6/c/6car9rqrptwd0md4bppt1jr9vpf9r4b77a74wzqc1e0pt b/implementation/fuzz/inputs-genvm-modules-complete-json/6/c/6car9rqrptwd0md4bppt1jr9vpf9r4b77a74wzqc1e0pt new file mode 100644 index 00000000..b2d09e62 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/6/c/6car9rqrptwd0md4bppt1jr9vpf9r4b77a74wzqc1e0pt @@ -0,0 +1 @@ +[113333333333333.3333331e1 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/6/e/6ebwq1t7yq53h7rpn35n04399zqvqkar42gebmqmbefmc b/implementation/fuzz/inputs-genvm-modules-complete-json/6/e/6ebwq1t7yq53h7rpn35n04399zqvqkar42gebmqmbefmc new file mode 100644 index 00000000..7de4855a --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/6/e/6ebwq1t7yq53h7rpn35n04399zqvqkar42gebmqmbefmc @@ -0,0 +1,2 @@ + +"\u0000Ρ‹\u0000 r" diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/6/h/6h4krjdvvw5pmgfnbbh5tyhw5hfpbtfpk72x35bcvyb0c b/implementation/fuzz/inputs-genvm-modules-complete-json/6/h/6h4krjdvvw5pmgfnbbh5tyhw5hfpbtfpk72x35bcvyb0c new file mode 100644 index 00000000..8a0ff978 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/6/h/6h4krjdvvw5pmgfnbbh5tyhw5hfpbtfpk72x35bcvyb0c @@ -0,0 +1,12 @@ + { + "":0, + "": { "":0, + "": { + " + "": + " + + + + + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/6/h/6he21spbt5fm52anbs9a3b7dj3s86f40gg6cz81acrt7t b/implementation/fuzz/inputs-genvm-modules-complete-json/6/h/6he21spbt5fm52anbs9a3b7dj3s86f40gg6cz81acrt7t new file mode 100644 index 00000000..e71642b2 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/6/h/6he21spbt5fm52anbs9a3b7dj3s86f40gg6cz81acrt7t @@ -0,0 +1,23 @@ +{ + "foo" + + + + + + + + + + + + + + + + + + + + +! \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/6/h/6hwdx92zmptj4h2j7zf3r3e42w3zh6bvbrz5z2p2e3h2c b/implementation/fuzz/inputs-genvm-modules-complete-json/6/h/6hwdx92zmptj4h2j7zf3r3e42w3zh6bvbrz5z2p2e3h2c new file mode 100644 index 00000000..4de2b806 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/6/h/6hwdx92zmptj4h2j7zf3r3e42w3zh6bvbrz5z2p2e3h2c @@ -0,0 +1 @@ + " \uDbbb\uDDbbbFDcuDcb \uDbbb\uDDcb \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/6/k/6kjg78eczceasp09agz77gav04w2cmqttcb1a1z89k7rg b/implementation/fuzz/inputs-genvm-modules-complete-json/6/k/6kjg78eczceasp09agz77gav04w2cmqttcb1a1z89k7rg new file mode 100644 index 00000000..681d0ce2 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/6/k/6kjg78eczceasp09agz77gav04w2cmqttcb1a1z89k7rg @@ -0,0 +1,2 @@ + +"\u0000\bσ—₯—[8本\u0000Ρ‹\u00ba1 dr" diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/6/m/6m6xkpezne5pp3m6z8age604q7cpv1x920qnkvz1bkwyp b/implementation/fuzz/inputs-genvm-modules-complete-json/6/m/6m6xkpezne5pp3m6z8age604q7cpv1x920qnkvz1bkwyp new file mode 100644 index 00000000..1f9d1d74 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/6/m/6m6xkpezne5pp3m6z8age604q7cpv1x920qnkvz1bkwyp @@ -0,0 +1,8 @@ +{ + "foo": [ + "bar", + "baz\n\\{", + 123, + -123, + -1e10, +1e20,[[[[[[[[[[[[[[[[[[[[ nulx, fa \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/6/m/6mqgag5enzvbfj1jj3p1h9vnbqe748dq2ppejheajd5ng b/implementation/fuzz/inputs-genvm-modules-complete-json/6/m/6mqgag5enzvbfj1jj3p1h9vnbqe748dq2ppejheajd5ng new file mode 100644 index 00000000..8b56da12 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/6/m/6mqgag5enzvbfj1jj3p1h9vnbqe748dq2ppejheajd5ng @@ -0,0 +1 @@ +"UUUTmbΒ  TmbΒ  bbbbb," diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/6/m/6mrbsr64qxtbfwjkrca2mcwecqydf7daxy28pc0vfe7ga b/implementation/fuzz/inputs-genvm-modules-complete-json/6/m/6mrbsr64qxtbfwjkrca2mcwecqydf7daxy28pc0vfe7ga new file mode 100644 index 00000000..17d167ea --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/6/m/6mrbsr64qxtbfwjkrca2mcwecqydf7daxy28pc0vfe7ga @@ -0,0 +1 @@ +nu1 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/6/n/6nh78x0s7vpt01c4jm93bcz1b34cy3fr0tqwv7tr549er b/implementation/fuzz/inputs-genvm-modules-complete-json/6/n/6nh78x0s7vpt01c4jm93bcz1b34cy3fr0tqwv7tr549er new file mode 100644 index 00000000..c539e548 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/6/n/6nh78x0s7vpt01c4jm93bcz1b34cy3fr0tqwv7tr549er @@ -0,0 +1,5 @@ +[[null, +1113,[[[[[[[[[[[[[[[[[[[[[[[[[[ [[[[[[[[[[[[[ [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ [[[[[[ 11111111 + + +E1 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/6/n/6nndeg4erafb4pzb2901wkjqtdds2c1xdb7fv3hxmfy08 b/implementation/fuzz/inputs-genvm-modules-complete-json/6/n/6nndeg4erafb4pzb2901wkjqtdds2c1xdb7fv3hxmfy08 new file mode 100644 index 00000000..af7fc103 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/6/n/6nndeg4erafb4pzb2901wkjqtdds2c1xdb7fv3hxmfy08 @@ -0,0 +1 @@ +1e-11111 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/6/p/6pr35dbxwq0p8n3ms3tvtmkqeb60fef4pnkwz0fvagavg b/implementation/fuzz/inputs-genvm-modules-complete-json/6/p/6pr35dbxwq0p8n3ms3tvtmkqeb60fef4pnkwz0fvagavg new file mode 100644 index 00000000..7305c1ee --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/6/p/6pr35dbxwq0p8n3ms3tvtmkqeb60fef4pnkwz0fvagavg @@ -0,0 +1,7 @@ +{ "Β€\\u ( d foo": 12" + + + + + + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/6/p/6py5gqqwqynzs9x0hw765emeg89cewjz87h01btpt6hsm b/implementation/fuzz/inputs-genvm-modules-complete-json/6/p/6py5gqqwqynzs9x0hw765emeg89cewjz87h01btpt6hsm new file mode 100644 index 00000000..fa000fb7 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/6/p/6py5gqqwqynzs9x0hw765emeg89cewjz87h01btpt6hsm @@ -0,0 +1,15 @@ +{ "foo": [ + "bar", + "baz\n\\{", + -1e10, + 1e20, + null, + true, + [1, 2, 3] + ], + "bar": [ ], + "bar": [ + "русскиС mbbbbbbbbbbbbbbapppppppppppppz\t\uabbbbb", + "mbbbbbbbbbbbbbbaz\t\\uabbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/6/q/6qywzpqkyc3m2nervjxc7xrff5by8m334by2ehynfa566 b/implementation/fuzz/inputs-genvm-modules-complete-json/6/q/6qywzpqkyc3m2nervjxc7xrff5by8m334by2ehynfa566 new file mode 100644 index 00000000..f66672c7 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/6/q/6qywzpqkyc3m2nervjxc7xrff5by8m334by2ehynfa566 @@ -0,0 +1,17 @@ +{ + "foo": [ + "bar", + "baz\n\\z", + 123, + -123, + 111e-11111111111111111110, + null, + true, + false, + [11111111111111111, 2, 3] + ], + "bar": [ + "русскиС Π±ΡƒΠΊΠ²Ρ‹ ζ—₯本", + "mbbbbbbbbbbbbbbaz\t\uabbbbt\\uabbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/9e76479946b62a0bcd38c64e6362921ffc3e5c9d88c50c8eb3dd77529793625d b/implementation/fuzz/inputs-genvm-modules-complete-json/6/s/6s8c2d0dafx3tczd8ba0d3q08tdben51y533v9nfbp9tt similarity index 85% rename from implementation/fuzz/inputs-genvm-modules-normalize-whitespace/9e76479946b62a0bcd38c64e6362921ffc3e5c9d88c50c8eb3dd77529793625d rename to implementation/fuzz/inputs-genvm-modules-complete-json/6/s/6s8c2d0dafx3tczd8ba0d3q08tdben51y533v9nfbp9tt index a3ead84d..2e3fe814 100644 --- a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/9e76479946b62a0bcd38c64e6362921ffc3e5c9d88c50c8eb3dd77529793625d +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/6/s/6s8c2d0dafx3tczd8ba0d3q08tdben51y533v9nfbp9tt @@ -1,4 +1,3 @@ - @@ -7,6 +6,8 @@ + + @@ -16,4 +17,3 @@ - \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/6/s/6saxjzt801waqf7pynga8dtztvf33cxav6gnakf3drksj b/implementation/fuzz/inputs-genvm-modules-complete-json/6/s/6saxjzt801waqf7pynga8dtztvf33cxav6gnakf3drksj new file mode 100644 index 00000000..20d0bd62 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/6/s/6saxjzt801waqf7pynga8dtztvf33cxav6gnakf3drksj @@ -0,0 +1 @@ + "mbbbbbbbbbb\\\\\\\\\\\\\b diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/6/s/6srqet48avcywyfmwdxfve36a9ghbr9wsa51d35cydp2c b/implementation/fuzz/inputs-genvm-modules-complete-json/6/s/6srqet48avcywyfmwdxfve36a9ghbr9wsa51d35cydp2c new file mode 100644 index 00000000..a96f3ccf --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/6/s/6srqet48avcywyfmwdxfve36a9ghbr9wsa51d35cydp2c @@ -0,0 +1,3 @@ +{ + +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/6/s/6ss1rznx8wf9tt6fyrps4510fq9zj0d5d4634jyyevnq0 b/implementation/fuzz/inputs-genvm-modules-complete-json/6/s/6ss1rznx8wf9tt6fyrps4510fq9zj0d5d4634jyyevnq0 new file mode 100644 index 00000000..1fc1b744 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/6/s/6ss1rznx8wf9tt6fyrps4510fq9zj0d5d4634jyyevnq0 @@ -0,0 +1,2 @@ + [111112, 1111.11111116112, 1111.1e10, + 1e+2, 111E111, 2, 111111111111e+2, 1111, 2, 1111.11, 11111.1110111111E111, -1111111611111111] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/6/t/6tx3r8mjwycqk3cc19tk6vz6v0mpx5b1skh4619860kcy b/implementation/fuzz/inputs-genvm-modules-complete-json/6/t/6tx3r8mjwycqk3cc19tk6vz6v0mpx5b1skh4619860kcy new file mode 100644 index 00000000..e588a892 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/6/t/6tx3r8mjwycqk3cc19tk6vz6v0mpx5b1skh4619860kcy @@ -0,0 +1 @@ +ff \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/6/v/6vkrd7rbzfmpqy99m87rakngp591kxkjb6npwzhyetr0g b/implementation/fuzz/inputs-genvm-modules-complete-json/6/v/6vkrd7rbzfmpqy99m87rakngp591kxkjb6npwzhyetr0g new file mode 100644 index 00000000..9257f642 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/6/v/6vkrd7rbzfmpqy99m87rakngp591kxkjb6npwzhyetr0g @@ -0,0 +1,2 @@ + +"Ρ€bbbbIbbγ€» r" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/6/x/6x04fjc4187d9vgnra9sgq9mms214txwpsp3kvqgd3cvj b/implementation/fuzz/inputs-genvm-modules-complete-json/6/x/6x04fjc4187d9vgnra9sgq9mms214txwpsp3kvqgd3cvj new file mode 100644 index 00000000..02393625 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/6/x/6x04fjc4187d9vgnra9sgq9mms214txwpsp3kvqgd3cvj @@ -0,0 +1,2 @@ + [{ "o": [ ] +}] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/6/y/6yvt19scjh8kgqscps8x2f1rz3e4jbc9aasct5hsk1s72 b/implementation/fuzz/inputs-genvm-modules-complete-json/6/y/6yvt19scjh8kgqscps8x2f1rz3e4jbc9aasct5hsk1s72 new file mode 100644 index 00000000..571117af --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/6/y/6yvt19scjh8kgqscps8x2f1rz3e4jbc9aasct5hsk1s72 @@ -0,0 +1 @@ +1111111111111111111111 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/6/z/6z0a57zfe11k63pk5mdzqwqh9q4wnhr3rjsqsvpsd56b2 b/implementation/fuzz/inputs-genvm-modules-complete-json/6/z/6z0a57zfe11k63pk5mdzqwqh9q4wnhr3rjsqsvpsd56b2 new file mode 100644 index 00000000..28355e98 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/6/z/6z0a57zfe11k63pk5mdzqwqh9q4wnhr3rjsqsvpsd56b2 @@ -0,0 +1,3 @@ +{ + "o": 9 , + "" "fdo": [[[[[zn[[[ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/6/z/6zkx97mnxq79jxx3p6v1ash40kmtc2sg8rb0pkxga505p b/implementation/fuzz/inputs-genvm-modules-complete-json/6/z/6zkx97mnxq79jxx3p6v1ash40kmtc2sg8rb0pkxga505p new file mode 100644 index 00000000..abf2a5fd --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/6/z/6zkx97mnxq79jxx3p6v1ash40kmtc2sg8rb0pkxga505p @@ -0,0 +1 @@ +"\nb11.11112E-2222ba ,," \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/6/z/6zm7xgc4drd8e67mde018kypayx07c8vb8vh8j7vpd57r b/implementation/fuzz/inputs-genvm-modules-complete-json/6/z/6zm7xgc4drd8e67mde018kypayx07c8vb8vh8j7vpd57r new file mode 100644 index 00000000..31fc68cb --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/6/z/6zm7xgc4drd8e67mde018kypayx07c8vb8vh8j7vpd57r @@ -0,0 +1 @@ +229191121611111111111111111 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/7/0/700hxytcf4491x4ydx46vqpyacj3205hsb3brs7p7n4ty b/implementation/fuzz/inputs-genvm-modules-complete-json/7/0/700hxytcf4491x4ydx46vqpyacj3205hsb3brs7p7n4ty new file mode 100644 index 00000000..ea428b5e --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/7/0/700hxytcf4491x4ydx46vqpyacj3205hsb3brs7p7n4ty @@ -0,0 +1,6 @@ +{ + "foo": [ + [[[[ "bar", + "bazdn\\z", + 123, + -123, "222 -1e nu \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/7/3/732e6g98xw74xt2m44tza0kca6t24qsaqm5cmrgw0xf86 b/implementation/fuzz/inputs-genvm-modules-complete-json/7/3/732e6g98xw74xt2m44tza0kca6t24qsaqm5cmrgw0xf86 new file mode 100644 index 00000000..d908aae6 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/7/3/732e6g98xw74xt2m44tza0kca6t24qsaqm5cmrgw0xf86 @@ -0,0 +1 @@ +"f\\\\\\\\\\uabbbbb" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/7/4/740p7skkfw1b68nrs3sq5zxtr9axxaxd01nrkkgbqebw0 b/implementation/fuzz/inputs-genvm-modules-complete-json/7/4/740p7skkfw1b68nrs3sq5zxtr9axxaxd01nrkkgbqebw0 new file mode 100644 index 00000000..cf593d28 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/7/4/740p7skkfw1b68nrs3sq5zxtr9axxaxd01nrkkgbqebw0 @@ -0,0 +1,4 @@ +"f0 \uDbbb\uDcb000000\ro 1a+\ro 1 ( (000ΠΊΤΈ \uDbbb\uDcb000000\ro 1` ( (000 bbaa+000000000000000000Ρ€\\uab 1bb" + + + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/7/6/761bhyqcknm6pyt1f7j8jd61q4yrzhmvhy353v23257tr b/implementation/fuzz/inputs-genvm-modules-complete-json/7/6/761bhyqcknm6pyt1f7j8jd61q4yrzhmvhy353v23257tr new file mode 100644 index 00000000..46c8a118 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/7/6/761bhyqcknm6pyt1f7j8jd61q4yrzhmvhy353v23257tr @@ -0,0 +1 @@ +"fqgzd.\uDbbb\ t \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/7/7/779swdcbj9e7djcah0pmjvtxsd7m588966stts93xg9yg b/implementation/fuzz/inputs-genvm-modules-complete-json/7/7/779swdcbj9e7djcah0pmjvtxsd7m588966stts93xg9yg new file mode 100644 index 00000000..4cb5aea3 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/7/7/779swdcbj9e7djcah0pmjvtxsd7m588966stts93xg9yg @@ -0,0 +1,17 @@ +{ + "foo": [ + "bar", + -123, + -1e10, + 1e20, + null, + true, + false, + [1, 2, 3] + ], + "bar": [ + "русс иС Π±ΡƒΠΊΠ²Ρ‹ ζ—₯本", + "mbbbbbbbbbbbbbbaz\t\uabbbbb", + "mbbbbbbbbbbbbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/7/a/7aey027n4v93kp4s0qh20fx4gkved3fw15n7r18yq07ha b/implementation/fuzz/inputs-genvm-modules-complete-json/7/a/7aey027n4v93kp4s0qh20fx4gkved3fw15n7r18yq07ha new file mode 100644 index 00000000..4d1ae35b --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/7/a/7aey027n4v93kp4s0qh20fx4gkved3fw15n7r18yq07ha @@ -0,0 +1 @@ +f \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/7/b/7bpbt1he8ntbggfw9tse48emwb2kbycj77jgz76ddeyfg b/implementation/fuzz/inputs-genvm-modules-complete-json/7/b/7bpbt1he8ntbggfw9tse48emwb2kbycj77jgz76ddeyfg new file mode 100644 index 00000000..0f00eaf0 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/7/b/7bpbt1he8ntbggfw9tse48emwb2kbycj77jgz76ddeyfg @@ -0,0 +1 @@ +22222212222222222222E+2 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/7/c/7c777qz80hpxtgnynsqmqj74byzc144gtxq76kwyrmad6 b/implementation/fuzz/inputs-genvm-modules-complete-json/7/c/7c777qz80hpxtgnynsqmqj74byzc144gtxq76kwyrmad6 new file mode 100644 index 00000000..4bcf55c2 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/7/c/7c777qz80hpxtgnynsqmqj74byzc144gtxq76kwyrmad6 @@ -0,0 +1,2 @@ +{ + "Moo" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/7/c/7cqjbvhqx45fv2n0r972app5mqy1hrq1xdk2bnq3he5qj b/implementation/fuzz/inputs-genvm-modules-complete-json/7/c/7cqjbvhqx45fv2n0r972app5mqy1hrq1xdk2bnq3he5qj new file mode 100644 index 00000000..17948a72 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/7/c/7cqjbvhqx45fv2n0r972app5mqy1hrq1xdk2bnq3he5qj @@ -0,0 +1,2 @@ +3e-922 + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/7/f/7f6vedsdq879bn98wspeyrsn6dvzagxmryh2y5r1d1c9w b/implementation/fuzz/inputs-genvm-modules-complete-json/7/f/7f6vedsdq879bn98wspeyrsn6dvzagxmryh2y5r1d1c9w new file mode 100644 index 00000000..0e903ed1 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/7/f/7f6vedsdq879bn98wspeyrsn6dvzagxmryh2y5r1d1c9w @@ -0,0 +1,2 @@ +{" ! [[UUΠ±C ": [ 5] +} \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/7/g/7g15fmyk9fhwvtzx0qcm8p3hc0z75ys0dtvtpqnvcyw52 b/implementation/fuzz/inputs-genvm-modules-complete-json/7/g/7g15fmyk9fhwvtzx0qcm8p3hc0z75ys0dtvtpqnvcyw52 new file mode 100644 index 00000000..403c3e50 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/7/g/7g15fmyk9fhwvtzx0qcm8p3hc0z75ys0dtvtpqnvcyw52 @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/7/g/7gpcdbfdsm22fw79j8b0h99d158d59af312qf7xembhzr b/implementation/fuzz/inputs-genvm-modules-complete-json/7/g/7gpcdbfdsm22fw79j8b0h99d158d59af312qf7xembhzr new file mode 100644 index 00000000..04e89e51 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/7/g/7gpcdbfdsm22fw79j8b0h99d158d59af312qf7xembhzr @@ -0,0 +1,2 @@ + [ +[[[[[[[[[[[[[[[[ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/7/j/7jhrja2fjxnemgfffhcbwms91t3nct5hha8x28py326y8 b/implementation/fuzz/inputs-genvm-modules-complete-json/7/j/7jhrja2fjxnemgfffhcbwms91t3nct5hha8x28py326y8 new file mode 100644 index 00000000..6df8dcef --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/7/j/7jhrja2fjxnemgfffhcbwms91t3nct5hha8x28py326y8 @@ -0,0 +1 @@ + "\f{obj ζ—₯ub%[" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/7/m/7mpygmpq420qqgrshs10nenpk20mnzqtqz3mt22jhm9zw b/implementation/fuzz/inputs-genvm-modules-complete-json/7/m/7mpygmpq420qqgrshs10nenpk20mnzqtqz3mt22jhm9zw new file mode 100644 index 00000000..05ee0f43 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/7/m/7mpygmpq420qqgrshs10nenpk20mnzqtqz3mt22jhm9zw @@ -0,0 +1,2 @@ +{ + "foo" " \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/7/n/7n4tmxbfh43pa7xerzq5gzg15ad9hz007etcbz5fm0g7c b/implementation/fuzz/inputs-genvm-modules-complete-json/7/n/7n4tmxbfh43pa7xerzq5gzg15ad9hz007etcbz5fm0g7c new file mode 100644 index 00000000..71a357c1 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/7/n/7n4tmxbfh43pa7xerzq5gzg15ad9hz007etcbz5fm0g7c @@ -0,0 +1 @@ + [11.1110111111E111,[[[[[[[[ 111111211111111111111111111111e+2, 111E111, 2, 1111e+2,1, 2, [[[[[[[[[[[[ 11, 211.11, 2, 1111.11111116112, 1111.11111116111111S \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/7/n/7ncz7fcaqtq179b6m9pjb88yd9ge48fcq98mz0z7z04jm b/implementation/fuzz/inputs-genvm-modules-complete-json/7/n/7ncz7fcaqtq179b6m9pjb88yd9ge48fcq98mz0z7z04jm new file mode 100644 index 00000000..1ed24d39 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/7/n/7ncz7fcaqtq179b6m9pjb88yd9ge48fcq98mz0z7z04jm @@ -0,0 +1 @@ + "bξ₯˜5 %UwwUo]σ—₯£" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/7/r/7rm41e9f692tk3awzkp4y3mfm8tz4tzk9mqdrydn4jwh8 b/implementation/fuzz/inputs-genvm-modules-complete-json/7/r/7rm41e9f692tk3awzkp4y3mfm8tz4tzk9mqdrydn4jwh8 new file mode 100644 index 0000000000000000000000000000000000000000..63ccef8a19957bfbd91fddbd7f9d36e8fe8c0235 GIT binary patch literal 33 ZcmY#Th(-qr3JOXJNl8gDr7jFC&H$;<32*=a literal 0 HcmV?d00001 diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/7/s/7s0bar4weepxqm36gjtmngd0s4s87t0zs71gras1azkja b/implementation/fuzz/inputs-genvm-modules-complete-json/7/s/7s0bar4weepxqm36gjtmngd0s4s87t0zs71gras1azkja new file mode 100644 index 00000000..042c0536 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/7/s/7s0bar4weepxqm36gjtmngd0s4s87t0zs71gras1azkja @@ -0,0 +1 @@ + [1, 2,[1, 2, \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/7/t/7t6wync6tykyywar2shdyswn9r1re8ta4x7kxc21586p0 b/implementation/fuzz/inputs-genvm-modules-complete-json/7/t/7t6wync6tykyywar2shdyswn9r1re8ta4x7kxc21586p0 new file mode 100644 index 00000000..19ac14f3 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/7/t/7t6wync6tykyywar2shdyswn9r1re8ta4x7kxc21586p0 @@ -0,0 +1 @@ +"UUUTmbbbbb00Ρ‹\\\\\\\\\\\\\\u0Wbb\u0bbbbbb," diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/7/y/7y7bc8ad66w2zzn4rpff9ntkjw5r6g7rsqa16hgy3at1g b/implementation/fuzz/inputs-genvm-modules-complete-json/7/y/7y7bc8ad66w2zzn4rpff9ntkjw5r6g7rsqa16hgy3at1g new file mode 100644 index 00000000..c2c7498a --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/7/y/7y7bc8ad66w2zzn4rpff9ntkjw5r6g7rsqa16hgy3at1g @@ -0,0 +1 @@ +61111111.1116116111111e-525 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/8/3/83rbwdmzca793m5jvdh96qbx9kmsq4ydvyg3e6yqny8k2 b/implementation/fuzz/inputs-genvm-modules-complete-json/8/3/83rbwdmzca793m5jvdh96qbx9kmsq4ydvyg3e6yqny8k2 new file mode 100644 index 00000000..f9b9b4bf --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/8/3/83rbwdmzca793m5jvdh96qbx9kmsq4ydvyg3e6yqny8k2 @@ -0,0 +1 @@ + [ -11111111111111111111, -11111111111111111111, 2, 3] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/8/5/85rhes7r132gs7dr83w6w7q2x33ce0h82vffctfn6eqv0 b/implementation/fuzz/inputs-genvm-modules-complete-json/8/5/85rhes7r132gs7dr83w6w7q2x33ce0h82vffctfn6eqv0 new file mode 100644 index 00000000..becdce45 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/8/5/85rhes7r132gs7dr83w6w7q2x33ce0h82vffctfn6eqv0 @@ -0,0 +1 @@ + " d\uDbbb\uDebb \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/8/6/867rvjnahynn25ac14cwvt9t6jzvggz372ydp8fqswv2a b/implementation/fuzz/inputs-genvm-modules-complete-json/8/6/867rvjnahynn25ac14cwvt9t6jzvggz372ydp8fqswv2a new file mode 100644 index 00000000..78d81846 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/8/6/867rvjnahynn25ac14cwvt9t6jzvggz372ydp8fqswv2a @@ -0,0 +1,2 @@ +"\t % -1" + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/8/6/86k3v2zfg6c9n6n90t48dxem8619svqp59wxtvej6c5ha b/implementation/fuzz/inputs-genvm-modules-complete-json/8/6/86k3v2zfg6c9n6n90t48dxem8619svqp59wxtvej6c5ha new file mode 100644 index 00000000..30de4617 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/8/6/86k3v2zfg6c9n6n90t48dxem8619svqp59wxtvej6c5ha @@ -0,0 +1,2 @@ + +"\u0000\bσ—₯—u0000\bσ—₯—[8ζœ¬Ρ‹bσ—₯—\u00000000\bσ—₯—0\bσ—₯—t0000\bσ—₯—[8ζœ¬Ρ‹b0u0000\bσ—₯—[Ρ‹[8ζœ¬Ρ‹bσ—₯—\u0000Ρ‹\u00ba1 r" diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/8/7/8772ghfrhxs1ahgdn3fw4m2y492x836wf7k46sr5err0m b/implementation/fuzz/inputs-genvm-modules-complete-json/8/7/8772ghfrhxs1ahgdn3fw4m2y492x836wf7k46sr5err0m new file mode 100644 index 00000000..8434cf31 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/8/7/8772ghfrhxs1ahgdn3fw4m2y492x836wf7k46sr5err0m @@ -0,0 +1 @@ +[[[[[[[[[[[[[[[[ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/8/7/87qbtm14hgyxeyn8ey5dgnxtggr212adw4kwvzbnybdct b/implementation/fuzz/inputs-genvm-modules-complete-json/8/7/87qbtm14hgyxeyn8ey5dgnxtggr212adw4kwvzbnybdct new file mode 100644 index 00000000..350ba3a0 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/8/7/87qbtm14hgyxeyn8ey5dgnxtggr212adw4kwvzbnybdct @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/8/b/8b7pzscez3h4q8sxq7dmxcqvnfn5pgkdfxrqq1gr9csvt b/implementation/fuzz/inputs-genvm-modules-complete-json/8/b/8b7pzscez3h4q8sxq7dmxcqvnfn5pgkdfxrqq1gr9csvt new file mode 100644 index 00000000..a69e68bd --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/8/b/8b7pzscez3h4q8sxq7dmxcqvnfn5pgkdfxrqq1gr9csvt @@ -0,0 +1 @@ +33333333333.333333333333 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/8/b/8bky8ft3qegkfnscpfrx977aq71fvfkfggktv1q8f7f78 b/implementation/fuzz/inputs-genvm-modules-complete-json/8/b/8bky8ft3qegkfnscpfrx977aq71fvfkfggktv1q8f7f78 new file mode 100644 index 00000000..a1963301 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/8/b/8bky8ft3qegkfnscpfrx977aq71fvfkfggktv1q8f7f78 @@ -0,0 +1,5 @@ + + + [1, { + "": { + , 2]] diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/8/b/8bxa62pfbntw5ftd5xs2v8rrcdsmt2s84vc608cfn9efy b/implementation/fuzz/inputs-genvm-modules-complete-json/8/b/8bxa62pfbntw5ftd5xs2v8rrcdsmt2s84vc608cfn9efy new file mode 100644 index 00000000..5f4d2369 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/8/b/8bxa62pfbntw5ftd5xs2v8rrcdsmt2s84vc608cfn9efy @@ -0,0 +1 @@ +" \/Χ€ Π±ΡƒΠΊ1111112111111" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/8/d/8d55edm56x6y688pfpkvrm2m67q4j9xxaw2twbsbkbyec b/implementation/fuzz/inputs-genvm-modules-complete-json/8/d/8d55edm56x6y688pfpkvrm2m67q4j9xxaw2twbsbkbyec new file mode 100644 index 00000000..a7fa929d --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/8/d/8d55edm56x6y688pfpkvrm2m67q4j9xxaw2twbsbkbyec @@ -0,0 +1,16 @@ +{ + "foo": [ + "bar", + "baz\n\\{", + -1e10, + 1e20, + null, + true, + [1, 2, 3] + ], + "bar": [ ], + "bar": [ + "русскиС mbbbbbbbbb", + "mbbbbbbbbbbbbbbaz\t\\uabbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/8/e/8eqck26hk0rmgzpwed603cmmqm6wzhp1e3aarwg34r7xg b/implementation/fuzz/inputs-genvm-modules-complete-json/8/e/8eqck26hk0rmgzpwed603cmmqm6wzhp1e3aarwg34r7xg new file mode 100644 index 00000000..c3f1ccfb --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/8/e/8eqck26hk0rmgzpwed603cmmqm6wzhp1e3aarwg34r7xg @@ -0,0 +1,6 @@ +[[[1, + + + + + a, \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/8/g/8gdah1hcnmr8captbmxmrvjxs1dn8gbn0v7a10jf9msfp b/implementation/fuzz/inputs-genvm-modules-complete-json/8/g/8gdah1hcnmr8captbmxmrvjxs1dn8gbn0v7a10jf9msfp new file mode 100644 index 00000000..f5265fe0 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/8/g/8gdah1hcnmr8captbmxmrvjxs1dn8gbn0v7a10jf9msfp @@ -0,0 +1,16 @@ +{ + "foo nul": [ + "bar", + "baz\n\\z", + 123, + null, + true, + false, + [1, 2, 3] + ], + "bar": [ + "русскиС Π±ΡƒΠΊΠ²Ρ‹ ζ—₯本", + "mbbbbbbbbbbbbbbaz\t\uabbbbb", + "mbbbbbbbbbbbbbbaz\t\\ua]bbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/8/g/8gezvhamg4x4p2dc22wt60w0stt38wpk0re71drs2dajr b/implementation/fuzz/inputs-genvm-modules-complete-json/8/g/8gezvhamg4x4p2dc22wt60w0stt38wpk0re71drs2dajr new file mode 100644 index 00000000..506ec005 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/8/g/8gezvhamg4x4p2dc22wt60w0stt38wpk0re71drs2dajr @@ -0,0 +1 @@ + "bσ—‘—5 UwIoUσ—­£" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/8/g/8gt98q0hhqgn55z6vczfxgp42qb5awzyjd131cz51zwbg b/implementation/fuzz/inputs-genvm-modules-complete-json/8/g/8gt98q0hhqgn55z6vczfxgp42qb5awzyjd131cz51zwbg new file mode 100644 index 00000000..142b1f77 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/8/g/8gt98q0hhqgn55z6vczfxgp42qb5awzyjd131cz51zwbg @@ -0,0 +1 @@ +222222222252222252121e12 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/8/j/8jqj3fy8nbnk194bxprz80a7yfmdhdkqtgtd6s8zk138w b/implementation/fuzz/inputs-genvm-modules-complete-json/8/j/8jqj3fy8nbnk194bxprz80a7yfmdhdkqtgtd6s8zk138w new file mode 100644 index 00000000..98cdcb9b --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/8/j/8jqj3fy8nbnk194bxprz80a7yfmdhdkqtgtd6s8zk138w @@ -0,0 +1 @@ +61111111.1116116111111e-525 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/8/m/8m2qdka9j6w1fg7mg0wn0tb355jkbatrnzv6n7nmw4skt b/implementation/fuzz/inputs-genvm-modules-complete-json/8/m/8m2qdka9j6w1fg7mg0wn0tb355jkbatrnzv6n7nmw4skt new file mode 100644 index 00000000..1af563d8 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/8/m/8m2qdka9j6w1fg7mg0wn0tb355jkbatrnzv6n7nmw4skt @@ -0,0 +1 @@ +1111111111111111111111111E-111111111901111 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/8/n/8n4hrxsbftvpfmavg3smrgjc4hh6ezkawrfcepxk2jmde b/implementation/fuzz/inputs-genvm-modules-complete-json/8/n/8n4hrxsbftvpfmavg3smrgjc4hh6ezkawrfcepxk2jmde new file mode 100644 index 00000000..534ef538 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/8/n/8n4hrxsbftvpfmavg3smrgjc4hh6ezkawrfcepxk2jmde @@ -0,0 +1,2 @@ +1e-515 + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/8/p/8psw56dbhmxvhk9fhkmk8y0p0gwa88vszgnchgenysxh6 b/implementation/fuzz/inputs-genvm-modules-complete-json/8/p/8psw56dbhmxvhk9fhkmk8y0p0gwa88vszgnchgenysxh6 new file mode 100644 index 00000000..1100d71d --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/8/p/8psw56dbhmxvhk9fhkmk8y0p0gwa88vszgnchgenysxh6 @@ -0,0 +1 @@ +"\"\t" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/8/q/8qk3zk4k8vtbbkpxa5q0sp8fch71gka8tv8bny9kdktm2 b/implementation/fuzz/inputs-genvm-modules-complete-json/8/q/8qk3zk4k8vtbbkpxa5q0sp8fch71gka8tv8bny9kdktm2 new file mode 100644 index 00000000..81917f13 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/8/q/8qk3zk4k8vtbbkpxa5q0sp8fch71gka8tv8bny9kdktm2 @@ -0,0 +1,2 @@ +1e-515 + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/8/r/8r0hp2xf71j0b3sxjm9mkbwvpyf8jff8n64n6wkjr6368 b/implementation/fuzz/inputs-genvm-modules-complete-json/8/r/8r0hp2xf71j0b3sxjm9mkbwvpyf8jff8n64n6wkjr6368 new file mode 100644 index 00000000..4a248f73 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/8/r/8r0hp2xf71j0b3sxjm9mkbwvpyf8jff8n64n6wkjr6368 @@ -0,0 +1,2 @@ + +"Ρ€bbbbIbbγ€» % r" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/8/r/8rxmh4hmmgpgm7dxwht1cp1amvsd0qxnc23tdgfdh8k9g b/implementation/fuzz/inputs-genvm-modules-complete-json/8/r/8rxmh4hmmgpgm7dxwht1cp1amvsd0qxnc23tdgfdh8k9g new file mode 100644 index 00000000..3809136f --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/8/r/8rxmh4hmmgpgm7dxwht1cp1amvsd0qxnc23tdgfdh8k9g @@ -0,0 +1 @@ +0e5555555515355 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/8/r/8rzd98pan04nx6q9q9n5mkqtz86cg7hnkm1mkcq44qjmt b/implementation/fuzz/inputs-genvm-modules-complete-json/8/r/8rzd98pan04nx6q9q9n5mkqtz86cg7hnkm1mkcq44qjmt new file mode 100644 index 00000000..6b7e2b5d --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/8/r/8rzd98pan04nx6q9q9n5mkqtz86cg7hnkm1mkcq44qjmt @@ -0,0 +1,4 @@ +{ + "foo": [ + ], + ] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/8/s/8s2r3qc3h1anzm3hkwr6j3r6rmad853tc0ds25hrc2fb2 b/implementation/fuzz/inputs-genvm-modules-complete-json/8/s/8s2r3qc3h1anzm3hkwr6j3r6rmad853tc0ds25hrc2fb2 new file mode 100644 index 00000000..e102ecba --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/8/s/8s2r3qc3h1anzm3hkwr6j3r6rmad853tc0ds25hrc2fb2 @@ -0,0 +1,21 @@ +{ + "foo": [ + "bar", + "baz\n\\{", + 123, + -123, + -1e10, + 1e20, + null, + true, + false, + [1, 2, 3] + ], + "mbb -barar": [ ], + "": [ ], + "bar": [ + "русскиС Π±ΡƒΠΊΠ²Π‡ ζ—₯本", + "mbbbbbbbbbbbbbbaz\t\uabbbbb", + "mbbbbbbbbaz\tbbbbbbbbbbbaz\t\\uabbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/8/v/8vdr6yembj5xj2bm2cyg20q57cj4hrf1p8x80ndymj4aw b/implementation/fuzz/inputs-genvm-modules-complete-json/8/v/8vdr6yembj5xj2bm2cyg20q57cj4hrf1p8x80ndymj4aw new file mode 100644 index 00000000..1eece913 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/8/v/8vdr6yembj5xj2bm2cyg20q57cj4hrf1p8x80ndymj4aw @@ -0,0 +1 @@ + [11111,1111111111111119111.1111,1111311111111.11111111111111111111111111E111,11113111111111.111119E111, 2, 31111111111111111111E111, 2, 3] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/8/w/8wy643cjwk8wfh0004edbbst8wa1b73aywaqfc0mcf0kp b/implementation/fuzz/inputs-genvm-modules-complete-json/8/w/8wy643cjwk8wfh0004edbbst8wa1b73aywaqfc0mcf0kp new file mode 100644 index 00000000..a4d7c422 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/8/w/8wy643cjwk8wfh0004edbbst8wa1b73aywaqfc0mcf0kp @@ -0,0 +1,5 @@ +{ + "foo": 1e20, + "\\uabb@bb" + + "baz\n\ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/8/x/8xbj6pprejy2v5shedeb0xq7dqfchsdcjy9tt523dz5mc b/implementation/fuzz/inputs-genvm-modules-complete-json/8/x/8xbj6pprejy2v5shedeb0xq7dqfchsdcjy9tt523dz5mc new file mode 100644 index 00000000..deca8b3d --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/8/x/8xbj6pprejy2v5shedeb0xq7dqfchsdcjy9tt523dz5mc @@ -0,0 +1 @@ +33333333333333333323e3 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/8/y/8ykr1bp9z55bqng865rp8s6wx7w7hc6n2jk0wanp61062 b/implementation/fuzz/inputs-genvm-modules-complete-json/8/y/8ykr1bp9z55bqng865rp8s6wx7w7hc6n2jk0wanp61062 new file mode 100644 index 00000000..2da5c009 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/8/y/8ykr1bp9z55bqng865rp8s6wx7w7hc6n2jk0wanp61062 @@ -0,0 +1 @@ +[[[[[[[[[[[[[[[[[[[[[[[ [[[[[[[[[[[[[ [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/9/0/900w61hvabmpf0a96abnadvjsnn6q53zwxatx2t7d0tw6 b/implementation/fuzz/inputs-genvm-modules-complete-json/9/0/900w61hvabmpf0a96abnadvjsnn6q53zwxatx2t7d0tw6 new file mode 100644 index 00000000..3f966638 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/9/0/900w61hvabmpf0a96abnadvjsnn6q53zwxatx2t7d0tw6 @@ -0,0 +1 @@ + [ {}] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/9/6/969ard33ab13sqxf820qr4xb9cgscj55spqa0neg7epe4 b/implementation/fuzz/inputs-genvm-modules-complete-json/9/6/969ard33ab13sqxf820qr4xb9cgscj55spqa0neg7epe4 new file mode 100644 index 00000000..53dad9ed --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/9/6/969ard33ab13sqxf820qr4xb9cgscj55spqa0neg7epe4 @@ -0,0 +1 @@ +1922222222222222222222121e122 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/9/6/96samzadf8rnaf6j7g1t39dft2r9tm3b9rkj28d3edhhp b/implementation/fuzz/inputs-genvm-modules-complete-json/9/6/96samzadf8rnaf6j7g1t39dft2r9tm3b9rkj28d3edhhp new file mode 100644 index 00000000..89522335 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/9/6/96samzadf8rnaf6j7g1t39dft2r9tm3b9rkj28d3edhhp @@ -0,0 +1,2 @@ +111111111111111111111111e104 + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/9/8/98854yqfez4decm5rhrdrs2bxfdjmcge4tgd9wygtgh10 b/implementation/fuzz/inputs-genvm-modules-complete-json/9/8/98854yqfez4decm5rhrdrs2bxfdjmcge4tgd9wygtgh10 new file mode 100644 index 00000000..1f7d6c5a --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/9/8/98854yqfez4decm5rhrdrs2bxfdjmcge4tgd9wygtgh10 @@ -0,0 +1 @@ +{ } \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/9/a/9ajc9e1wemy51xq2xpwm52cc7pjda593acq5vtwd1bfd4 b/implementation/fuzz/inputs-genvm-modules-complete-json/9/a/9ajc9e1wemy51xq2xpwm52cc7pjda593acq5vtwd1bfd4 new file mode 100644 index 00000000..70d300d2 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/9/a/9ajc9e1wemy51xq2xpwm52cc7pjda593acq5vtwd1bfd4 @@ -0,0 +1 @@ + " \b
 t]u" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/9/a/9arqrp6srq0pd6vccjfkfymafvrh4qg32cx1br2exnd5a b/implementation/fuzz/inputs-genvm-modules-complete-json/9/a/9arqrp6srq0pd6vccjfkfymafvrh4qg32cx1br2exnd5a new file mode 100644 index 00000000..5ced72d9 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/9/a/9arqrp6srq0pd6vccjfkfymafvrh4qg32cx1br2exnd5a @@ -0,0 +1,20 @@ +11111111111111 + + + + + + + + + + + + + + + + + + +1 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/9/d/9dt8e15aj1a7yp124h139794c8we0wdnmp7jrtepj9zzt b/implementation/fuzz/inputs-genvm-modules-complete-json/9/d/9dt8e15aj1a7yp124h139794c8we0wdnmp7jrtepj9zzt new file mode 100644 index 00000000..a643e6bc --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/9/d/9dt8e15aj1a7yp124h139794c8we0wdnmp7jrtepj9zzt @@ -0,0 +1,5 @@ +[[[[[[[[ + + +3,[[[[[[[[[[[[[[[[[[[[[[[[[[ [[[[[[[[[[[[[ [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ null, +111E1 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/9/f/9f06vx30habsa70mpy6c1mtqscca4q0q70695yqxx3578 b/implementation/fuzz/inputs-genvm-modules-complete-json/9/f/9f06vx30habsa70mpy6c1mtqscca4q0q70695yqxx3578 new file mode 100644 index 00000000..d0d6d308 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/9/f/9f06vx30habsa70mpy6c1mtqscca4q0q70695yqxx3578 @@ -0,0 +1 @@ + " \uDbbb\uDDbB @21V'Dcuaz\tMu" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/9/f/9ftgssx6eqdc4ktnadb6t92gz3g3veep8a5fqkeve8jsy b/implementation/fuzz/inputs-genvm-modules-complete-json/9/f/9ftgssx6eqdc4ktnadb6t92gz3g3veep8a5fqkeve8jsy new file mode 100644 index 00000000..b31cc9ae --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/9/f/9ftgssx6eqdc4ktnadb6t92gz3g3veep8a5fqkeve8jsy @@ -0,0 +1 @@ +1  \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/9/m/9m5b5xm7mmt0whe7p2yqrr46d84j7tphrj202tazvzgej b/implementation/fuzz/inputs-genvm-modules-complete-json/9/m/9m5b5xm7mmt0whe7p2yqrr46d84j7tphrj202tazvzgej new file mode 100644 index 00000000..1cf5f0f2 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/9/m/9m5b5xm7mmt0whe7p2yqrr46d84j7tphrj202tazvzgej @@ -0,0 +1 @@ +22222222222222222222222 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/9/p/9pa7me5gr10qg7qmhs49aqy7ymgpef4b6y8j5tp56p6ke b/implementation/fuzz/inputs-genvm-modules-complete-json/9/p/9pa7me5gr10qg7qmhs49aqy7ymgpef4b6y8j5tp56p6ke new file mode 100644 index 00000000..afb4d195 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/9/p/9pa7me5gr10qg7qmhs49aqy7ymgpef4b6y8j5tp56p6ke @@ -0,0 +1 @@ + " &Ρƒ\u00A0" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/9/p/9pt2pz5srn6gthdq019taegmadr7pfwxv24e5qajd5cbe b/implementation/fuzz/inputs-genvm-modules-complete-json/9/p/9pt2pz5srn6gthdq019taegmadr7pfwxv24e5qajd5cbe new file mode 100644 index 00000000..fea41278 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/9/p/9pt2pz5srn6gthdq019taegmadr7pfwxv24e5qajd5cbe @@ -0,0 +1 @@ + [111111111111111111111111E111, 2,111111111111E111, 2, 3] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/9/p/9pyzvf0969q6r8nwnmr7yhwyegx8m8vffgnbjh8darjcj b/implementation/fuzz/inputs-genvm-modules-complete-json/9/p/9pyzvf0969q6r8nwnmr7yhwyegx8m8vffgnbjh8darjcj new file mode 100644 index 00000000..1c9cf715 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/9/p/9pyzvf0969q6r8nwnmr7yhwyegx8m8vffgnbjh8darjcj @@ -0,0 +1 @@ +[[ ]] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/9/r/9rccrh1twhkb756ves35p72ydzvds4v9a58ckxv4ccjhy b/implementation/fuzz/inputs-genvm-modules-complete-json/9/r/9rccrh1twhkb756ves35p72ydzvds4v9a58ckxv4ccjhy new file mode 100644 index 0000000000000000000000000000000000000000..8dfde12d2ca1458e54defd67afb717aee119444f GIT binary patch literal 20 Xcmb<cO3TkzvQmf!gFFUK&Xi~XK(qzF literal 0 HcmV?d00001 diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/9/r/9rwcjbaa7kxnvt61ysr5aa9a1htsgsc5akc4w41k3y6ht b/implementation/fuzz/inputs-genvm-modules-complete-json/9/r/9rwcjbaa7kxnvt61ysr5aa9a1htsgsc5akc4w41k3y6ht new file mode 100644 index 00000000..fcc4aca8 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/9/r/9rwcjbaa7kxnvt61ysr5aa9a1htsgsc5akc4w41k3y6ht @@ -0,0 +1 @@ +"Ρ€\\σ—₯ΊFbb r" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/9/s/9saeytr2fzvg7ep4x96tzbq20tv1b9mfv6q6q9bbrakrr b/implementation/fuzz/inputs-genvm-modules-complete-json/9/s/9saeytr2fzvg7ep4x96tzbq20tv1b9mfv6q6q9bbrakrr new file mode 100644 index 00000000..7de57ce0 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/9/s/9saeytr2fzvg7ep4x96tzbq20tv1b9mfv6q6q9bbrakrr @@ -0,0 +1,16 @@ + +"Ρ€\r" + + + + + + + + + + + + + + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/9/s/9sry4weyyekrmefx9xx4jcsfnf1dj0vy1e5vp1zsyyzny b/implementation/fuzz/inputs-genvm-modules-complete-json/9/s/9sry4weyyekrmefx9xx4jcsfnf1dj0vy1e5vp1zsyyzny new file mode 100644 index 00000000..ce208334 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/9/s/9sry4weyyekrmefx9xx4jcsfnf1dj0vy1e5vp1zsyyzny @@ -0,0 +1 @@ +"σ—₯Ίbbb" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/9/t/9t1jhwp88r17fndf8a6mw93spgxbzcv3bgyvnhv70ca5g b/implementation/fuzz/inputs-genvm-modules-complete-json/9/t/9t1jhwp88r17fndf8a6mw93spgxbzcv3bgyvnhv70ca5g new file mode 100644 index 00000000..f1861e53 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/9/t/9t1jhwp88r17fndf8a6mw93spgxbzcv3bgyvnhv70ca5g @@ -0,0 +1,3 @@ +{ + "1111114 -12222": [ ] +} \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/9/t/9tv8bc75k3bzqhjeqpwkyv29kzt7p8bazzx3ty7artsk8 b/implementation/fuzz/inputs-genvm-modules-complete-json/9/t/9tv8bc75k3bzqhjeqpwkyv29kzt7p8bazzx3ty7artsk8 new file mode 100644 index 00000000..37b49b91 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/9/t/9tv8bc75k3bzqhjeqpwkyv29kzt7p8bazzx3ty7artsk8 @@ -0,0 +1 @@ +111115111111111111111e-52111111222 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/9/v/9vhw4120f7nv6dvvhmsyb1khrxmwbqkgs22k838edwedj b/implementation/fuzz/inputs-genvm-modules-complete-json/9/v/9vhw4120f7nv6dvvhmsyb1khrxmwbqkgs22k838edwedj new file mode 100644 index 00000000..592dc850 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/9/v/9vhw4120f7nv6dvvhmsyb1khrxmwbqkgs22k838edwedj @@ -0,0 +1 @@ +111515111111112111.191E-52222262222222222 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/9/w/9wbbvcxcy4p2925nytewgyq3wh3tjbn6mp8d0cap1n7jw b/implementation/fuzz/inputs-genvm-modules-complete-json/9/w/9wbbvcxcy4p2925nytewgyq3wh3tjbn6mp8d0cap1n7jw new file mode 100644 index 00000000..524a4030 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/9/w/9wbbvcxcy4p2925nytewgyq3wh3tjbn6mp8d0cap1n7jw @@ -0,0 +1,17 @@ +{ + "foo": [ + "bar", + "baz\n\\z", + 123, + -123, + null, + true, null, + true, + false, + [1, 2, 3] + ], + "bar": [ + "русÁкиС Π±ΡƒΠΊΠ²Ρ‹ ζ—₯本", + "mbbbbbbbbbbbbbbabbbbbbaz\t\\uabbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/9/w/9wdtbsr7qeshkz69p2kgshe0xsa45ntjw6ypx6a3142by b/implementation/fuzz/inputs-genvm-modules-complete-json/9/w/9wdtbsr7qeshkz69p2kgshe0xsa45ntjw6ypx6a3142by new file mode 100644 index 00000000..18cdf484 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/9/w/9wdtbsr7qeshkz69p2kgshe0xsa45ntjw6ypx6a3142by @@ -0,0 +1,2 @@ +{ + "foo bbbbbbbbbbbbbbbbbbbbbbba -12222": 22": \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/9/x/9x2e52cvp0d2ebjhvbb9gfcyjss12qtv2kr0cdjkcang4 b/implementation/fuzz/inputs-genvm-modules-complete-json/9/x/9x2e52cvp0d2ebjhvbb9gfcyjss12qtv2kr0cdjkcang4 new file mode 100644 index 00000000..7d1bd0be --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/9/x/9x2e52cvp0d2ebjhvbb9gfcyjss12qtv2kr0cdjkcang4 @@ -0,0 +1 @@ + " Ρƒ\u01bb \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/9/x/9xn2apb41f7ev2vzwfh5002wp8jq1t9hgycz9b913znx2 b/implementation/fuzz/inputs-genvm-modules-complete-json/9/x/9xn2apb41f7ev2vzwfh5002wp8jq1t9hgycz9b913znx2 new file mode 100644 index 00000000..3cf51750 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/9/x/9xn2apb41f7ev2vzwfh5002wp8jq1t9hgycz9b913znx2 @@ -0,0 +1,5 @@ +{ + "foo": false, + + + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/9/x/9xnfg7md78205facm1zrbcwazzg87wrxkg6nhhyc7242p b/implementation/fuzz/inputs-genvm-modules-complete-json/9/x/9xnfg7md78205facm1zrbcwazzg87wrxkg6nhhyc7242p new file mode 100644 index 00000000..02e4a84d --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/9/x/9xnfg7md78205facm1zrbcwazzg87wrxkg6nhhyc7242p @@ -0,0 +1 @@ +false \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/9/x/9xrbx5dcssbk9fgs3250tt06s57txd90ddpdevz64y7kt b/implementation/fuzz/inputs-genvm-modules-complete-json/9/x/9xrbx5dcssbk9fgs3250tt06s57txd90ddpdevz64y7kt new file mode 100644 index 00000000..f3679596 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/9/x/9xrbx5dcssbk9fgs3250tt06s57txd90ddpdevz64y7kt @@ -0,0 +1,21 @@ +{ + "foo": [ + "bar", + "baz\n\\{", + 123, + -123, + -1e10, + 1e20, + null, + true, + false, + [1, 2, 3] + ], + "mbb -barar": [ ], + "": [ ], + "bar": [ + "mbbbbbbbbbbbbbbΠ±ΡƒΠΊΠ²Π‡ ζ—₯本", + "mbbbbbbbbbbbbbbaz\t\uabbbbb", + "mbbbbbbbbbbbbbbaz\t\\uabbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/9/y/9yywvhjaq25kg1r9489qdcrbdp051mxnyexd4r98zcxw8 b/implementation/fuzz/inputs-genvm-modules-complete-json/9/y/9yywvhjaq25kg1r9489qdcrbdp051mxnyexd4r98zcxw8 new file mode 100644 index 00000000..319d580c --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/9/y/9yywvhjaq25kg1r9489qdcrbdp051mxnyexd4r98zcxw8 @@ -0,0 +1 @@ +333333333333333333.333 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/a/2/a2cd3q0q8nty2kamzc30yd3yrsca6cv5p1qwzb5jkkh2m b/implementation/fuzz/inputs-genvm-modules-complete-json/a/2/a2cd3q0q8nty2kamzc30yd3yrsca6cv5p1qwzb5jkkh2m new file mode 100644 index 00000000..cdf04eff --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/a/2/a2cd3q0q8nty2kamzc30yd3yrsca6cv5p1qwzb5jkkh2m @@ -0,0 +1 @@ + [1111111111111111111111111111111111111111.1111111111E111,1111111111111111111111111.1111111111E111, 2, 1111111111, 2, 1111,11111111111111111111111111111111111E111, 2, 3] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/a/4/a44jjd8tqccyt151zdc50zv32f9v7e9x39qmxrf50pswy b/implementation/fuzz/inputs-genvm-modules-complete-json/a/4/a44jjd8tqccyt151zdc50zv32f9v7e9x39qmxrf50pswy new file mode 100644 index 00000000..34f8ab51 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/a/4/a44jjd8tqccyt151zdc50zv32f9v7e9x39qmxrf50pswy @@ -0,0 +1 @@ +3333333133333333333.3 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/a/4/a4r64smhx7gpf3pf0ztfjdp4cqr4fjdcgdwjzgdzxnbf2 b/implementation/fuzz/inputs-genvm-modules-complete-json/a/4/a4r64smhx7gpf3pf0ztfjdp4cqr4fjdcgdwjzgdzxnbf2 new file mode 100644 index 00000000..93645522 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/a/4/a4r64smhx7gpf3pf0ztfjdp4cqr4fjdcgdwjzgdzxnbf2 @@ -0,0 +1,2 @@ +{ + "foo" ~ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/a/6/a61dsw5z4095vy5a9nrzbbmctqj1s60t8a4nx4z5masbw b/implementation/fuzz/inputs-genvm-modules-complete-json/a/6/a61dsw5z4095vy5a9nrzbbmctqj1s60t8a4nx4z5masbw new file mode 100644 index 00000000..95aee945 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/a/6/a61dsw5z4095vy5a9nrzbbmctqj1s60t8a4nx4z5masbw @@ -0,0 +1,2 @@ + + 22222222222222222222222222E-22222222222222212222222222222 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/a/6/a6sv323xdcfz69fkq3hdepy0014265agv95f8v9685n56 b/implementation/fuzz/inputs-genvm-modules-complete-json/a/6/a6sv323xdcfz69fkq3hdepy0014265agv95f8v9685n56 new file mode 100644 index 00000000..6ea4504a --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/a/6/a6sv323xdcfz69fkq3hdepy0014265agv95f8v9685n56 @@ -0,0 +1 @@ +666666666666666666666666666 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/a/7/a78f2j2cp06v3pxabfw8a6qfz1gzxpvnzm06d4rt8qtmw b/implementation/fuzz/inputs-genvm-modules-complete-json/a/7/a78f2j2cp06v3pxabfw8a6qfz1gzxpvnzm06d4rt8qtmw new file mode 100644 index 00000000..307cce42 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/a/7/a78f2j2cp06v3pxabfw8a6qfz1gzxpvnzm06d4rt8qtmw @@ -0,0 +1,2 @@ +{ + "foo": [1111111111111111 -Π΅ Π₯本 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/a/7/a7k16nf5284aa24r0spdgrph37mpb4d7ws7ez4qwqzt3t b/implementation/fuzz/inputs-genvm-modules-complete-json/a/7/a7k16nf5284aa24r0spdgrph37mpb4d7ws7ez4qwqzt3t new file mode 100644 index 00000000..41398434 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/a/7/a7k16nf5284aa24r0spdgrph37mpb4d7ws7ez4qwqzt3t @@ -0,0 +1 @@ +"uj\\Ρ€ " \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/a/9/a9ygmyzxnswhedm95ey6jmpa2egzn9jceyd74zjj8ep8t b/implementation/fuzz/inputs-genvm-modules-complete-json/a/9/a9ygmyzxnswhedm95ey6jmpa2egzn9jceyd74zjj8ep8t new file mode 100644 index 00000000..38b95ef2 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/a/9/a9ygmyzxnswhedm95ey6jmpa2egzn9jceyd74zjj8ep8t @@ -0,0 +1,2 @@ + [ + - bbbp \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/a/d/adqgep5rgg42hdagzzcaqcwkq8dnq5xywq41p8yzhjhxr b/implementation/fuzz/inputs-genvm-modules-complete-json/a/d/adqgep5rgg42hdagzzcaqcwkq8dnq5xywq41p8yzhjhxr new file mode 100644 index 00000000..e83a8ca9 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/a/d/adqgep5rgg42hdagzzcaqcwkq8dnq5xywq41p8yzhjhxr @@ -0,0 +1 @@ +2222222222222222222222 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/a/f/afg3jgfgacqbyp0e0hw5w3bw7ne6fqnsbymaw99qvym9a b/implementation/fuzz/inputs-genvm-modules-complete-json/a/f/afg3jgfgacqbyp0e0hw5w3bw7ne6fqnsbymaw99qvym9a new file mode 100644 index 00000000..1142f922 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/a/f/afg3jgfgacqbyp0e0hw5w3bw7ne6fqnsbymaw99qvym9a @@ -0,0 +1 @@ + "с\fffff`fffffffffffff\fffffffffvabba\f/////////////////ffffff@ffffffffffffffffffabba\//////fffffffffffffffffffffffabba\/////////////////////////////bbbb" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/a/g/ag3rh55v0g8j79j5bb7267tzqgdjrqzhpbvde5qp98sxt b/implementation/fuzz/inputs-genvm-modules-complete-json/a/g/ag3rh55v0g8j79j5bb7267tzqgdjrqzhpbvde5qp98sxt new file mode 100644 index 00000000..294f54e6 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/a/g/ag3rh55v0g8j79j5bb7267tzqgdjrqzhpbvde5qp98sxt @@ -0,0 +1,2 @@ + [1,3] + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/a/j/aj2dkkddzftkjddb7ztsdbjz6dmpjjn6x3y295gq39n98 b/implementation/fuzz/inputs-genvm-modules-complete-json/a/j/aj2dkkddzftkjddb7ztsdbjz6dmpjjn6x3y295gq39n98 new file mode 100644 index 00000000..ad062a18 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/a/j/aj2dkkddzftkjddb7ztsdbjz6dmpjjn6x3y295gq39n98 @@ -0,0 +1 @@ +[[[[[[[[[[[[[[[ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/a/n/anhr0t9ghya12j6htrh77pjte2jeq23ftqyw4b2qjej52 b/implementation/fuzz/inputs-genvm-modules-complete-json/a/n/anhr0t9ghya12j6htrh77pjte2jeq23ftqyw4b2qjej52 new file mode 100644 index 00000000..e655bab1 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/a/n/anhr0t9ghya12j6htrh77pjte2jeq23ftqyw4b2qjej52 @@ -0,0 +1 @@ +true \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/a/p/ap712egc4y89qq05pep6z4f2y6mxw2xrc5yw0q9b25nd4 b/implementation/fuzz/inputs-genvm-modules-complete-json/a/p/ap712egc4y89qq05pep6z4f2y6mxw2xrc5yw0q9b25nd4 new file mode 100644 index 00000000..75f418ee --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/a/p/ap712egc4y89qq05pep6z4f2y6mxw2xrc5yw0q9b25nd4 @@ -0,0 +1,13 @@ +0.6 + + + + + + + + + + + + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/a/q/aqpdhzv05hc0sxkg8vgjmvymaqw3f6q26mwmty8vygs48 b/implementation/fuzz/inputs-genvm-modules-complete-json/a/q/aqpdhzv05hc0sxkg8vgjmvymaqw3f6q26mwmty8vygs48 new file mode 100644 index 00000000..8259e437 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/a/q/aqpdhzv05hc0sxkg8vgjmvymaqw3f6q26mwmty8vygs48 @@ -0,0 +1,31 @@ +{ + "foo": [ + "bar", + "baz\n\\{", + 123, + -123, + -1e10, + 1e20, + null, + true, + + + + + + + + + + + + + + + + [1, 2, 3] + ], + "mbb -bar": [ ], + "bar": [ + "русскиС Π±ΡƒΠΊΠ²Π‡ ζ—₯n\/22\\uba \n\/22\\ubue, + f ζ—₯ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/a/r/arfgc8zcqrbh0f5yzpjrqts7ganxrw1b6bpcvm9d3g1wm b/implementation/fuzz/inputs-genvm-modules-complete-json/a/r/arfgc8zcqrbh0f5yzpjrqts7ganxrw1b6bpcvm9d3g1wm new file mode 100644 index 00000000..52e512d6 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/a/r/arfgc8zcqrbh0f5yzpjrqts7ganxrw1b6bpcvm9d3g1wm @@ -0,0 +1 @@ + [11111,111131111111111111E-123111, 2, 3111111111111111111111E111,11,111111111111111E111, 2, 3] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/a/t/atmpzxjkayxkqzjsefhdxc96qbm94etg252p41fx214pt b/implementation/fuzz/inputs-genvm-modules-complete-json/a/t/atmpzxjkayxkqzjsefhdxc96qbm94etg252p41fx214pt new file mode 100644 index 00000000..44c280bc --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/a/t/atmpzxjkayxkqzjsefhdxc96qbm94etg252p41fx214pt @@ -0,0 +1 @@ +33333333.3333333333333 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/a/v/avvknp9mtyb13s8xzb43p22w4g6hasm6s9xn96z93hhh0 b/implementation/fuzz/inputs-genvm-modules-complete-json/a/v/avvknp9mtyb13s8xzb43p22w4g6hasm6s9xn96z93hhh0 new file mode 100644 index 00000000..7fdd577b --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/a/v/avvknp9mtyb13s8xzb43p22w4g6hasm6s9xn96z93hhh0 @@ -0,0 +1,2 @@ +{ + " "  \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/a/x/ax3adfvkr5t4afqhfb6tbtgg4heqcv34z860ftazx8f1e b/implementation/fuzz/inputs-genvm-modules-complete-json/a/x/ax3adfvkr5t4afqhfb6tbtgg4heqcv34z860ftazx8f1e new file mode 100644 index 00000000..67535a8b --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/a/x/ax3adfvkr5t4afqhfb6tbtgg4heqcv34z860ftazx8f1e @@ -0,0 +1 @@ +0e \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/a/x/ax87204a9gvnrwhsfs6gw3wd9dgp53k6wcj5v7t5vt39g b/implementation/fuzz/inputs-genvm-modules-complete-json/a/x/ax87204a9gvnrwhsfs6gw3wd9dgp53k6wcj5v7t5vt39g new file mode 100644 index 00000000..ac0f1eff --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/a/x/ax87204a9gvnrwhsfs6gw3wd9dgp53k6wcj5v7t5vt39g @@ -0,0 +1,2 @@ +{ " 111111111111111e-52222 f 8 o": [ ] +} \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/b/5/b5xkhvdfpwcw9t9dhmdggn32aycqwaftsqjc93ph6f9mr b/implementation/fuzz/inputs-genvm-modules-complete-json/b/5/b5xkhvdfpwcw9t9dhmdggn32aycqwaftsqjc93ph6f9mr new file mode 100644 index 00000000..0e50b408 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/b/5/b5xkhvdfpwcw9t9dhmdggn32aycqwaftsqjc93ph6f9mr @@ -0,0 +1 @@ + "Ρ€111 z" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/b/7/b72dm8vrs55qdq3mbmh9s1rr4x1qyhfb9hz91w0fg6am6 b/implementation/fuzz/inputs-genvm-modules-complete-json/b/7/b72dm8vrs55qdq3mbmh9s1rr4x1qyhfb9hz91w0fg6am6 new file mode 100644 index 00000000..54c025c9 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/b/7/b72dm8vrs55qdq3mbmh9s1rr4x1qyhfb9hz91w0fg6am6 @@ -0,0 +1 @@ + "ς€»ƒ сскڸʡ6Π²ΡƒΠΊbbcbbbbb:bbb Bxbz" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/b/7/b792ade597qyvp0q44d1pnbj9yrxc2aegmjtzxw2g6cwr b/implementation/fuzz/inputs-genvm-modules-complete-json/b/7/b792ade597qyvp0q44d1pnbj9yrxc2aegmjtzxw2g6cwr new file mode 100644 index 00000000..65ee998f --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/b/7/b792ade597qyvp0q44d1pnbj9yrxc2aegmjtzxw2g6cwr @@ -0,0 +1 @@ +122222222222222222.111e1 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/b/9/b922m26p9tjmqpcgjmm15a87y3q6ge5vr12vfd5cegjza b/implementation/fuzz/inputs-genvm-modules-complete-json/b/9/b922m26p9tjmqpcgjmm15a87y3q6ge5vr12vfd5cegjza new file mode 100644 index 00000000..d2a643cd --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/b/9/b922m26p9tjmqpcgjmm15a87y3q6ge5vr12vfd5cegjza @@ -0,0 +1,2 @@ +{ + "\f{o" "b \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/b/9/b9zjk64ptxd1rqjsh0g7yg9bwc3vr6jay0rqjnb2cgq3g b/implementation/fuzz/inputs-genvm-modules-complete-json/b/9/b9zjk64ptxd1rqjsh0g7yg9bwc3vr6jay0rqjnb2cgq3g new file mode 100644 index 00000000..76156a35 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/b/9/b9zjk64ptxd1rqjsh0g7yg9bwc3vr6jay0rqjnb2cgq3g @@ -0,0 +1,7 @@ + "Β€ 12" + + + + + + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/b/c/bcnnahc097sf7wfdz84yntp9h8xhb1cyntp5mtfbh0bcr b/implementation/fuzz/inputs-genvm-modules-complete-json/b/c/bcnnahc097sf7wfdz84yntp9h8xhb1cyntp5mtfbh0bcr new file mode 100644 index 00000000..7b59f79d --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/b/c/bcnnahc097sf7wfdz84yntp9h8xhb1cyntp5mtfbh0bcr @@ -0,0 +1 @@ +9 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/b/c/bcvpdd7znb1h88w52fyf4ss16j6g5x6y1xdrbch3f2498 b/implementation/fuzz/inputs-genvm-modules-complete-json/b/c/bcvpdd7znb1h88w52fyf4ss16j6g5x6y1xdrbch3f2498 new file mode 100644 index 00000000..8c8733b2 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/b/c/bcvpdd7znb1h88w52fyf4ss16j6g5x6y1xdrbch3f2498 @@ -0,0 +1 @@ +1.11e-11 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/b/f/bfdm55ahtbz3bstsb7wa1zc5cwe8j0kf1ac6kn2b3c9e0 b/implementation/fuzz/inputs-genvm-modules-complete-json/b/f/bfdm55ahtbz3bstsb7wa1zc5cwe8j0kf1ac6kn2b3c9e0 new file mode 100644 index 00000000..ab1d6fd5 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/b/f/bfdm55ahtbz3bstsb7wa1zc5cwe8j0kf1ac6kn2b3c9e0 @@ -0,0 +1 @@ +1E1 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/b/g/bgnptbbyxjbtzx5r683vg13k1gv4c428jbfk27m58vwca b/implementation/fuzz/inputs-genvm-modules-complete-json/b/g/bgnptbbyxjbtzx5r683vg13k1gv4c428jbfk27m58vwca new file mode 100644 index 00000000..e5e20ad6 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/b/g/bgnptbbyxjbtzx5r683vg13k1gv4c428jbfk27m58vwca @@ -0,0 +1,2 @@ + [ { + " ( QQ": [ -11111111111111111111111111111111111111111111111111111111131111111, -11111111111 , 3] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/b/h/bh5pj4v790fdb76zgnp1h74skr5nv7mcty01stnyj86dc b/implementation/fuzz/inputs-genvm-modules-complete-json/b/h/bh5pj4v790fdb76zgnp1h74skr5nv7mcty01stnyj86dc new file mode 100644 index 00000000..57fb3157 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/b/h/bh5pj4v790fdb76zgnp1h74skr5nv7mcty01stnyj86dc @@ -0,0 +1,21 @@ +{ + "foo": [ + "bar", + "baz\n\\{", + 123, + -123, + -1e10, + 1e20, + null, + true, + false, + [1, 2, 3] + ], + "mbb -barar": [ ], + "": [ ], + "bar": [ + "русскиС Π±ΡƒΠΊΠ²Π‡ ζ—₯本", + "mbbbbbbbbbbbbbbbz\t\uabbbbb", + "mbbbbbbbbbbbbbbbbbbbbaz\t\\uabbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/b/h/bhjjkqvqr89xn9f94n91n25rqzs58s7gkq12c4qb19n82 b/implementation/fuzz/inputs-genvm-modules-complete-json/b/h/bhjjkqvqr89xn9f94n91n25rqzs58s7gkq12c4qb19n82 new file mode 100644 index 00000000..1868abf8 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/b/h/bhjjkqvqr89xn9f94n91n25rqzs58s7gkq12c4qb19n82 @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/b/h/bhyhcrg11qye6fygr8ve1jrv75z6vxzgwpc10ykare34p b/implementation/fuzz/inputs-genvm-modules-complete-json/b/h/bhyhcrg11qye6fygr8ve1jrv75z6vxzgwpc10ykare34p new file mode 100644 index 00000000..8d1c21e4 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/b/h/bhyhcrg11qye6fygr8ve1jrv75z6vxzgwpc10ykare34p @@ -0,0 +1 @@ +nul \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/b/j/bjf5jcbq1bh9rmwq2arjsd3nbkfj9cabtkjc6rytb707y b/implementation/fuzz/inputs-genvm-modules-complete-json/b/j/bjf5jcbq1bh9rmwq2arjsd3nbkfj9cabtkjc6rytb707y new file mode 100644 index 00000000..c2bff90b --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/b/j/bjf5jcbq1bh9rmwq2arjsd3nbkfj9cabtkjc6rytb707y @@ -0,0 +1 @@ +51111111111111111111111211.11 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/b/j/bjw8y3t1ypcbv2r80jsymm5yqtkbx41kqb7yax0dag4rp b/implementation/fuzz/inputs-genvm-modules-complete-json/b/j/bjw8y3t1ypcbv2r80jsymm5yqtkbx41kqb7yax0dag4rp new file mode 100644 index 00000000..ef32ca5a --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/b/j/bjw8y3t1ypcbv2r80jsymm5yqtkbx41kqb7yax0dag4rp @@ -0,0 +1,4 @@ +1111511111111111111111111111.11e-111111111 + + + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/b/k/bk7tg4v8q8489zy60xa9kjwfn9dtnf6f5841aw9mbz84j b/implementation/fuzz/inputs-genvm-modules-complete-json/b/k/bk7tg4v8q8489zy60xa9kjwfn9dtnf6f5841aw9mbz84j new file mode 100644 index 00000000..0d9261cc --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/b/k/bk7tg4v8q8489zy60xa9kjwfn9dtnf6f5841aw9mbz84j @@ -0,0 +1 @@ +"\f:X" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/b/m/bmckydcqpd8k65kkc18gahhkr8ckmc2tfs4fsc3rx54mw b/implementation/fuzz/inputs-genvm-modules-complete-json/b/m/bmckydcqpd8k65kkc18gahhkr8ckmc2tfs4fsc3rx54mw new file mode 100644 index 00000000..7d60ea5e --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/b/m/bmckydcqpd8k65kkc18gahhkr8ckmc2tfs4fsc3rx54mw @@ -0,0 +1 @@ + "\\\\\\\\\\\\\\\\\bbΡ€ΠΊu[1\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\bbbbbbbbub bbcΠ΅ bbcΠ΅ Π±ΡƒΡ„\\\\\\\\\\\\\\\\с Bq{z" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/b/m/bmwh1asmc9q9qptsthtjgw247pyeasgpcy17gx32ka5kc b/implementation/fuzz/inputs-genvm-modules-complete-json/b/m/bmwh1asmc9q9qptsthtjgw247pyeasgpcy17gx32ka5kc new file mode 100644 index 00000000..33187e68 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/b/m/bmwh1asmc9q9qptsthtjgw247pyeasgpcy17gx32ka5kc @@ -0,0 +1 @@ +"\nb}`H" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/b/n/bn21bfa4kh47929dmgnee4d1f05hgv6js583hxnzacqat b/implementation/fuzz/inputs-genvm-modules-complete-json/b/n/bn21bfa4kh47929dmgnee4d1f05hgv6js583hxnzacqat new file mode 100644 index 00000000..6295fdc3 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/b/n/bn21bfa4kh47929dmgnee4d1f05hgv6js583hxnzacqat @@ -0,0 +1 @@ +[, \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/b/p/bp3te430cppsj8f0n1tr69zgme8zm3zdctwkts89tqkgj b/implementation/fuzz/inputs-genvm-modules-complete-json/b/p/bp3te430cppsj8f0n1tr69zgme8zm3zdctwkts89tqkgj new file mode 100644 index 00000000..09441456 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/b/p/bp3te430cppsj8f0n1tr69zgme8zm3zdctwkts89tqkgj @@ -0,0 +1,2 @@ + +[[[[[[[[[ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/b/q/bqtd1xq1cbjw4bv28vrdaptenxn31d03qfycsxd94dztw b/implementation/fuzz/inputs-genvm-modules-complete-json/b/q/bqtd1xq1cbjw4bv28vrdaptenxn31d03qfycsxd94dztw new file mode 100644 index 00000000..41c52d37 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/b/q/bqtd1xq1cbjw4bv28vrdaptenxn31d03qfycsxd94dztw @@ -0,0 +1 @@ +222222222222.2222222222 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/b/s/bsbp7ykhgax3jbh9pm9v832kxgfpc8yj2wdqnj0dnbshc b/implementation/fuzz/inputs-genvm-modules-complete-json/b/s/bsbp7ykhgax3jbh9pm9v832kxgfpc8yj2wdqnj0dnbshc new file mode 100644 index 00000000..e79b4410 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/b/s/bsbp7ykhgax3jbh9pm9v832kxgfpc8yj2wdqnj0dnbshc @@ -0,0 +1 @@ + "\bkbb \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/b/s/bsvnbfz83cjp98eh9qrz4e7baqbcfr8w07c9mr8h8h76r b/implementation/fuzz/inputs-genvm-modules-complete-json/b/s/bsvnbfz83cjp98eh9qrz4e7baqbcfr8w07c9mr8h8h76r new file mode 100644 index 00000000..dd7c9233 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/b/s/bsvnbfz83cjp98eh9qrz4e7baqbcfr8w07c9mr8h8h76r @@ -0,0 +1 @@ + " Ρƒ\u01bA0" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/b/t/btpd8pnyd810hns19dkvp0pm6mbctydr6nxh2h5bj28f2 b/implementation/fuzz/inputs-genvm-modules-complete-json/b/t/btpd8pnyd810hns19dkvp0pm6mbctydr6nxh2h5bj28f2 new file mode 100644 index 00000000..ee27b7aa --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/b/t/btpd8pnyd810hns19dkvp0pm6mbctydr6nxh2h5bj28f2 @@ -0,0 +1 @@ +3333333333333333333333333333 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/b/v/bv4dcsdt837x02eq88kajab91nq0ctkmc5fja6s14x49g b/implementation/fuzz/inputs-genvm-modules-complete-json/b/v/bv4dcsdt837x02eq88kajab91nq0ctkmc5fja6s14x49g new file mode 100644 index 00000000..ccf2679c --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/b/v/bv4dcsdt837x02eq88kajab91nq0ctkmc5fja6s14x49g @@ -0,0 +1,19 @@ +{ + "": [ + "bar", + "baz\n\\{", + 123, + -1e10, + 1e20, + null, + true, + false, + [1, 2, 3] + ], + " ar": [ ], + "bar": [ + "русскиС Π±ΡƒΠΊΠ²Π‡ ζ—₯本", + "mbbbbbbbbbbbbbbaz\t\uabbb b", + "mmmmmmmbbbbbbbbaz\t\\uabbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/b/y/by0vf31j6shg87ajmc19gh3f36tc4vr6b7v81n73f21cg b/implementation/fuzz/inputs-genvm-modules-complete-json/b/y/by0vf31j6shg87ajmc19gh3f36tc4vr6b7v81n73f21cg new file mode 100644 index 00000000..adf0de8c --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/b/y/by0vf31j6shg87ajmc19gh3f36tc4vr6b7v81n73f21cg @@ -0,0 +1 @@ +-1e-1 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/b/y/by404cjxa879cfr3mkp8j5v99ewqdh1ka3bqptc3r7248 b/implementation/fuzz/inputs-genvm-modules-complete-json/b/y/by404cjxa879cfr3mkp8j5v99ewqdh1ka3bqptc3r7248 new file mode 100644 index 00000000..284c3a9a --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/b/y/by404cjxa879cfr3mkp8j5v99ewqdh1ka3bqptc3r7248 @@ -0,0 +1 @@ + "\bσ—₯—Eσ—₯—1 " \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/c/0/c00d75zm9sz89vs3pnwfs94sn7rxdxg25tyqgx5jbdbp0 b/implementation/fuzz/inputs-genvm-modules-complete-json/c/0/c00d75zm9sz89vs3pnwfs94sn7rxdxg25tyqgx5jbdbp0 new file mode 100644 index 00000000..27628bca --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/c/0/c00d75zm9sz89vs3pnwfs94sn7rxdxg25tyqgx5jbdbp0 @@ -0,0 +1 @@ + "русскиʡ Π±ΡƒΠΊ12 > B\bz" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/c/1/c1xfgsxp69zhyth69xdj5wvrsj2v898bb12r1xvcpg390 b/implementation/fuzz/inputs-genvm-modules-complete-json/c/1/c1xfgsxp69zhyth69xdj5wvrsj2v898bb12r1xvcpg390 new file mode 100644 index 00000000..998384b2 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/c/1/c1xfgsxp69zhyth69xdj5wvrsj2v898bb12r1xvcpg390 @@ -0,0 +1,2 @@ +{ + "foo" " \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/c/3/c3t4rfdgtkkcy6efj3fyz1excf7pww2x15239qk3etjpy b/implementation/fuzz/inputs-genvm-modules-complete-json/c/3/c3t4rfdgtkkcy6efj3fyz1excf7pww2x15239qk3etjpy new file mode 100644 index 00000000..8878e547 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/c/3/c3t4rfdgtkkcy6efj3fyz1excf7pww2x15239qk3etjpy @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/c/4/c4szvzad8fpgnp25t9ea96c4nbkmts8gxsdkq301t1sbr b/implementation/fuzz/inputs-genvm-modules-complete-json/c/4/c4szvzad8fpgnp25t9ea96c4nbkmts8gxsdkq301t1sbr new file mode 100644 index 00000000..c61f7f7e --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/c/4/c4szvzad8fpgnp25t9ea96c4nbkmts8gxsdkq301t1sbr @@ -0,0 +1 @@ +"\t" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/708713afdb215a7fdd1b3246ec662a3d992648dc262ca1d5360be94b6d75313b b/implementation/fuzz/inputs-genvm-modules-complete-json/c/4/c4wkbgpf2hrga2ja9gan85nve6vs8ndygv9atc4gvx9zr similarity index 50% rename from implementation/fuzz/inputs-genvm-modules-normalize-whitespace/708713afdb215a7fdd1b3246ec662a3d992648dc262ca1d5360be94b6d75313b rename to implementation/fuzz/inputs-genvm-modules-complete-json/c/4/c4wkbgpf2hrga2ja9gan85nve6vs8ndygv9atc4gvx9zr index 59f144ee093d5bcf17b6de64d11c2ba0b53eacec..139597f9cb07c5d48bed18984ec4747f4b4f3438 100644 GIT binary patch literal 2 Jcmd<$0ssIY02KfL literal 2 Jcmd;L0000M01E&B diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/c/5/c5gqzg4fntq5azg6h5f1n3k89v7c3zkd79a68gvqpa9jy b/implementation/fuzz/inputs-genvm-modules-complete-json/c/5/c5gqzg4fntq5azg6h5f1n3k89v7c3zkd79a68gvqpa9jy new file mode 100644 index 00000000..ebbfecc7 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/c/5/c5gqzg4fntq5azg6h5f1n3k89v7c3zkd79a68gvqpa9jy @@ -0,0 +1 @@ +22202222222222222222E22222222222 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/c/b/cb2jgg28j8qmpnrxvgwqwhy9rsa3qzrnqmzj9ppdafjag b/implementation/fuzz/inputs-genvm-modules-complete-json/c/b/cb2jgg28j8qmpnrxvgwqwhy9rsa3qzrnqmzj9ppdafjag new file mode 100644 index 00000000..f4c9f6c9 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/c/b/cb2jgg28j8qmpnrxvgwqwhy9rsa3qzrnqmzj9ppdafjag @@ -0,0 +1 @@ + " \uDAAb\ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/c/c/ccn5t75s30qs8f06japgnm7sc9vq1m1qwrqs634edemx6 b/implementation/fuzz/inputs-genvm-modules-complete-json/c/c/ccn5t75s30qs8f06japgnm7sc9vq1m1qwrqs634edemx6 new file mode 100644 index 00000000..bbfb7ba0 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/c/c/ccn5t75s30qs8f06japgnm7sc9vq1m1qwrqs634edemx6 @@ -0,0 +1,3 @@ + [0, + + 1111111111111] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/c/d/cdbjcc0akvqmxres0ak235a7j1g5x8af42gvewk0w0ary b/implementation/fuzz/inputs-genvm-modules-complete-json/c/d/cdbjcc0akvqmxres0ak235a7j1g5x8af42gvewk0w0ary new file mode 100644 index 00000000..2e15497b --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/c/d/cdbjcc0akvqmxres0ak235a7j1g5x8af42gvewk0w0ary @@ -0,0 +1,5 @@ + { + "":0, + + + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/c/d/cdswa6f4rx04dgwghpam5zkxp9c4yvnry1a9dm6708hcr b/implementation/fuzz/inputs-genvm-modules-complete-json/c/d/cdswa6f4rx04dgwghpam5zkxp9c4yvnry1a9dm6708hcr new file mode 100644 index 00000000..9db2ae42 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/c/d/cdswa6f4rx04dgwghpam5zkxp9c4yvnry1a9dm6708hcr @@ -0,0 +1,16 @@ + [1, 2, + + + + + + "рус + + + + + + + + + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/c/e/ce6q5vbgw31nc3nvdrve6xhx519nntpfp466y772cd562 b/implementation/fuzz/inputs-genvm-modules-complete-json/c/e/ce6q5vbgw31nc3nvdrve6xhx519nntpfp466y772cd562 new file mode 100644 index 00000000..67efaf23 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/c/e/ce6q5vbgw31nc3nvdrve6xhx519nntpfp466y772cd562 @@ -0,0 +1 @@ +"{" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/c/f/cf6wctsh5htv0fhkw46ab49ws2dqrng17h3panefejg3c b/implementation/fuzz/inputs-genvm-modules-complete-json/c/f/cf6wctsh5htv0fhkw46ab49ws2dqrng17h3panefejg3c new file mode 100644 index 00000000..a4053c5c --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/c/f/cf6wctsh5htv0fhkw46ab49ws2dqrng17h3panefejg3c @@ -0,0 +1,15 @@ +{ + "mb": [ + "bar", + "baz\n\\{", + 123, + 1e20, + null, + true, + false, + [1, 2, 3] + ], + "mbb -barar": [ ], + "": [ ], + "bar": 11112, 3Π΅ Π₯本", + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/c/f/cfm826zbpsn5r4wex6spd3043c4y5gcnbvyxmfpgj97w2 b/implementation/fuzz/inputs-genvm-modules-complete-json/c/f/cfm826zbpsn5r4wex6spd3043c4y5gcnbvyxmfpgj97w2 new file mode 100644 index 00000000..670e0c14 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/c/f/cfm826zbpsn5r4wex6spd3043c4y5gcnbvyxmfpgj97w2 @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/c/h/ch7v9q3a817bjvgqk19qhsrmre8hr82mhrjc2zq2m19gr b/implementation/fuzz/inputs-genvm-modules-complete-json/c/h/ch7v9q3a817bjvgqk19qhsrmre8hr82mhrjc2zq2m19gr new file mode 100644 index 00000000..69294512 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/c/h/ch7v9q3a817bjvgqk19qhsrmre8hr82mhrjc2zq2m19gr @@ -0,0 +1,7 @@ + "Ρ€\\u < 12CabbbMb" + + + + + + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/c/k/ck2mhak8fj2023hsbnd500aknqbzpvx1gghaxdfbmx9mr b/implementation/fuzz/inputs-genvm-modules-complete-json/c/k/ck2mhak8fj2023hsbnd500aknqbzpvx1gghaxdfbmx9mr new file mode 100644 index 00000000..b5020ebf --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/c/k/ck2mhak8fj2023hsbnd500aknqbzpvx1gghaxdfbmx9mr @@ -0,0 +1,14 @@ +{ + "foo": [ + "bar", + "baz\n\\z", + 123, + -123, + [[[[[[[[[[[[[[[[[[[[[{ + + + + + + + -1 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/c/n/cnnd3zk59t022ennk6sne1h1m5jzkv16e7tcqaaeh4ngc b/implementation/fuzz/inputs-genvm-modules-complete-json/c/n/cnnd3zk59t022ennk6sne1h1m5jzkv16e7tcqaaeh4ngc new file mode 100644 index 00000000..74d31ceb --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/c/n/cnnd3zk59t022ennk6sne1h1m5jzkv16e7tcqaaeh4ngc @@ -0,0 +1 @@ +[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/c/q/cqg62jc16td8pgmwkmpb887sccg89r05ph3n0k2emj4jw b/implementation/fuzz/inputs-genvm-modules-complete-json/c/q/cqg62jc16td8pgmwkmpb887sccg89r05ph3n0k2emj4jw new file mode 100644 index 00000000..0337474c --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/c/q/cqg62jc16td8pgmwkmpb887sccg89r05ph3n0k2emj4jw @@ -0,0 +1 @@ +111e-5111111111111111111 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/c/q/cqvndvgd652rw5bfgenmkatrxbypkxm658ykfnq29h4g0 b/implementation/fuzz/inputs-genvm-modules-complete-json/c/q/cqvndvgd652rw5bfgenmkatrxbypkxm658ykfnq29h4g0 new file mode 100644 index 00000000..1b6cd4f4 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/c/q/cqvndvgd652rw5bfgenmkatrxbypkxm658ykfnq29h4g0 @@ -0,0 +1 @@ +"bAz\n\/b" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/c/s/csk3f07z6bfh43433v3t4sfrg22ybgsyfy42fqm6x1v3r b/implementation/fuzz/inputs-genvm-modules-complete-json/c/s/csk3f07z6bfh43433v3t4sfrg22ybgsyfy42fqm6x1v3r new file mode 100644 index 00000000..5b9ecb8a --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/c/s/csk3f07z6bfh43433v3t4sfrg22ybgsyfy42fqm6x1v3r @@ -0,0 +1,18 @@ +{ + "foo": [ + "bar", + "baz\n\\{", + 123, + 111111111911111111111.11111e-1120, + null, + true, + false, + [1, 2, 3] + ], + "mbb b -barar": [ ], + "": [ ], + "bar": [ + "рۃсскиС Π±ΡƒΠΊΠ²Π‡ ζ—₯本", + "mbbbbbbbbmbbbbbbbbbbb bbaz\t\\uabbbb1b" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/c/t/ctj41vh54xd59bpg48hmrjgz890q69jmejh756a0q8nnt b/implementation/fuzz/inputs-genvm-modules-complete-json/c/t/ctj41vh54xd59bpg48hmrjgz890q69jmejh756a0q8nnt new file mode 100644 index 00000000..91042ef7 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/c/t/ctj41vh54xd59bpg48hmrjgz890q69jmejh756a0q8nnt @@ -0,0 +1 @@ +[ , , \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/c/t/ctqthskpwwnmrm8g6khj3j8hxs5yfvhxgf086pq30jhtr b/implementation/fuzz/inputs-genvm-modules-complete-json/c/t/ctqthskpwwnmrm8g6khj3j8hxs5yfvhxgf086pq30jhtr new file mode 100644 index 00000000..e12fc7f5 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/c/t/ctqthskpwwnmrm8g6khj3j8hxs5yfvhxgf086pq30jhtr @@ -0,0 +1 @@ +"UUUIbcγ€€ rbbbb," diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/c/w/cw1cfp9bdvcqkqg5tm08ggrc5bda7zmqz8xq3k3d6ysam b/implementation/fuzz/inputs-genvm-modules-complete-json/c/w/cw1cfp9bdvcqkqg5tm08ggrc5bda7zmqz8xq3k3d6ysam new file mode 100644 index 00000000..351a0eee --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/c/w/cw1cfp9bdvcqkqg5tm08ggrc5bda7zmqz8xq3k3d6ysam @@ -0,0 +1,2 @@ + +"Ρ€[ bΠ²‡ σ—₯Ίu b r" diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/c/w/cw1v9r7a7eysaxd0fpm6d58tbdg981d7z1jdjvcy9xmj6 b/implementation/fuzz/inputs-genvm-modules-complete-json/c/w/cw1v9r7a7eysaxd0fpm6d58tbdg981d7z1jdjvcy9xmj6 new file mode 100644 index 00000000..5d5bb8dc --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/c/w/cw1v9r7a7eysaxd0fpm6d58tbdg981d7z1jdjvcy9xmj6 @@ -0,0 +1 @@ +"UUUTmbΒ  $ \u8bbbbbb," diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/c/w/cwa33vsd7h36xx5pzk5wzyk4hnn9ha7y87eh90vszdtwc b/implementation/fuzz/inputs-genvm-modules-complete-json/c/w/cwa33vsd7h36xx5pzk5wzyk4hnn9ha7y87eh90vszdtwc new file mode 100644 index 00000000..85ee7d2d --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/c/w/cwa33vsd7h36xx5pzk5wzyk4hnn9ha7y87eh90vszdtwc @@ -0,0 +1 @@ + " Ρƒ0000000000000/000\u0000" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/c/w/cwmpyftwdby1bddzawz9ee6bavbfagjwem3n3yvsa5936 b/implementation/fuzz/inputs-genvm-modules-complete-json/c/w/cwmpyftwdby1bddzawz9ee6bavbfagjwem3n3yvsa5936 new file mode 100644 index 00000000..d3d13c72 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/c/w/cwmpyftwdby1bddzawz9ee6bavbfagjwem3n3yvsa5936 @@ -0,0 +1 @@ +"Π±ΡƒΠΊ12C \f\\ " \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/c/z/czsfsvp41g06qv09d387sz4kknemgat7pv72apmrvav5r b/implementation/fuzz/inputs-genvm-modules-complete-json/c/z/czsfsvp41g06qv09d387sz4kknemgat7pv72apmrvav5r new file mode 100644 index 00000000..319c9aab --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/c/z/czsfsvp41g06qv09d387sz4kknemgat7pv72apmrvav5r @@ -0,0 +1,2 @@ + [ + 1111.61e-111111111111] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/d/1/d13vbtddm9r6x9jq3wdgpckfe4xzxgp19bh2x9ncjqwnc b/implementation/fuzz/inputs-genvm-modules-complete-json/d/1/d13vbtddm9r6x9jq3wdgpckfe4xzxgp19bh2x9ncjqwnc new file mode 100644 index 00000000..0a9e1344 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/d/1/d13vbtddm9r6x9jq3wdgpckfe4xzxgp19bh2x9ncjqwnc @@ -0,0 +1,2 @@ + +"\u0000Ρ‹\u0000њ\u0000a1 r" diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/d/1/d1q9ctz9pyrpym136n3yra4fhxqvzjks22hjm9y2errxy b/implementation/fuzz/inputs-genvm-modules-complete-json/d/1/d1q9ctz9pyrpym136n3yra4fhxqvzjks22hjm9y2errxy new file mode 100644 index 00000000..44591c85 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/d/1/d1q9ctz9pyrpym136n3yra4fhxqvzjks22hjm9y2errxy @@ -0,0 +1,16 @@ +{ + "foo": [ + "bar", + "baz\n\\{", + -1e10, + 1e20, + null, + true, + [1, 2, 3] + ], + "bar": [ ], + "bar" : [ + "русскиС mbbbbbbbbbbbbbbaz\t\uabbbbb", + "mbbbbbbbbbbbbbaz\t\\uabbaz\t\\uabbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/d/2/d27grm16kjp4rfzwbzk3p7sda804ykfma4wmb53ar4998 b/implementation/fuzz/inputs-genvm-modules-complete-json/d/2/d27grm16kjp4rfzwbzk3p7sda804ykfma4wmb53ar4998 new file mode 100644 index 00000000..029a8e72 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/d/2/d27grm16kjp4rfzwbzk3p7sda804ykfma4wmb53ar4998 @@ -0,0 +1,3 @@ + [ [{ + "emo": true, + m \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/d/4/d46ttf0zq9pvmgs3qjpdegznawqw1ndzbtbvs40the3h6 b/implementation/fuzz/inputs-genvm-modules-complete-json/d/4/d46ttf0zq9pvmgs3qjpdegznawqw1ndzbtbvs40the3h6 new file mode 100644 index 00000000..a47fffd2 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/d/4/d46ttf0zq9pvmgs3qjpdegznawqw1ndzbtbvs40the3h6 @@ -0,0 +1 @@ + 111111111111111111111111 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/d/6/d6dqgr1jgbkmebh826wq0heyv6071h8j84qw4xmjdcrap b/implementation/fuzz/inputs-genvm-modules-complete-json/d/6/d6dqgr1jgbkmebh826wq0heyv6071h8j84qw4xmjdcrap new file mode 100644 index 00000000..e18bd45f --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/d/6/d6dqgr1jgbkmebh826wq0heyv6071h8j84qw4xmjdcrap @@ -0,0 +1,4 @@ + [ + +[ [[[[[[[[[[[[[ 123, + ]]]]]]]]]] u \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/d/7/d7hsbgnjd6mtrdw1nsn4p31s2arbkw7gy8manf8vp4tz4 b/implementation/fuzz/inputs-genvm-modules-complete-json/d/7/d7hsbgnjd6mtrdw1nsn4p31s2arbkw7gy8manf8vp4tz4 new file mode 100644 index 00000000..d3a91180 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/d/7/d7hsbgnjd6mtrdw1nsn4p31s2arbkw7gy8manf8vp4tz4 @@ -0,0 +1,2 @@ + [, +  \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/d/8/d82hxbrtkmkz7zt2vg3zq107ez9g017egpmcac6w50vbm b/implementation/fuzz/inputs-genvm-modules-complete-json/d/8/d82hxbrtkmkz7zt2vg3zq107ez9g017egpmcac6w50vbm new file mode 100644 index 00000000..ea9684a3 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/d/8/d82hxbrtkmkz7zt2vg3zq107ez9g017egpmcac6w50vbm @@ -0,0 +1,2 @@ +tr + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/d/9/d91jv8nn4e18xh44q9kswsggmy91vy11ykx85ttpc6a7g b/implementation/fuzz/inputs-genvm-modules-complete-json/d/9/d91jv8nn4e18xh44q9kswsggmy91vy11ykx85ttpc6a7g new file mode 100644 index 00000000..4b98a45d --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/d/9/d91jv8nn4e18xh44q9kswsggmy91vy11ykx85ttpc6a7g @@ -0,0 +1 @@ +"w\ubba6,\ubba6" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/d/9/d93mqvmwkazrrm0gzftmzw2604x4pkpner8a4g52rgnhc b/implementation/fuzz/inputs-genvm-modules-complete-json/d/9/d93mqvmwkazrrm0gzftmzw2604x4pkpner8a4g52rgnhc new file mode 100644 index 00000000..6ce08bd2 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/d/9/d93mqvmwkazrrm0gzftmzw2604x4pkpner8a4g52rgnhc @@ -0,0 +1,53 @@ +{ + + + + + + + + + + "foo": "bIr", + + + + + + + + + + + + + + + + + + + + + + + + + "Ρ€\\uuuuuuuuuuu + + + + + + + + + + + + + + + + +  \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/d/c/dctpvb2zt2hrkg19mkfqc9h9sk8yz3rxj95z6y2n6tbee b/implementation/fuzz/inputs-genvm-modules-complete-json/d/c/dctpvb2zt2hrkg19mkfqc9h9sk8yz3rxj95z6y2n6tbee new file mode 100644 index 00000000..5ebb4767 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/d/c/dctpvb2zt2hrkg19mkfqc9h9sk8yz3rxj95z6y2n6tbee @@ -0,0 +1 @@ +0e-5555555515 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/d/e/de70hek7tx48svhkcmgtr4jp6cp0nmzsgzqdp0qkrbjz0 b/implementation/fuzz/inputs-genvm-modules-complete-json/d/e/de70hek7tx48svhkcmgtr4jp6cp0nmzsgzqdp0qkrbjz0 new file mode 100644 index 00000000..593257e4 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/d/e/de70hek7tx48svhkcmgtr4jp6cp0nmzsgzqdp0qkrbjz0 @@ -0,0 +1,3 @@ +{ + "foo": -123, + [[[ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/d/e/dewbqrqshc4a4e544gy7bra0cc7ngwb8nk98zwexz4ek2 b/implementation/fuzz/inputs-genvm-modules-complete-json/d/e/dewbqrqshc4a4e544gy7bra0cc7ngwb8nk98zwexz4ek2 new file mode 100644 index 00000000..7dd46edb --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/d/e/dewbqrqshc4a4e544gy7bra0cc7ngwb8nk98zwexz4ek2 @@ -0,0 +1 @@ + "\f" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/d/h/dhk6b8jk1e8d8gg6n97s15b4h6y9q8n0r5yp0r9wsmzgc b/implementation/fuzz/inputs-genvm-modules-complete-json/d/h/dhk6b8jk1e8d8gg6n97s15b4h6y9q8n0r5yp0r9wsmzgc new file mode 100644 index 00000000..afcf0476 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/d/h/dhk6b8jk1e8d8gg6n97s15b4h6y9q8n0r5yp0r9wsmzgc @@ -0,0 +1 @@ +11.11 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/d/k/dk9ye8hftb4hf441w1rsnrk9zhps2tfwfm6xdz6v8497a b/implementation/fuzz/inputs-genvm-modules-complete-json/d/k/dk9ye8hftb4hf441w1rsnrk9zhps2tfwfm6xdz6v8497a new file mode 100644 index 00000000..18bf327e --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/d/k/dk9ye8hftb4hf441w1rsnrk9zhps2tfwfm6xdz6v8497a @@ -0,0 +1 @@ +3. \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/d/k/dkba0gfw61g6810w4p3njyk0d0wh0xf498g53ntcj4rjt b/implementation/fuzz/inputs-genvm-modules-complete-json/d/k/dkba0gfw61g6810w4p3njyk0d0wh0xf498g53ntcj4rjt new file mode 100644 index 00000000..9af238c1 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/d/k/dkba0gfw61g6810w4p3njyk0d0wh0xf498g53ntcj4rjt @@ -0,0 +1 @@ +"Ρ€\\γ—₯11bb r" diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/d/k/dkmhenq6tnsbsncefwrtcxcj2zc3h3jkdmcb7f95skgny b/implementation/fuzz/inputs-genvm-modules-complete-json/d/k/dkmhenq6tnsbsncefwrtcxcj2zc3h3jkdmcb7f95skgny new file mode 100644 index 0000000000000000000000000000000000000000..2bcb16779a7f9ae2592d89c6bac8274d4c7a3f8e GIT binary patch literal 47 kcmY%3tX5D^P|8=bQiujJP<aXpv9(-aLJ}y)5UZdB0N2b2*#H0l literal 0 HcmV?d00001 diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/d/m/dm4nvqbr2pm9vaagx1nkz146334pgeaqhwkvdsw3pv9wy b/implementation/fuzz/inputs-genvm-modules-complete-json/d/m/dm4nvqbr2pm9vaagx1nkz146334pgeaqhwkvdsw3pv9wy new file mode 100644 index 00000000..b72be9ea --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/d/m/dm4nvqbr2pm9vaagx1nkz146334pgeaqhwkvdsw3pv9wy @@ -0,0 +1,2 @@ +{ + "f5o": [fals \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/d/n/dnw4ywfh2ctkm2tm83xfhj4nv0a6hph0220eznt31rf9t b/implementation/fuzz/inputs-genvm-modules-complete-json/d/n/dnw4ywfh2ctkm2tm83xfhj4nv0a6hph0220eznt31rf9t new file mode 100644 index 00000000..d58006f5 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/d/n/dnw4ywfh2ctkm2tm83xfhj4nv0a6hph0220eznt31rf9t @@ -0,0 +1,10 @@ +{ + + + + + + + + + "foo ": \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/d/p/dpafw3gwg4st7g6jsby4tqa9b6b8qz8yfsy3wxcqa8m9m b/implementation/fuzz/inputs-genvm-modules-complete-json/d/p/dpafw3gwg4st7g6jsby4tqa9b6b8qz8yfsy3wxcqa8m9m new file mode 100644 index 00000000..334e5a6d --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/d/p/dpafw3gwg4st7g6jsby4tqa9b6b8qz8yfsy3wxcqa8m9m @@ -0,0 +1,3 @@ + [1111111111111111111111111111111111.1111111111E111,1111111111111111111111111.1111111111E111, 2, 1111111111, 2, 111111111111E111,1111111111111111, 2, 3] + ], + 3] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/d/q/dqbjhhgnmdp394wyrsrdhhaarc93anpyxdb722st357r0 b/implementation/fuzz/inputs-genvm-modules-complete-json/d/q/dqbjhhgnmdp394wyrsrdhhaarc93anpyxdb722st357r0 new file mode 100644 index 00000000..b3c82227 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/d/q/dqbjhhgnmdp394wyrsrdhhaarc93anpyxdb722st357r0 @@ -0,0 +1 @@ + 1 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/d/v/dv5tqa29astpjxqb9qb9nqy7a9p2yp7jff7ewbyqj4jjt b/implementation/fuzz/inputs-genvm-modules-complete-json/d/v/dv5tqa29astpjxqb9qb9nqy7a9p2yp7jff7ewbyqj4jjt new file mode 100644 index 00000000..05a9c01f --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/d/v/dv5tqa29astpjxqb9qb9nqy7a9p2yp7jff7ewbyqj4jjt @@ -0,0 +1 @@ + " Ρƒ\u00000" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/d/v/dv8djb7j6ykqpqp5xkznbzx0fx4j0fv9eeb86t21ngqc8 b/implementation/fuzz/inputs-genvm-modules-complete-json/d/v/dv8djb7j6ykqpqp5xkznbzx0fx4j0fv9eeb86t21ngqc8 new file mode 100644 index 00000000..a3ec2ed7 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/d/v/dv8djb7j6ykqpqp5xkznbzx0fx4j0fv9eeb86t21ngqc8 @@ -0,0 +1 @@ +"\"" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/d/v/dvm11rn82hs4z3c99f5kvs8e08skq9frctwpbtgj6vxzw b/implementation/fuzz/inputs-genvm-modules-complete-json/d/v/dvm11rn82hs4z3c99f5kvs8e08skq9frctwpbtgj6vxzw new file mode 100644 index 00000000..9e4ae730 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/d/v/dvm11rn82hs4z3c99f5kvs8e08skq9frctwpbtgj6vxzw @@ -0,0 +1 @@ + 8 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/d/v/dvp6az1bt52tr38crbmgm9ffvsaqm9me6x8w8z62qca6w b/implementation/fuzz/inputs-genvm-modules-complete-json/d/v/dvp6az1bt52tr38crbmgm9ffvsaqm9me6x8w8z62qca6w new file mode 100644 index 00000000..44266bf2 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/d/v/dvp6az1bt52tr38crbmgm9ffvsaqm9me6x8w8z62qca6w @@ -0,0 +1 @@ +tr \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/d/v/dvpzes6wrn2rt75rrgmvsx54ep3zb3e2psa7fvyghhd28 b/implementation/fuzz/inputs-genvm-modules-complete-json/d/v/dvpzes6wrn2rt75rrgmvsx54ep3zb3e2psa7fvyghhd28 new file mode 100644 index 00000000..3f4d71ed --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/d/v/dvpzes6wrn2rt75rrgmvsx54ep3zb3e2psa7fvyghhd28 @@ -0,0 +1 @@ + 22222222222222222222. \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/d/z/dzr40rh8bnnba51n7mg5cgxjhd6m83qazmq7w4yp7b13r b/implementation/fuzz/inputs-genvm-modules-complete-json/d/z/dzr40rh8bnnba51n7mg5cgxjhd6m83qazmq7w4yp7b13r new file mode 100644 index 00000000..d8c33a70 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/d/z/dzr40rh8bnnba51n7mg5cgxjhd6m83qazmq7w4yp7b13r @@ -0,0 +1 @@ +"σ—₯Ίbb b" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/e/0/e02vsbkhabpazvdv8bwezwexe2e88n4dn8ve59wy3dkfp b/implementation/fuzz/inputs-genvm-modules-complete-json/e/0/e02vsbkhabpazvdv8bwezwexe2e88n4dn8ve59wy3dkfp new file mode 100644 index 00000000..1fd2567c --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/e/0/e02vsbkhabpazvdv8bwezwexe2e88n4dn8ve59wy3dkfp @@ -0,0 +1 @@ +11.11211E312 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/e/0/e0c7785fpwp0z6r9he4bbkpk3xqjndx2yanmy2d4qdvcp b/implementation/fuzz/inputs-genvm-modules-complete-json/e/0/e0c7785fpwp0z6r9he4bbkpk3xqjndx2yanmy2d4qdvcp new file mode 100644 index 00000000..27404285 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/e/0/e0c7785fpwp0z6r9he4bbkpk3xqjndx2yanmy2d4qdvcp @@ -0,0 +1,2 @@ +[[[[[[[[[[[[[[[[[[[[[[ +. \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/e/1/e1epvtd5v0g86azye9vbwtdfeq00p1sseg096m3j9cs80 b/implementation/fuzz/inputs-genvm-modules-complete-json/e/1/e1epvtd5v0g86azye9vbwtdfeq00p1sseg096m3j9cs80 new file mode 100644 index 00000000..fc00ba8e --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/e/1/e1epvtd5v0g86azye9vbwtdfeq00p1sseg096m3j9cs80 @@ -0,0 +1,2 @@ + +"Ρ€jξ—¨750 1111Ρ€jξ—¨750 1\uDbbb\uDc11111111\uDbbb\uDc1111111111Ρ€jξ—¨750 1\uDbbb\uDc11111111p" diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/e/1/e1mmg4qtd0ezmtb4bf6beemmaqh9h76hkfx8th8y94mey b/implementation/fuzz/inputs-genvm-modules-complete-json/e/1/e1mmg4qtd0ezmtb4bf6beemmaqh9h76hkfx8th8y94mey new file mode 100644 index 00000000..9934738b --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/e/1/e1mmg4qtd0ezmtb4bf6beemmaqh9h76hkfx8th8y94mey @@ -0,0 +1,2 @@ + +"Ρ€\\uba1, B\bΓ΅ r" diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/e/2/e20857bs4stbmpedvjgn6avmv6weje9x8gk386v9jrspt b/implementation/fuzz/inputs-genvm-modules-complete-json/e/2/e20857bs4stbmpedvjgn6avmv6weje9x8gk386v9jrspt new file mode 100644 index 00000000..ccc4756f --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/e/2/e20857bs4stbmpedvjgn6avmv6weje9x8gk386v9jrspt @@ -0,0 +1 @@ +1.11e-1 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/e/2/e2qxf7j5hfttj5g4d8e2hzb90pmtfb16brhwm4v6gx11p b/implementation/fuzz/inputs-genvm-modules-complete-json/e/2/e2qxf7j5hfttj5g4d8e2hzb90pmtfb16brhwm4v6gx11p new file mode 100644 index 00000000..cf255418 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/e/2/e2qxf7j5hfttj5g4d8e2hzb90pmtfb16brhwm4v6gx11p @@ -0,0 +1,19 @@ +{ + " foo" : [ + "bar", + "baz\n\\z", + 123, + -123, + -1e10, + 1e20, + null, + true, + false, + [1, 2, 3] + ], + "bar": [ + "русскиС Π±ΡƒΠΊΠ²Ρ‹ ζ—¬ζœ¬", + "mbbbbbbbb}bbbbbaz\t\uabbbbb", + "mbbbbbbbbbbbbbbaz\t\\uabbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/e/4/e42jccxftrntwr3se574eq9e7ykcpcapyewr1rsa5kf6g b/implementation/fuzz/inputs-genvm-modules-complete-json/e/4/e42jccxftrntwr3se574eq9e7ykcpcapyewr1rsa5kf6g new file mode 100644 index 00000000..e8da29e5 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/e/4/e42jccxftrntwr3se574eq9e7ykcpcapyewr1rsa5kf6g @@ -0,0 +1,2 @@ +{ + "foo": [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ [[[[[[[[[[[[ "bar", 33333333333333333333333333333333 [ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/e/5/e59046tvy72ss2g7vv16ppqtgy8nrq349ajkt5esk7j60 b/implementation/fuzz/inputs-genvm-modules-complete-json/e/5/e59046tvy72ss2g7vv16ppqtgy8nrq349ajkt5esk7j60 new file mode 100644 index 00000000..adc657f8 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/e/5/e59046tvy72ss2g7vv16ppqtgy8nrq349ajkt5esk7j60 @@ -0,0 +1 @@ +1e-515 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/e/5/e5pd5j0jb6rj91rnf8vdfv5mj7frgnh6b3q6ty1nff452 b/implementation/fuzz/inputs-genvm-modules-complete-json/e/5/e5pd5j0jb6rj91rnf8vdfv5mj7frgnh6b3q6ty1nff452 new file mode 100644 index 00000000..36b0b16d --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/e/5/e5pd5j0jb6rj91rnf8vdfv5mj7frgnh6b3q6ty1nff452 @@ -0,0 +1 @@ +"ζ·₯" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/e/6/e6h4svg8jmemyt4177w80xen3bs5fstgc8jbkb6ttcrqg b/implementation/fuzz/inputs-genvm-modules-complete-json/e/6/e6h4svg8jmemyt4177w80xen3bs5fstgc8jbkb6ttcrqg new file mode 100644 index 00000000..fac3eb9a --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/e/6/e6h4svg8jmemyt4177w80xen3bs5fstgc8jbkb6ttcrqg @@ -0,0 +1 @@ +[1.1 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/e/6/e6tzw9ba8amkm74msqq0012txz5aqq9q9qe99f8csqr1c b/implementation/fuzz/inputs-genvm-modules-complete-json/e/6/e6tzw9ba8amkm74msqq0012txz5aqq9q9qe99f8csqr1c new file mode 100644 index 00000000..898c6b10 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/e/6/e6tzw9ba8amkm74msqq0012txz5aqq9q9qe99f8csqr1c @@ -0,0 +1 @@ +"UUUT3UUUUUTUUUUUUUUTUU3UUUUUTUUUUUwUoUUUUUwUo3UUUUUUUUUUUUUUUUUUUUUUUUUUU3UUUUUTUUUUUUTUU3UUUUUTUUUUUwUoUUUUUwUo33333UUUUUTUUUUUwUoUUUUUwUo333UUUUUUTUU3UUUUUTUUU\n]\\\\\\\\\\" diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/e/7/e7cx24jfe7r8jg011g5j6z871zjjqc0hvtcsf8bksf9rp b/implementation/fuzz/inputs-genvm-modules-complete-json/e/7/e7cx24jfe7r8jg011g5j6z871zjjqc0hvtcsf8bksf9rp new file mode 100644 index 00000000..35007d4c --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/e/7/e7cx24jfe7r8jg011g5j6z871zjjqc0hvtcsf8bksf9rp @@ -0,0 +1,10 @@ + { + "":0, + "" : { + "": + " + + + + + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/e/8/e8erj3me5g573cnrzgk30d8350x3e8tdtkqn6banswm2y b/implementation/fuzz/inputs-genvm-modules-complete-json/e/8/e8erj3me5g573cnrzgk30d8350x3e8tdtkqn6banswm2y new file mode 100644 index 00000000..802ae488 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/e/8/e8erj3me5g573cnrzgk30d8350x3e8tdtkqn6banswm2y @@ -0,0 +1 @@ +"UUUT3UUUUUTUU3UUUUUTUUUUUUTUU3UUwUoUUUUUwUUUUUTUUUUUwUoUUUUUwUoUUUUUUUUUUUTUUUUUUTUU3UUUUUTUUU\n]\\\\\\\\\\" diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/e/8/e8g8n6g0e3b9b31ptzy2zdmf8q7xp9txsqhqhkhj78hp8 b/implementation/fuzz/inputs-genvm-modules-complete-json/e/8/e8g8n6g0e3b9b31ptzy2zdmf8q7xp9txsqhqhkhj78hp8 new file mode 100644 index 00000000..9a1aa7f3 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/e/8/e8g8n6g0e3b9b31ptzy2zdmf8q7xp9txsqhqhkhj78hp8 @@ -0,0 +1,3 @@ +{ + +} \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/e/9/e9s4tg672179s8epq9vjc1t8xc7p426bjcz5arcz9p9py b/implementation/fuzz/inputs-genvm-modules-complete-json/e/9/e9s4tg672179s8epq9vjc1t8xc7p426bjcz5arcz9p9py new file mode 100644 index 00000000..50048170 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/e/9/e9s4tg672179s8epq9vjc1t8xc7p426bjcz5arcz9p9py @@ -0,0 +1 @@ + " ζ—₯u 52C BΟΊΠΈΟ΅ бссϺиϡ ΠΊbbbbbbbbu bуЃссруссϺиϡ ΠΊbbbbbbbbu bbbbbbbbΠ²Π‹ σ—₯ƒΡΡΡ€ΡƒΡΡΟΊΠ₯Ο΅ Π±ΡƒΠΊbbbb 52C Bxbz" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/e/b/eb127pt6r50fwsjxvtcvtgrqqyjskvzraqnmbz5c4nhs8 b/implementation/fuzz/inputs-genvm-modules-complete-json/e/b/eb127pt6r50fwsjxvtcvtgrqqyjskvzraqnmbz5c4nhs8 new file mode 100644 index 00000000..c6256ecf --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/e/b/eb127pt6r50fwsjxvtcvtgrqqyjskvzraqnmbz5c4nhs8 @@ -0,0 +1 @@ + "с\\ffffffffff2fcccccccccccccccccccffffffffffffff1111ffffffffffffffffffffffffffffffffffffffff11111e-ΠΊbbcbbbbbIfffffffffffffffffffffffffffffff111fffffabba\/: [bb [bbbb" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/e/b/ebppt36hmn1raq16m83ywr2v1hfksbc677j05jz3a5cta b/implementation/fuzz/inputs-genvm-modules-complete-json/e/b/ebppt36hmn1raq16m83ywr2v1hfksbc677j05jz3a5cta new file mode 100644 index 00000000..150c1b5e --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/e/b/ebppt36hmn1raq16m83ywr2v1hfksbc677j05jz3a5cta @@ -0,0 +1 @@ +{ " "r" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/e/b/ebt503723z2emxs30rr989mgg7jcw5597bgjzdtbsn43y b/implementation/fuzz/inputs-genvm-modules-complete-json/e/b/ebt503723z2emxs30rr989mgg7jcw5597bgjzdtbsn43y new file mode 100644 index 00000000..4fdd6cd6 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/e/b/ebt503723z2emxs30rr989mgg7jcw5597bgjzdtbsn43y @@ -0,0 +1,9 @@ + { + "":0, + "ZMo" "": { + "": + " + + + + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/e/b/ebv0ny12mz2f64ah46mqt4h1244vn25gtjdh4rcp40yxr b/implementation/fuzz/inputs-genvm-modules-complete-json/e/b/ebv0ny12mz2f64ah46mqt4h1244vn25gtjdh4rcp40yxr new file mode 100644 index 00000000..fd2504b1 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/e/b/ebv0ny12mz2f64ah46mqt4h1244vn25gtjdh4rcp40yxr @@ -0,0 +1,14 @@ +{ + "feeeeeedeeeeeeeeeVeeeeeEeeeeoo": [ + + -123, + null, + true, + false, + [1111111111111111111111111111, 2, 3] + ], + "bar": [ + "русскиС Π±ΡƒΠΊΠ²έ‹ ζ—₯本uabbbbb", + "ΠΊΠΈΠ΅ Π±ΡƒΠΊΠ²Ρ‹ ζ—₯uabbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/e/c/ec6n23yd7cbe5zk7fasp1c8h7h6r9pqx84kf59kkq9t6j b/implementation/fuzz/inputs-genvm-modules-complete-json/e/c/ec6n23yd7cbe5zk7fasp1c8h7h6r9pqx84kf59kkq9t6j new file mode 100644 index 00000000..f750bfd7 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/e/c/ec6n23yd7cbe5zk7fasp1c8h7h6r9pqx84kf59kkq9t6j @@ -0,0 +1 @@ +2E-2221111111 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/e/c/ecadpx0fdp3t2b9f0dq8tkyjz7bb17wp7qek6c5j2wbpy b/implementation/fuzz/inputs-genvm-modules-complete-json/e/c/ecadpx0fdp3t2b9f0dq8tkyjz7bb17wp7qek6c5j2wbpy new file mode 100644 index 00000000..7c6c6755 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/e/c/ecadpx0fdp3t2b9f0dq8tkyjz7bb17wp7qek6c5j2wbpy @@ -0,0 +1 @@ +fals \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/e/c/ecbfma1cype1bcfa17gp8y8mzvsqb7r09f6trh09sv72y b/implementation/fuzz/inputs-genvm-modules-complete-json/e/c/ecbfma1cype1bcfa17gp8y8mzvsqb7r09f6trh09sv72y new file mode 100644 index 00000000..d258019c --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/e/c/ecbfma1cype1bcfa17gp8y8mzvsqb7r09f6trh09sv72y @@ -0,0 +1 @@ +{" {" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/e/d/edj2jxcnb53f4y97p9a8rgypqezdfap98c3wgneg7k3fg b/implementation/fuzz/inputs-genvm-modules-complete-json/e/d/edj2jxcnb53f4y97p9a8rgypqezdfap98c3wgneg7k3fg new file mode 100644 index 00000000..5047d508 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/e/d/edj2jxcnb53f4y97p9a8rgypqezdfap98c3wgneg7k3fg @@ -0,0 +1,3 @@ + [ + ", + 12! \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/e/d/edpwnq7kzhv187n0eefn36fxjqqkh811hpe5mschmt682 b/implementation/fuzz/inputs-genvm-modules-complete-json/e/d/edpwnq7kzhv187n0eefn36fxjqqkh811hpe5mschmt682 new file mode 100644 index 00000000..323a93f1 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/e/d/edpwnq7kzhv187n0eefn36fxjqqkh811hpe5mschmt682 @@ -0,0 +1,12 @@ +{ + "oo" + + + + + + + + + + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/e/e/eegsza8ysntt59aawddf8nrjfz11yr5avv3mfzjpa4cct b/implementation/fuzz/inputs-genvm-modules-complete-json/e/e/eegsza8ysntt59aawddf8nrjfz11yr5avv3mfzjpa4cct new file mode 100644 index 00000000..3060a7b4 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/e/e/eegsza8ysntt59aawddf8nrjfz11yr5avv3mfzjpa4cct @@ -0,0 +1,2 @@ +22222222222222222222E+2 + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/e/f/efqfmbd7qgdmfcmx9gnpypcabepgxw28cyk07c90xqzkg b/implementation/fuzz/inputs-genvm-modules-complete-json/e/f/efqfmbd7qgdmfcmx9gnpypcabepgxw28cyk07c90xqzkg new file mode 100644 index 00000000..28c7047e --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/e/f/efqfmbd7qgdmfcmx9gnpypcabepgxw28cyk07c90xqzkg @@ -0,0 +1 @@ + "русскиϡ Π±ΡƒΠΊbbbbbbbcbbbbΠ²Π‹ ζ—₯u 52C Bxbz" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/e/f/efthb1cez1ecc2v4ft3g9br7hgm9pn19295v93vj7xd98 b/implementation/fuzz/inputs-genvm-modules-complete-json/e/f/efthb1cez1ecc2v4ft3g9br7hgm9pn19295v93vj7xd98 new file mode 100644 index 00000000..8767388d --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/e/f/efthb1cez1ecc2v4ft3g9br7hgm9pn19295v93vj7xd98 @@ -0,0 +1,2 @@ + +"Ρ€\\ub` d [[1, B\bΓ΅ r" diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/e/g/egvnv6b036px26c1pfxdyetr2ke0t5k19q9x0697h0620 b/implementation/fuzz/inputs-genvm-modules-complete-json/e/g/egvnv6b036px26c1pfxdyetr2ke0t5k19q9x0697h0620 new file mode 100644 index 00000000..fdbc6e7a --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/e/g/egvnv6b036px26c1pfxdyetr2ke0t5k19q9x0697h0620 @@ -0,0 +1 @@ + [] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/e/k/ekw1vwn92g566pqhj4psd63yh1t2mmrty1xvfba4v2dmg b/implementation/fuzz/inputs-genvm-modules-complete-json/e/k/ekw1vwn92g566pqhj4psd63yh1t2mmrty1xvfba4v2dmg new file mode 100644 index 00000000..919aec92 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/e/k/ekw1vwn92g566pqhj4psd63yh1t2mmrty1xvfba4v2dmg @@ -0,0 +1,27 @@ +{ + "fz\t\uabbbbboo": [ + "bar", + "baz\n\\z", + -1e10, + + -123, + 123, -123, + + -123, + -1e10, + + -123, + 1e20, + -123, + -1e10, + null, + true, + false, + [1, 2, 3] + ], + "bar": [ + "русскиС Π±ΡƒΠΊΠ²Ρ‹ ζ—₯本", + "mbbbbbbbbbbbbbbaz\t\uabbbbb", + "mbbbbbbbbbbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/e/k/ekw8e6jejmqxgs5feynsczrd46hhyhh4w8kr21xx78fs2 b/implementation/fuzz/inputs-genvm-modules-complete-json/e/k/ekw8e6jejmqxgs5feynsczrd46hhyhh4w8kr21xx78fs2 new file mode 100644 index 00000000..30d81fa7 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/e/k/ekw8e6jejmqxgs5feynsczrd46hhyhh4w8kr21xx78fs2 @@ -0,0 +1 @@ +"\u0000Ρ‹\u0 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/e/q/eqw32e17fbw05ezyr972xx9kp3yv30jgk5fk31yhwhhye b/implementation/fuzz/inputs-genvm-modules-complete-json/e/q/eqw32e17fbw05ezyr972xx9kp3yv30jgk5fk31yhwhhye new file mode 100644 index 00000000..5a59c0da --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/e/q/eqw32e17fbw05ezyr972xx9kp3yv30jgk5fk31yhwhhye @@ -0,0 +1 @@ +22222222222222222222E-2222 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/e/r/er4venxzh25x3nhg28jjsvkz3r9vdwrmn3mp51n9hjscc b/implementation/fuzz/inputs-genvm-modules-complete-json/e/r/er4venxzh25x3nhg28jjsvkz3r9vdwrmn3mp51n9hjscc new file mode 100644 index 00000000..e5517b81 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/e/r/er4venxzh25x3nhg28jjsvkz3r9vdwrmn3mp51n9hjscc @@ -0,0 +1 @@ +"\n}u`H" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/e/r/er8fpyb5ted5r0ydph71yvwk92w9ad8jc7cwdey5d1wt8 b/implementation/fuzz/inputs-genvm-modules-complete-json/e/r/er8fpyb5ted5r0ydph71yvwk92w9ad8jc7cwdey5d1wt8 new file mode 100644 index 00000000..926e5fd2 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/e/r/er8fpyb5ted5r0ydph71yvwk92w9ad8jc7cwdey5d1wt8 @@ -0,0 +1 @@ +{"": [ 5]} \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/e/r/erbzxzy6c0se61kh6fw955s9q7c74r6dqrn29pq7j7nva b/implementation/fuzz/inputs-genvm-modules-complete-json/e/r/erbzxzy6c0se61kh6fw955s9q7c74r6dqrn29pq7j7nva new file mode 100644 index 00000000..fc4b9c37 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/e/r/erbzxzy6c0se61kh6fw955s9q7c74r6dqrn29pq7j7nva @@ -0,0 +1 @@ +1116116211333133333333331E-13 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/e/t/etng0z2n7emt92qnkhvn9e40mre0cm7fj6cavcg3fa8pt b/implementation/fuzz/inputs-genvm-modules-complete-json/e/t/etng0z2n7emt92qnkhvn9e40mre0cm7fj6cavcg3fa8pt new file mode 100644 index 00000000..c9e0708d --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/e/t/etng0z2n7emt92qnkhvn9e40mre0cm7fj6cavcg3fa8pt @@ -0,0 +1 @@ + [1, 2, b11111111 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/e/x/ex8ebm6z9e8603wab4pna1mf1cdec3wrp32jem3aw64zy b/implementation/fuzz/inputs-genvm-modules-complete-json/e/x/ex8ebm6z9e8603wab4pna1mf1cdec3wrp32jem3aw64zy new file mode 100644 index 0000000000000000000000000000000000000000..4dbc79dbff71020e3629309bf25d991fe930b781 GIT binary patch literal 291 zcmb>CQUC&_wETP}D}`t<2P}}3Sfr$b%&khvi;1a15;HV1)``Xq;CeyE;Z}w$qhOe7 xWPq?VuQVq|2c#P-P%<(y0!ru`rec-GCkV4sK_MwA5ynBHp?b?06hNNP1pu1zL3;oI literal 0 HcmV?d00001 diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/e/y/ey6frz98yn558h2wzg6a1q9m9mna6h73tfa0bazs7sn8j b/implementation/fuzz/inputs-genvm-modules-complete-json/e/y/ey6frz98yn558h2wzg6a1q9m9mna6h73tfa0bazs7sn8j new file mode 100644 index 00000000..413d7be8 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/e/y/ey6frz98yn558h2wzg6a1q9m9mna6h73tfa0bazs7sn8j @@ -0,0 +1,2 @@ +{ + "foo" a \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/f/0/f0kp1rtsrfr2e8b3kkp860q87jekvsmh09s5yh461kebm b/implementation/fuzz/inputs-genvm-modules-complete-json/f/0/f0kp1rtsrfr2e8b3kkp860q87jekvsmh09s5yh461kebm new file mode 100644 index 00000000..b2f0eea2 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/f/0/f0kp1rtsrfr2e8b3kkp860q87jekvsmh09s5yh461kebm @@ -0,0 +1,2 @@ + +"\u0000\bσ—₯—[8ζœ¬Ρ‹bσ—₯—\u0000Ρ‹\u00ba1 r" diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/f/0/f0vkjt3y2enqdjd79k21h1c0knams8se5fehqmjz3ezmj b/implementation/fuzz/inputs-genvm-modules-complete-json/f/0/f0vkjt3y2enqdjd79k21h1c0knams8se5fehqmjz3ezmj new file mode 100644 index 00000000..def511f4 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/f/0/f0vkjt3y2enqdjd79k21h1c0knams8se5fehqmjz3ezmj @@ -0,0 +1,12 @@ +{ + "fz\t\uabbbbboo": [ + "bar", + "baz\n\\z", + 123, + -123, + -1e10, + 1e20, + null, + true, + f[ + "bar \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/f/0/f0w1wp4f4g2j3eev35nwd693ztg9rywxbf2xz2sw97efc b/implementation/fuzz/inputs-genvm-modules-complete-json/f/0/f0w1wp4f4g2j3eev35nwd693ztg9rywxbf2xz2sw97efc new file mode 100644 index 00000000..b370520a --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/f/0/f0w1wp4f4g2j3eev35nwd693ztg9rywxbf2xz2sw97efc @@ -0,0 +1,2 @@ +{ "fooo":": [ + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/f/2/f2977sr15mekm241hw6307vdxw6jfpvtzcv96e0kpb30t b/implementation/fuzz/inputs-genvm-modules-complete-json/f/2/f2977sr15mekm241hw6307vdxw6jfpvtzcv96e0kpb30t new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/f/2/f2977sr15mekm241hw6307vdxw6jfpvtzcv96e0kpb30t @@ -0,0 +1 @@ + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/f/2/f2mw7jkye43zq3m8tcjfavsyxb0ydpfvhecr88cct7qqy b/implementation/fuzz/inputs-genvm-modules-complete-json/f/2/f2mw7jkye43zq3m8tcjfavsyxb0ydpfvhecr88cct7qqy new file mode 100644 index 00000000..18653291 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/f/2/f2mw7jkye43zq3m8tcjfavsyxb0ydpfvhecr88cct7qqy @@ -0,0 +1 @@ +3. \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/f/5/f5mxash0dv9x92eban9n4bja4a9f02re6s0cfdvgxggsg b/implementation/fuzz/inputs-genvm-modules-complete-json/f/5/f5mxash0dv9x92eban9n4bja4a9f02re6s0cfdvgxggsg new file mode 100644 index 00000000..adccbbe2 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/f/5/f5mxash0dv9x92eban9n4bja4a9f02re6s0cfdvgxggsg @@ -0,0 +1 @@ +3E2 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/f/7/f722yfy9gpzwpkyd441q5bxa9d6q0znw8cn5c38yakb8j b/implementation/fuzz/inputs-genvm-modules-complete-json/f/7/f722yfy9gpzwpkyd441q5bxa9d6q0znw8cn5c38yakb8j new file mode 100644 index 00000000..75a0687b --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/f/7/f722yfy9gpzwpkyd441q5bxa9d6q0znw8cn5c38yakb8j @@ -0,0 +1,2 @@ + +":" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/f/8/f894kk1aszm9zrt7r2rk33emns5jh7mt8v9ma5atbwd3t b/implementation/fuzz/inputs-genvm-modules-complete-json/f/8/f894kk1aszm9zrt7r2rk33emns5jh7mt8v9ma5atbwd3t new file mode 100644 index 00000000..fdd178b4 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/f/8/f894kk1aszm9zrt7r2rk33emns5jh7mt8v9ma5atbwd3t @@ -0,0 +1 @@ +{ "foo" [[[ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/f/8/f8f838tn0zrencezbxme6vs2k4ehm2m25vpvaakhz7ger b/implementation/fuzz/inputs-genvm-modules-complete-json/f/8/f8f838tn0zrencezbxme6vs2k4ehm2m25vpvaakhz7ger new file mode 100644 index 00000000..f2f7eee3 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/f/8/f8f838tn0zrencezbxme6vs2k4ehm2m25vpvaakhz7ger @@ -0,0 +1 @@ + [[ [[1, , \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/f/9/f9qj2956qabnb1858sgp1x5fqec9f81j796pe84cd0fnr b/implementation/fuzz/inputs-genvm-modules-complete-json/f/9/f9qj2956qabnb1858sgp1x5fqec9f81j796pe84cd0fnr new file mode 100644 index 00000000..f47a7fa7 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/f/9/f9qj2956qabnb1858sgp1x5fqec9f81j796pe84cd0fnr @@ -0,0 +1,62 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + "Ρ€bb\\b" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/f/b/fbq5975x2xbt7q49nejc75g2gkgq1sazyfz2g4s7dbza0 b/implementation/fuzz/inputs-genvm-modules-complete-json/f/b/fbq5975x2xbt7q49nejc75g2gkgq1sazyfz2g4s7dbza0 new file mode 100644 index 00000000..5e1dc0b4 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/f/b/fbq5975x2xbt7q49nejc75g2gkgq1sazyfz2g4s7dbza0 @@ -0,0 +1 @@ + "}k" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/f/d/fdarbet0j38wtdpacqvcedcp5nskf95r02gek39rmg8qw b/implementation/fuzz/inputs-genvm-modules-complete-json/f/d/fdarbet0j38wtdpacqvcedcp5nskf95r02gek39rmg8qw new file mode 100644 index 00000000..604b5cc9 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/f/d/fdarbet0j38wtdpacqvcedcp5nskf95r02gek39rmg8qw @@ -0,0 +1 @@ +22222222222235222121e+22 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/f/d/fdxw7s5sstzbwkzhcmrd55mrgw5ydy3kwys6cpx1znspy b/implementation/fuzz/inputs-genvm-modules-complete-json/f/d/fdxw7s5sstzbwkzhcmrd55mrgw5ydy3kwys6cpx1znspy new file mode 100644 index 00000000..7acd0cfd --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/f/d/fdxw7s5sstzbwkzhcmrd55mrgw5ydy3kwys6cpx1znspy @@ -0,0 +1,34 @@ + [1.6 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/f/e/fe908rz29x6xjyhzyt3cdtz1btr6tgatnyhnc07a2d5vg b/implementation/fuzz/inputs-genvm-modules-complete-json/f/e/fe908rz29x6xjyhzyt3cdtz1btr6tgatnyhnc07a2d5vg new file mode 100644 index 00000000..e1039c28 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/f/e/fe908rz29x6xjyhzyt3cdtz1btr6tgatnyhnc07a2d5vg @@ -0,0 +1 @@ +22222222222222222222 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/f/f/ffabam1yxkm107he6rdbwpthe7npazfph718gv3802qy8 b/implementation/fuzz/inputs-genvm-modules-complete-json/f/f/ffabam1yxkm107he6rdbwpthe7npazfph718gv3802qy8 new file mode 100644 index 00000000..4265151e --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/f/f/ffabam1yxkm107he6rdbwpthe7npazfph718gv3802qy8 @@ -0,0 +1,2 @@ +{" ] fo": [ 5] +} \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/f/f/ffzbr34j2epdjf3hm6gs0bn44dkv64q71s8vxrwn4ghfy b/implementation/fuzz/inputs-genvm-modules-complete-json/f/f/ffzbr34j2epdjf3hm6gs0bn44dkv64q71s8vxrwn4ghfy new file mode 100644 index 00000000..1a242859 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/f/f/ffzbr34j2epdjf3hm6gs0bn44dkv64q71s8vxrwn4ghfy @@ -0,0 +1,17 @@ +{ + "foo": [ + "bar", + "baz\n\\z" ], + "bar": [ + -123, + -1e10, + null, + true, + false, + [1, 2, 3] + ], + "bar": [ + "Ρ€ΡƒΡ¬az\t\uabbbbb", + "mbbbbbbbbbbbbbbaz\t\\uabbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/f/k/fkeqwc9ng2a7y42dae5ax8bv7zvms6s4jqb3ms9q1062a b/implementation/fuzz/inputs-genvm-modules-complete-json/f/k/fkeqwc9ng2a7y42dae5ax8bv7zvms6s4jqb3ms9q1062a new file mode 100644 index 00000000..dba95bcc --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/f/k/fkeqwc9ng2a7y42dae5ax8bv7zvms6s4jqb3ms9q1062a @@ -0,0 +1 @@ + [11111111,1111311111111111111.11e-51111,111131111111111111E111, 2, 3] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/f/m/fmw01nxpr25fjkpb44v57kfxy22jbhre9jk9xmz903st6 b/implementation/fuzz/inputs-genvm-modules-complete-json/f/m/fmw01nxpr25fjkpb44v57kfxy22jbhre9jk9xmz903st6 new file mode 100644 index 00000000..b77cf959 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/f/m/fmw01nxpr25fjkpb44v57kfxy22jbhre9jk9xmz903st6 @@ -0,0 +1 @@ +1e-3333333333333333333333333333333333333333333 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/f/r/frr2tx60pfkvq45nsmz9b71pp579pdqd5g9njtv0bnw90 b/implementation/fuzz/inputs-genvm-modules-complete-json/f/r/frr2tx60pfkvq45nsmz9b71pp579pdqd5g9njtv0bnw90 new file mode 100644 index 00000000..65b0a109 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/f/r/frr2tx60pfkvq45nsmz9b71pp579pdqd5g9njtv0bnw90 @@ -0,0 +1,3 @@ + [[ [ + [[[[[ [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ [ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/f/t/ft635tsas864bn4tkwtkbxf623k66fnffb27x07czte6t b/implementation/fuzz/inputs-genvm-modules-complete-json/f/t/ft635tsas864bn4tkwtkbxf623k66fnffb27x07czte6t new file mode 100644 index 00000000..89d6ff21 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/f/t/ft635tsas864bn4tkwtkbxf623k66fnffb27x07czte6t @@ -0,0 +1,2 @@ + + "\"\/" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/f/v/fvgd11vja9g02rct4rgk4sbyz2pq8g6ce8sv4hc7wwbtr b/implementation/fuzz/inputs-genvm-modules-complete-json/f/v/fvgd11vja9g02rct4rgk4sbyz2pq8g6ce8sv4hc7wwbtr new file mode 100644 index 00000000..6a864745 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/f/v/fvgd11vja9g02rct4rgk4sbyz2pq8g6ce8sv4hc7wwbtr @@ -0,0 +1,19 @@ +{ + "foo": [ + "bar", + "baz\n\\z", + 123, + -123, + -1e10, + 1e20, + null, + true, + false, + [1, 2, 3] + ], + "bar": [ + "русскиС Π±ΡƒΠΊΠ²Ρ‹ ζ—₯本", + "mbbbbbbbbbbbbbbaz\t\uabbbbb", + "mbbbb + ] +} \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/f/v/fvn42ng8jwjq002dmsatnmn21as2fd1g5h4srjywym9g2 b/implementation/fuzz/inputs-genvm-modules-complete-json/f/v/fvn42ng8jwjq002dmsatnmn21as2fd1g5h4srjywym9g2 new file mode 100644 index 00000000..5a373848 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/f/v/fvn42ng8jwjq002dmsatnmn21as2fd1g5h4srjywym9g2 @@ -0,0 +1 @@ +"\n" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/f/v/fvqnwfm2teefsk6cqhaahqvmaeppjcdzgfyph7y64cs06 b/implementation/fuzz/inputs-genvm-modules-complete-json/f/v/fvqnwfm2teefsk6cqhaahqvmaeppjcdzgfyph7y64cs06 new file mode 100644 index 00000000..446e291f --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/f/v/fvqnwfm2teefsk6cqhaahqvmaeppjcdzgfyph7y64cs06 @@ -0,0 +1,6 @@ +[[[[[[[[ + + +3,[[[[[[[[[[[[[[[[[[[[[[[[[[ [[[[[[[[[[[[[ [[[[[[[[[[[[[[[[[[[[[[[[[[111123, + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[{ + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/f/x/fxv0dxa6py99hk9vmne6g5w317jh1spjt90vf5pkq904c b/implementation/fuzz/inputs-genvm-modules-complete-json/f/x/fxv0dxa6py99hk9vmne6g5w317jh1spjt90vf5pkq904c new file mode 100644 index 00000000..3f5a8068 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/f/x/fxv0dxa6py99hk9vmne6g5w317jh1spjt90vf5pkq904c @@ -0,0 +1,2 @@ + +"Ρ€\\uΠ²Ρ‹ [[1, 1 ζ—₯本B\bΓ΅ " diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/g/1/g1fsd04pp0ek735a5d3fzbxnxhpvbafw9hvh99kvbwz8p b/implementation/fuzz/inputs-genvm-modules-complete-json/g/1/g1fsd04pp0ek735a5d3fzbxnxhpvbafw9hvh99kvbwz8p new file mode 100644 index 00000000..32e219e8 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/g/1/g1fsd04pp0ek735a5d3fzbxnxhpvbafw9hvh99kvbwz8p @@ -0,0 +1,2 @@ + + "Ρ€\\uabbb ] " \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/g/3/g3htj4zjgtrxfvw4hmnbkzdp6bdh1j0x9rrj0804xz2g6 b/implementation/fuzz/inputs-genvm-modules-complete-json/g/3/g3htj4zjgtrxfvw4hmnbkzdp6bdh1j0x9rrj0804xz2g6 new file mode 100644 index 00000000..31d070e7 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/g/3/g3htj4zjgtrxfvw4hmnbkzdp6bdh1j0x9rrj0804xz2g6 @@ -0,0 +1 @@ + "\"\ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/g/3/g3m2wz64dne91ntf7e90dhbf9srktb5p29aq26jzwk0bg b/implementation/fuzz/inputs-genvm-modules-complete-json/g/3/g3m2wz64dne91ntf7e90dhbf9srktb5p29aq26jzwk0bg new file mode 100644 index 0000000000000000000000000000000000000000..6ac837047360d438ec3a43bf2ae7a9e96f5baf10 GIT binary patch literal 76 zcmb>CQUC%aLqo%0Acg`ZD+L9^R3if&kff5KalH~R2yj9Hm;)g|n!$iUfn5Or#H|U) literal 0 HcmV?d00001 diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/g/4/g4eq0tjf2yp04axpjztgdrg2q1m1rvzsvp351nc80dpam b/implementation/fuzz/inputs-genvm-modules-complete-json/g/4/g4eq0tjf2yp04axpjztgdrg2q1m1rvzsvp351nc80dpam new file mode 100644 index 00000000..3626f409 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/g/4/g4eq0tjf2yp04axpjztgdrg2q1m1rvzsvp351nc80dpam @@ -0,0 +1,19 @@ +{ "fo" + + + + + + + + + + + + + + + + +: [ ] +} \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/g/5/g5pjkd2759bpn5f461dkws3h2j15yy7keerbyspd857b0 b/implementation/fuzz/inputs-genvm-modules-complete-json/g/5/g5pjkd2759bpn5f461dkws3h2j15yy7keerbyspd857b0 new file mode 100644 index 00000000..a53da60b --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/g/5/g5pjkd2759bpn5f461dkws3h2j15yy7keerbyspd857b0 @@ -0,0 +1 @@ +"Ρ€ΠΊU[ ; с11 111111," \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/g/5/g5v21bx76yg9g3vd6kmxcnkh2dg6q0pctj5bhngfannba b/implementation/fuzz/inputs-genvm-modules-complete-json/g/5/g5v21bx76yg9g3vd6kmxcnkh2dg6q0pctj5bhngfannba new file mode 100644 index 00000000..749e1e20 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/g/5/g5v21bx76yg9g3vd6kmxcnkh2dg6q0pctj5bhngfannba @@ -0,0 +1 @@ +{ + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/g/6/g6t74w7s5gapgnshxyn15401mxhzm5hh3fve4cdmjwncr b/implementation/fuzz/inputs-genvm-modules-complete-json/g/6/g6t74w7s5gapgnshxyn15401mxhzm5hh3fve4cdmjwncr new file mode 100644 index 00000000..f535e2ab --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/g/6/g6t74w7s5gapgnshxyn15401mxhzm5hh3fve4cdmjwncr @@ -0,0 +1,16 @@ +{ + "foo": [ + "bar", + "baz\n\\z", + 1.3, + -123, + -1e10, + 1e20, + null, + true, + false, + [ "русскиС Π±ΡƒΠΊΠ²Ρ‹ ζ₯本", + "mbbbbbbbbbbbbbbaz\t\uabbbbb", + "mbbbbbbbbbbbbb Π±Ρƒ\\uabbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/g/7/g79z13a134phq13pmsjfmm4a8jzb5n2nbzjaq0bvj78ja b/implementation/fuzz/inputs-genvm-modules-complete-json/g/7/g79z13a134phq13pmsjfmm4a8jzb5n2nbzjaq0bvj78ja new file mode 100644 index 00000000..9ff98df1 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/g/7/g79z13a134phq13pmsjfmm4a8jzb5n2nbzjaq0bvj78ja @@ -0,0 +1 @@ +{ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/g/7/g7bwdm2hqypr8xfsgwq6cgekw1wqsrzpww5nwscttq46p b/implementation/fuzz/inputs-genvm-modules-complete-json/g/7/g7bwdm2hqypr8xfsgwq6cgekw1wqsrzpww5nwscttq46p new file mode 100644 index 00000000..4c741acc --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/g/7/g7bwdm2hqypr8xfsgwq6cgekw1wqsrzpww5nwscttq46p @@ -0,0 +1 @@ + "\"((((((((((((((((" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/g/8/g8w3at38nnnj5pbjqbfs84vekef7cz8w0hppz1egw5dwg b/implementation/fuzz/inputs-genvm-modules-complete-json/g/8/g8w3at38nnnj5pbjqbfs84vekef7cz8w0hppz1egw5dwg new file mode 100644 index 00000000..20db39d1 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/g/8/g8w3at38nnnj5pbjqbfs84vekef7cz8w0hppz1egw5dwg @@ -0,0 +1 @@ + fal2 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/g/9/g9qa3ry1jg09wd47m2xwpvgae90wnxtr2rvwf8e146qg6 b/implementation/fuzz/inputs-genvm-modules-complete-json/g/9/g9qa3ry1jg09wd47m2xwpvgae90wnxtr2rvwf8e146qg6 new file mode 100644 index 00000000..97c1aec3 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/g/9/g9qa3ry1jg09wd47m2xwpvgae90wnxtr2rvwf8e146qg6 @@ -0,0 +1 @@ +1111111111111111111111111 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/g/9/g9qtzypr7b7x9c00zgpnw90pch381j378ttszp1jfb658 b/implementation/fuzz/inputs-genvm-modules-complete-json/g/9/g9qtzypr7b7x9c00zgpnw90pch381j378ttszp1jfb658 new file mode 100644 index 00000000..f0cf5d4e --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/g/9/g9qtzypr7b7x9c00zgpnw90pch381j378ttszp1jfb658 @@ -0,0 +1,3 @@ +{ + "111E11111111111111111.11111111353331111xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx1111111.111111111133.3foo 4 -12222": 44444444444444444444444444 +} \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/g/b/gb62dbqk4z17473cs581byf9df8452h86tz7r7vjja3er b/implementation/fuzz/inputs-genvm-modules-complete-json/g/b/gb62dbqk4z17473cs581byf9df8452h86tz7r7vjja3er new file mode 100644 index 00000000..27b88173 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/g/b/gb62dbqk4z17473cs581byf9df8452h86tz7r7vjja3er @@ -0,0 +1,9 @@ +{ + "": { + "":{ + "": { + "": { + "": { + "":{ + "": { +, \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/g/b/gb6q6gtnpe7dtegw8qsney5km59ma0yn4690y7qg2hq04 b/implementation/fuzz/inputs-genvm-modules-complete-json/g/b/gb6q6gtnpe7dtegw8qsney5km59ma0yn4690y7qg2hq04 new file mode 100644 index 00000000..e971cb03 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/g/b/gb6q6gtnpe7dtegw8qsney5km59ma0yn4690y7qg2hq04 @@ -0,0 +1 @@ +22222222222222222222 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/g/d/gdawmvp49h1dt7259q310ypjasyxkq3fxs1svg1rr3c18 b/implementation/fuzz/inputs-genvm-modules-complete-json/g/d/gdawmvp49h1dt7259q310ypjasyxkq3fxs1svg1rr3c18 new file mode 100644 index 00000000..6e24c9c1 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/g/d/gdawmvp49h1dt7259q310ypjasyxkq3fxs1svg1rr3c18 @@ -0,0 +1 @@ + [11111,111111E 111, \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/g/d/gdrt3ewgwe2zgrax02f99a2mz17c7mzej0zkhmc5sxz60 b/implementation/fuzz/inputs-genvm-modules-complete-json/g/d/gdrt3ewgwe2zgrax02f99a2mz17c7mzej0zkhmc5sxz60 new file mode 100644 index 00000000..68a26cbe --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/g/d/gdrt3ewgwe2zgrax02f99a2mz17c7mzej0zkhmc5sxz60 @@ -0,0 +1 @@ + "Ρ€Ρƒ2C _qbz" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/g/e/gejprghszwxv6ppp3mrranea2ptkgwenp6h2gpyn03rsm b/implementation/fuzz/inputs-genvm-modules-complete-json/g/e/gejprghszwxv6ppp3mrranea2ptkgwenp6h2gpyn03rsm new file mode 100644 index 00000000..0aa7ade6 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/g/e/gejprghszwxv6ppp3mrranea2ptkgwenp6h2gpyn03rsm @@ -0,0 +1 @@ +111111311111111111111111111E111 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/g/m/gm4zy0ph9dzdwvdf02pvmmnb3z71fxqdzmk5tfcd95016 b/implementation/fuzz/inputs-genvm-modules-complete-json/g/m/gm4zy0ph9dzdwvdf02pvmmnb3z71fxqdzmk5tfcd95016 new file mode 100644 index 00000000..3f3eefe6 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/g/m/gm4zy0ph9dzdwvdf02pvmmnb3z71fxqdzmk5tfcd95016 @@ -0,0 +1 @@ +0E5555555515 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/g/m/gmbdhbg2n4jtfvgpzkdrvag402km42qm3cvykssn1f7c2 b/implementation/fuzz/inputs-genvm-modules-complete-json/g/m/gmbdhbg2n4jtfvgpzkdrvag402km42qm3cvykssn1f7c2 new file mode 100644 index 00000000..78c1311a --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/g/m/gmbdhbg2n4jtfvgpzkdrvag402km42qm3cvykssn1f7c2 @@ -0,0 +1 @@ +333333333.33333333333333333333333333 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/g/m/gmgzrf6rdq7jav2mc6v518jtcs63dr389zfqj1awnmx82 b/implementation/fuzz/inputs-genvm-modules-complete-json/g/m/gmgzrf6rdq7jav2mc6v518jtcs63dr389zfqj1awnmx82 new file mode 100644 index 00000000..7d4677b9 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/g/m/gmgzrf6rdq7jav2mc6v518jtcs63dr389zfqj1awnmx82 @@ -0,0 +1 @@ +111112222222222222222222222 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/g/m/gmyj4521be0c23ktwdyqs75esqbby40y7tc4ee40mg33e b/implementation/fuzz/inputs-genvm-modules-complete-json/g/m/gmyj4521be0c23ktwdyqs75esqbby40y7tc4ee40mg33e new file mode 100644 index 00000000..4fa45610 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/g/m/gmyj4521be0c23ktwdyqs75esqbby40y7tc4ee40mg33e @@ -0,0 +1,5 @@ +{ + "foo": [ + 0 , + 123, + -1e * nu \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/g/n/gn161457rhjg8q1gtgwrxwmhv7fe4hd36n16x15e3zjmm b/implementation/fuzz/inputs-genvm-modules-complete-json/g/n/gn161457rhjg8q1gtgwrxwmhv7fe4hd36n16x15e3zjmm new file mode 100644 index 00000000..98015413 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/g/n/gn161457rhjg8q1gtgwrxwmhv7fe4hd36n16x15e3zjmm @@ -0,0 +1,4 @@ + [ + [ + [[[ [ + [[[ ,,,,] ,,,,] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/g/q/gqnz8wygwa45hanm7s90tvaw890psc9tg15712zb3p7j4 b/implementation/fuzz/inputs-genvm-modules-complete-json/g/q/gqnz8wygwa45hanm7s90tvaw890psc9tg15712zb3p7j4 new file mode 100644 index 00000000..583b66ad --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/g/q/gqnz8wygwa45hanm7s90tvaw890psc9tg15712zb3p7j4 @@ -0,0 +1 @@ + [1111111111111111111111111111111111.1111111111E111,11111111111111111111111111.1111111111E111, 2,11111111E111,11111111111111111111111E111, 2, 3] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/g/s/gs2391srmekajtdzkg6xvyr4rd6ndvp5279vwc0xn2bcp b/implementation/fuzz/inputs-genvm-modules-complete-json/g/s/gs2391srmekajtdzkg6xvyr4rd6ndvp5279vwc0xn2bcp new file mode 100644 index 00000000..21e4fc78 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/g/s/gs2391srmekajtdzkg6xvyr4rd6ndvp5279vwc0xn2bcp @@ -0,0 +1 @@ +[[[[[[[[,,,,,,,,,, \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/g/s/gsxfwzkf15t5ab0xfzq7kwvzf9c3ej1vk3sdz9jpnykm8 b/implementation/fuzz/inputs-genvm-modules-complete-json/g/s/gsxfwzkf15t5ab0xfzq7kwvzf9c3ej1vk3sdz9jpnykm8 new file mode 100644 index 00000000..aa9643b4 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/g/s/gsxfwzkf15t5ab0xfzq7kwvzf9c3ej1vk3sdz9jpnykm8 @@ -0,0 +1,2 @@ +{ + "foo" & \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/g/t/gtnxy4531pmxv9847jz7perpg0cpz4wnkdkxgtydg3v60 b/implementation/fuzz/inputs-genvm-modules-complete-json/g/t/gtnxy4531pmxv9847jz7perpg0cpz4wnkdkxgtydg3v60 new file mode 100644 index 00000000..ba544124 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/g/t/gtnxy4531pmxv9847jz7perpg0cpz4wnkdkxgtydg3v60 @@ -0,0 +1,17 @@ +{ + "foo": [ + "bar", + "baz\n\\{", + -0, + 1e20, + null, + true, + [1, 2, 3] + ], + "barrrrrrrrrr": [ ], + "brrrr": [ ], + "bar": [ + "русскиС mbbbbbbbbb", + "mbbbbbbbbbbbbbbaz\t\\uabbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/g/v/gvh7p1xkxz9brkee1ea2ywawfgxqn5x8t9svnqt9244ea b/implementation/fuzz/inputs-genvm-modules-complete-json/g/v/gvh7p1xkxz9brkee1ea2ywawfgxqn5x8t9svnqt9244ea new file mode 100644 index 00000000..edb52046 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/g/v/gvh7p1xkxz9brkee1ea2ywawfgxqn5x8t9svnqt9244ea @@ -0,0 +1 @@ + "bσ—₯—1111σ—₯—111本" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/g/w/gw0baa8yz1ty6dywpg4xjeawgmehkk287m40ygkx6f6zp b/implementation/fuzz/inputs-genvm-modules-complete-json/g/w/gw0baa8yz1ty6dywpg4xjeawgmehkk287m40ygkx6f6zp new file mode 100644 index 00000000..b04a28ed --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/g/w/gw0baa8yz1ty6dywpg4xjeawgmehkk287m40ygkx6f6zp @@ -0,0 +1,3 @@ +{ + "": { + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/g/x/gx98f4rk68hcfbeqdx4pk7bzjz07vg6e7a3cxzx3v9wft b/implementation/fuzz/inputs-genvm-modules-complete-json/g/x/gx98f4rk68hcfbeqdx4pk7bzjz07vg6e7a3cxzx3v9wft new file mode 100644 index 00000000..a03bdb9d --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/g/x/gx98f4rk68hcfbeqdx4pk7bzjz07vg6e7a3cxzx3v9wft @@ -0,0 +1,2 @@ +[, + ] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/g/x/gxwg8stb3a070a7d4pg61a7rveqv0ccr5bhfpn7a4rstg b/implementation/fuzz/inputs-genvm-modules-complete-json/g/x/gxwg8stb3a070a7d4pg61a7rveqv0ccr5bhfpn7a4rstg new file mode 100644 index 00000000..6b787f72 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/g/x/gxwg8stb3a070a7d4pg61a7rveqv0ccr5bhfpn7a4rstg @@ -0,0 +1 @@ +1.1111 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/g/y/gyqgq9ge6v81dz4ah04p7s2wmgpxxvzqwytpzbmwz7b3a b/implementation/fuzz/inputs-genvm-modules-complete-json/g/y/gyqgq9ge6v81dz4ah04p7s2wmgpxxvzqwytpzbmwz7b3a new file mode 100644 index 00000000..baaa5311 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/g/y/gyqgq9ge6v81dz4ah04p7s2wmgpxxvzqwytpzbmwz7b3a @@ -0,0 +1,4 @@ +{ + "foo" + + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/g/z/gzj084c49th6y4ryysvh0027yvt77cfmxxn79qcyyhzgg b/implementation/fuzz/inputs-genvm-modules-complete-json/g/z/gzj084c49th6y4ryysvh0027yvt77cfmxxn79qcyyhzgg new file mode 100644 index 00000000..86a3e43d --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/g/z/gzj084c49th6y4ryysvh0027yvt77cfmxxn79qcyyhzgg @@ -0,0 +1 @@ + 1 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/h/1/h1ybayrg290j0f6myh3tt1p63zzjx1mf29k0rwewqqchw b/implementation/fuzz/inputs-genvm-modules-complete-json/h/1/h1ybayrg290j0f6myh3tt1p63zzjx1mf29k0rwewqqchw new file mode 100644 index 00000000..e0e1ad41 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/h/1/h1ybayrg290j0f6myh3tt1p63zzjx1mf29k0rwewqqchw @@ -0,0 +1,3 @@ + + +"Ρ€\r" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/h/2/h213vzjvnvfx83c7abfycjh14x57afa1ceyznkx4tg0k8 b/implementation/fuzz/inputs-genvm-modules-complete-json/h/2/h213vzjvnvfx83c7abfycjh14x57afa1ceyznkx4tg0k8 new file mode 100644 index 00000000..bc430114 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/h/2/h213vzjvnvfx83c7abfycjh14x57afa1ceyznkx4tg0k8 @@ -0,0 +1,2 @@ +"\r -1" + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/h/2/h2wcbzae9g1dae8mbrgtcph3phzvn6xs0p4hgwcxnzeea b/implementation/fuzz/inputs-genvm-modules-complete-json/h/2/h2wcbzae9g1dae8mbrgtcph3phzvn6xs0p4hgwcxnzeea new file mode 100644 index 00000000..e4b6bad7 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/h/2/h2wcbzae9g1dae8mbrgtcph3phzvn6xs0p4hgwcxnzeea @@ -0,0 +1,20 @@ +{ + "foo": [ + "bar", + "baz\n\\z", + 1.3, + 1.3, + -123, + -1e10, + 1e20, + null, + true, + false, + [1, 2, 3] + ], + "bar": [ + "русскиС Π±ΡƒΠΊΠ²Ρ‹ ζ₯本", + { "foo": [ + "b]r", + ", + " \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/h/4/h4ep3m70qztxm8dd5ftste6jege5zgy5t6j021833j7ej b/implementation/fuzz/inputs-genvm-modules-complete-json/h/4/h4ep3m70qztxm8dd5ftste6jege5zgy5t6j021833j7ej new file mode 100644 index 00000000..19626109 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/h/4/h4ep3m70qztxm8dd5ftste6jege5zgy5t6j021833j7ej @@ -0,0 +1 @@ +222222222222222222222222222 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/h/4/h4jhyy4pf9dk9sm960mm91qpnw8draq2q4mwvdngp56d2 b/implementation/fuzz/inputs-genvm-modules-complete-json/h/4/h4jhyy4pf9dk9sm960mm91qpnw8draq2q4mwvdngp56d2 new file mode 100644 index 00000000..048c40e1 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/h/4/h4jhyy4pf9dk9sm960mm91qpnw8draq2q4mwvdngp56d2 @@ -0,0 +1 @@ +1.611e00 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/h/5/h5wxz58hzw7pmvdgyt09dyarfk10n5g1rfxf5nep1531r b/implementation/fuzz/inputs-genvm-modules-complete-json/h/5/h5wxz58hzw7pmvdgyt09dyarfk10n5g1rfxf5nep1531r new file mode 100644 index 0000000000000000000000000000000000000000..d98443ff77e7fe101a4f5fbc24c79cc98a298e80 GIT binary patch literal 260 zcmb>CQUC&_wETP}D}`t<2P}}3Sfr!_kxNRfO390fse<xB(uPLH5J?aR1fn58!7$ax z03nlCnv(<O>4G#sR3b<vBO@aqPuDONA%=xzhA9TMEgDk>;u5HM83Q&2sA93PvAF<M Chb>(I literal 0 HcmV?d00001 diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/h/6/h62x0c4w7p7jsrz1h1ezka109ryjqetjtm9n6bth6ea4e b/implementation/fuzz/inputs-genvm-modules-complete-json/h/6/h62x0c4w7p7jsrz1h1ezka109ryjqetjtm9n6bth6ea4e new file mode 100644 index 00000000..bbeba644 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/h/6/h62x0c4w7p7jsrz1h1ezka109ryjqetjtm9n6bth6ea4e @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/h/8/h8dk8bqhfqc4qy61zpzberwejg3b5evkeq3dma3js2t0e b/implementation/fuzz/inputs-genvm-modules-complete-json/h/8/h8dk8bqhfqc4qy61zpzberwejg3b5evkeq3dma3js2t0e new file mode 100644 index 00000000..7dfe9512 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/h/8/h8dk8bqhfqc4qy61zpzberwejg3b5evkeq3dma3js2t0e @@ -0,0 +1 @@ +" \uDbbb\uDDbBVuu" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/h/8/h8eegm2adakfzwk3dh4hwkepwt67gpkyfw5x6ywrrs9jg b/implementation/fuzz/inputs-genvm-modules-complete-json/h/8/h8eegm2adakfzwk3dh4hwkepwt67gpkyfw5x6ywrrs9jg new file mode 100644 index 00000000..538f7bf9 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/h/8/h8eegm2adakfzwk3dh4hwkepwt67gpkyfw5x6ywrrs9jg @@ -0,0 +1 @@ + [1, [1, [1, [1, [1, [1, [1, [1, \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/h/8/h8s754wqsxw871nt24v192x2eck1jz9w9s8bk89m7657m b/implementation/fuzz/inputs-genvm-modules-complete-json/h/8/h8s754wqsxw871nt24v192x2eck1jz9w9s8bk89m7657m new file mode 100644 index 00000000..249a78ec --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/h/8/h8s754wqsxw871nt24v192x2eck1jz9w9s8bk89m7657m @@ -0,0 +1,20 @@ +{ + "bar": [ + "рус", + true, + 123, + -123, + -1e10, + 1e20, + null, + "baz\n\\{", + false, + [1, 2, 3] + ], + "bar": [ ], + "bar": [ + " русскиС Π±ΡƒΠΊΠ²Π‡ ζ—₯本", + "mbbbbbbbbbbbbbbaz\t\uabbbbb", + "mbbbbbbbbbbbbbbaz\t\\uabbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/h/a/ha1k3z20wf9ahpv6ttewhna2m0251hx8tvtsk13pdazb8 b/implementation/fuzz/inputs-genvm-modules-complete-json/h/a/ha1k3z20wf9ahpv6ttewhna2m0251hx8tvtsk13pdazb8 new file mode 100644 index 00000000..c2520b29 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/h/a/ha1k3z20wf9ahpv6ttewhna2m0251hx8tvtsk13pdazb8 @@ -0,0 +1 @@ +111111111111111111111111111 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/h/a/haagrvyam74hae6x8wsw3an148qa9caw8cjx2jwxpyznr b/implementation/fuzz/inputs-genvm-modules-complete-json/h/a/haagrvyam74hae6x8wsw3an148qa9caw8cjx2jwxpyznr new file mode 100644 index 00000000..382a60ca --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/h/a/haagrvyam74hae6x8wsw3an148qa9caw8cjx2jwxpyznr @@ -0,0 +1 @@ + 3333333333333333333333333 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/h/c/hc7wg34atrg7afqky1ajnxprcdzbbdqfg09fb92m063r2 b/implementation/fuzz/inputs-genvm-modules-complete-json/h/c/hc7wg34atrg7afqky1ajnxprcdzbbdqfg09fb92m063r2 new file mode 100644 index 00000000..226d9e67 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/h/c/hc7wg34atrg7afqky1ajnxprcdzbbdqfg09fb92m063r2 @@ -0,0 +1,2 @@ + +"Ρ€bbbb\\ua1$, r" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/h/f/hfh1j3r352wvahzbgnh8t449j3pm9kb4d1ax0ezky2b7j b/implementation/fuzz/inputs-genvm-modules-complete-json/h/f/hfh1j3r352wvahzbgnh8t449j3pm9kb4d1ax0ezky2b7j new file mode 100644 index 00000000..1cbaef2f --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/h/f/hfh1j3r352wvahzbgnh8t449j3pm9kb4d1ax0ezky2b7j @@ -0,0 +1 @@ + [1111111111.1110111111, 2, 1111.11, 2, 1111.111, 1111.11, 2, 1111.11111111111e+2, 1111, 2, 1111.11, 2, 1111.11111116112, 1111.1111111611111111] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/h/g/hgw6n5jgrsxe3aydj10cvq35ma4wsg7qczhz1r64d7hn0 b/implementation/fuzz/inputs-genvm-modules-complete-json/h/g/hgw6n5jgrsxe3aydj10cvq35ma4wsg7qczhz1r64d7hn0 new file mode 100644 index 00000000..9d012302 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/h/g/hgw6n5jgrsxe3aydj10cvq35ma4wsg7qczhz1r64d7hn0 @@ -0,0 +1 @@ +1111.11111111111111111E311, \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/h/h/hhfkz92b2h54dsc8s1vzbhfzenxw9r7hx740z41fexja0 b/implementation/fuzz/inputs-genvm-modules-complete-json/h/h/hhfkz92b2h54dsc8s1vzbhfzenxw9r7hx740z41fexja0 new file mode 100644 index 00000000..24abda44 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/h/h/hhfkz92b2h54dsc8s1vzbhfzenxw9r7hx740z41fexja0 @@ -0,0 +1 @@ + [1111111111111111111111E111, 2,11111111111111111111111111111111111111E111,1111E111, 2,1111111111111111E111, 1, 3] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/h/h/hhkddf8e6nbae84a2vgwbs0dcm847zgw3vctbsjd6j2ag b/implementation/fuzz/inputs-genvm-modules-complete-json/h/h/hhkddf8e6nbae84a2vgwbs0dcm847zgw3vctbsjd6j2ag new file mode 100644 index 00000000..dd34e663 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/h/h/hhkddf8e6nbae84a2vgwbs0dcm847zgw3vctbsjd6j2ag @@ -0,0 +1,2 @@ + [[ [[{ + "fz1T1111111111F1111111111111111111111111111111111111111111111115q1111111111A1111111111111111.11111111E111.111111,o": "bar", , \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/h/h/hhxkfkn00xvw37f3t551t5tcemzfqe9jegt9wgftas6c2 b/implementation/fuzz/inputs-genvm-modules-complete-json/h/h/hhxkfkn00xvw37f3t551t5tcemzfqe9jegt9wgftas6c2 new file mode 100644 index 00000000..31a1eccb --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/h/h/hhxkfkn00xvw37f3t551t5tcemzfqe9jegt9wgftas6c2 @@ -0,0 +1,16 @@ +{ + "foo": [ + + 123, + -123, + -1e10, + null, + true, + false, + [1, 2, 3] + ], + "bar": [ + "русскиС Π±ΪƒΠΊΨ²Ρ‹ ζ—₯本az\t\uabbb\b", + "bbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/h/j/hjrr8ycqfat197v4k1hxtkt0zcyrvhgx8tmzx326zrd0r b/implementation/fuzz/inputs-genvm-modules-complete-json/h/j/hjrr8ycqfat197v4k1hxtkt0zcyrvhgx8tmzx326zrd0r new file mode 100644 index 00000000..9b97d2c1 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/h/j/hjrr8ycqfat197v4k1hxtkt0zcyrvhgx8tmzx326zrd0r @@ -0,0 +1,2 @@ + +"Ρ€[ rΠ²‡ σ—₯Ίub" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/h/n/hn2gy0dcn4srykfn7x9vabhne57j2x7y8dg4mhrjdw7wj b/implementation/fuzz/inputs-genvm-modules-complete-json/h/n/hn2gy0dcn4srykfn7x9vabhne57j2x7y8dg4mhrjdw7wj new file mode 100644 index 00000000..218b1dbb --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/h/n/hn2gy0dcn4srykfn7x9vabhne57j2x7y8dg4mhrjdw7wj @@ -0,0 +1 @@ +"Ρ€\\σ—₯Ίr" diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/h/n/hn9qfpe5y3st6etwykvvb01v697hvpsbzgpq98e44xvpy b/implementation/fuzz/inputs-genvm-modules-complete-json/h/n/hn9qfpe5y3st6etwykvvb01v697hvpsbzgpq98e44xvpy new file mode 100644 index 00000000..81750b96 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/h/n/hn9qfpe5y3st6etwykvvb01v697hvpsbzgpq98e44xvpy @@ -0,0 +1 @@ +{ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/h/n/hnamj82p5zpjpyzzvxe1ztgte2spnpdqet0gm82frhmda b/implementation/fuzz/inputs-genvm-modules-complete-json/h/n/hnamj82p5zpjpyzzvxe1ztgte2spnpdqet0gm82frhmda new file mode 100644 index 00000000..96741794 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/h/n/hnamj82p5zpjpyzzvxe1ztgte2spnpdqet0gm82frhmda @@ -0,0 +1,5 @@ +{ + "foo" + +% + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/h/q/hq75ncgkjyx1j6h3h5vw9x9m3nw6f6h5rnkv1e1qx55qp b/implementation/fuzz/inputs-genvm-modules-complete-json/h/q/hq75ncgkjyx1j6h3h5vw9x9m3nw6f6h5rnkv1e1qx55qp new file mode 100644 index 00000000..02a2488e --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/h/q/hq75ncgkjyx1j6h3h5vw9x9m3nw6f6h5rnkv1e1qx55qp @@ -0,0 +1,2 @@ +"\"\t" + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/h/q/hqmnjzsw0nb75195g8emmhk4tjw68n44xwvgkz4w8063p b/implementation/fuzz/inputs-genvm-modules-complete-json/h/q/hqmnjzsw0nb75195g8emmhk4tjw68n44xwvgkz4w8063p new file mode 100644 index 0000000000000000000000000000000000000000..98a6f3466a1cefd839b33502b511eec8eb39259c GIT binary patch literal 74 scmb>CQczG*vZ_{q(O@1IL<}SWWr78u{Iq<fw0s!gQmAKOPyiYN07N_xrvLx| literal 0 HcmV?d00001 diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/h/r/hrbxm7ybsmr740a6hbpeaavxczg1yq6eza16jvst2ppg0 b/implementation/fuzz/inputs-genvm-modules-complete-json/h/r/hrbxm7ybsmr740a6hbpeaavxczg1yq6eza16jvst2ppg0 new file mode 100644 index 00000000..d14796e9 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/h/r/hrbxm7ybsmr740a6hbpeaavxczg1yq6eza16jvst2ppg0 @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + + + + + + + +111 + + +\uDbbC \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/h/s/hsznzbxvyc2zgevg0344280fmx3ahkk1hnjzr1715g3j6 b/implementation/fuzz/inputs-genvm-modules-complete-json/h/s/hsznzbxvyc2zgevg0344280fmx3ahkk1hnjzr1715g3j6 new file mode 100644 index 00000000..d92022c9 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/h/s/hsznzbxvyc2zgevg0344280fmx3ahkk1hnjzr1715g3j6 @@ -0,0 +1 @@ +"Ρ€ΠΊ\"\"VnG\"ra[1:" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/h/t/ht4v0zqe12ekgy8jzscfbwprhaj8nwpz70p11k1zwsmzg b/implementation/fuzz/inputs-genvm-modules-complete-json/h/t/ht4v0zqe12ekgy8jzscfbwprhaj8nwpz70p11k1zwsmzg new file mode 100644 index 00000000..b3d1a3cf --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/h/t/ht4v0zqe12ekgy8jzscfbwprhaj8nwpz70p11k1zwsmzg @@ -0,0 +1,6 @@ +{ + "foo": [ + "bar", + "ba", + "mbbbbbbbbbbbbbz\n\\z", + 123333333333333.3333333333333333333333333333 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/h/v/hvcgvt6d2g6613m3w203h288dzd2k6d92wn4kfvzbd7je b/implementation/fuzz/inputs-genvm-modules-complete-json/h/v/hvcgvt6d2g6613m3w203h288dzd2k6d92wn4kfvzbd7je new file mode 100644 index 00000000..49318512 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/h/v/hvcgvt6d2g6613m3w203h288dzd2k6d92wn4kfvzbd7je @@ -0,0 +1,2 @@ +11.222E+2 + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/h/y/hyc888ywyvy5fr83bdyj7dxjeq62mc10s4xshxsqhw7wp b/implementation/fuzz/inputs-genvm-modules-complete-json/h/y/hyc888ywyvy5fr83bdyj7dxjeq62mc10s4xshxsqhw7wp new file mode 100644 index 00000000..054936f0 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/h/y/hyc888ywyvy5fr83bdyj7dxjeq62mc10s4xshxsqhw7wp @@ -0,0 +1,5 @@ + { + + "": {"": { + "#: { +," \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/h/z/hz604cp3v0gt2pjkkzmqd84nx5qfv1d90x6qtre0n9qcg b/implementation/fuzz/inputs-genvm-modules-complete-json/h/z/hz604cp3v0gt2pjkkzmqd84nx5qfv1d90x6qtre0n9qcg new file mode 100644 index 00000000..2deed627 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/h/z/hz604cp3v0gt2pjkkzmqd84nx5qfv1d90x6qtre0n9qcg @@ -0,0 +1,3 @@ +{ + "fooo#:{ + " \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/j/0/j00hadzgnrkbdch4ja8rjvc6yc6fdk5canytgzgr7qhs2 b/implementation/fuzz/inputs-genvm-modules-complete-json/j/0/j00hadzgnrkbdch4ja8rjvc6yc6fdk5canytgzgr7qhs2 new file mode 100644 index 00000000..59b55ade --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/j/0/j00hadzgnrkbdch4ja8rjvc6yc6fdk5canytgzgr7qhs2 @@ -0,0 +1 @@ +"UUUTmbb\u0000bbb," diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/j/1/j1wvajs26edvpejqkc27syy0dqxttg0epvf1dbzfvce18 b/implementation/fuzz/inputs-genvm-modules-complete-json/j/1/j1wvajs26edvpejqkc27syy0dqxttg0epvf1dbzfvce18 new file mode 100644 index 00000000..c346c2e3 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/j/1/j1wvajs26edvpejqkc27syy0dqxttg0epvf1dbzfvce18 @@ -0,0 +1,2 @@ +[[[[[[[[[[[[[[[[ + t \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/j/4/j4nzmgpapfbw9q5gd1r99hzchwqmnc130bv3e102xeekw b/implementation/fuzz/inputs-genvm-modules-complete-json/j/4/j4nzmgpapfbw9q5gd1r99hzchwqmnc130bv3e102xeekw new file mode 100644 index 00000000..b85dcfef --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/j/4/j4nzmgpapfbw9q5gd1r99hzchwqmnc130bv3e102xeekw @@ -0,0 +1,14 @@ +{ + "1" +: [ { + "1": [ ] + + + + + + + + +} ] +} \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/j/4/j4zgfdkbdfjrbhyctrkkdwzyy6tv0xx7xdjd3syvq9pte b/implementation/fuzz/inputs-genvm-modules-complete-json/j/4/j4zgfdkbdfjrbhyctrkkdwzyy6tv0xx7xdjd3syvq9pte new file mode 100644 index 00000000..ec8f9eb9 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/j/4/j4zgfdkbdfjrbhyctrkkdwzyy6tv0xx7xdjd3syvq9pte @@ -0,0 +1 @@ +1111111111111111111111911 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/j/5/j5a2m63pssmgd46emw3s4mrk88agm1pr3ykwkpxx5nhna b/implementation/fuzz/inputs-genvm-modules-complete-json/j/5/j5a2m63pssmgd46emw3s4mrk88agm1pr3ykwkpxx5nhna new file mode 100644 index 00000000..bfacfd81 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/j/5/j5a2m63pssmgd46emw3s4mrk88agm1pr3ykwkpxx5nhna @@ -0,0 +1,3 @@ + { + "":0, + "" false "bar", diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/j/7/j7gc0kzcgk2v3hq0ehgh1fcs2272ryyan83wm7p3gxa78 b/implementation/fuzz/inputs-genvm-modules-complete-json/j/7/j7gc0kzcgk2v3hq0ehgh1fcs2272ryyan83wm7p3gxa78 new file mode 100644 index 00000000..979a890a --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/j/7/j7gc0kzcgk2v3hq0ehgh1fcs2272ryyan83wm7p3gxa78 @@ -0,0 +1,2 @@ + [111111111111111111.1110111111E111, -1e11, + 1e+2, 111E111, 2, 111111111111e+2, 1111, 2, 1111.11, 2, 1111.11111116112, 11111.1111111611111111] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/j/8/j83hfvekz2y5g1jhg1ykyemphdhfyarhmwg7smgvy28w8 b/implementation/fuzz/inputs-genvm-modules-complete-json/j/8/j83hfvekz2y5g1jhg1ykyemphdhfyarhmwg7smgvy28w8 new file mode 100644 index 00000000..3621e87d --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/j/8/j83hfvekz2y5g1jhg1ykyemphdhfyarhmwg7smgvy28w8 @@ -0,0 +1,7 @@ +{ + "foo": [ + "bazdn\\z", + 123, + -123, + , +  nu \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/j/8/j8rb6d6578z22xtepbzrc67xhp4cr6c5azxex2p3m841y b/implementation/fuzz/inputs-genvm-modules-complete-json/j/8/j8rb6d6578z22xtepbzrc67xhp4cr6c5azxex2p3m841y new file mode 100644 index 00000000..9852dea0 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/j/8/j8rb6d6578z22xtepbzrc67xhp4cr6c5azxex2p3m841y @@ -0,0 +1,2 @@ + [ { +}] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/j/8/j8ytdmjfsm33pdkx1vneefqterd9ajy4w4caztg49m49r b/implementation/fuzz/inputs-genvm-modules-complete-json/j/8/j8ytdmjfsm33pdkx1vneefqterd9ajy4w4caztg49m49r new file mode 100644 index 00000000..21f54f3c --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/j/8/j8ytdmjfsm33pdkx1vneefqterd9ajy4w4caztg49m49r @@ -0,0 +1,2 @@ + +"Ρ€\\σ—₯—U" diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/j/b/jb8e27vb4h8m64gn1qepw3kgrzm6ygdtgrr03nb685w3y b/implementation/fuzz/inputs-genvm-modules-complete-json/j/b/jb8e27vb4h8m64gn1qepw3kgrzm6ygdtgrr03nb685w3y new file mode 100644 index 00000000..13d23914 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/j/b/jb8e27vb4h8m64gn1qepw3kgrzm6ygdtgrr03nb685w3y @@ -0,0 +1,10 @@ + { + "":0, + "" 22E-: { + "": + " + + + + + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/j/b/jbqnh283dae76dtmhfkvq7nmd6qt8rva7y2hbz51fmdzy b/implementation/fuzz/inputs-genvm-modules-complete-json/j/b/jbqnh283dae76dtmhfkvq7nmd6qt8rva7y2hbz51fmdzy new file mode 100644 index 00000000..3758af5c --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/j/b/jbqnh283dae76dtmhfkvq7nmd6qt8rva7y2hbz51fmdzy @@ -0,0 +1,3 @@ + [0, + + ] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/j/d/jdats9sxdjzpxdfkb3kape5azt6yde0se4f664h6tefke b/implementation/fuzz/inputs-genvm-modules-complete-json/j/d/jdats9sxdjzpxdfkb3kape5azt6yde0se4f664h6tefke new file mode 100644 index 00000000..c3b6af2b --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/j/d/jdats9sxdjzpxdfkb3kape5azt6yde0se4f664h6tefke @@ -0,0 +1 @@ +"\f Ρƒ\u01b1111111\\" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/j/d/jdqkeac33py9haf2ervq2ggmcwhen40aj4sc291x22xv6 b/implementation/fuzz/inputs-genvm-modules-complete-json/j/d/jdqkeac33py9haf2ervq2ggmcwhen40aj4sc291x22xv6 new file mode 100644 index 00000000..208c1dee --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/j/d/jdqkeac33py9haf2ervq2ggmcwhen40aj4sc291x22xv6 @@ -0,0 +1,2 @@ +22222222222222222222E-22222222222222222222222 + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/j/d/jdshqt7jp74v9e149zrv089cb812zcdqz9nsrvzyn54p4 b/implementation/fuzz/inputs-genvm-modules-complete-json/j/d/jdshqt7jp74v9e149zrv089cb812zcdqz9nsrvzyn54p4 new file mode 100644 index 00000000..416458dd --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/j/d/jdshqt7jp74v9e149zrv089cb812zcdqz9nsrvzyn54p4 @@ -0,0 +1 @@ + " \b
 " \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/j/e/je6jbergqyrxdm309dnn4x1t085j601kvppjycgfam982 b/implementation/fuzz/inputs-genvm-modules-complete-json/j/e/je6jbergqyrxdm309dnn4x1t085j601kvppjycgfam982 new file mode 100644 index 00000000..a035f9aa --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/j/e/je6jbergqyrxdm309dnn4x1t085j601kvppjycgfam982 @@ -0,0 +1 @@ + " Ρƒ\u0000" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/j/k/jk1wdf80q75qrbb8zbzgycmahpcbtyjzgm502764ee8tp b/implementation/fuzz/inputs-genvm-modules-complete-json/j/k/jk1wdf80q75qrbb8zbzgycmahpcbtyjzgm502764ee8tp new file mode 100644 index 00000000..1364c010 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/j/k/jk1wdf80q75qrbb8zbzgycmahpcbtyjzgm502764ee8tp @@ -0,0 +1 @@ + [11111,11113111131111111111111E111, 2, 3111111111111111111111E111,11113111E111, 2, 3] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/j/p/jpqfx60td314ksyxv666k9awxntamx4zcnbxbyw8pvsrc b/implementation/fuzz/inputs-genvm-modules-complete-json/j/p/jpqfx60td314ksyxv666k9awxntamx4zcnbxbyw8pvsrc new file mode 100644 index 00000000..2607e3fc --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/j/p/jpqfx60td314ksyxv666k9awxntamx4zcnbxbyw8pvsrc @@ -0,0 +1 @@ +"\f\\ ζ—₯12 " \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/j/r/jr60zpc4dd7hc1cac6jydf05dyf59kv9g86pyygsmq76m b/implementation/fuzz/inputs-genvm-modules-complete-json/j/r/jr60zpc4dd7hc1cac6jydf05dyf59kv9g86pyygsmq76m new file mode 100644 index 00000000..592e35a2 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/j/r/jr60zpc4dd7hc1cac6jydf05dyf59kv9g86pyygsmq76m @@ -0,0 +1,17 @@ +{ + "foo": [ + "bar", + "baz\n\\{", + -1e10, + 1e20, + null, + true, + [1, 2, 3] + ], + "barrrrrrrrrr": [ ], + "brrrr": [ ], + "bar": [ + "русски 1e20bbbbbb", + "mbbbbbbbbbbbbbbaz\t\\uabbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/j/s/jsgx8wb9rbc02j98tz450a1xkst7j3v185atgmtsbcvfm b/implementation/fuzz/inputs-genvm-modules-complete-json/j/s/jsgx8wb9rbc02j98tz450a1xkst7j3v185atgmtsbcvfm new file mode 100644 index 00000000..9e26dfee --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/j/s/jsgx8wb9rbc02j98tz450a1xkst7j3v185atgmtsbcvfm @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/j/t/jtd53xadg9jt8grsvsh9a8bkceye8zr1rxx4wn1qh5gwp b/implementation/fuzz/inputs-genvm-modules-complete-json/j/t/jtd53xadg9jt8grsvsh9a8bkceye8zr1rxx4wn1qh5gwp new file mode 100644 index 00000000..42d64a02 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/j/t/jtd53xadg9jt8grsvsh9a8bkceye8zr1rxx4wn1qh5gwp @@ -0,0 +1,2 @@ + +"Ρ€\\uba d [[1, B\bΓ΅ r" diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/j/v/jvm7y97hwf2hespz771jydy0mxe1gbgqqj86q67z3bxw8 b/implementation/fuzz/inputs-genvm-modules-complete-json/j/v/jvm7y97hwf2hespz771jydy0mxe1gbgqqj86q67z3bxw8 new file mode 100644 index 00000000..e8e4c3b8 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/j/v/jvm7y97hwf2hespz771jydy0mxe1gbgqqj86q67z3bxw8 @@ -0,0 +1,44 @@ +{ + + + + + + + + + + "foo": "bar", + + + + + + + + + (Ρ€\\uuabbbbb" + + + + + + + + + + + + +1111111 + + + + +" + + + + + + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/j/w/jw14ta37vnr37zdj46hcvkxg3tzdbvv9hrkcmj98rhhy2 b/implementation/fuzz/inputs-genvm-modules-complete-json/j/w/jw14ta37vnr37zdj46hcvkxg3tzdbvv9hrkcmj98rhhy2 new file mode 100644 index 00000000..b789b07a --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/j/w/jw14ta37vnr37zdj46hcvkxg3tzdbvv9hrkcmj98rhhy2 @@ -0,0 +1 @@ + "bσ—₯—50 " \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/j/w/jwtrd5e0gr5evfwqz6n983xmpr1zqcpg53ap1gftvkx3p b/implementation/fuzz/inputs-genvm-modules-complete-json/j/w/jwtrd5e0gr5evfwqz6n983xmpr1zqcpg53ap1gftvkx3p new file mode 100644 index 00000000..13ad946c --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/j/w/jwtrd5e0gr5evfwqz6n983xmpr1zqcpg53ap1gftvkx3p @@ -0,0 +1 @@ +[[[[[[[[[[[[[[ diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/j/y/jy2dwkz0s5qa16xke0n9d4zkaefmp349kbwyjpd6nj1ry b/implementation/fuzz/inputs-genvm-modules-complete-json/j/y/jy2dwkz0s5qa16xke0n9d4zkaefmp349kbwyjpd6nj1ry new file mode 100644 index 00000000..65b37ef0 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/j/y/jy2dwkz0s5qa16xke0n9d4zkaefmp349kbwyjpd6nj1ry @@ -0,0 +1 @@ +"\"/" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/j/y/jy8tv9rzs0ewfa9c71aqc4t7qym7mjyywwxb0cjkm39tc b/implementation/fuzz/inputs-genvm-modules-complete-json/j/y/jy8tv9rzs0ewfa9c71aqc4t7qym7mjyywwxb0cjkm39tc new file mode 100644 index 00000000..92aad6b2 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/j/y/jy8tv9rzs0ewfa9c71aqc4t7qym7mjyywwxb0cjkm39tc @@ -0,0 +1,4 @@ +{" + + + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/k/0/k0adjkygzy5vt8e1yhtb9r0dhg19hxzxcdtp5whmpe6a0 b/implementation/fuzz/inputs-genvm-modules-complete-json/k/0/k0adjkygzy5vt8e1yhtb9r0dhg19hxzxcdtp5whmpe6a0 new file mode 100644 index 00000000..4a210dd1 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/k/0/k0adjkygzy5vt8e1yhtb9r0dhg19hxzxcdtp5whmpe6a0 @@ -0,0 +1,19 @@ +{ + "foo": [ + 123, + -123, + -1e10, + 1e20, + null, + true, + false, + [1, 2, 3] + ], + "bar": [ + "русскиС Π±ΡƒΠΊΠ²Ρ‹ ζ—₯本", + [1, 2, 3] + ], + "bau": "mbbb", + "mau": "mbbbbbbbbbbb\\rabbbbb" + +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/k/0/k0s9ekbqz6ts2av7eb8n3yjv4wytdaarys3ed9gvnz5b4 b/implementation/fuzz/inputs-genvm-modules-complete-json/k/0/k0s9ekbqz6ts2av7eb8n3yjv4wytdaarys3ed9gvnz5b4 new file mode 100644 index 00000000..13086e6a --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/k/0/k0s9ekbqz6ts2av7eb8n3yjv4wytdaarys3ed9gvnz5b4 @@ -0,0 +1,4 @@ +2222222222222222222222222E-2 + + + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/k/0/k0wkm3rtc76y4amxbq6a3bvj28wt28t9127hf7cverh92 b/implementation/fuzz/inputs-genvm-modules-complete-json/k/0/k0wkm3rtc76y4amxbq6a3bvj28wt28t9127hf7cverh92 new file mode 100644 index 00000000..b94016a4 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/k/0/k0wkm3rtc76y4amxbq6a3bvj28wt28t9127hf7cverh92 @@ -0,0 +1,3 @@ +{ + "11111111111111": 1e20, + "111123o" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/k/1/k1rjtsqeqb0xm894bjt7vy73prmyhyhzpe9hg1nk349n4 b/implementation/fuzz/inputs-genvm-modules-complete-json/k/1/k1rjtsqeqb0xm894bjt7vy73prmyhyhzpe9hg1nk349n4 new file mode 100644 index 00000000..56546f4f --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/k/1/k1rjtsqeqb0xm894bjt7vy73prmyhyhzpe9hg1nk349n4 @@ -0,0 +1 @@ + "+\ro \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/k/1/k1wz7cgm8ra1sarnn2w689q42sed8xb9ve78mvj85knry b/implementation/fuzz/inputs-genvm-modules-complete-json/k/1/k1wz7cgm8ra1sarnn2w689q42sed8xb9ve78mvj85knry new file mode 100644 index 00000000..aa93bc6b --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/k/1/k1wz7cgm8ra1sarnn2w689q42sed8xb9ve78mvj85knry @@ -0,0 +1,4 @@ +{ + "":{ + "":{ "":{ "bIr", + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/k/4/k4838jmv5xazrbrknbzv2w01bzfh74j3s7pw5wntdgzcg b/implementation/fuzz/inputs-genvm-modules-complete-json/k/4/k4838jmv5xazrbrknbzv2w01bzfh74j3s7pw5wntdgzcg new file mode 100644 index 00000000..38a40dc0 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/k/4/k4838jmv5xazrbrknbzv2w01bzfh74j3s7pw5wntdgzcg @@ -0,0 +1 @@ +2222222222222222222222222 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/k/6/k6nka9cctyna6x83rethzyb8d7vy6915xmv1xa90snb2c b/implementation/fuzz/inputs-genvm-modules-complete-json/k/6/k6nka9cctyna6x83rethzyb8d7vy6915xmv1xa90snb2c new file mode 100644 index 00000000..59c7df2b --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/k/6/k6nka9cctyna6x83rethzyb8d7vy6915xmv1xa90snb2c @@ -0,0 +1,2 @@ + +"\u0000Ρ‹\u00F0Ρ‹\u00000Ρ€\u00000Ρ€\\wb00Ρ‹\u00000Ρ€\u00a1 00Ρ‹\u0000Ρ‹\u00000Ρ€\u00000Ρ€\\ζ—₯ur" diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/k/8/k83gwrajz3mzbq960z8vb6sq1mf72n39dbja6a93mfqsy b/implementation/fuzz/inputs-genvm-modules-complete-json/k/8/k83gwrajz3mzbq960z8vb6sq1mf72n39dbja6a93mfqsy new file mode 100644 index 00000000..329aa4e5 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/k/8/k83gwrajz3mzbq960z8vb6sq1mf72n39dbja6a93mfqsy @@ -0,0 +1 @@ +111116111111111111113e-522 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/k/a/karq2vdertx5s0530v4cc7hf5932h40dhsz5wbkh6ac38 b/implementation/fuzz/inputs-genvm-modules-complete-json/k/a/karq2vdertx5s0530v4cc7hf5932h40dhsz5wbkh6ac38 new file mode 100644 index 00000000..a725fedd --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/k/a/karq2vdertx5s0530v4cc7hf5932h40dhsz5wbkh6ac38 @@ -0,0 +1 @@ +"Ρ€\r p" diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/k/b/kbmxbt4vz15rxtvy75a2c7zrcqq80ywr682e36936shdy b/implementation/fuzz/inputs-genvm-modules-complete-json/k/b/kbmxbt4vz15rxtvy75a2c7zrcqq80ywr682e36936shdy new file mode 100644 index 00000000..fb8fc12c --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/k/b/kbmxbt4vz15rxtvy75a2c7zrcqq80ywr682e36936shdy @@ -0,0 +1,3 @@ +{ + "": { + "": {", \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/k/d/kd1dfh3hfn0b7cn92xd8fyrx6jx96e1sp3pwdq64wvpc8 b/implementation/fuzz/inputs-genvm-modules-complete-json/k/d/kd1dfh3hfn0b7cn92xd8fyrx6jx96e1sp3pwdq64wvpc8 new file mode 100644 index 00000000..f58b86b6 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/k/d/kd1dfh3hfn0b7cn92xd8fyrx6jx96e1sp3pwdq64wvpc8 @@ -0,0 +1 @@ +"{p" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/k/d/kdbgas0n2fctkfdx4m1bevmrk333drnreerk4scwtmb0t b/implementation/fuzz/inputs-genvm-modules-complete-json/k/d/kdbgas0n2fctkfdx4m1bevmrk333drnreerk4scwtmb0t new file mode 100644 index 00000000..3e0d4b40 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/k/d/kdbgas0n2fctkfdx4m1bevmrk333drnreerk4scwtmb0t @@ -0,0 +1,17 @@ +{ + "foo": [ + "bar", + "baz\n\\{", + -1e10, + 1e20, + null, + true, + [1, 2, 3] + ], + "bar": [ ], + "brrrr": [ ], + "bar": [ + "єусскиС mbbbbbbbbb", + "mbbbbbbbbbbbbbbaz\t\\uabbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/k/d/kddc8w3t0sa6bgt4hqfmk0s702fhw3aj2s23vrf113vhj b/implementation/fuzz/inputs-genvm-modules-complete-json/k/d/kddc8w3t0sa6bgt4hqfmk0s702fhw3aj2s23vrf113vhj new file mode 100644 index 00000000..10637529 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/k/d/kddc8w3t0sa6bgt4hqfmk0s702fhw3aj2s23vrf113vhj @@ -0,0 +1,17 @@ +{ + "mb": [ + "bar", + "baz\n\\{", + 123, + 123, + 1e20, + false, + [1, 2, 3] + ], + "mbb -barar": [ ], + "": [ ], + "bar": [ + "русскиС Π±ΡƒΠΊΠ²Π‡ ζ—₯本", + "mbbbbbbbbbbbbbbaz\tbbbbbaz\t\\uabbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/k/d/kdkd1aeh2c5te4wzhqg19vgrt8xay96svegd8190h53fa b/implementation/fuzz/inputs-genvm-modules-complete-json/k/d/kdkd1aeh2c5te4wzhqg19vgrt8xay96svegd8190h53fa new file mode 100644 index 00000000..acafb3fe --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/k/d/kdkd1aeh2c5te4wzhqg19vgrt8xay96svegd8190h53fa @@ -0,0 +1,2 @@ + +"Ρ€jᛨ750 1\uDbbb\uDc11.1111111111p" diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/k/g/kgaj8p56a7r8jrt40152sd7fcc4c90j1y2890rqat47kw b/implementation/fuzz/inputs-genvm-modules-complete-json/k/g/kgaj8p56a7r8jrt40152sd7fcc4c90j1y2890rqat47kw new file mode 100644 index 00000000..df19984b --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/k/g/kgaj8p56a7r8jrt40152sd7fcc4c90j1y2890rqat47kw @@ -0,0 +1,2 @@ + +"Ρ€bbbb\\ualbb r" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/k/g/kgmgxgmb7s36z7t5w0c13v2kz5m7bf4qw7bgcys380f08 b/implementation/fuzz/inputs-genvm-modules-complete-json/k/g/kgmgxgmb7s36z7t5w0c13v2kz5m7bf4qw7bgcys380f08 new file mode 100644 index 00000000..5e71d323 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/k/g/kgmgxgmb7s36z7t5w0c13v2kz5m7bf4qw7bgcys380f08 @@ -0,0 +1,33 @@ +[" + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/k/j/kj25phm9g9r65k34nf2bnbvktjcxarpnshyhss1hgzrcm b/implementation/fuzz/inputs-genvm-modules-complete-json/k/j/kj25phm9g9r65k34nf2bnbvktjcxarpnshyhss1hgzrcm new file mode 100644 index 00000000..23934054 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/k/j/kj25phm9g9r65k34nf2bnbvktjcxarpnshyhss1hgzrcm @@ -0,0 +1,2 @@ +[ + - \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/k/j/kjsqsn694mxs46wv9pxk5wr0v7w32h7yq1am1qxnrq3a8 b/implementation/fuzz/inputs-genvm-modules-complete-json/k/j/kjsqsn694mxs46wv9pxk5wr0v7w32h7yq1am1qxnrq3a8 new file mode 100644 index 00000000..9937376a --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/k/j/kjsqsn694mxs46wv9pxk5wr0v7w32h7yq1am1qxnrq3a8 @@ -0,0 +1,2 @@ + [1111111111.1111111e+2, 111E111, 2, 111111111111111111E111, -1e10, + 1e+2, 111E111, 2, 111111111111e+2, 111E+11, 2, 111111111111611111111] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/k/k/kk2z2tx173gmjdhbh0tkeg62s8n7kcsv9gktyt864mhnm b/implementation/fuzz/inputs-genvm-modules-complete-json/k/k/kk2z2tx173gmjdhbh0tkeg62s8n7kcsv9gktyt864mhnm new file mode 100644 index 00000000..12f11141 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/k/k/kk2z2tx173gmjdhbh0tkeg62s8n7kcsv9gktyt864mhnm @@ -0,0 +1,2 @@ + [1111111111.1110111111111e+2, 1111, 2, 1111111111E111, -1e10, + 1e+2, 111E111, 2, 111111111111e+2, 1111, 2, 1111.11, 2, 1111.11111116112, 1111.1111111611111111] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/k/m/km351mdw2h6myk3rettmc5nhmye9h8k67d8rd5s747qx2 b/implementation/fuzz/inputs-genvm-modules-complete-json/k/m/km351mdw2h6myk3rettmc5nhmye9h8k67d8rd5s747qx2 new file mode 100644 index 00000000..168d582a --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/k/m/km351mdw2h6myk3rettmc5nhmye9h8k67d8rd5s747qx2 @@ -0,0 +1,26 @@ + { + "":0, + "" + + + + + + + + + + + + + + + + : { + "": + " + + + + + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/k/n/kn0szcfjdy98emzg60xkmccxk3qdcsk6ads62zsf4vyqm b/implementation/fuzz/inputs-genvm-modules-complete-json/k/n/kn0szcfjdy98emzg60xkmccxk3qdcsk6ads62zsf4vyqm new file mode 100644 index 00000000..52389214 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/k/n/kn0szcfjdy98emzg60xkmccxk3qdcsk6ads62zsf4vyqm @@ -0,0 +1 @@ +2291911216111111111111111 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/k/p/kpeapxmj9qkwvszvqsrktmkcrkvss8wq7dfm9p462p064 b/implementation/fuzz/inputs-genvm-modules-complete-json/k/p/kpeapxmj9qkwvszvqsrktmkcrkvss8wq7dfm9p462p064 new file mode 100644 index 00000000..eb90332a --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/k/p/kpeapxmj9qkwvszvqsrktmkcrkvss8wq7dfm9p462p064 @@ -0,0 +1,2 @@ +{ + "foo": [[[[[[[ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/k/p/kpv7nrbmhdc6556cy33hd5d3fqcw518m68q4gcbsmzpma b/implementation/fuzz/inputs-genvm-modules-complete-json/k/p/kpv7nrbmhdc6556cy33hd5d3fqcw518m68q4gcbsmzpma new file mode 100644 index 00000000..0ba0793e --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/k/p/kpv7nrbmhdc6556cy33hd5d3fqcw518m68q4gcbsmzpma @@ -0,0 +1 @@ +tru \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/k/q/kqkz5qnj4nkj1esk3m2v3tw4hzgcvd4aq2zcv0a0vv90a b/implementation/fuzz/inputs-genvm-modules-complete-json/k/q/kqkz5qnj4nkj1esk3m2v3tw4hzgcvd4aq2zcv0a0vv90a new file mode 100644 index 00000000..b94c32d3 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/k/q/kqkz5qnj4nkj1esk3m2v3tw4hzgcvd4aq2zcv0a0vv90a @@ -0,0 +1 @@ + 22222232222222222222E2 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/k/r/kr3jxhewcm04e4px2m2znvdxjbr686pmewxrstvaxz9he b/implementation/fuzz/inputs-genvm-modules-complete-json/k/r/kr3jxhewcm04e4px2m2znvdxjbr686pmewxrstvaxz9he new file mode 100644 index 0000000000000000000000000000000000000000..21252d769b0914a7d3bde0a4ec55c89c03f4b3fd GIT binary patch literal 139 zcmb>CQUC&_wETP}D}`t<2P}}3Sfr$b%&qdti;1a15;HV3)B|E{z{nV?H5xwv+N8_C M$iTp*AOd6p0Qe>zGXMYp literal 0 HcmV?d00001 diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/k/r/krh4hj84agz7txk2hsapr4r7gbpxac5ttbdefme657ysg b/implementation/fuzz/inputs-genvm-modules-complete-json/k/r/krh4hj84agz7txk2hsapr4r7gbpxac5ttbdefme657ysg new file mode 100644 index 00000000..48d77787 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/k/r/krh4hj84agz7txk2hsapr4r7gbpxac5ttbdefme657ysg @@ -0,0 +1 @@ + [ { } ] diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/k/s/ksqdnet3e2y8ztax53q7hddaej4xqjgaz5kveh5eeyxbw b/implementation/fuzz/inputs-genvm-modules-complete-json/k/s/ksqdnet3e2y8ztax53q7hddaej4xqjgaz5kveh5eeyxbw new file mode 100644 index 00000000..c27ee231 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/k/s/ksqdnet3e2y8ztax53q7hddaej4xqjgaz5kveh5eeyxbw @@ -0,0 +1 @@ +111.1115 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/k/t/kt8wttecepmpdjfnc0ht0x0gwb6a1jq03s3yj444kqjcy b/implementation/fuzz/inputs-genvm-modules-complete-json/k/t/kt8wttecepmpdjfnc0ht0x0gwb6a1jq03s3yj444kqjcy new file mode 100644 index 00000000..cd15dfcb --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/k/t/kt8wttecepmpdjfnc0ht0x0gwb6a1jq03s3yj444kqjcy @@ -0,0 +1,15 @@ +{ + "fo + + + + + + + + + + + + +[[ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/k/t/ktafsc8tmnxj0cxxmr3jes8gz76p54fs8wzfqh7pz9274 b/implementation/fuzz/inputs-genvm-modules-complete-json/k/t/ktafsc8tmnxj0cxxmr3jes8gz76p54fs8wzfqh7pz9274 new file mode 100644 index 00000000..dd013145 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/k/t/ktafsc8tmnxj0cxxmr3jes8gz76p54fs8wzfqh7pz9274 @@ -0,0 +1 @@ + "с\fffffffffffffffffff\fffffffffvabba\f/////////////////ffffff@ffffffffffffffffffabba\//////fffffffffffffffffffffffabba\//////////////////////-/111111111111111/////bbbb" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/k/y/ky0p59y4jd3yq8j6gz8zz668capv25c5ya0s58qydxcst b/implementation/fuzz/inputs-genvm-modules-complete-json/k/y/ky0p59y4jd3yq8j6gz8zz668capv25c5ya0s58qydxcst new file mode 100644 index 00000000..a0aeede1 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/k/y/ky0p59y4jd3yq8j6gz8zz668capv25c5ya0s58qydxcst @@ -0,0 +1 @@ +fa \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/k/y/ky4mbkfs19k1fmkfhx1ej4jqns8h1v8dyj58eg6j33dmt b/implementation/fuzz/inputs-genvm-modules-complete-json/k/y/ky4mbkfs19k1fmkfhx1ej4jqns8h1v8dyj58eg6j33dmt new file mode 100644 index 00000000..cd67668e --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/k/y/ky4mbkfs19k1fmkfhx1ej4jqns8h1v8dyj58eg6j33dmt @@ -0,0 +1 @@ +1111111111121105111111e-625 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/k/y/kyst24ensy38wm2t0at68frg52g4g812hm20ws9ne1mde b/implementation/fuzz/inputs-genvm-modules-complete-json/k/y/kyst24ensy38wm2t0at68frg52g4g812hm20ws9ne1mde new file mode 100644 index 00000000..1fbf0d2d --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/k/y/kyst24ensy38wm2t0at68frg52g4g812hm20ws9ne1mde @@ -0,0 +1,19 @@ +{ + + "barrrrrrrrrr": [ ], + "foo": [ + "bar", + "baz\n\\{", + -1e10, + 1e20, + null, + true, + [1, 2, 3] + ], + "barrrrrrrrrr": [ ], + "brrrr": [ ], + "bar": [ + "русскиС mbbbbbbbbb", + "mbbbbbbbbbbbbbbaz\t\\uabbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/k/y/kyx60vyjpbs117mf2gz69h6enprvzm0hcwt3z641kf5xe b/implementation/fuzz/inputs-genvm-modules-complete-json/k/y/kyx60vyjpbs117mf2gz69h6enprvzm0hcwt3z641kf5xe new file mode 100644 index 00000000..c8eddce8 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/k/y/kyx60vyjpbs117mf2gz69h6enprvzm0hcwt3z641kf5xe @@ -0,0 +1 @@ +[1,22,, ' \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/k/z/kz3m9qrp2a5axfa9jy5h8ewtxanwygafwhx4r5fwbztce b/implementation/fuzz/inputs-genvm-modules-complete-json/k/z/kz3m9qrp2a5axfa9jy5h8ewtxanwygafwhx4r5fwbztce new file mode 100644 index 00000000..03c502e6 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/k/z/kz3m9qrp2a5axfa9jy5h8ewtxanwygafwhx4r5fwbztce @@ -0,0 +1,5 @@ +{"a + + + +, \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/m/0/m0wmr0cmdm54pr7gsbxyskvymrvkdyh003phfftngp9dj b/implementation/fuzz/inputs-genvm-modules-complete-json/m/0/m0wmr0cmdm54pr7gsbxyskvymrvkdyh003phfftngp9dj new file mode 100644 index 00000000..54eb83c6 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/m/0/m0wmr0cmdm54pr7gsbxyskvymrvkdyh003phfftngp9dj @@ -0,0 +1,3 @@ + [1111111111.1111111111111e+2, 1111, 2, 1111.111111E111, -1e10, + 1e+2, 111E111, 2, 111111111111e+2, 1111, 2, 1111.11, 2, 1111.11111116112, -1e10, + 1111] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/m/1/m1qhqarrsg7mnv6d49q2ksgb6q2afqe8z16ct7dkxy9aw b/implementation/fuzz/inputs-genvm-modules-complete-json/m/1/m1qhqarrsg7mnv6d49q2ksgb6q2afqe8z16ct7dkxy9aw new file mode 100644 index 00000000..7f2a262e --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/m/1/m1qhqarrsg7mnv6d49q2ksgb6q2afqe8z16ct7dkxy9aw @@ -0,0 +1 @@ + "bIbbς€»ƒz\n\/222222ссς€»ƒU2222222222сскиʡ6Π±ΡƒΠΊbbcbbbbbIbbς€»ƒz\n\/222222сскиʡ6Π±bΪ²Ρ‹ ζ—₯12 & Bxbz" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/m/3/m32t8fg331gpkvmxp9mkgyxyk8r1ndbqtfxxh7904mkda b/implementation/fuzz/inputs-genvm-modules-complete-json/m/3/m32t8fg331gpkvmxp9mkgyxyk8r1ndbqtfxxh7904mkda new file mode 100644 index 00000000..d51fe04c --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/m/3/m32t8fg331gpkvmxp9mkgyxyk8r1ndbqtfxxh7904mkda @@ -0,0 +1,21 @@ +{ + "foo": [ + "bar", + "baz\n\\z", + 123, + -123, + -1e10, + 1e20, + null, + true, + false, + [1, 2, 3] + +], + "bar" + : [ + "руссºиС Π±ΡƒΠΊΠ²Ρ‹ ζ—₯本\uabbbbb", + "m1111111111111111111111111bbbbbbaz\t", + "mbbΠ²Ρ‹ ζ—₯本\uabbbbbbaz\t\\uabbbbb" + ] +} \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/m/3/m3ahpq4mgferty6rhpdp79xvte4cfjmzwtf53kmsxaz4g b/implementation/fuzz/inputs-genvm-modules-complete-json/m/3/m3ahpq4mgferty6rhpdp79xvte4cfjmzwtf53kmsxaz4g new file mode 100644 index 00000000..1dcd97c3 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/m/3/m3ahpq4mgferty6rhpdp79xvte4cfjmzwtf53kmsxaz4g @@ -0,0 +1,3 @@ +{ + "f[ + " \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/m/3/m3mzkfk06j8qbanxvsdd0ctez8e8js8qq86w1n8s85kzj b/implementation/fuzz/inputs-genvm-modules-complete-json/m/3/m3mzkfk06j8qbanxvsdd0ctez8e8js8qq86w1n8s85kzj new file mode 100644 index 00000000..5f075631 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/m/3/m3mzkfk06j8qbanxvsdd0ctez8e8js8qq86w1n8s85kzj @@ -0,0 +1 @@ + [1, 2, 3] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/m/4/m493qx6671gfk42fjcpzapxgjv932b4najhcdcvb87vcy b/implementation/fuzz/inputs-genvm-modules-complete-json/m/4/m493qx6671gfk42fjcpzapxgjv932b4najhcdcvb87vcy new file mode 100644 index 00000000..8ed2b59c --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/m/4/m493qx6671gfk42fjcpzapxgjv932b4najhcdcvb87vcy @@ -0,0 +1,5 @@ +[[[[[[[[ + + +3,[[[[[[[[[[[[[[[[[[[[[[[[[[ [[[[[[[[[[[[[ [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ +[[[[[[[[[[[[[[[[ null,[ 11111111111E1 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/m/5/m5972ey1t36ymxs03rc2yz80xavs5dfjfc8gy1zgsm2yc b/implementation/fuzz/inputs-genvm-modules-complete-json/m/5/m5972ey1t36ymxs03rc2yz80xavs5dfjfc8gy1zgsm2yc new file mode 100644 index 00000000..5c281734 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/m/5/m5972ey1t36ymxs03rc2yz80xavs5dfjfc8gy1zgsm2yc @@ -0,0 +1 @@ + 22222222222222222222.Q \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/m/6/m68has31kcvzdmf3rmzmc54tja0yk4ebs21kt9v1htm16 b/implementation/fuzz/inputs-genvm-modules-complete-json/m/6/m68has31kcvzdmf3rmzmc54tja0yk4ebs21kt9v1htm16 new file mode 100644 index 0000000000000000000000000000000000000000..2cf4ff3d4da5ac78b2190d90fcbc9d8de017b425 GIT binary patch literal 20 Vcma!!=2GD0<x&7b2w;eg0RS0T0!#n^ literal 0 HcmV?d00001 diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/m/6/m6abvdrjj88h71m40pamp9x2ty7p1v0h5afjcscnhfjvm b/implementation/fuzz/inputs-genvm-modules-complete-json/m/6/m6abvdrjj88h71m40pamp9x2ty7p1v0h5afjcscnhfjvm new file mode 100644 index 00000000..ca71c442 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/m/6/m6abvdrjj88h71m40pamp9x2ty7p1v0h5afjcscnhfjvm @@ -0,0 +1,21 @@ + { + "":0, + "" + + + + + + + + + + + : { + "": + " + + + + + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/m/7/m7dpad9g37xfp5826mapya0j9fjfasbja8fsz721k7hse b/implementation/fuzz/inputs-genvm-modules-complete-json/m/7/m7dpad9g37xfp5826mapya0j9fjfasbja8fsz721k7hse new file mode 100644 index 00000000..f6f25307 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/m/7/m7dpad9g37xfp5826mapya0j9fjfasbja8fsz721k7hse @@ -0,0 +1 @@ +2E+22 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/m/7/m7k6a08a240xtex9khqvm8zkj6v4ane2yqvajq13a64q8 b/implementation/fuzz/inputs-genvm-modules-complete-json/m/7/m7k6a08a240xtex9khqvm8zkj6v4ane2yqvajq13a64q8 new file mode 100644 index 00000000..29de0d71 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/m/7/m7k6a08a240xtex9khqvm8zkj6v4ane2yqvajq13a64q8 @@ -0,0 +1,2 @@ + +"Ρ€bbbb\\ua11, bb\\ua11A r" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/m/9/m9ypnkqd0fd7b89g1r8zw2sn6t1ffhc34chtkb4zt4g6p b/implementation/fuzz/inputs-genvm-modules-complete-json/m/9/m9ypnkqd0fd7b89g1r8zw2sn6t1ffhc34chtkb4zt4g6p new file mode 100644 index 00000000..218b9b07 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/m/9/m9ypnkqd0fd7b89g1r8zw2sn6t1ffhc34chtkb4zt4g6p @@ -0,0 +1 @@ + "b50 " \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/m/a/mabk4cpr62tte8fpee0h93ej82ebadtdmzev02ankgxxc b/implementation/fuzz/inputs-genvm-modules-complete-json/m/a/mabk4cpr62tte8fpee0h93ej82ebadtdmzev02ankgxxc new file mode 100644 index 00000000..501f5aba --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/m/a/mabk4cpr62tte8fpee0h93ej82ebadtdmzev02ankgxxc @@ -0,0 +1,4 @@ +{ + "gggggggggggggggggg224222gggggggggoo": [ + 123, +[[[[[[[ ]]]]]]], "222 -1e  [[ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/m/b/mb6qw3wvss3wv90ty3689ayg9bh3hm1yj1v81d7mmkmrm b/implementation/fuzz/inputs-genvm-modules-complete-json/m/b/mb6qw3wvss3wv90ty3689ayg9bh3hm1yj1v81d7mmkmrm new file mode 100644 index 00000000..3900c232 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/m/b/mb6qw3wvss3wv90ty3689ayg9bh3hm1yj1v81d7mmkmrm @@ -0,0 +1,2 @@ + +"Ρ€bbbb\\ua11, r" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/m/b/mbn6cytj7h2etyzm73q23gzwb3r3zzn46a26147c3ngtt b/implementation/fuzz/inputs-genvm-modules-complete-json/m/b/mbn6cytj7h2etyzm73q23gzwb3r3zzn46a26147c3ngtt new file mode 100644 index 00000000..874452fa --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/m/b/mbn6cytj7h2etyzm73q23gzwb3r3zzn46a26147c3ngtt @@ -0,0 +1,20 @@ + [, + + + + + + + + + + + + + + + + + + + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/m/c/mc2g13cjhwvkmnjkzhap1wv65s0d6mrsfnpvgqjd4d43t b/implementation/fuzz/inputs-genvm-modules-complete-json/m/c/mc2g13cjhwvkmnjkzhap1wv65s0d6mrsfnpvgqjd4d43t new file mode 100644 index 00000000..052fca9f --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/m/c/mc2g13cjhwvkmnjkzhap1wv65s0d6mrsfnpvgqjd4d43t @@ -0,0 +1 @@ +[[[[[[[ az\ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/m/c/mcvp61x7qvhm54q7venkq2j0rxvcj8pkz54tktsmz696m b/implementation/fuzz/inputs-genvm-modules-complete-json/m/c/mcvp61x7qvhm54q7venkq2j0rxvcj8pkz54tktsmz696m new file mode 100644 index 00000000..ce0882a3 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/m/c/mcvp61x7qvhm54q7venkq2j0rxvcj8pkz54tktsmz696m @@ -0,0 +1,18 @@ +{ + "foo": [ + "bar", + 123, + -123, + -1e10, + 1e-0, + null, + true, + false, + [1111111111111e-71512, 3] + ], + "bar": [ + "русскиС Π±ΡƒΠΊΠ²Ρ‹ ζ—₯本", + "mbbbbbbbbbbbbbbaz\t\uabbbbb", + "mbbbbbbabbbbbbbaz\t\\uabbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/m/d/md3nvsew9f6adkessshmv0jp07cs9t1ammp05wfapy95w b/implementation/fuzz/inputs-genvm-modules-complete-json/m/d/md3nvsew9f6adkessshmv0jp07cs9t1ammp05wfapy95w new file mode 100644 index 00000000..f0629a68 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/m/d/md3nvsew9f6adkessshmv0jp07cs9t1ammp05wfapy95w @@ -0,0 +1,2 @@ + true + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/m/d/mdzbn1tkad4dcz38gynvv3b9vjjxa753a6smp4w0gq8fw b/implementation/fuzz/inputs-genvm-modules-complete-json/m/d/mdzbn1tkad4dcz38gynvv3b9vjjxa753a6smp4w0gq8fw new file mode 100644 index 00000000..b8aeb8bd --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/m/d/mdzbn1tkad4dcz38gynvv3b9vjjxa753a6smp4w0gq8fw @@ -0,0 +1 @@ +333333333333333333333333333333333 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/m/f/mfe555xm1czdmsz55emnt639nhve81rb305236qf5cvw2 b/implementation/fuzz/inputs-genvm-modules-complete-json/m/f/mfe555xm1czdmsz55emnt639nhve81rb305236qf5cvw2 new file mode 100644 index 00000000..6122d1ce --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/m/f/mfe555xm1czdmsz55emnt639nhve81rb305236qf5cvw2 @@ -0,0 +1 @@ +0e5555555515 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/m/f/mfp4jnm2pgnprzney40fatb7pp46t42kb5fk5bwagzksy b/implementation/fuzz/inputs-genvm-modules-complete-json/m/f/mfp4jnm2pgnprzney40fatb7pp46t42kb5fk5bwagzksy new file mode 100644 index 00000000..b963128c --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/m/f/mfp4jnm2pgnprzney40fatb7pp46t42kb5fk5bwagzksy @@ -0,0 +1 @@ +[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/m/f/mftybwf9g3rm5g31dmygq0ksw3hdxwpmr2msqhz46581g b/implementation/fuzz/inputs-genvm-modules-complete-json/m/f/mftybwf9g3rm5g31dmygq0ksw3hdxwpmr2msqhz46581g new file mode 100644 index 00000000..822d04f7 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/m/f/mftybwf9g3rm5g31dmygq0ksw3hdxwpmr2msqhz46581g @@ -0,0 +1,20 @@ +{ + "foo": [ + "bar" + ], + "bar": [ + 123, + -123, + -1e10, + 1e20, + null, + true, + false, + [1, 2, 3] + ], + "bar": [ + "русскиС Π±ΡƒΠΊΠ²Ρ‹ ζ—₯本", + "mbbbbbbbbbbbbbbaz\t\uabbbbb", + "mbbbbbbbbbbbbbbaz\t\\uabbbbb" + ] +} \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/m/g/mg5f0w63svmgw0k720qpmarnn3b7x1jfd6sxzfjrv0qsj b/implementation/fuzz/inputs-genvm-modules-complete-json/m/g/mg5f0w63svmgw0k720qpmarnn3b7x1jfd6sxzfjrv0qsj new file mode 100644 index 00000000..8633d5fd --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/m/g/mg5f0w63svmgw0k720qpmarnn3b7x1jfd6sxzfjrv0qsj @@ -0,0 +1,3 @@ +{ + "11111111111111": 1e20, + "13o" * \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/m/h/mh89vhjanjap5bqrg5n0y4awn9ey3b9hv04y0505ycf8m b/implementation/fuzz/inputs-genvm-modules-complete-json/m/h/mh89vhjanjap5bqrg5n0y4awn9ey3b9hv04y0505ycf8m new file mode 100644 index 00000000..1ab4a6c4 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/m/h/mh89vhjanjap5bqrg5n0y4awn9ey3b9hv04y0505ycf8m @@ -0,0 +1,2 @@ + [{ "\\uba o": [ ] +} ] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/m/h/mhkkk8x1hczb24rycg0wk93k0n71vrkd129shee2kb770 b/implementation/fuzz/inputs-genvm-modules-complete-json/m/h/mhkkk8x1hczb24rycg0wk93k0n71vrkd129shee2kb770 new file mode 100644 index 00000000..2540ef53 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/m/h/mhkkk8x1hczb24rycg0wk93k0n71vrkd129shee2kb770 @@ -0,0 +1,2 @@ + [1, 7, 3] + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/m/j/mj61caxe8khbx3femfbaeeqyf8rtjxxnvh7peyzvr6nr8 b/implementation/fuzz/inputs-genvm-modules-complete-json/m/j/mj61caxe8khbx3femfbaeeqyf8rtjxxnvh7peyzvr6nr8 new file mode 100644 index 00000000..9dce60cb --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/m/j/mj61caxe8khbx3femfbaeeqyf8rtjxxnvh7peyzvr6nr8 @@ -0,0 +1 @@ +"\fζ—₯12 \f\\ ζ—₯1 " \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/m/q/mq2w64nvaq0ek0d3n4trpsnkrx13bh73eexhqy3qz1n50 b/implementation/fuzz/inputs-genvm-modules-complete-json/m/q/mq2w64nvaq0ek0d3n4trpsnkrx13bh73eexhqy3qz1n50 new file mode 100644 index 00000000..87c7412f --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/m/q/mq2w64nvaq0ek0d3n4trpsnkrx13bh73eexhqy3qz1n50 @@ -0,0 +1 @@ +"<&\f]}" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/m/s/msjqvqw6nxfk7ccsbfqbphcbtjm3chag6qdy7t667dvxw b/implementation/fuzz/inputs-genvm-modules-complete-json/m/s/msjqvqw6nxfk7ccsbfqbphcbtjm3chag6qdy7t667dvxw new file mode 100644 index 00000000..3ba26305 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/m/s/msjqvqw6nxfk7ccsbfqbphcbtjm3chag6qdy7t667dvxw @@ -0,0 +1,4 @@ +[[[[[[[[[[[[[ + + +3,[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ [[[[[[[[[[[[[ [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/m/w/mw17vyjk6cj3yh3k8f0rdzt65b7j6tp7vzhht4e8g3ed6 b/implementation/fuzz/inputs-genvm-modules-complete-json/m/w/mw17vyjk6cj3yh3k8f0rdzt65b7j6tp7vzhht4e8g3ed6 new file mode 100644 index 00000000..6cd9e197 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/m/w/mw17vyjk6cj3yh3k8f0rdzt65b7j6tp7vzhht4e8g3ed6 @@ -0,0 +1,2 @@ +{ " :111111111111111e-52222 f %o": [ ] +} \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/m/w/mw3bwkxt1sgswfqnxb2ksghfs930jn8wt94e4ftsrh84g b/implementation/fuzz/inputs-genvm-modules-complete-json/m/w/mw3bwkxt1sgswfqnxb2ksghfs930jn8wt94e4ftsrh84g new file mode 100644 index 00000000..0796eef5 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/m/w/mw3bwkxt1sgswfqnxb2ksghfs930jn8wt94e4ftsrh84g @@ -0,0 +1 @@ + [1111e-1611116111111112, 311, 11E1] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/m/w/mw95h82x0xmdasfnqp69gzrv0kag7fyytg72g7wsqp1qe b/implementation/fuzz/inputs-genvm-modules-complete-json/m/w/mw95h82x0xmdasfnqp69gzrv0kag7fyytg72g7wsqp1qe new file mode 100644 index 00000000..fc2f3ad2 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/m/w/mw95h82x0xmdasfnqp69gzrv0kag7fyytg72g7wsqp1qe @@ -0,0 +1,30 @@ +{ "foo": [ + "bar", + "baz\n\\{", + -1e10, + 1e20, + null, + true, + [1, 2, 3] + ], + "bar": [ ], + + + + + + + + + + + + + + + + "bar": [ + "русскиС mbbbbbbbbbbbbbbaz\t\uabbbbb", + "mbbbbbbbbbbbbbbaz\t\\uabbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/m/w/mwvwy5wqq8a78r9krk8bdrkc6dgy4rfwcjjjm0d6n04zm b/implementation/fuzz/inputs-genvm-modules-complete-json/m/w/mwvwy5wqq8a78r9krk8bdrkc6dgy4rfwcjjjm0d6n04zm new file mode 100644 index 00000000..cc69198b --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/m/w/mwvwy5wqq8a78r9krk8bdrkc6dgy4rfwcjjjm0d6n04zm @@ -0,0 +1,3 @@ +{ + "foo": [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + [[[[[[[[[[[[ "bar", ]]]]]]]]]]]]]]]]]]]] ,,,,,,,,,,,,,,,,,,,,,,,,,,,M,,,,,&! \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/m/x/mxys9rn0bks5a8stzkxj5z2e617qr56dn9r6y4amjf6aw b/implementation/fuzz/inputs-genvm-modules-complete-json/m/x/mxys9rn0bks5a8stzkxj5z2e617qr56dn9r6y4amjf6aw new file mode 100644 index 00000000..33908ee7 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/m/x/mxys9rn0bks5a8stzkxj5z2e617qr56dn9r6y4amjf6aw @@ -0,0 +1 @@ +"f\\\"f ζ—₯" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/n/1/n1dtcpmdyg4mv2echk5hmx1gw435cy74yzqe63rv3w2xe b/implementation/fuzz/inputs-genvm-modules-complete-json/n/1/n1dtcpmdyg4mv2echk5hmx1gw435cy74yzqe63rv3w2xe new file mode 100644 index 00000000..fcc2eaa8 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/n/1/n1dtcpmdyg4mv2echk5hmx1gw435cy74yzqe63rv3w2xe @@ -0,0 +1 @@ +"\/ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/n/1/n1r5h883zt39y2k6qb2s6t6aj88pdr0f3vfs1fw18y5gr b/implementation/fuzz/inputs-genvm-modules-complete-json/n/1/n1r5h883zt39y2k6qb2s6t6aj88pdr0f3vfs1fw18y5gr new file mode 100644 index 00000000..1e3d036b --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/n/1/n1r5h883zt39y2k6qb2s6t6aj88pdr0f3vfs1fw18y5gr @@ -0,0 +1,15 @@ +{ + "foo": [ + "bar", + "baz\n\\z", + 12222222222222222222222222222e10, + 1e20, + null, + true, + false, + [1, 2, 3] + ], + "bar": [[[[[[[[[[[[[[[[[[[[[[[[[ + "русскиС Π±ΡƒΠΊΠ²Ρ‹ ζ—₯本", + "mgbbbbbbbbbbbbbaz\t\uabbbbb", +8 "mb \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/n/1/n1v7ztryqh0pqzj368wwbxqkvm7qbvs33n66hxx7nrqwm b/implementation/fuzz/inputs-genvm-modules-complete-json/n/1/n1v7ztryqh0pqzj368wwbxqkvm7qbvs33n66hxx7nrqwm new file mode 100644 index 00000000..eb3abdc2 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/n/1/n1v7ztryqh0pqzj368wwbxqkvm7qbvs33n66hxx7nrqwm @@ -0,0 +1 @@ +1.11111e \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/n/2/n2hmwrdvrc4nsc74704jejejj7axdre87g6zpggvrbd3r b/implementation/fuzz/inputs-genvm-modules-complete-json/n/2/n2hmwrdvrc4nsc74704jejejj7axdre87g6zpggvrbd3r new file mode 100644 index 00000000..2d72a13d --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/n/2/n2hmwrdvrc4nsc74704jejejj7axdre87g6zpggvrbd3r @@ -0,0 +1,3 @@ + +[ { +} ] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/n/2/n2j89f9aqmhsv9bnyeqzn91bpdt7b98hfef1h55f3svva b/implementation/fuzz/inputs-genvm-modules-complete-json/n/2/n2j89f9aqmhsv9bnyeqzn91bpdt7b98hfef1h55f3svva new file mode 100644 index 00000000..6369edf7 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/n/2/n2j89f9aqmhsv9bnyeqzn91bpdt7b98hfef1h55f3svva @@ -0,0 +1 @@ +11111222112222222222222 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/n/3/n3g9vf59rm5hmm5f7jxhc5st3mtygp36p0x9wwj2qtnbw b/implementation/fuzz/inputs-genvm-modules-complete-json/n/3/n3g9vf59rm5hmm5f7jxhc5st3mtygp36p0x9wwj2qtnbw new file mode 100644 index 00000000..0395deee --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/n/3/n3g9vf59rm5hmm5f7jxhc5st3mtygp36p0x9wwj2qtnbw @@ -0,0 +1,2 @@ + [1111111111.1110111111E111, -1e11, + 1e+2, 111E111, 2, 111111111111e+2, 1111, 2, 1111.11, 2, 1111.11111116112, 11111.111111110111111E1 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/n/5/n5159m6kqnn4x3frndv6zm7acsqtt2abaknc1zk5px8ce b/implementation/fuzz/inputs-genvm-modules-complete-json/n/5/n5159m6kqnn4x3frndv6zm7acsqtt2abaknc1zk5px8ce new file mode 100644 index 00000000..3acfd58a --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/n/5/n5159m6kqnn4x3frndv6zm7acsqtt2abaknc1zk5px8ce @@ -0,0 +1,2 @@ +{ + " "fo \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/n/7/n7dy1dsabxynmvt3x0c6gehwykk90p57sm0dw4yv30wn2 b/implementation/fuzz/inputs-genvm-modules-complete-json/n/7/n7dy1dsabxynmvt3x0c6gehwykk90p57sm0dw4yv30wn2 new file mode 100644 index 00000000..3330307f --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/n/7/n7dy1dsabxynmvt3x0c6gehwykk90p57sm0dw4yv30wn2 @@ -0,0 +1 @@ +"}f\\\\\\ζ·₯" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/n/8/n83y3d9a13g5dmxd7kahqes3eqy60fcv6v0xd2dhd783j b/implementation/fuzz/inputs-genvm-modules-complete-json/n/8/n83y3d9a13g5dmxd7kahqes3eqy60fcv6v0xd2dhd783j new file mode 100644 index 00000000..bf79039c --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/n/8/n83y3d9a13g5dmxd7kahqes3eqy60fcv6v0xd2dhd783j @@ -0,0 +1 @@ +-123 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/n/9/n9330495hyfzbhes4m5b24jh013b6wkyrb1nv59takz1e b/implementation/fuzz/inputs-genvm-modules-complete-json/n/9/n9330495hyfzbhes4m5b24jh013b6wkyrb1nv59takz1e new file mode 100644 index 00000000..51a89349 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/n/9/n9330495hyfzbhes4m5b24jh013b6wkyrb1nv59takz1e @@ -0,0 +1 @@ +{ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/n/9/n9afwk9mv069s48jzh27mv334kwn483b343g05psgg1h0 b/implementation/fuzz/inputs-genvm-modules-complete-json/n/9/n9afwk9mv069s48jzh27mv334kwn483b343g05psgg1h0 new file mode 100644 index 00000000..a7c31961 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/n/9/n9afwk9mv069s48jzh27mv334kwn483b343g05psgg1h0 @@ -0,0 +1,2 @@ +{ " 111111111111111e-52222 f g": [ ] +} \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/n/g/ng5dswqsnt0nz180w1njgh84rsevdd04qvzx737tqp4ny b/implementation/fuzz/inputs-genvm-modules-complete-json/n/g/ng5dswqsnt0nz180w1njgh84rsevdd04qvzx737tqp4ny new file mode 100644 index 00000000..9c93de0d --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/n/g/ng5dswqsnt0nz180w1njgh84rsevdd04qvzx737tqp4ny @@ -0,0 +1 @@ +fal \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/n/g/ng7y0bw3s81dq313y936cngh5v3vbxst6c4s91v444jr2 b/implementation/fuzz/inputs-genvm-modules-complete-json/n/g/ng7y0bw3s81dq313y936cngh5v3vbxst6c4s91v444jr2 new file mode 100644 index 00000000..77b9a119 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/n/g/ng7y0bw3s81dq313y936cngh5v3vbxst6c4s91v444jr2 @@ -0,0 +1 @@ +[[1, ,,,,, \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/n/g/ngk6cm42qnjr54s0gwv2gehm9t0v2kq8p3fb84fdbq6wm b/implementation/fuzz/inputs-genvm-modules-complete-json/n/g/ngk6cm42qnjr54s0gwv2gehm9t0v2kq8p3fb84fdbq6wm new file mode 100644 index 00000000..e1e7e9e9 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/n/g/ngk6cm42qnjr54s0gwv2gehm9t0v2kq8p3fb84fdbq6wm @@ -0,0 +1,2 @@ + +"Ρ€]r" diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/n/j/njm94atgazn4sqp0y06bfer99f5n9d0z47251b0t1nbrc b/implementation/fuzz/inputs-genvm-modules-complete-json/n/j/njm94atgazn4sqp0y06bfer99f5n9d0z47251b0t1nbrc new file mode 100644 index 00000000..f1c192f4 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/n/j/njm94atgazn4sqp0y06bfer99f5n9d0z47251b0t1nbrc @@ -0,0 +1 @@ +111116111111111111113e-5221 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/n/j/njx918nmttxgteyaykpskyqswf27byd6dv7v3c6mr9ce8 b/implementation/fuzz/inputs-genvm-modules-complete-json/n/j/njx918nmttxgteyaykpskyqswf27byd6dv7v3c6mr9ce8 new file mode 100644 index 00000000..cd5c8fdf --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/n/j/njx918nmttxgteyaykpskyqswf27byd6dv7v3c6mr9ce8 @@ -0,0 +1 @@ + "bξ—¨750 :UwUoUσ—₯£" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/n/m/nmysrgc94jph3xbps9f1zmwdyb5z68rtbqmhx2j43rv54 b/implementation/fuzz/inputs-genvm-modules-complete-json/n/m/nmysrgc94jph3xbps9f1zmwdyb5z68rtbqmhx2j43rv54 new file mode 100644 index 00000000..5576a040 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/n/m/nmysrgc94jph3xbps9f1zmwdyb5z68rtbqmhx2j43rv54 @@ -0,0 +1,6 @@ +{ + "fo + + + +o": \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/n/s/nsvc3ayxsqq26r2y7ehk3t0x139cmem9fyjmx7q07yr70 b/implementation/fuzz/inputs-genvm-modules-complete-json/n/s/nsvc3ayxsqq26r2y7ehk3t0x139cmem9fyjmx7q07yr70 new file mode 100644 index 00000000..6b47d665 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/n/s/nsvc3ayxsqq26r2y7ehk3t0x139cmem9fyjmx7q07yr70 @@ -0,0 +1 @@ + {" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/n/t/ntftmawhwhs2ha6c76dxh02gswf2h4vh6vham7m3sfdct b/implementation/fuzz/inputs-genvm-modules-complete-json/n/t/ntftmawhwhs2ha6c76dxh02gswf2h4vh6vham7m3sfdct new file mode 100644 index 00000000..ae943fc4 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/n/t/ntftmawhwhs2ha6c76dxh02gswf2h4vh6vham7m3sfdct @@ -0,0 +1,2 @@ +{ +  \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/n/t/ntg935cet85vqftahbkvazp0hy57ee73y1gccm9hf8gqc b/implementation/fuzz/inputs-genvm-modules-complete-json/n/t/ntg935cet85vqftahbkvazp0hy57ee73y1gccm9hf8gqc new file mode 100644 index 00000000..c91ef8cb --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/n/t/ntg935cet85vqftahbkvazp0hy57ee73y1gccm9hf8gqc @@ -0,0 +1 @@ +22222222222222222222222E2 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/n/t/ntq3r5tp52qea2p58k13t20fp1y8z4y6sctkjf8bhmyec b/implementation/fuzz/inputs-genvm-modules-complete-json/n/t/ntq3r5tp52qea2p58k13t20fp1y8z4y6sctkjf8bhmyec new file mode 100644 index 00000000..14f5d4d7 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/n/t/ntq3r5tp52qea2p58k13t20fp1y8z4y6sctkjf8bhmyec @@ -0,0 +1 @@ +666666666668666666666 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/n/v/nvav67fs6raebm8yy7tnxdrfn4nstrdcc7xtwj7ec4864 b/implementation/fuzz/inputs-genvm-modules-complete-json/n/v/nvav67fs6raebm8yy7tnxdrfn4nstrdcc7xtwj7ec4864 new file mode 100644 index 00000000..a8dd4ab7 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/n/v/nvav67fs6raebm8yy7tnxdrfn4nstrdcc7xtwj7ec4864 @@ -0,0 +1 @@ +111.11 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/n/x/nx9ndd6wvxxe9pztv02yrhqsk0qhkk03hhsb6z4xqpqg6 b/implementation/fuzz/inputs-genvm-modules-complete-json/n/x/nx9ndd6wvxxe9pztv02yrhqsk0qhkk03hhsb6z4xqpqg6 new file mode 100644 index 00000000..5dd8b28b --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/n/x/nx9ndd6wvxxe9pztv02yrhqsk0qhkk03hhsb6z4xqpqg6 @@ -0,0 +1 @@ +[[ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/n/x/nxhz15vspeftz6hvkdstfgdnmv7q9bh5v5tpasqshkx74 b/implementation/fuzz/inputs-genvm-modules-complete-json/n/x/nxhz15vspeftz6hvkdstfgdnmv7q9bh5v5tpasqshkx74 new file mode 100644 index 00000000..1b983528 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/n/x/nxhz15vspeftz6hvkdstfgdnmv7q9bh5v5tpasqshkx74 @@ -0,0 +1 @@ +222222222.52222252121e12 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/n/y/nysnaj2py5tjqqj394c5ndb89ycet34x94fw8dygjd4ew b/implementation/fuzz/inputs-genvm-modules-complete-json/n/y/nysnaj2py5tjqqj394c5ndb89ycet34x94fw8dygjd4ew new file mode 100644 index 00000000..36b421d4 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/n/y/nysnaj2py5tjqqj394c5ndb89ycet34x94fw8dygjd4ew @@ -0,0 +1,16 @@ +{ + "foo": [ + "bar", + "baz\n\\z", + 123, + -123, + -1e10, + null, + true, + false, + [1, 2, 3] + ], + "bar": [ + "mbbbbbbbbbbbbbbaz\t\\uabbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/n/z/nzf01kbbdv8hwahc86r3tjncbce3gynjtrz5t02p99bqy b/implementation/fuzz/inputs-genvm-modules-complete-json/n/z/nzf01kbbdv8hwahc86r3tjncbce3gynjtrz5t02p99bqy new file mode 100644 index 00000000..b06b8302 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/n/z/nzf01kbbdv8hwahc86r3tjncbce3gynjtrz5t02p99bqy @@ -0,0 +1 @@ +"\/" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/p/0/p0p3c0xzs6tr5c3y2meepqx344ra5nnwzj5k4p3w197yw b/implementation/fuzz/inputs-genvm-modules-complete-json/p/0/p0p3c0xzs6tr5c3y2meepqx344ra5nnwzj5k4p3w197yw new file mode 100644 index 00000000..c4c4bd73 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/p/0/p0p3c0xzs6tr5c3y2meepqx344ra5nnwzj5k4p3w197yw @@ -0,0 +1,17 @@ +{ + "foo": [ + "bar", + "baz\n\\{", + -1e10, + 1e20, + null, + true, + [1, 2, 3] + ], + "barrrrrrrrrr": [ ], + "brrrr": [ ], + "bar": [ + "русскиС mbbbbbbbbb", + "mbbbbbbbbbbbbbbaz\t\\uabbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/p/1/p1kszrpddaqfpf849d2mmygc3pa2w2j7canbcmvbvns8y b/implementation/fuzz/inputs-genvm-modules-complete-json/p/1/p1kszrpddaqfpf849d2mmygc3pa2w2j7canbcmvbvns8y new file mode 100644 index 00000000..16dcb65f --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/p/1/p1kszrpddaqfpf849d2mmygc3pa2w2j7canbcmvbvns8y @@ -0,0 +1,2 @@ +22919112161111111111 + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/p/2/p287mydv35s2xwtnrdnyc3pd56awqzh67157x8y6mpw9e b/implementation/fuzz/inputs-genvm-modules-complete-json/p/2/p287mydv35s2xwtnrdnyc3pd56awqzh67157x8y6mpw9e new file mode 100644 index 00000000..39e10c5d --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/p/2/p287mydv35s2xwtnrdnyc3pd56awqzh67157x8y6mpw9e @@ -0,0 +1,2 @@ +{ + "foo": false, "fo \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/p/2/p2fx30snm4efr2k1x8mqpej6xdv2hpwc13vv2njapmt3c b/implementation/fuzz/inputs-genvm-modules-complete-json/p/2/p2fx30snm4efr2k1x8mqpej6xdv2hpwc13vv2njapmt3c new file mode 100644 index 00000000..cbf74e83 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/p/2/p2fx30snm4efr2k1x8mqpej6xdv2hpwc13vv2njapmt3c @@ -0,0 +1 @@ + [111191e-11111111111113] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/p/3/p32zpavsm0j6fx9mbn21b8126993m8mc8fkpmheapshac b/implementation/fuzz/inputs-genvm-modules-complete-json/p/3/p32zpavsm0j6fx9mbn21b8126993m8mc8fkpmheapshac new file mode 100644 index 00000000..3cf20d57 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/p/3/p32zpavsm0j6fx9mbn21b8126993m8mc8fkpmheapshac @@ -0,0 +1 @@ +- \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/p/3/p368t8wtgrt740a0aeyfjjts65pyaw6w3sjpazhpt2h94 b/implementation/fuzz/inputs-genvm-modules-complete-json/p/3/p368t8wtgrt740a0aeyfjjts65pyaw6w3sjpazhpt2h94 new file mode 100644 index 00000000..46a5d7d9 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/p/3/p368t8wtgrt740a0aeyfjjts65pyaw6w3sjpazhpt2h94 @@ -0,0 +1,3 @@ +{ + "fosss,+G,,,,+,1111": 1e20, + "11111111": 1,,,G2, 3 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/p/5/p5b4pv72mn0h9ekhe92x8zhn6r03n7j6fh0w6vz1spe9g b/implementation/fuzz/inputs-genvm-modules-complete-json/p/5/p5b4pv72mn0h9ekhe92x8zhn6r03n7j6fh0w6vz1spe9g new file mode 100644 index 00000000..f9df2675 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/p/5/p5b4pv72mn0h9ekhe92x8zhn6r03n7j6fh0w6vz1spe9g @@ -0,0 +1 @@ + "\rrrrrrrrrrrrrrrrrrrru" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/p/5/p5wwkgjqjyja1vak6tv7yxyaygm73g4psv9pexeqr9v1c b/implementation/fuzz/inputs-genvm-modules-complete-json/p/5/p5wwkgjqjyja1vak6tv7yxyaygm73g4psv9pexeqr9v1c new file mode 100644 index 00000000..df12d6df --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/p/5/p5wwkgjqjyja1vak6tv7yxyaygm73g4psv9pexeqr9v1c @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/p/8/p868kv3f7ap1y7frhkfzjcxs3p4ja3za0e46vj4e245g6 b/implementation/fuzz/inputs-genvm-modules-complete-json/p/8/p868kv3f7ap1y7frhkfzjcxs3p4ja3za0e46vj4e245g6 new file mode 100644 index 00000000..a76be9e6 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/p/8/p868kv3f7ap1y7frhkfzjcxs3p4ja3za0e46vj4e245g6 @@ -0,0 +1,3 @@ + [1111111111.1110111111E111, -1e10, + 1e+2, 111E111, 2, 111111111111e+2, 11111e+2, 1111, 2, 1111.11, 2, 1, 2, 1111.11, 2, 1111 + [11116112, 1111.1111111611111111] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/p/a/patxxt64rh1qcbhctjpsbnhx1s8h1kt28aejhvs4nt71a b/implementation/fuzz/inputs-genvm-modules-complete-json/p/a/patxxt64rh1qcbhctjpsbnhx1s8h1kt28aejhvs4nt71a new file mode 100644 index 00000000..b448a8dc --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/p/a/patxxt64rh1qcbhctjpsbnhx1s8h1kt28aejhvs4nt71a @@ -0,0 +1 @@ +33 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/p/b/pb7f6wcyfjdk2c23myaqx4sqbef6wsy8j3dvz3m449fzr b/implementation/fuzz/inputs-genvm-modules-complete-json/p/b/pb7f6wcyfjdk2c23myaqx4sqbef6wsy8j3dvz3m449fzr new file mode 100644 index 00000000..d75916e8 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/p/b/pb7f6wcyfjdk2c23myaqx4sqbef6wsy8j3dvz3m449fzr @@ -0,0 +1 @@ + "Yσ—₯—111 " \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/p/b/pbejrk8cbm11vcgfv0k9vbc3v6mhntmfevse94ddtx87j b/implementation/fuzz/inputs-genvm-modules-complete-json/p/b/pbejrk8cbm11vcgfv0k9vbc3v6mhntmfevse94ddtx87j new file mode 100644 index 00000000..31e244cc --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/p/b/pbejrk8cbm11vcgfv0k9vbc3v6mhntmfevse94ddtx87j @@ -0,0 +1 @@ +1111122222222222221112222 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/p/e/pek78ck9a0kyxemegbagc2rjfedsha7z80gt6fztyht2w b/implementation/fuzz/inputs-genvm-modules-complete-json/p/e/pek78ck9a0kyxemegbagc2rjfedsha7z80gt6fztyht2w new file mode 100644 index 00000000..f800b0a9 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/p/e/pek78ck9a0kyxemegbagc2rjfedsha7z80gt6fztyht2w @@ -0,0 +1,2 @@ + +"Ρ€[\bΠ²€ σ—₯Ίub r" diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/p/f/pfp74myh8qrgwatp0tvksqb3r6rze4am7zcerdfpbt95g b/implementation/fuzz/inputs-genvm-modules-complete-json/p/f/pfp74myh8qrgwatp0tvksqb3r6rze4am7zcerdfpbt95g new file mode 100644 index 00000000..0a011e24 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/p/f/pfp74myh8qrgwatp0tvksqb3r6rze4am7zcerdfpbt95g @@ -0,0 +1 @@ +11.1111111111111111111111111111111111 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/p/f/pftwz4qs3hsbf428b1mkem5f51drw08afybzybxz4kz24 b/implementation/fuzz/inputs-genvm-modules-complete-json/p/f/pftwz4qs3hsbf428b1mkem5f51drw08afybzybxz4kz24 new file mode 100644 index 00000000..c1c2152a --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/p/f/pftwz4qs3hsbf428b1mkem5f51drw08afybzybxz4kz24 @@ -0,0 +1,3 @@ +[ + "", + ", e-515 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/p/g/pg8h16tn6nzr6s54fqys8twzwvt4nf3r1jnpjz1abaj6y b/implementation/fuzz/inputs-genvm-modules-complete-json/p/g/pg8h16tn6nzr6s54fqys8twzwvt4nf3r1jnpjz1abaj6y new file mode 100644 index 00000000..fc1a105c --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/p/g/pg8h16tn6nzr6s54fqys8twzwvt4nf3r1jnpjz1abaj6y @@ -0,0 +1,21 @@ +{ + "foo": [ + "bar", + "baz\n\\{", + 123, + -123, + -1e10, + 1e20, + null, + true, + false, + [1, 2, 3] + ], + "mbb -barar": [ ] , + "": [ ], + "bar": [ + "русскиС Π±ΡƒΠΊΠ²Π‡ ζ—₯本", + "mbbbbbbbbbbbbbbaz\t\uabbbbb", + "mbbbbbbbbbbbbbbaz\t\\uabbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/p/h/phrk1hwdpd06mahek2rzzrfqsmhng70qv6ttv0zfkva50 b/implementation/fuzz/inputs-genvm-modules-complete-json/p/h/phrk1hwdpd06mahek2rzzrfqsmhng70qv6ttv0zfkva50 new file mode 100644 index 00000000..ee3c1187 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/p/h/phrk1hwdpd06mahek2rzzrfqsmhng70qv6ttv0zfkva50 @@ -0,0 +1,6 @@ +{ + "foo": [[[[[[[[[[[[[[[[ + + +3, [[[[[[[[[[[[[[[[[123, [[[[[[[[[[[[[[[[[[[[n\\zt\uabbbbb", + "mbbbbb !11uuuuuuuuuuuu[[[[[[[[[[[[[[[111111 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/p/j/pj481h7q6wx5z6s7b4mrma2mj26awddz79ve8mttxh7fg b/implementation/fuzz/inputs-genvm-modules-complete-json/p/j/pj481h7q6wx5z6s7b4mrma2mj26awddz79ve8mttxh7fg new file mode 100644 index 00000000..8cdbec1f --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/p/j/pj481h7q6wx5z6s7b4mrma2mj26awddz79ve8mttxh7fg @@ -0,0 +1 @@ + [11111,111131111111111111E111, 2, 3111111111E111, 2, 311E111,11113111E111, 2, 3] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/p/j/pj75key1yebs99qhj5qr9paqvy0ck0w2y7ph9e63n4xyc b/implementation/fuzz/inputs-genvm-modules-complete-json/p/j/pj75key1yebs99qhj5qr9paqvy0ck0w2y7ph9e63n4xyc new file mode 100644 index 00000000..f24d2b5a --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/p/j/pj75key1yebs99qhj5qr9paqvy0ck0w2y7ph9e63n4xyc @@ -0,0 +1,15 @@ +"\\ubba ,\uabbbbb" + + + + + + + + + + + + + +" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/p/m/pm1rsjzm5ffxrb7tjz9g2ttmbsg79bd399fgh3qffaxr0 b/implementation/fuzz/inputs-genvm-modules-complete-json/p/m/pm1rsjzm5ffxrb7tjz9g2ttmbsg79bd399fgh3qffaxr0 new file mode 100644 index 00000000..39613cf9 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/p/m/pm1rsjzm5ffxrb7tjz9g2ttmbsg79bd399fgh3qffaxr0 @@ -0,0 +1,15 @@ +{ + + "1": [ + { + "1": [ ] + + + + + + + + +} ] +} \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/p/r/prdm0t56pfry6xsf9mvc4w1rkrzy7crwff5srqf7zz2a6 b/implementation/fuzz/inputs-genvm-modules-complete-json/p/r/prdm0t56pfry6xsf9mvc4w1rkrzy7crwff5srqf7zz2a6 new file mode 100644 index 00000000..4a6acf3e --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/p/r/prdm0t56pfry6xsf9mvc4w1rkrzy7crwff5srqf7zz2a6 @@ -0,0 +1 @@ +2111111111611111111111111E11 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/p/s/psq0xew56vpdtmecqyd965zxgzke9tmb0tgj5nsbr6hke b/implementation/fuzz/inputs-genvm-modules-complete-json/p/s/psq0xew56vpdtmecqyd965zxgzke9tmb0tgj5nsbr6hke new file mode 100644 index 00000000..4b3b4910 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/p/s/psq0xew56vpdtmecqyd965zxgzke9tmb0tgj5nsbr6hke @@ -0,0 +1,20 @@ +{ + "foo": [ + "bar", + "baz\n\\{", + 123, + -123, + -1e10, + 1e20, + null, + true, + false, + [1, 2, 3] + ], + "mbb t\uabbbbbbbbbbbaz111151111111111161111.11112\t -bar": [ ], + "\\\\r": [ + "Π‡ ζ—₯本", + "mbbbbbbbbbbbcbbaz\t\uabbbbb", + "mbbbbbbbbbbbbbbaz\t\\uab" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/p/v/pvk1zkyxmscw8gfnprjyjz3s4mq07nb1ndg0bf4ptr5x6 b/implementation/fuzz/inputs-genvm-modules-complete-json/p/v/pvk1zkyxmscw8gfnprjyjz3s4mq07nb1ndg0bf4ptr5x6 new file mode 100644 index 00000000..83a8c69b --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/p/v/pvk1zkyxmscw8gfnprjyjz3s4mq07nb1ndg0bf4ptr5x6 @@ -0,0 +1,16 @@ +{ + "fo: [ ],o": [ + "bar", + "b?z\n\\{", + -1e10, + 1e20, + null, + true, + [1, 2, 3] + ], + "bar": [ ], + "bar": [ + "русскиС mbbbbbbbbbbbbbbaz\t\uabbbbb", + "mbbbbbbbbbbbbbbaz\t\\uabbbbp" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/p/w/pwps76p2mngwb2g0s36t87nfw9771gnkbh88x64v3b9za b/implementation/fuzz/inputs-genvm-modules-complete-json/p/w/pwps76p2mngwb2g0s36t87nfw9771gnkbh88x64v3b9za new file mode 100644 index 00000000..d6a23011 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/p/w/pwps76p2mngwb2g0s36t87nfw9771gnkbh88x64v3b9za @@ -0,0 +1 @@ + [9.11161e-111111101111133333] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/p/w/pwxm2140kkdd3y52kv6da4r8r7vk6bhxk2nhhj2wrrx90 b/implementation/fuzz/inputs-genvm-modules-complete-json/p/w/pwxm2140kkdd3y52kv6da4r8r7vk6bhxk2nhhj2wrrx90 new file mode 100644 index 00000000..450c6a42 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/p/w/pwxm2140kkdd3y52kv6da4r8r7vk6bhxk2nhhj2wrrx90 @@ -0,0 +1,2 @@ + +"Ρ€\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\uba [[1, B\bΓ΅ r" diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/p/z/pzdkt9mh86k8rq8ksknbzsvptc3274n2n1zjgswg1ehqm b/implementation/fuzz/inputs-genvm-modules-complete-json/p/z/pzdkt9mh86k8rq8ksknbzsvptc3274n2n1zjgswg1ehqm new file mode 100644 index 00000000..810c539b --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/p/z/pzdkt9mh86k8rq8ksknbzsvptc3274n2n1zjgswg1ehqm @@ -0,0 +1 @@ +[3] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/q/0/q0nf8cpayztamyr7evqjfy6wga3zngxhk4kvx0dza4mv8 b/implementation/fuzz/inputs-genvm-modules-complete-json/q/0/q0nf8cpayztamyr7evqjfy6wga3zngxhk4kvx0dza4mv8 new file mode 100644 index 00000000..aa831c58 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/q/0/q0nf8cpayztamyr7evqjfy6wga3zngxhk4kvx0dza4mv8 @@ -0,0 +1 @@ +"\"\"\tt" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/q/0/q0prpmnjbk9d5ymty9jcby1093s57dkm118qcs250k05y b/implementation/fuzz/inputs-genvm-modules-complete-json/q/0/q0prpmnjbk9d5ymty9jcby1093s57dkm118qcs250k05y new file mode 100644 index 00000000..eee00c5f --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/q/0/q0prpmnjbk9d5ymty9jcby1093s57dkm118qcs250k05y @@ -0,0 +1,4 @@ +"f\\\\\\\\\\uab\\\\" + + + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/q/2/q2m3epw20qqgp7shh42j9kbbeq79znh82r03bmv9jq832 b/implementation/fuzz/inputs-genvm-modules-complete-json/q/2/q2m3epw20qqgp7shh42j9kbbeq79znh82r03bmv9jq832 new file mode 100644 index 00000000..251d15e9 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/q/2/q2m3epw20qqgp7shh42j9kbbeq79znh82r03bmv9jq832 @@ -0,0 +1,16 @@ +{ + "foo": [ + "bar", + "baz\n\\z", + 123, + true, + false, + -1e10, + -123, + 1e20, + null, + true, + false, + [11111111111111111111111111111111111111111111171111111111111, 2, [[[{ "o": [ { "o": [ +} +3] " \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/q/3/q3p690xw3m4yzy85b9vb4t4cq42158d96rfd923m8aa2p b/implementation/fuzz/inputs-genvm-modules-complete-json/q/3/q3p690xw3m4yzy85b9vb4t4cq42158d96rfd923m8aa2p new file mode 100644 index 00000000..1931346b --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/q/3/q3p690xw3m4yzy85b9vb4t4cq42158d96rfd923m8aa2p @@ -0,0 +1,32 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/q/4/q45my1w9wtefsgpzdk6nvsz98z23k9wmz63w1ej6xe766 b/implementation/fuzz/inputs-genvm-modules-complete-json/q/4/q45my1w9wtefsgpzdk6nvsz98z23k9wmz63w1ej6xe766 new file mode 100644 index 00000000..1fd33ce7 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/q/4/q45my1w9wtefsgpzdk6nvsz98z23k9wmz63w1ej6xe766 @@ -0,0 +1 @@ +1.1 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/q/5/q5df95x81dj1gn9tpbj8w8r0ht7c3w60hskcn6mgjtrme b/implementation/fuzz/inputs-genvm-modules-complete-json/q/5/q5df95x81dj1gn9tpbj8w8r0ht7c3w60hskcn6mgjtrme new file mode 100644 index 00000000..e1598f6e --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/q/5/q5df95x81dj1gn9tpbj8w8r0ht7c3w60hskcn6mgjtrme @@ -0,0 +1 @@ + " \uDbbb \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/q/6/q6gebjxz3vwfj4pd0dc1kerm6gq0egmy1mxr4g82zjg64 b/implementation/fuzz/inputs-genvm-modules-complete-json/q/6/q6gebjxz3vwfj4pd0dc1kerm6gq0egmy1mxr4g82zjg64 new file mode 100644 index 00000000..23bacb18 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/q/6/q6gebjxz3vwfj4pd0dc1kerm6gq0egmy1mxr4g82zjg64 @@ -0,0 +1 @@ +11111111111191111.11111e-1111111111111111111111111111 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/q/9/q9a59mn46kmhq246znnmh6wzajqcmngexwn2qhbepg02a b/implementation/fuzz/inputs-genvm-modules-complete-json/q/9/q9a59mn46kmhq246znnmh6wzajqcmngexwn2qhbepg02a new file mode 100644 index 00000000..a45d1408 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/q/9/q9a59mn46kmhq246znnmh6wzajqcmngexwn2qhbepg02a @@ -0,0 +1 @@ +[[[[[[[[[[[ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/q/9/q9d9n5fkfwsfdery7z1e7gsh5a9qdqmt0vfmkf185rp7p b/implementation/fuzz/inputs-genvm-modules-complete-json/q/9/q9d9n5fkfwsfdery7z1e7gsh5a9qdqmt0vfmkf185rp7p new file mode 100644 index 00000000..7d6dc432 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/q/9/q9d9n5fkfwsfdery7z1e7gsh5a9qdqmt0vfmkf185rp7p @@ -0,0 +1,16 @@ +{ + "foo": [ + + 123, + -123, + -1e10, + null, + true, + false, + [1, 2, 3] + ], + "bar": [ + "русскиС Π±ΡƒΠΊΨ²Ρ‹ ζ—₯本az\t\uabbb\b", + "bbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/q/9/q9e1emsm4enja8d8gdxb0173t52zah1pm9rbbatghtg6a b/implementation/fuzz/inputs-genvm-modules-complete-json/q/9/q9e1emsm4enja8d8gdxb0173t52zah1pm9rbbatghtg6a new file mode 100644 index 00000000..06fa6fa0 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/q/9/q9e1emsm4enja8d8gdxb0173t52zah1pm9rbbatghtg6a @@ -0,0 +1,17 @@ +{ + "foo": [ + "bar", + "baz\n\\z", + 123, + -123, + -1e10, + 1e20, + null, +true, + false, + [1, 2, 3] + ], "bar": [ + ], "Ρ€ΡƒΠ΅ \\": "barΠ±ΡƒΠΊΠ²Ρ‹ ζ—₯本", + "mbbbbbbbbbbbbbbaz\t\uabbbbb", + "mb + #b \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/q/b/qb1dg1yxdy9fbywxyn689e3n8s17jqvs8vj3dm4c09f6p b/implementation/fuzz/inputs-genvm-modules-complete-json/q/b/qb1dg1yxdy9fbywxyn689e3n8s17jqvs8vj3dm4c09f6p new file mode 100644 index 00000000..f04fbaa3 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/q/b/qb1dg1yxdy9fbywxyn689e3n8s17jqvs8vj3dm4c09f6p @@ -0,0 +1 @@ +[21E-2 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/q/c/qcq8fxxg0886x5118vcn9hn9re95kjbm64m1mnv1v2yv8 b/implementation/fuzz/inputs-genvm-modules-complete-json/q/c/qcq8fxxg0886x5118vcn9hn9re95kjbm64m1mnv1v2yv8 new file mode 100644 index 00000000..eae2f4a2 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/q/c/qcq8fxxg0886x5118vcn9hn9re95kjbm64m1mnv1v2yv8 @@ -0,0 +1 @@ +1.6E11 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/q/d/qd5dzxmff201w13fv07e00kgx7y6xzk80dmf6h0n6e8he b/implementation/fuzz/inputs-genvm-modules-complete-json/q/d/qd5dzxmff201w13fv07e00kgx7y6xzk80dmf6h0n6e8he new file mode 100644 index 00000000..fbcd9a82 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/q/d/qd5dzxmff201w13fv07e00kgx7y6xzk80dmf6h0n6e8he @@ -0,0 +1,8 @@ +{ + "foo": [ + "bar", + "bazdn\\z", + 123, + [[[[[[[ 1e20, + null, + -123, " 4 222 6 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/q/d/qdarjc9gm33ymfwkpsw3pdm4bm7e38hyakp6m5rd3hdf6 b/implementation/fuzz/inputs-genvm-modules-complete-json/q/d/qdarjc9gm33ymfwkpsw3pdm4bm7e38hyakp6m5rd3hdf6 new file mode 100644 index 00000000..c1b47714 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/q/d/qdarjc9gm33ymfwkpsw3pdm4bm7e38hyakp6m5rd3hdf6 @@ -0,0 +1 @@ +22222222222222222222222 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/q/d/qdjjdjb5ztvry0ky10ryyx6j3a0v84f4jcfwa8v3swn3j b/implementation/fuzz/inputs-genvm-modules-complete-json/q/d/qdjjdjb5ztvry0ky10ryyx6j3a0v84f4jcfwa8v3swn3j new file mode 100644 index 00000000..39565a68 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/q/d/qdjjdjb5ztvry0ky10ryyx6j3a0v84f4jcfwa8v3swn3j @@ -0,0 +1 @@ + " \uDbbb\uDbbb \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/q/e/qebc2ds26vf511s9ev9e0ymjbmw01an7s43nypap1yd2j b/implementation/fuzz/inputs-genvm-modules-complete-json/q/e/qebc2ds26vf511s9ev9e0ymjbmw01an7s43nypap1yd2j new file mode 100644 index 00000000..2b425d4b --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/q/e/qebc2ds26vf511s9ev9e0ymjbmw01an7s43nypap1yd2j @@ -0,0 +1,2 @@ + [ + "bar", diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/q/g/qg7xyqbzsmnv7z2r12gtre42jtnn0evnz8sghsbzfbvty b/implementation/fuzz/inputs-genvm-modules-complete-json/q/g/qg7xyqbzsmnv7z2r12gtre42jtnn0evnz8sghsbzfbvty new file mode 100644 index 00000000..a822575a --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/q/g/qg7xyqbzsmnv7z2r12gtre42jtnn0evnz8sghsbzfbvty @@ -0,0 +1 @@ +"Ρ€\\\\\\}\\\\111111.11\\\\\\\\\\\\σ—₯ΊF`b r" diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/q/g/qgqa29n6vmqy1cmsdgn259sa2gear6ansyytb7dsnj380 b/implementation/fuzz/inputs-genvm-modules-complete-json/q/g/qgqa29n6vmqy1cmsdgn259sa2gear6ansyytb7dsnj380 new file mode 100644 index 00000000..d8a2c315 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/q/g/qgqa29n6vmqy1cmsdgn259sa2gear6ansyytb7dsnj380 @@ -0,0 +1 @@ +"σ—₯Ίb" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/q/g/qgw5gj9bemrbyaptz86xrsvkgrz43e5fd766v8sptck92 b/implementation/fuzz/inputs-genvm-modules-complete-json/q/g/qgw5gj9bemrbyaptz86xrsvkgrz43e5fd766v8sptck92 new file mode 100644 index 00000000..32f64f4d --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/q/g/qgw5gj9bemrbyaptz86xrsvkgrz43e5fd766v8sptck92 @@ -0,0 +1 @@ +t \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/q/m/qmsqzbe79qkxy5zv5daw6m1x7kh2f68gnxz4k8rxr7bet b/implementation/fuzz/inputs-genvm-modules-complete-json/q/m/qmsqzbe79qkxy5zv5daw6m1x7kh2f68gnxz4k8rxr7bet new file mode 100644 index 0000000000000000000000000000000000000000..18aec8be686f7e590860f3404b0de4ce34a27704 GIT binary patch literal 256 zcmb>CQUC&_wETP}D}`t<2P}}3Sfr$b%&khvi;1a15;HV1hAD`~1PX?!Mg}l>1%<rQ qoE$Ju7o-hrgOQODkk&O!1@TA%FxwOql9Cc(93&cQSQ!IhQ}X~@gEJ2R literal 0 HcmV?d00001 diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/q/m/qmtws2y6pk25cgmf8fg74p3z1e8tw9g7h72m2x5am92tc b/implementation/fuzz/inputs-genvm-modules-complete-json/q/m/qmtws2y6pk25cgmf8fg74p3z1e8tw9g7h72m2x5am92tc new file mode 100644 index 00000000..c68d4e7b --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/q/m/qmtws2y6pk25cgmf8fg74p3z1e8tw9g7h72m2x5am92tc @@ -0,0 +1,18 @@ +{ + "foo": [ + "bar", + 11611111111111111111111111111111113e-923, + -123, + -1e10, + 1e20, + null, + -1e10, + false, + [1, 2, 3] + ], + "bar": [ + "руссºиС Π±ΡƒΠΊΠ²Ρ‹ ζ—₯本", + "mbbbbbbbbbbbbbbaz\t\uabbbbb", + "mbbbbbbbbbbbbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/q/n/qn0t88k7rhyt3jyq9174j4k059espkf5nnnvt1wn254ea b/implementation/fuzz/inputs-genvm-modules-complete-json/q/n/qn0t88k7rhyt3jyq9174j4k059espkf5nnnvt1wn254ea new file mode 100644 index 00000000..1fe4bd36 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/q/n/qn0t88k7rhyt3jyq9174j4k059espkf5nnnvt1wn254ea @@ -0,0 +1 @@ +11111111111191111.1111e-625 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/q/y/qyecheaygp5n2htj123hvsgfyfjdpmwb46v7yv8k9kbsp b/implementation/fuzz/inputs-genvm-modules-complete-json/q/y/qyecheaygp5n2htj123hvsgfyfjdpmwb46v7yv8k9kbsp new file mode 100644 index 00000000..62be4a73 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/q/y/qyecheaygp5n2htj123hvsgfyfjdpmwb46v7yv8k9kbsp @@ -0,0 +1 @@ +"\"\"\"\fofoB" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/q/z/qz9v320wcgc0ydng7frvmm58x4az6r1w8j5knk9c8ejma b/implementation/fuzz/inputs-genvm-modules-complete-json/q/z/qz9v320wcgc0ydng7frvmm58x4az6r1w8j5knk9c8ejma new file mode 100644 index 00000000..09ea9bde --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/q/z/qz9v320wcgc0ydng7frvmm58x4az6r1w8j5knk9c8ejma @@ -0,0 +1 @@ +"Ρ€\\ub " \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/r/0/r005wtb7jsahx35v0fgnd0d5cp3gnas1fv24jsbyt7fet b/implementation/fuzz/inputs-genvm-modules-complete-json/r/0/r005wtb7jsahx35v0fgnd0d5cp3gnas1fv24jsbyt7fet new file mode 100644 index 00000000..7cc3f535 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/r/0/r005wtb7jsahx35v0fgnd0d5cp3gnas1fv24jsbyt7fet @@ -0,0 +1,3 @@ +{ + "foo": [ + "ba [[Π΅ Π₯本 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/r/1/r1b4z4m42s02q4ab3gezsvp1ytyfhhmxag9sdxdrsh3cr b/implementation/fuzz/inputs-genvm-modules-complete-json/r/1/r1b4z4m42s02q4ab3gezsvp1ytyfhhmxag9sdxdrsh3cr new file mode 100644 index 00000000..37d97ea5 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/r/1/r1b4z4m42s02q4ab3gezsvp1ytyfhhmxag9sdxdrsh3cr @@ -0,0 +1 @@ +"[ou" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/r/1/r1cdd04qajrp85sczjm8s2y26c7xyns3mwtypkx1n19y8 b/implementation/fuzz/inputs-genvm-modules-complete-json/r/1/r1cdd04qajrp85sczjm8s2y26c7xyns3mwtypkx1n19y8 new file mode 100644 index 00000000..83ebb20c --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/r/1/r1cdd04qajrp85sczjm8s2y26c7xyns3mwtypkx1n19y8 @@ -0,0 +1,2 @@ + +"\u0000Ρ‹\u0000Ρ‹\u00ba r" diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/r/2/r254rs232s782j37a38v7f3cbgmxr8gw2450eq26f9732 b/implementation/fuzz/inputs-genvm-modules-complete-json/r/2/r254rs232s782j37a38v7f3cbgmxr8gw2450eq26f9732 new file mode 100644 index 00000000..280fffb7 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/r/2/r254rs232s782j37a38v7f3cbgmxr8gw2450eq26f9732 @@ -0,0 +1 @@ +" {abr" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/r/2/r29f3etsn2vgy2342p8ezq2bbbn5mgwqx5mb67eyre9hg b/implementation/fuzz/inputs-genvm-modules-complete-json/r/2/r29f3etsn2vgy2342p8ezq2bbbn5mgwqx5mb67eyre9hg new file mode 100644 index 00000000..fb97add1 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/r/2/r29f3etsn2vgy2342p8ezq2bbbn5mgwqx5mb67eyre9hg @@ -0,0 +1,2 @@ + +"ά€ά€\r\r" diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/r/3/r32mdfagf8w7x7tfzqsgb5k4abcfmk3p1kmggc8ya37dg b/implementation/fuzz/inputs-genvm-modules-complete-json/r/3/r32mdfagf8w7x7tfzqsgb5k4abcfmk3p1kmggc8ya37dg new file mode 100644 index 00000000..82e30f75 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/r/3/r32mdfagf8w7x7tfzqsgb5k4abcfmk3p1kmggc8ya37dg @@ -0,0 +1,3 @@ +{ + "foo": [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + [[[[[[[[[[[[ "bar", ,,,,,,,,,,,,,,,,,,, ,,,,,,,,,,,,,,,,,,,,,,,,,,,,, ! \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/r/3/r3y803bhd7qf17s9aq8q5rxrh6384f5q48h7e3pbmn94y b/implementation/fuzz/inputs-genvm-modules-complete-json/r/3/r3y803bhd7qf17s9aq8q5rxrh6384f5q48h7e3pbmn94y new file mode 100644 index 00000000..191e46be --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/r/3/r3y803bhd7qf17s9aq8q5rxrh6384f5q48h7e3pbmn94y @@ -0,0 +1,2 @@ +{ " 111111111111111z-52222 f o": [ ] +} \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/r/7/r76wr3jct03x1hcdfyr9zc5de8kjzhqdftatb9p0txjy2 b/implementation/fuzz/inputs-genvm-modules-complete-json/r/7/r76wr3jct03x1hcdfyr9zc5de8kjzhqdftatb9p0txjy2 new file mode 100644 index 00000000..b945ede3 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/r/7/r76wr3jct03x1hcdfyr9zc5de8kjzhqdftatb9p0txjy2 @@ -0,0 +1,4 @@ +{ "o": [ { "o": [ { "o": [ { "o": [ +} ] +} +} \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/r/9/r9gm6axcx8mpe1qp1whakh1y0t0wgfahsvgwth5p7mwy4 b/implementation/fuzz/inputs-genvm-modules-complete-json/r/9/r9gm6axcx8mpe1qp1whakh1y0t0wgfahsvgwth5p7mwy4 new file mode 100644 index 00000000..defe0be3 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/r/9/r9gm6axcx8mpe1qp1whakh1y0t0wgfahsvgwth5p7mwy4 @@ -0,0 +1 @@ + tru \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/r/a/ranvb980qjve6cak095mg75awgrd5p14s3bqnccs3d9vp b/implementation/fuzz/inputs-genvm-modules-complete-json/r/a/ranvb980qjve6cak095mg75awgrd5p14s3bqnccs3d9vp new file mode 100644 index 00000000..a8ae3f7f --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/r/a/ranvb980qjve6cak095mg75awgrd5p14s3bqnccs3d9vp @@ -0,0 +1 @@ +33333333333333333333.333 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/r/a/rar68phs41hzs9gfadgbhm8jp91n9zbyb73cbmvhc8mhy b/implementation/fuzz/inputs-genvm-modules-complete-json/r/a/rar68phs41hzs9gfadgbhm8jp91n9zbyb73cbmvhc8mhy new file mode 100644 index 00000000..8b999060 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/r/a/rar68phs41hzs9gfadgbhm8jp91n9zbyb73cbmvhc8mhy @@ -0,0 +1 @@ +111.111 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/r/d/rd9b48n0xygtvkx3nckbc9p4ky0kphkxj1b0zzpfa9bg4 b/implementation/fuzz/inputs-genvm-modules-complete-json/r/d/rd9b48n0xygtvkx3nckbc9p4ky0kphkxj1b0zzpfa9bg4 new file mode 100644 index 00000000..0a031d7c --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/r/d/rd9b48n0xygtvkx3nckbc9p4ky0kphkxj1b0zzpfa9bg4 @@ -0,0 +1 @@ +66666666666666666666 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/r/f/rfcj6z0sr8yz063zzaxameqvq8kntp05bh1cp3tnefr06 b/implementation/fuzz/inputs-genvm-modules-complete-json/r/f/rfcj6z0sr8yz063zzaxameqvq8kntp05bh1cp3tnefr06 new file mode 100644 index 0000000000000000000000000000000000000000..9ca637f200bc6d861e9a27187e73f9940058d559 GIT binary patch literal 11 Scma!6;^N}s<>fVGPyhf2t^pJP literal 0 HcmV?d00001 diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/r/g/rg8pj4jk38xv6fhpf1bbn2398bkvcgmkqq1crd55ja9ke b/implementation/fuzz/inputs-genvm-modules-complete-json/r/g/rg8pj4jk38xv6fhpf1bbn2398bkvcgmkqq1crd55ja9ke new file mode 100644 index 00000000..fffe9153 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/r/g/rg8pj4jk38xv6fhpf1bbn2398bkvcgmkqq1crd55ja9ke @@ -0,0 +1 @@ +11111111111111111111111111e-11111111111111 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/r/g/rg8q6t50jp1c9gst4ek2j8r245e2hw5a7gwvvpgnfhf1t b/implementation/fuzz/inputs-genvm-modules-complete-json/r/g/rg8q6t50jp1c9gst4ek2j8r245e2hw5a7gwvvpgnfhf1t new file mode 100644 index 00000000..009a4498 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/r/g/rg8q6t50jp1c9gst4ek2j8r245e2hw5a7gwvvpgnfhf1t @@ -0,0 +1 @@ +"Ρ€}}}}}}" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/r/h/rhtmpfv58qes02wt4ktjxrwveypfwns9hne6hvk9mrd44 b/implementation/fuzz/inputs-genvm-modules-complete-json/r/h/rhtmpfv58qes02wt4ktjxrwveypfwns9hne6hvk9mrd44 new file mode 100644 index 00000000..374405ff --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/r/h/rhtmpfv58qes02wt4ktjxrwveypfwns9hne6hvk9mrd44 @@ -0,0 +1 @@ +"Ρ€ΒΊ\\\\bba[1a[1Ρ€ΒΊΡ€ΒΊa[1!" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/r/j/rj71xj66vxs4qczheqy8ne3rtf5jsa0e84d787cfeh8xe b/implementation/fuzz/inputs-genvm-modules-complete-json/r/j/rj71xj66vxs4qczheqy8ne3rtf5jsa0e84d787cfeh8xe new file mode 100644 index 00000000..6ba0e99d --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/r/j/rj71xj66vxs4qczheqy8ne3rtf5jsa0e84d787cfeh8xe @@ -0,0 +1,2 @@ + + 525 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/r/k/rks8yw6t9nqwhn9ygqqjfn42egqbmz9tvxwjj4wtvvp7j b/implementation/fuzz/inputs-genvm-modules-complete-json/r/k/rks8yw6t9nqwhn9ygqqjfn42egqbmz9tvxwjj4wtvvp7j new file mode 100644 index 0000000000000000000000000000000000000000..6f99806f7df2c6effa5cde54dfdc79522a9b01d8 GIT binary patch literal 278 zcmb>CQUC&_wETP}D}`t<2P}}3Sfr!_=YoU~AetBeQ=V6vlLIzL*U-pV2PO)mm5hvx gfNWjERG0{<G|Vju3Q0+cFb)z8HLr|;c!Tpw0e&twbpQYW literal 0 HcmV?d00001 diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/r/k/rkwm6vptf0pxp94z7dseek6j01pcsaxa669xmnj0gpy68 b/implementation/fuzz/inputs-genvm-modules-complete-json/r/k/rkwm6vptf0pxp94z7dseek6j01pcsaxa669xmnj0gpy68 new file mode 100644 index 00000000..ae7201e8 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/r/k/rkwm6vptf0pxp94z7dseek6j01pcsaxa669xmnj0gpy68 @@ -0,0 +1 @@ +222222222222222222222222222 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/r/n/rngntzfd0n1t66793erxeqbgbp8etqkyfsg0m56gb19xe b/implementation/fuzz/inputs-genvm-modules-complete-json/r/n/rngntzfd0n1t66793erxeqbgbp8etqkyfsg0m56gb19xe new file mode 100644 index 00000000..97dbc147 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/r/n/rngntzfd0n1t66793erxeqbgbp8etqkyfsg0m56gb19xe @@ -0,0 +1 @@ +"\r[ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/r/n/rnvm0dbv2tezpv8wtgzehd84x35bbqecq4r3pp1ss5s7y b/implementation/fuzz/inputs-genvm-modules-complete-json/r/n/rnvm0dbv2tezpv8wtgzehd84x35bbqecq4r3pp1ss5s7y new file mode 100644 index 00000000..80d1f550 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/r/n/rnvm0dbv2tezpv8wtgzehd84x35bbqecq4r3pp1ss5s7y @@ -0,0 +1 @@ +11111111211111121111.111 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/r/p/rphqjmqw0ersskqb60kag18j2xj99gcz1caszeapaxv82 b/implementation/fuzz/inputs-genvm-modules-complete-json/r/p/rphqjmqw0ersskqb60kag18j2xj99gcz1caszeapaxv82 new file mode 100644 index 00000000..72b704fe --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/r/p/rphqjmqw0ersskqb60kag18j2xj99gcz1caszeapaxv82 @@ -0,0 +1,7 @@ +[" + + + + [ + + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/r/p/rpjqfegw5ybpp8qakdfc3k8gwf3jwx1e9jpm8nvyjaa2g b/implementation/fuzz/inputs-genvm-modules-complete-json/r/p/rpjqfegw5ybpp8qakdfc3k8gwf3jwx1e9jpm8nvyjaa2g new file mode 100644 index 00000000..f12b1eab --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/r/p/rpjqfegw5ybpp8qakdfc3k8gwf3jwx1e9jpm8nvyjaa2g @@ -0,0 +1 @@ + 222222222222222222222222 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/r/q/rq57j344ypcyv8zrwr98g3g5n8h3pr19p8mcmmg6n1avc b/implementation/fuzz/inputs-genvm-modules-complete-json/r/q/rq57j344ypcyv8zrwr98g3g5n8h3pr19p8mcmmg6n1avc new file mode 100644 index 00000000..b1f7375e --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/r/q/rq57j344ypcyv8zrwr98g3g5n8h3pr19p8mcmmg6n1avc @@ -0,0 +1,21 @@ +{ + "foo": [ + "bar", + "baz\n\\{", + 123, + -123, + -1e10, + 1e20, + null, + true, + false, + [1, 2, 3] + ], + "mbb -barar": [ ], + "": [ ], + "bar": [ + "русскиС Π±ΡƒΠΊΠ²Π‡ ζ—₯本", + "mbbbbbbbbbbbbbbaz\t\\\\\\\uabbbbb", + "mbbbbbbbbbbbbbbaz\t\\uabbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/r/v/rvfj1zs8cr9sxh03r64qr0y9rkmatt2qs2772k49fk6qt b/implementation/fuzz/inputs-genvm-modules-complete-json/r/v/rvfj1zs8cr9sxh03r64qr0y9rkmatt2qs2772k49fk6qt new file mode 100644 index 00000000..19d1c1a4 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/r/v/rvfj1zs8cr9sxh03r64qr0y9rkmatt2qs2772k49fk6qt @@ -0,0 +1 @@ +[1, 211 11. \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/r/v/rvh6nrpbnsfjd33qy67epbwhnvf8rjr75dhcz9w85t8ta b/implementation/fuzz/inputs-genvm-modules-complete-json/r/v/rvh6nrpbnsfjd33qy67epbwhnvf8rjr75dhcz9w85t8ta new file mode 100644 index 00000000..72b2e4c7 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/r/v/rvh6nrpbnsfjd33qy67epbwhnvf8rjr75dhcz9w85t8ta @@ -0,0 +1,7 @@ +{ + "foo": [ + "bar", true, + "bar", true, + true, + fa + 1111 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/r/v/rvnx9h9xjstjgzmmj18r8g515kwvre8wrb9zr8btxhpyj b/implementation/fuzz/inputs-genvm-modules-complete-json/r/v/rvnx9h9xjstjgzmmj18r8g515kwvre8wrb9zr8btxhpyj new file mode 100644 index 00000000..ea740b2e --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/r/v/rvnx9h9xjstjgzmmj18r8g515kwvre8wrb9zr8btxhpyj @@ -0,0 +1 @@ +"\"\"\"\ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/r/w/rw6hk88m8twgs79ndsypq12yb3ab7ae4n2t3z45gy2ey8 b/implementation/fuzz/inputs-genvm-modules-complete-json/r/w/rw6hk88m8twgs79ndsypq12yb3ab7ae4n2t3z45gy2ey8 new file mode 100644 index 00000000..8fbd03fc --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/r/w/rw6hk88m8twgs79ndsypq12yb3ab7ae4n2t3z45gy2ey8 @@ -0,0 +1,4 @@ +"f (ѝ\\u @ nuabbbbb" + + + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/r/x/rx7phk58zbbyr4dcgzymj9fvdwpyazrc9f0qkpgb17p5w b/implementation/fuzz/inputs-genvm-modules-complete-json/r/x/rx7phk58zbbyr4dcgzymj9fvdwpyazrc9f0qkpgb17p5w new file mode 100644 index 00000000..c6916a5f --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/r/x/rx7phk58zbbyr4dcgzymj9fvdwpyazrc9f0qkpgb17p5w @@ -0,0 +1,8 @@ + { + "":0, + "(1.1116116111111 " + + + + + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/r/y/ry06bs5fkzz4rmvkf6p1sv8mhyafa5bgaar290ae37kxc b/implementation/fuzz/inputs-genvm-modules-complete-json/r/y/ry06bs5fkzz4rmvkf6p1sv8mhyafa5bgaar290ae37kxc new file mode 100644 index 00000000..6bb3aa70 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/r/y/ry06bs5fkzz4rmvkf6p1sv8mhyafa5bgaar290ae37kxc @@ -0,0 +1,3 @@ + [0, + +3] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/r/z/rzes82sf0h9mheve4rtr7et2yahhaapjhjfajk33b8kbc b/implementation/fuzz/inputs-genvm-modules-complete-json/r/z/rzes82sf0h9mheve4rtr7et2yahhaapjhjfajk33b8kbc new file mode 100644 index 00000000..705ca61c --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/r/z/rzes82sf0h9mheve4rtr7et2yahhaapjhjfajk33b8kbc @@ -0,0 +1 @@ +"\///r" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/r/z/rzktdgqz67sfj5ykcwv0fr8e3ssaesj6zxnfbb2pgjczw b/implementation/fuzz/inputs-genvm-modules-complete-json/r/z/rzktdgqz67sfj5ykcwv0fr8e3ssaesj6zxnfbb2pgjczw new file mode 100644 index 00000000..292d9c60 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/r/z/rzktdgqz67sfj5ykcwv0fr8e3ssaesj6zxnfbb2pgjczw @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/r/z/rzvcecva3rt0z8852gz1f9tgq7a7eebcfbs5dvhbp3hr4 b/implementation/fuzz/inputs-genvm-modules-complete-json/r/z/rzvcecva3rt0z8852gz1f9tgq7a7eebcfbs5dvhbp3hr4 new file mode 100644 index 00000000..d8cea7db --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/r/z/rzvcecva3rt0z8852gz1f9tgq7a7eebcfbs5dvhbp3hr4 @@ -0,0 +1 @@ +33333333333333333333E333 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/s/0/s0z3smqnjfb2way9n0n9praqmjxjz6j87a2t6n04gzxha b/implementation/fuzz/inputs-genvm-modules-complete-json/s/0/s0z3smqnjfb2way9n0n9praqmjxjz6j87a2t6n04gzxha new file mode 100644 index 00000000..4e3fe0c8 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/s/0/s0z3smqnjfb2way9n0n9praqmjxjz6j87a2t6n04gzxha @@ -0,0 +1 @@ +":fz" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/s/3/s3as9nevz7ys78r8cyz6jhd7z2e50c8ve5npey393prya b/implementation/fuzz/inputs-genvm-modules-complete-json/s/3/s3as9nevz7ys78r8cyz6jhd7z2e50c8ve5npey393prya new file mode 100644 index 00000000..39e40873 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/s/3/s3as9nevz7ys78r8cyz6jhd7z2e50c8ve5npey393prya @@ -0,0 +1,34 @@ +[" + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/s/5/s5dpj46txknxehtjbtm2mf8xe42m810vdfw9ngf5e2p6j b/implementation/fuzz/inputs-genvm-modules-complete-json/s/5/s5dpj46txknxehtjbtm2mf8xe42m810vdfw9ngf5e2p6j new file mode 100644 index 00000000..e0e0f88c --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/s/5/s5dpj46txknxehtjbtm2mf8xe42m810vdfw9ngf5e2p6j @@ -0,0 +1,19 @@ +{ + "foo": [ + "bar", + "baz\n\\z", + -1e10, + 1e20, + null, + true, + false, + [1, 2, 3] + ], + "bar": [ + "Ρ€ΡƒΡζœ¬", + [1, 2, 3] + ], + "bau": "mbbbbbbbbbbbbbbaz\t\uabbbbb", + "mau": "\t\\rabbbbb" + +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/s/7/s7wz172q8pprzf70wxcd76nvqz816abpercv1m6d0tsje b/implementation/fuzz/inputs-genvm-modules-complete-json/s/7/s7wz172q8pprzf70wxcd76nvqz816abpercv1m6d0tsje new file mode 100644 index 00000000..a8632d14 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/s/7/s7wz172q8pprzf70wxcd76nvqz816abpercv1m6d0tsje @@ -0,0 +1 @@ +111.116111116116111111e-515 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/s/8/s84jvn9nfee3kzs0w03f45814khxz0m70hns9mf3kbhrt b/implementation/fuzz/inputs-genvm-modules-complete-json/s/8/s84jvn9nfee3kzs0w03f45814khxz0m70hns9mf3kbhrt new file mode 100644 index 00000000..e6fe0606 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/s/8/s84jvn9nfee3kzs0w03f45814khxz0m70hns9mf3kbhrt @@ -0,0 +1,2 @@ + +"Ρ€bbbb\\ua1$, ' r" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/s/a/sab21yexdccy2rhr3w32q832kk62dbwj6rmj7wnnfsyj0 b/implementation/fuzz/inputs-genvm-modules-complete-json/s/a/sab21yexdccy2rhr3w32q832kk62dbwj6rmj7wnnfsyj0 new file mode 100644 index 00000000..4352e8f9 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/s/a/sab21yexdccy2rhr3w32q832kk62dbwj6rmj7wnnfsyj0 @@ -0,0 +1 @@ +" \/ם" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/s/a/sah5nna4pvs9azyhqsq3rqz5dxsr2m9n1a557zm3ejx1a b/implementation/fuzz/inputs-genvm-modules-complete-json/s/a/sah5nna4pvs9azyhqsq3rqz5dxsr2m9n1a557zm3ejx1a new file mode 100644 index 00000000..b0eaa6e1 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/s/a/sah5nna4pvs9azyhqsq3rqz5dxsr2m9n1a557zm3ejx1a @@ -0,0 +1 @@ +";\\ζ³›a?" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/s/a/saswj19zc6xv7kmsb087yrcrhdzb4hhvf3w65pxkmbkh8 b/implementation/fuzz/inputs-genvm-modules-complete-json/s/a/saswj19zc6xv7kmsb087yrcrhdzb4hhvf3w65pxkmbkh8 new file mode 100644 index 00000000..ea0980a8 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/s/a/saswj19zc6xv7kmsb087yrcrhdzb4hhvf3w65pxkmbkh8 @@ -0,0 +1 @@ +[[[,,,[[,,,[ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/s/b/sbepevrk0tzwspssqan4ycb1pk0scekrjak5y8t6jm9e6 b/implementation/fuzz/inputs-genvm-modules-complete-json/s/b/sbepevrk0tzwspssqan4ycb1pk0scekrjak5y8t6jm9e6 new file mode 100644 index 00000000..10da8b57 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/s/b/sbepevrk0tzwspssqan4ycb1pk0scekrjak5y8t6jm9e6 @@ -0,0 +1,3 @@ +[ + "ba222222222222 r"& + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/s/b/sbj1y9pevygw3shngyczkcfcxbmjv4fmf0tqzcgy9w5wy b/implementation/fuzz/inputs-genvm-modules-complete-json/s/b/sbj1y9pevygw3shngyczkcfcxbmjv4fmf0tqzcgy9w5wy new file mode 100644 index 00000000..2a6204d9 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/s/b/sbj1y9pevygw3shngyczkcfcxbmjv4fmf0tqzcgy9w5wy @@ -0,0 +1,3 @@ +{ + "1": [ {} ] +} \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/s/c/sch1bx4mpgstvnv1fns60d9hrzswpt6s8fgmvr6pjtzd6 b/implementation/fuzz/inputs-genvm-modules-complete-json/s/c/sch1bx4mpgstvnv1fns60d9hrzswpt6s8fgmvr6pjtzd6 new file mode 100644 index 00000000..fbfc843d --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/s/c/sch1bx4mpgstvnv1fns60d9hrzswpt6s8fgmvr6pjtzd6 @@ -0,0 +1,2 @@ +222222212222222222223222 + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/s/e/sepyp22kyv11cm1njcbsxr0d0p9cnyaqjn8cpheqapv4c b/implementation/fuzz/inputs-genvm-modules-complete-json/s/e/sepyp22kyv11cm1njcbsxr0d0p9cnyaqjn8cpheqapv4c new file mode 100644 index 00000000..8b002264 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/s/e/sepyp22kyv11cm1njcbsxr0d0p9cnyaqjn8cpheqapv4c @@ -0,0 +1 @@ +222E+2 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/s/e/sewpv417w2tcwf9p9n6sj8c5aa14eryvjt55q6m61byf0 b/implementation/fuzz/inputs-genvm-modules-complete-json/s/e/sewpv417w2tcwf9p9n6sj8c5aa14eryvjt55q6m61byf0 new file mode 100644 index 00000000..a9e4de08 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/s/e/sewpv417w2tcwf9p9n6sj8c5aa14eryvjt55q6m61byf0 @@ -0,0 +1 @@ + [111511e-111111111111111111,111.11131e-111111111, 3] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/s/h/shbzkkv7z5sq8ejty3d85npfsypfnrtktm61ce75wm8gm b/implementation/fuzz/inputs-genvm-modules-complete-json/s/h/shbzkkv7z5sq8ejty3d85npfsypfnrtktm61ce75wm8gm new file mode 100644 index 00000000..bfae0c80 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/s/h/shbzkkv7z5sq8ejty3d85npfsypfnrtktm61ce75wm8gm @@ -0,0 +1 @@ +1e-1 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/s/k/sk04ga1sjg4xj9mvqfbxq5a8g0rczv1p5hmpjj7m25fw6 b/implementation/fuzz/inputs-genvm-modules-complete-json/s/k/sk04ga1sjg4xj9mvqfbxq5a8g0rczv1p5hmpjj7m25fw6 new file mode 100644 index 00000000..0d09a46f --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/s/k/sk04ga1sjg4xj9mvqfbxq5a8g0rczv1p5hmpjj7m25fw6 @@ -0,0 +1,3 @@ +{ + "foo" ", + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/s/k/skw1n9487pkkys6qjqnwh3dxwnnfwwd3yrgg6cb49jjdc b/implementation/fuzz/inputs-genvm-modules-complete-json/s/k/skw1n9487pkkys6qjqnwh3dxwnnfwwd3yrgg6cb49jjdc new file mode 100644 index 00000000..6c651fcb --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/s/k/skw1n9487pkkys6qjqnwh3dxwnnfwwd3yrgg6cb49jjdc @@ -0,0 +1 @@ +"fζ—₯1\\\\\\β€—" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/s/m/smed9gvse0wb83r7aqgkstv6gdcck4h94tej49q39tp7m b/implementation/fuzz/inputs-genvm-modules-complete-json/s/m/smed9gvse0wb83r7aqgkstv6gdcck4h94tej49q39tp7m new file mode 100644 index 00000000..bd182a85 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/s/m/smed9gvse0wb83r7aqgkstv6gdcck4h94tej49q39tp7m @@ -0,0 +1 @@ +[2, 3] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/s/m/smj6k6yx46bcymgfhcmn1hbhej3km6yfr3c9wvh5xe63t b/implementation/fuzz/inputs-genvm-modules-complete-json/s/m/smj6k6yx46bcymgfhcmn1hbhej3km6yfr3c9wvh5xe63t new file mode 100644 index 0000000000000000000000000000000000000000..b7037acb485617f55eab2fc8363b263c24135d0b GIT binary patch literal 127 zcmb>CQUC%aLlmH7rJ!J#YGj}Tl2kG@&R61v0Z!aN0jdUMAPB&;@M71ZzyMSO0KV4? AwEzGB literal 0 HcmV?d00001 diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/s/n/sn8ydke73da5rnxnmpdn1sywrw0e9qw1z39rm8vw1875p b/implementation/fuzz/inputs-genvm-modules-complete-json/s/n/sn8ydke73da5rnxnmpdn1sywrw0e9qw1z39rm8vw1875p new file mode 100644 index 00000000..d9dd6a1c --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/s/n/sn8ydke73da5rnxnmpdn1sywrw0e9qw1z39rm8vw1875p @@ -0,0 +1,15 @@ +{ "foo": [ + "bar", + "baz\n\\{", + -1e10, + 1e20, + null, + true, + [1, 2, 3] + ], + "bapppppppppppppppppppppppppr": [ ], + "bar": [ + "русскиС mbbbbbbbbbbbbbbaz\t\uabbbbb", + "mbbbEbbbbbbbbbbaz\t\\uabbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/s/p/spd83sdtasgb2mw9aaaqtd2ag8bd8r8sg02e78e5nrhqr b/implementation/fuzz/inputs-genvm-modules-complete-json/s/p/spd83sdtasgb2mw9aaaqtd2ag8bd8r8sg02e78e5nrhqr new file mode 100644 index 00000000..4ab5509d --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/s/p/spd83sdtasgb2mw9aaaqtd2ag8bd8r8sg02e78e5nrhqr @@ -0,0 +1,4 @@ +[[[[[[[[ +3,[[[[[[[[[[[[[[ + +3,[[[[[[[[[[[[[[[[ [[[[[[[[[[[[[ [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[S \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/s/p/spj3bsfmd8z424mnm6vnmqxxk2578bkzztxtxbw7td7a2 b/implementation/fuzz/inputs-genvm-modules-complete-json/s/p/spj3bsfmd8z424mnm6vnmqxxk2578bkzztxtxbw7td7a2 new file mode 100644 index 00000000..457022db --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/s/p/spj3bsfmd8z424mnm6vnmqxxk2578bkzztxtxbw7td7a2 @@ -0,0 +1,21 @@ +{ + "foo": [ + "bar", + "baz\n\\{", + 123, + -123, + -1e10, + 1e20, + null, + true, + false, + [1, 2, 3] + ], + "mbb -barar": [ ], + "": [ ], + " bar": [ + "русскиС Π±ΡƒΠΊΠ²Π‡ ζ—₯本", + "mbbbbbbbbbbbbbbaz\t\uabbbbb", + "mbbbbbbbbbbbbbbaz\t\\uabbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/s/s/ssqjmb7kcpmjdfbwtqxt8mmfqwacc1ag8yprymdb8snep b/implementation/fuzz/inputs-genvm-modules-complete-json/s/s/ssqjmb7kcpmjdfbwtqxt8mmfqwacc1ag8yprymdb8snep new file mode 100644 index 00000000..6e49eb1e --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/s/s/ssqjmb7kcpmjdfbwtqxt8mmfqwacc1ag8yprymdb8snep @@ -0,0 +1 @@ +"[[1, B\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ [\\\\\\\\\\\\\\\\@\\\\\\\\\\\\\\\\\\\\\\\\\\\\" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/s/s/sssc9q6nt579crjpv174n51wpjn43bn909zaxnaayb4sj b/implementation/fuzz/inputs-genvm-modules-complete-json/s/s/sssc9q6nt579crjpv174n51wpjn43bn909zaxnaayb4sj new file mode 100644 index 00000000..7c6f8311 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/s/s/sssc9q6nt579crjpv174n51wpjn43bn909zaxnaayb4sj @@ -0,0 +1,17 @@ +{ + "fon": [ + + "baz\n\\z", + 123, + -123, + -1e10, + true, + false, + [1, 2, 3] + ], + "bar": [ + "русскиС Π±ΡƒΠΊΠ²Ρ‹ ζ—₯本", + "mbbbbbbbb}bbbbbaz\t\uabbbbb", + "mbbbbbbbbbbbbbbaz\tΠΊΠ²bbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/s/s/ssw4ytb1yy66twvrgjyfvwhfnkxk2kx2b0tdx3pcbsxae b/implementation/fuzz/inputs-genvm-modules-complete-json/s/s/ssw4ytb1yy66twvrgjyfvwhfnkxk2kx2b0tdx3pcbsxae new file mode 100644 index 00000000..a327b3f2 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/s/s/ssw4ytb1yy66twvrgjyfvwhfnkxk2kx2b0tdx3pcbsxae @@ -0,0 +1,2 @@ + "Ρ€\\u + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/s/t/sty8pqkhmpd4zqgwev6da6th185z93cx5xdwgvjmwjtr8 b/implementation/fuzz/inputs-genvm-modules-complete-json/s/t/sty8pqkhmpd4zqgwev6da6th185z93cx5xdwgvjmwjtr8 new file mode 100644 index 00000000..e32dde2c --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/s/t/sty8pqkhmpd4zqgwev6da6th185z93cx5xdwgvjmwjtr8 @@ -0,0 +1 @@ +112201111111111111111e-12 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/s/w/swxbs2ndsbvwv69rf6yrma4478m5tbq1rxaew320spk8c b/implementation/fuzz/inputs-genvm-modules-complete-json/s/w/swxbs2ndsbvwv69rf6yrma4478m5tbq1rxaew320spk8c new file mode 100644 index 00000000..609b81eb --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/s/w/swxbs2ndsbvwv69rf6yrma4478m5tbq1rxaew320spk8c @@ -0,0 +1 @@ + [1, 2, ) b \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/s/x/sxgx73x7d4skxrhkga3978tv1ftdjwa13gf35gxpb4mz6 b/implementation/fuzz/inputs-genvm-modules-complete-json/s/x/sxgx73x7d4skxrhkga3978tv1ftdjwa13gf35gxpb4mz6 new file mode 100644 index 00000000..5bc6d8d9 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/s/x/sxgx73x7d4skxrhkga3978tv1ftdjwa13gf35gxpb4mz6 @@ -0,0 +1 @@ +"Ρ€\\σ—₯ΊFbb r" diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/s/y/syemw0n5g8krbh5mfk84gqvbstaw9hxayxjqf9q0zwsra b/implementation/fuzz/inputs-genvm-modules-complete-json/s/y/syemw0n5g8krbh5mfk84gqvbstaw9hxayxjqf9q0zwsra new file mode 100644 index 00000000..e18cd933 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/s/y/syemw0n5g8krbh5mfk84gqvbstaw9hxayxjqf9q0zwsra @@ -0,0 +1,4 @@ +"" + + + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/s/y/symmye7nza29sez2a9029kn3h0g0f51yvm5ertx0cppyr b/implementation/fuzz/inputs-genvm-modules-complete-json/s/y/symmye7nza29sez2a9029kn3h0g0f51yvm5ertx0cppyr new file mode 100644 index 00000000..158bf204 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/s/y/symmye7nza29sez2a9029kn3h0g0f51yvm5ertx0cppyr @@ -0,0 +1,16 @@ +[ + + + + + + + + + + + + + + +[[b[ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/s/z/szkkg31h97m32mqp6qrs2p8f41cvr6y7f6qfkr7bcpzke b/implementation/fuzz/inputs-genvm-modules-complete-json/s/z/szkkg31h97m32mqp6qrs2p8f41cvr6y7f6qfkr7bcpzke new file mode 100644 index 00000000..c12c8ae6 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/s/z/szkkg31h97m32mqp6qrs2p8f41cvr6y7f6qfkr7bcpzke @@ -0,0 +1 @@ +"\\\\ζ³’a" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/t/0/t0nc5jyr3pe6v9bs5qeaf4khz36sn00g713mypkjvb7me b/implementation/fuzz/inputs-genvm-modules-complete-json/t/0/t0nc5jyr3pe6v9bs5qeaf4khz36sn00g713mypkjvb7me new file mode 100644 index 00000000..5ed6c9d1 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/t/0/t0nc5jyr3pe6v9bs5qeaf4khz36sn00g713mypkjvb7me @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/t/3/t3gehabpnf1n5b1znxw3tssraag8cb1z8ff0zscdtsgwt b/implementation/fuzz/inputs-genvm-modules-complete-json/t/3/t3gehabpnf1n5b1znxw3tssraag8cb1z8ff0zscdtsgwt new file mode 100644 index 00000000..287f0e3b --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/t/3/t3gehabpnf1n5b1znxw3tssraag8cb1z8ff0zscdtsgwt @@ -0,0 +1 @@ +"Ρ€" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/t/4/t4c0kr0xyk4tgzr9xmzgk85mw4abc3f7qw7hjrgyjhv9r b/implementation/fuzz/inputs-genvm-modules-complete-json/t/4/t4c0kr0xyk4tgzr9xmzgk85mw4abc3f7qw7hjrgyjhv9r new file mode 100644 index 0000000000000000000000000000000000000000..f1a10f5e23b375eae340a55432a16b6cd6065ccf GIT binary patch literal 284 zcmb>CQUC&_wETP}D}`t<2P}}3Sfr$b%&khvi;1a15;HV1MkoO3iG~0L!&D;!giKy( tP7au-3(^2mhYXYyOpv*_*k+g#P}`z0WgsqrikC59Q-DJ(77SpLc>q4;FpvNM literal 0 HcmV?d00001 diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/t/7/t7mp7d81dhz89yhk5yk0pxn4jxj751g9yh1kvqwtpd5ww b/implementation/fuzz/inputs-genvm-modules-complete-json/t/7/t7mp7d81dhz89yhk5yk0pxn4jxj751g9yh1kvqwtpd5ww new file mode 100644 index 0000000000000000000000000000000000000000..1a348de3e2da6ccb628986062ba7158010dd5f5e GIT binary patch literal 32 Wcmb>CQUC&_wETQ748Xv^$N&IJK?5NG literal 0 HcmV?d00001 diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/t/b/tb20b0s3mn5gg6e6jjxn7fs8khj3jh9de89q75yjp2tt2 b/implementation/fuzz/inputs-genvm-modules-complete-json/t/b/tb20b0s3mn5gg6e6jjxn7fs8khj3jh9de89q75yjp2tt2 new file mode 100644 index 00000000..90d65a66 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/t/b/tb20b0s3mn5gg6e6jjxn7fs8khj3jh9de89q75yjp2tt2 @@ -0,0 +1 @@ +2222222222222222222222 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/t/b/tbh8ppt850729b51y9nq4wkbkeczhyr1qbxdyekh7ggcr b/implementation/fuzz/inputs-genvm-modules-complete-json/t/b/tbh8ppt850729b51y9nq4wkbkeczhyr1qbxdyekh7ggcr new file mode 100644 index 00000000..b581a106 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/t/b/tbh8ppt850729b51y9nq4wkbkeczhyr1qbxdyekh7ggcr @@ -0,0 +1,3 @@ +"f\\u\\\\\\\\\\\\" + + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/t/b/tbzd4c43m17kehn6f4gbvjv81512b7ma1ffaq1xm5e4vt b/implementation/fuzz/inputs-genvm-modules-complete-json/t/b/tbzd4c43m17kehn6f4gbvjv81512b7ma1ffaq1xm5e4vt new file mode 100644 index 00000000..ac81defc --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/t/b/tbzd4c43m17kehn6f4gbvjv81512b7ma1ffaq1xm5e4vt @@ -0,0 +1,33 @@ +111 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +1 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/t/c/tccnrkkqzmrd17eh91gmtanm2hqs9jva7xed2a54cayd0 b/implementation/fuzz/inputs-genvm-modules-complete-json/t/c/tccnrkkqzmrd17eh91gmtanm2hqs9jva7xed2a54cayd0 new file mode 100644 index 00000000..5f0fcdef --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/t/c/tccnrkkqzmrd17eh91gmtanm2hqs9jva7xed2a54cayd0 @@ -0,0 +1 @@ +111111111111111111111e-515151611111 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/t/c/tcqdv403zxegm5pgexxxknh3xzd09f9s1zw488xfzab4r b/implementation/fuzz/inputs-genvm-modules-complete-json/t/c/tcqdv403zxegm5pgexxxknh3xzd09f9s1zw488xfzab4r new file mode 100644 index 00000000..5370b604 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/t/c/tcqdv403zxegm5pgexxxknh3xzd09f9s1zw488xfzab4r @@ -0,0 +1,18 @@ +22222222222222222222 + + + + + + + + + + + + + + + + + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/t/e/tewt7a2mr57vzp3k7ydqxm7jdqg2xykadr6zbjkq1qys4 b/implementation/fuzz/inputs-genvm-modules-complete-json/t/e/tewt7a2mr57vzp3k7ydqxm7jdqg2xykadr6zbjkq1qys4 new file mode 100644 index 00000000..39a81d2e --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/t/e/tewt7a2mr57vzp3k7ydqxm7jdqg2xykadr6zbjkq1qys4 @@ -0,0 +1 @@ +2e+1722 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/t/f/tfjrjfrctqnbnhkwyyjgkcfx3e93q9g42xfx6prq8mqg0 b/implementation/fuzz/inputs-genvm-modules-complete-json/t/f/tfjrjfrctqnbnhkwyyjgkcfx3e93q9g42xfx6prq8mqg0 new file mode 100644 index 00000000..5b056ada --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/t/f/tfjrjfrctqnbnhkwyyjgkcfx3e93q9g42xfx6prq8mqg0 @@ -0,0 +1,6 @@ +{ + + + + + "Ca \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/t/g/tghvavtjyjphjx3r37e97cpexy5871p899xet1x7m3ht4 b/implementation/fuzz/inputs-genvm-modules-complete-json/t/g/tghvavtjyjphjx3r37e97cpexy5871p899xet1x7m3ht4 new file mode 100644 index 00000000..e5b653fd --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/t/g/tghvavtjyjphjx3r37e97cpexy5871p899xet1x7m3ht4 @@ -0,0 +1,2 @@ + +"Ρ€\\uba5R [[1, B\bΓ΅\\u|a5R r" diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/t/h/thzkrf36h9dg6rwsac79b4k61gw1vqv6kw8n84ambexj4 b/implementation/fuzz/inputs-genvm-modules-complete-json/t/h/thzkrf36h9dg6rwsac79b4k61gw1vqv6kw8n84ambexj4 new file mode 100644 index 00000000..a7ec67ea --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/t/h/thzkrf36h9dg6rwsac79b4k61gw1vqv6kw8n84ambexj4 @@ -0,0 +1 @@ +222222222222222222222222122222 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/t/j/tjz9ftrgnt21xexwhtaznm62y4d0n39sgcey08s4pqn32 b/implementation/fuzz/inputs-genvm-modules-complete-json/t/j/tjz9ftrgnt21xexwhtaznm62y4d0n39sgcey08s4pqn32 new file mode 100644 index 0000000000000000000000000000000000000000..58cf8c06d0807e9e1a90f96e028da05daa269edd GIT binary patch literal 3 KcmZRO{0{&DGyxd^ literal 0 HcmV?d00001 diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/t/k/tkqszdz6rkktpz6712rj3jfh5x4by23nm5tyrkgabbxqr b/implementation/fuzz/inputs-genvm-modules-complete-json/t/k/tkqszdz6rkktpz6712rj3jfh5x4by23nm5tyrkgabbxqr new file mode 100644 index 00000000..a3e4771e --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/t/k/tkqszdz6rkktpz6712rj3jfh5x4by23nm5tyrkgabbxqr @@ -0,0 +1,2 @@ +99999999999999999999999999999 + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/t/m/tm1sxtx5gy8fg6qqxy4fp89bd6j5p43r7k2rd53s0546m b/implementation/fuzz/inputs-genvm-modules-complete-json/t/m/tm1sxtx5gy8fg6qqxy4fp89bd6j5p43r7k2rd53s0546m new file mode 100644 index 00000000..24996f96 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/t/m/tm1sxtx5gy8fg6qqxy4fp89bd6j5p43r7k2rd53s0546m @@ -0,0 +1 @@ +{ "1": \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/t/q/tq0hzsh2sy3ddcsg40ew1mwzk7t3ndxnx76b7rk8pcbza b/implementation/fuzz/inputs-genvm-modules-complete-json/t/q/tq0hzsh2sy3ddcsg40ew1mwzk7t3ndxnx76b7rk8pcbza new file mode 100644 index 00000000..22eaddbf --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/t/q/tq0hzsh2sy3ddcsg40ew1mwzk7t3ndxnx76b7rk8pcbza @@ -0,0 +1,4 @@ +"\u0000Ρ‹\u00001115111\u00001115111111111111111Ρ‹\u00000000000000000000000000ΡΊΠ°Ρ‹ ε—₯Sabbbb)))))))}00Ρ‹\u000011151111111111111 + + + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/t/q/tqqm05gz4tfbkzy9d0x40p1cj1mpvv18yx53kpwk0hajm b/implementation/fuzz/inputs-genvm-modules-complete-json/t/q/tqqm05gz4tfbkzy9d0x40p1cj1mpvv18yx53kpwk0hajm new file mode 100644 index 00000000..cd6f68d9 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/t/q/tqqm05gz4tfbkzy9d0x40p1cj1mpvv18yx53kpwk0hajm @@ -0,0 +1 @@ + " Ρƒ00 0\u00000" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/t/q/tqsgy5jc4ey7xqhd0pqgv9peh363c6rtzpqgzxr2j370j b/implementation/fuzz/inputs-genvm-modules-complete-json/t/q/tqsgy5jc4ey7xqhd0pqgv9peh363c6rtzpqgzxr2j370j new file mode 100644 index 00000000..9d0a21e2 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/t/q/tqsgy5jc4ey7xqhd0pqgv9peh363c6rtzpqgzxr2j370j @@ -0,0 +1,9 @@ + { + "":0, + "mmmXmmmbd" + " + + + + + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/t/s/tsr9tph355x87s5cafd7358xfetzmna2jaje9tn63hnze b/implementation/fuzz/inputs-genvm-modules-complete-json/t/s/tsr9tph355x87s5cafd7358xfetzmna2jaje9tn63hnze new file mode 100644 index 00000000..f6c2f2fb --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/t/s/tsr9tph355x87s5cafd7358xfetzmna2jaje9tn63hnze @@ -0,0 +1 @@ +"Ρ€\\ub " \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/t/v/tvg7yr8tr3r56kpmyh06mgde04kmepy0y78fxwkqbvya2 b/implementation/fuzz/inputs-genvm-modules-complete-json/t/v/tvg7yr8tr3r56kpmyh06mgde04kmepy0y78fxwkqbvya2 new file mode 100644 index 0000000000000000000000000000000000000000..f9b078d60ec7f2729dfe211df7d0dacf969b6ae6 GIT binary patch literal 13 Ucmb<cP*6%rEK=g);!<D$02Q19a{vGU literal 0 HcmV?d00001 diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/t/v/tvmb6ayrwvh5vxfdq3xv5m2qvx77r48zezsjedz6mmae2 b/implementation/fuzz/inputs-genvm-modules-complete-json/t/v/tvmb6ayrwvh5vxfdq3xv5m2qvx77r48zezsjedz6mmae2 new file mode 100644 index 00000000..dea00efa --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/t/v/tvmb6ayrwvh5vxfdq3xv5m2qvx77r48zezsjedz6mmae2 @@ -0,0 +1 @@ +111115111111111111111e-5222222222222222222222222222222 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/t/w/tw670yp56sq147e0zedx9kpbkqgceykbz4v2y6ga8wvjw b/implementation/fuzz/inputs-genvm-modules-complete-json/t/w/tw670yp56sq147e0zedx9kpbkqgceykbz4v2y6ga8wvjw new file mode 100644 index 00000000..f2be8bf5 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/t/w/tw670yp56sq147e0zedx9kpbkqgceykbz4v2y6ga8wvjw @@ -0,0 +1,2 @@ + +"Ρ€bbbb\\ua1$, r" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/t/x/txnccemyszf7rt08ae038h1y4j3eangvrjm1f0qgfzw18 b/implementation/fuzz/inputs-genvm-modules-complete-json/t/x/txnccemyszf7rt08ae038h1y4j3eangvrjm1f0qgfzw18 new file mode 100644 index 00000000..a6cbca8e --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/t/x/txnccemyszf7rt08ae038h1y4j3eangvrjm1f0qgfzw18 @@ -0,0 +1,2 @@ +2222222222.2222222222E+2 + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/t/y/ty27wk4wrch8ce8vdw81g5azp09v462g422pxzsxn13jp b/implementation/fuzz/inputs-genvm-modules-complete-json/t/y/ty27wk4wrch8ce8vdw81g5azp09v462g422pxzsxn13jp new file mode 100644 index 00000000..b9fa5673 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/t/y/ty27wk4wrch8ce8vdw81g5azp09v462g422pxzsxn13jp @@ -0,0 +1 @@ +11111111111111111111111111 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/t/y/tyegkzzk97gb87zw8n0jxt3e1tm0af7ga1s127eg8y6g2 b/implementation/fuzz/inputs-genvm-modules-complete-json/t/y/tyegkzzk97gb87zw8n0jxt3e1tm0af7ga1s127eg8y6g2 new file mode 100644 index 00000000..2b2b21cf --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/t/y/tyegkzzk97gb87zw8n0jxt3e1tm0af7ga1s127eg8y6g2 @@ -0,0 +1 @@ + [111111111111111111111.3333E3] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/v/0/v01hpm7n0mr8w9zvfjj77ja3dds7d3t4eykavvjyb9m18 b/implementation/fuzz/inputs-genvm-modules-complete-json/v/0/v01hpm7n0mr8w9zvfjj77ja3dds7d3t4eykavvjyb9m18 new file mode 100644 index 00000000..d450d7b0 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/v/0/v01hpm7n0mr8w9zvfjj77ja3dds7d3t4eykavvjyb9m18 @@ -0,0 +1,21 @@ +{ + "foo": [ + "bar", + + + +1611111e-515 , + 1.3, + -123, + -1e10, + 1e20, + true, + false, + [1, 2, 3] + ], + "bar": [ + "русскиС Π±ΡƒΠΊΠ²Ρ‹ ζ₯本", + "mbbbbbbbbbbbbbbaz\t\uabbbbb", + "mbbbbbbbbbbbbb Π±Ρƒ\\uabbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/v/0/v0mpmhw2nvq116a2mq66w5ac0t0n0rh9n803ws906ezt6 b/implementation/fuzz/inputs-genvm-modules-complete-json/v/0/v0mpmhw2nvq116a2mq66w5ac0t0n0rh9n803ws906ezt6 new file mode 100644 index 00000000..b3814251 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/v/0/v0mpmhw2nvq116a2mq66w5ac0t0n0rh9n803ws906ezt6 @@ -0,0 +1,2 @@ + +> \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/v/0/v0n4ehya912q5endeyjvppc7xk4a3p4pywpme4e8cm0f2 b/implementation/fuzz/inputs-genvm-modules-complete-json/v/0/v0n4ehya912q5endeyjvppc7xk4a3p4pywpme4e8cm0f2 new file mode 100644 index 00000000..7c363989 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/v/0/v0n4ehya912q5endeyjvppc7xk4a3p4pywpme4e8cm0f2 @@ -0,0 +1,3 @@ + + +"Ρ€[\bΠ²‡ σ—₯Ί5 b r" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/v/1/v1bbhk3wqc0516fwh3znyrpb8ccx98nb67ytedg78vz4w b/implementation/fuzz/inputs-genvm-modules-complete-json/v/1/v1bbhk3wqc0516fwh3znyrpb8ccx98nb67ytedg78vz4w new file mode 100644 index 00000000..6e2b4bef --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/v/1/v1bbhk3wqc0516fwh3znyrpb8ccx98nb67ytedg78vz4w @@ -0,0 +1,18 @@ +{ + "fz\t\uabbbbboo": [ + "bar", + "baz\n\\z", + 123, + -123, + + 1e20, + null, + true, + false, + [1, 2, 3] + ], + "bar": [ + "русскиС Π±ΡƒΠΊΠ²Ρ‹ ζ—₯本", + "mbbbbbbbbbbbbbbabbbbbbbbbbb\\uabbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/v/2/v226w6c43jv18cfeb1bnxkjc1dv6yvb16ghhpksdtdqam b/implementation/fuzz/inputs-genvm-modules-complete-json/v/2/v226w6c43jv18cfeb1bnxkjc1dv6yvb16ghhpksdtdqam new file mode 100644 index 00000000..1cfbc51b --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/v/2/v226w6c43jv18cfeb1bnxkjc1dv6yvb16ghhpksdtdqam @@ -0,0 +1 @@ +"\"\fo\" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/v/2/v2ydwb8kqxsbsft3xqrjr6ph3ppj99yhenczadg1b6ate b/implementation/fuzz/inputs-genvm-modules-complete-json/v/2/v2ydwb8kqxsbsft3xqrjr6ph3ppj99yhenczadg1b6ate new file mode 100644 index 00000000..80ed29c8 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/v/2/v2ydwb8kqxsbsft3xqrjr6ph3ppj99yhenczadg1b6ate @@ -0,0 +1,8 @@ +{ + "foo": [ + "bar", +[[[[[[[[[[[[[[[[[[[[[[[[[[ 123, + -123, + -1e10, + 1e20, + null,,,,,,,,,,,,,,,,,,,,,,,,,a \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/v/3/v3fny0ttw0wsmmhf0tt3qwz2w8n455h6ty27qjq0pwqxe b/implementation/fuzz/inputs-genvm-modules-complete-json/v/3/v3fny0ttw0wsmmhf0tt3qwz2w8n455h6ty27qjq0pwqxe new file mode 100644 index 00000000..b7147fc5 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/v/3/v3fny0ttw0wsmmhf0tt3qwz2w8n455h6ty27qjq0pwqxe @@ -0,0 +1,20 @@ +[" + + + + + + + + + + + + + + + + + + + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/v/3/v3rf8grtvs24scm7eht2f38dwpeffbmnxptb81yszkbh4 b/implementation/fuzz/inputs-genvm-modules-complete-json/v/3/v3rf8grtvs24scm7eht2f38dwpeffbmnxptb81yszkbh4 new file mode 100644 index 00000000..4c723b2a --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/v/3/v3rf8grtvs24scm7eht2f38dwpeffbmnxptb81yszkbh4 @@ -0,0 +1 @@ +1.61e6 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/v/4/v4mx2wxpjc3ypdq1rfnjhg8jxg94pnfyzdsmkqq10vgz8 b/implementation/fuzz/inputs-genvm-modules-complete-json/v/4/v4mx2wxpjc3ypdq1rfnjhg8jxg94pnfyzdsmkqq10vgz8 new file mode 100644 index 00000000..62d5d70d --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/v/4/v4mx2wxpjc3ypdq1rfnjhg8jxg94pnfyzdsmkqq10vgz8 @@ -0,0 +1 @@ +1.111111211 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/v/5/v5j0gwb56jzrre8y93et4mcv9x71s3takyq3bkf8kwagg b/implementation/fuzz/inputs-genvm-modules-complete-json/v/5/v5j0gwb56jzrre8y93et4mcv9x71s3takyq3bkf8kwagg new file mode 100644 index 00000000..7ee11f64 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/v/5/v5j0gwb56jzrre8y93et4mcv9x71s3takyq3bkf8kwagg @@ -0,0 +1 @@ +"UUUVbcγ€» rIbcγ€» rbbbbbb," diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/v/5/v5sxkhpw15t9fyev7w2frdce2kzxtrbmpvrpm77cvz15e b/implementation/fuzz/inputs-genvm-modules-complete-json/v/5/v5sxkhpw15t9fyev7w2frdce2kzxtrbmpvrpm77cvz15e new file mode 100644 index 00000000..917e19ca --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/v/5/v5sxkhpw15t9fyev7w2frdce2kzxtrbmpvrpm77cvz15e @@ -0,0 +1,5 @@ +{ "Ρ€\\u 4 $$$$ 1111@1110": [ ] +} + + + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/v/6/v6qzwvrh6qzg0q78njxekj5xg587hfb3x88dytjc91tqy b/implementation/fuzz/inputs-genvm-modules-complete-json/v/6/v6qzwvrh6qzg0q78njxekj5xg587hfb3x88dytjc91tqy new file mode 100644 index 00000000..57a13fac --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/v/6/v6qzwvrh6qzg0q78njxekj5xg587hfb3x88dytjc91tqy @@ -0,0 +1,2 @@ + { + "":0, \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/v/b/vbtrzd9d9nszedymt6vd4b1nfyv5g3ne00k2cqf1d4zxe b/implementation/fuzz/inputs-genvm-modules-complete-json/v/b/vbtrzd9d9nszedymt6vd4b1nfyv5g3ne00k2cqf1d4zxe new file mode 100644 index 00000000..331141e5 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/v/b/vbtrzd9d9nszedymt6vd4b1nfyv5g3ne00k2cqf1d4zxe @@ -0,0 +1 @@ +{ "br" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/v/c/vc5ww77ga0fk0cdsxtckn180ensm0ns09dyavgvwky8r4 b/implementation/fuzz/inputs-genvm-modules-complete-json/v/c/vc5ww77ga0fk0cdsxtckn180ensm0ns09dyavgvwky8r4 new file mode 100644 index 00000000..3dda308b --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/v/c/vc5ww77ga0fk0cdsxtckn180ensm0ns09dyavgvwky8r4 @@ -0,0 +1,4 @@ +"f0(00bAz\"\"VnG\"rzDΠΊΤΈΚ± ΘšΡƒΓΊo00\ro 1a+\ro 1 ( (000 bbaa+O0D07Ρ€\\uabb{bb" + + + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/v/c/vcxkywx2ywv64776g163kf3cefzwz3694bxebr7svcyw8 b/implementation/fuzz/inputs-genvm-modules-complete-json/v/c/vcxkywx2ywv64776g163kf3cefzwz3694bxebr7svcyw8 new file mode 100644 index 00000000..bcf64b9d --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/v/c/vcxkywx2ywv64776g163kf3cefzwz3694bxebr7svcyw8 @@ -0,0 +1 @@ +2222222222222222222222 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/v/f/vffm0hhkv5f63jcf4j05zb3n78vd0qp63fex6qeq4xkp2 b/implementation/fuzz/inputs-genvm-modules-complete-json/v/f/vffm0hhkv5f63jcf4j05zb3n78vd0qp63fex6qeq4xkp2 new file mode 100644 index 00000000..9351e598 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/v/f/vffm0hhkv5f63jcf4j05zb3n78vd0qp63fex6qeq4xkp2 @@ -0,0 +1 @@ + "русскиʡ6Π±ΡƒΠΊbbcbbbbbIbbbΪ²Ρ‹ ζ—₯1 Bxbz" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/v/f/vfy3tt0qcnf44gpzmrtys7c6m0payywz0ebwd84xq28ky b/implementation/fuzz/inputs-genvm-modules-complete-json/v/f/vfy3tt0qcnf44gpzmrtys7c6m0payywz0ebwd84xq28ky new file mode 100644 index 00000000..e4eb59f0 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/v/f/vfy3tt0qcnf44gpzmrtys7c6m0payywz0ebwd84xq28ky @@ -0,0 +1 @@ +1.111 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/v/g/vgwa3yyr5zkyk7dg3xpn2rzrn42dgytz80he4m10560xy b/implementation/fuzz/inputs-genvm-modules-complete-json/v/g/vgwa3yyr5zkyk7dg3xpn2rzrn42dgytz80he4m10560xy new file mode 100644 index 00000000..0d707f26 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/v/g/vgwa3yyr5zkyk7dg3xpn2rzrn42dgytz80he4m10560xy @@ -0,0 +1,2 @@ +[[[[ +[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ [[[[[[[[[[[[[ [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/v/h/vh8rwxqf32axt398a08qrrvz9h90qzhjq4x86evknssbc b/implementation/fuzz/inputs-genvm-modules-complete-json/v/h/vh8rwxqf32axt398a08qrrvz9h90qzhjq4x86evknssbc new file mode 100644 index 00000000..c0988a9a --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/v/h/vh8rwxqf32axt398a08qrrvz9h90qzhjq4x86evknssbc @@ -0,0 +1,8 @@ + { + "":0, + " + + + + + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/v/m/vm4kkahbvyvb8sffh3rwmf7eq5bydnv3vx5gtap05a9z8 b/implementation/fuzz/inputs-genvm-modules-complete-json/v/m/vm4kkahbvyvb8sffh3rwmf7eq5bydnv3vx5gtap05a9z8 new file mode 100644 index 00000000..4d2c6902 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/v/m/vm4kkahbvyvb8sffh3rwmf7eq5bydnv3vx5gtap05a9z8 @@ -0,0 +1 @@ +2E+22 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/v/q/vqn8sf0fpgcf5nth6d4xkhjytxynyqpr8kp2z0nmxc0e6 b/implementation/fuzz/inputs-genvm-modules-complete-json/v/q/vqn8sf0fpgcf5nth6d4xkhjytxynyqpr8kp2z0nmxc0e6 new file mode 100644 index 00000000..8f92d76e --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/v/q/vqn8sf0fpgcf5nth6d4xkhjytxynyqpr8kp2z0nmxc0e6 @@ -0,0 +1 @@ +22222222222222222222222222222 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/v/q/vqpgxh7cnypmzrfphqxsy7qtxbqtsfnq1ep7ndksgww2e b/implementation/fuzz/inputs-genvm-modules-complete-json/v/q/vqpgxh7cnypmzrfphqxsy7qtxbqtsfnq1ep7ndksgww2e new file mode 100644 index 00000000..2ce695d2 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/v/q/vqpgxh7cnypmzrfphqxsy7qtxbqtsfnq1ep7ndksgww2e @@ -0,0 +1,2 @@ + +"Ρ€\\ub|5R [[1, B\bΓ΅ r" diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/v/s/vsdj5q9vxs4g30xzwvsbz7hg6nb9dn00tpdc6qk9tyhg2 b/implementation/fuzz/inputs-genvm-modules-complete-json/v/s/vsdj5q9vxs4g30xzwvsbz7hg6nb9dn00tpdc6qk9tyhg2 new file mode 100644 index 00000000..a63cf904 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/v/s/vsdj5q9vxs4g30xzwvsbz7hg6nb9dn00tpdc6qk9tyhg2 @@ -0,0 +1,4 @@ +{ + "11111111111111": 1e20, + +  \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/v/s/vsyxzq69qcqef54brey6fwpv7dh8fqmmgmbzq7a6n4j60 b/implementation/fuzz/inputs-genvm-modules-complete-json/v/s/vsyxzq69qcqef54brey6fwpv7dh8fqmmgmbzq7a6n4j60 new file mode 100644 index 00000000..a84fe555 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/v/s/vsyxzq69qcqef54brey6fwpv7dh8fqmmgmbzq7a6n4j60 @@ -0,0 +1,2 @@ +{ + " " \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/v/t/vt2g24hdjy5vwww4x7y5p44azbjrvzjn1717jas35cteg b/implementation/fuzz/inputs-genvm-modules-complete-json/v/t/vt2g24hdjy5vwww4x7y5p44azbjrvzjn1717jas35cteg new file mode 100644 index 0000000000000000000000000000000000000000..f5d7432810b7fa657fadc9538f7e6dcfa64c0ddf GIT binary patch literal 256 zcmb>CQUC&_wETP}PF_w<PAi3Iun1T*DX~aN2cjS;u_`4mCZ-C?2T2<m8ABvN91w^` z0t$wyMg|b+ywaQ;FiRJt1ELgYrIC>lkgaQ&3KJokhFPPakd%}N;~>#cgUT2Po0tax D$A>Wk literal 0 HcmV?d00001 diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/v/w/vwhwypz73cj0fkgaachvc33598qjb92j8ctd4j4ng3ndr b/implementation/fuzz/inputs-genvm-modules-complete-json/v/w/vwhwypz73cj0fkgaachvc33598qjb92j8ctd4j4ng3ndr new file mode 100644 index 00000000..ab0ba5fb --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/v/w/vwhwypz73cj0fkgaachvc33598qjb92j8ctd4j4ng3ndr @@ -0,0 +1,16 @@ +{ + "foo": [ + "bar", + "baz\n\\z", + 123, + -123, + -1e10, + 1e20, + null, +true, + false, + [1, 2, 3] + ], "baaz\n\\r": [ + ], "Ρ€ΡƒΠ΅ \\": "barΠ±ΡƒΠΊΠ²Ρ‹ ζ—₯本", + "mbbbbbbbbbbbbbbaz\t\uabbbbb", + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/v/w/vwtnpjteafpchkcd19tsd63z67d4m1fxbrwy46skjamq0 b/implementation/fuzz/inputs-genvm-modules-complete-json/v/w/vwtnpjteafpchkcd19tsd63z67d4m1fxbrwy46skjamq0 new file mode 100644 index 00000000..3f0ecf11 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/v/w/vwtnpjteafpchkcd19tsd63z67d4m1fxbrwy46skjamq0 @@ -0,0 +1 @@ +22919112161111111111111 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/v/y/vy8fz3gbm7k0mxgbew3vw5n5bmq6jhakqmttp3r5dbgrm b/implementation/fuzz/inputs-genvm-modules-complete-json/v/y/vy8fz3gbm7k0mxgbew3vw5n5bmq6jhakqmttp3r5dbgrm new file mode 100644 index 00000000..62e7d951 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/v/y/vy8fz3gbm7k0mxgbew3vw5n5bmq6jhakqmttp3r5dbgrm @@ -0,0 +1,4 @@ + [0, + "bar" + + ] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/w/0/w0225bd73w9dazbw6jb2pgh84cpw34z4aga4jzn7b82ap b/implementation/fuzz/inputs-genvm-modules-complete-json/w/0/w0225bd73w9dazbw6jb2pgh84cpw34z4aga4jzn7b82ap new file mode 100644 index 00000000..92f3a950 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/w/0/w0225bd73w9dazbw6jb2pgh84cpw34z4aga4jzn7b82ap @@ -0,0 +1 @@ + [11  \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/w/0/w07t44ytv2x4cagakbp8vyced6dxaffy35h6t2fsa3nk4 b/implementation/fuzz/inputs-genvm-modules-complete-json/w/0/w07t44ytv2x4cagakbp8vyced6dxaffy35h6t2fsa3nk4 new file mode 100644 index 0000000000000000000000000000000000000000..46bd69a4d3be5729be335102a9393dc7c8f7f086 GIT binary patch literal 76 jcmb>CQUC%aLlmH7rJ!J#YGj}Tl2kG@&R60k3>bI;w%`cs literal 0 HcmV?d00001 diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/w/1/w1229a8rvrkwbhyktew1xb3frek1atfrxzw33kq8ssgh2 b/implementation/fuzz/inputs-genvm-modules-complete-json/w/1/w1229a8rvrkwbhyktew1xb3frek1atfrxzw33kq8ssgh2 new file mode 100644 index 00000000..ef4eb2e4 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/w/1/w1229a8rvrkwbhyktew1xb3frek1atfrxzw33kq8ssgh2 @@ -0,0 +1 @@ + "\/r" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/w/2/w2abwa2c3bmhgjnc1qmt5xrhnr96aarma0tk1n2dz8sft b/implementation/fuzz/inputs-genvm-modules-complete-json/w/2/w2abwa2c3bmhgjnc1qmt5xrhnr96aarma0tk1n2dz8sft new file mode 100644 index 00000000..9c8371a2 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/w/2/w2abwa2c3bmhgjnc1qmt5xrhnr96aarma0tk1n2dz8sft @@ -0,0 +1 @@ + "bξ—¨750 4 :UwUoUσ—₯£" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/w/3/w3nar75v78qkx3d928h0he832sqbkctbmcrk5fe9ws8ba b/implementation/fuzz/inputs-genvm-modules-complete-json/w/3/w3nar75v78qkx3d928h0he832sqbkctbmcrk5fe9ws8ba new file mode 100644 index 00000000..6cc3083a --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/w/3/w3nar75v78qkx3d928h0he832sqbkctbmcrk5fe9ws8ba @@ -0,0 +1 @@ +222222222222222222222222222 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/w/8/w8j0vfcsy4ex7923x16y4wrhh0h70kenfhy215j09jf28 b/implementation/fuzz/inputs-genvm-modules-complete-json/w/8/w8j0vfcsy4ex7923x16y4wrhh0h70kenfhy215j09jf28 new file mode 100644 index 00000000..79132d73 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/w/8/w8j0vfcsy4ex7923x16y4wrhh0h70kenfhy215j09jf28 @@ -0,0 +1 @@ +2222222222222222222222E+2 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/w/8/w8rwmgvasgfdha3k13fyf7h2m6mg00kd77waj719nhbfe b/implementation/fuzz/inputs-genvm-modules-complete-json/w/8/w8rwmgvasgfdha3k13fyf7h2m6mg00kd77waj719nhbfe new file mode 100644 index 00000000..7894427f --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/w/8/w8rwmgvasgfdha3k13fyf7h2m6mg00kd77waj719nhbfe @@ -0,0 +1 @@ +111115111111111111111E-522122 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/w/a/wa556dm9x1mzfj5g2th5wmys862hry0ryn5rde5xg3be8 b/implementation/fuzz/inputs-genvm-modules-complete-json/w/a/wa556dm9x1mzfj5g2th5wmys862hry0ryn5rde5xg3be8 new file mode 100644 index 00000000..2f46758e --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/w/a/wa556dm9x1mzfj5g2th5wmys862hry0ryn5rde5xg3be8 @@ -0,0 +1,2 @@ + [ + 1111.61e-11111111111] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/w/a/waf2pz4cpn6fxk22zay0d1g8k7nzbhffyf573ajvg3te6 b/implementation/fuzz/inputs-genvm-modules-complete-json/w/a/waf2pz4cpn6fxk22zay0d1g8k7nzbhffyf573ajvg3te6 new file mode 100644 index 00000000..8b6c64fc --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/w/a/waf2pz4cpn6fxk22zay0d1g8k7nzbhffyf573ajvg3te6 @@ -0,0 +1,6 @@ +{ + + + + + } \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/w/a/wan3zs63jsevc9x72va1apz2t0ynsktxdxryvsf8bpt10 b/implementation/fuzz/inputs-genvm-modules-complete-json/w/a/wan3zs63jsevc9x72va1apz2t0ynsktxdxryvsf8bpt10 new file mode 100644 index 00000000..3b334923 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/w/a/wan3zs63jsevc9x72va1apz2t0ynsktxdxryvsf8bpt10 @@ -0,0 +1 @@ +"UUU\bΠΊ" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/w/b/wbn6hd1d29xjz910rz9haaxdj81atz1gpmk6a8910w5jj b/implementation/fuzz/inputs-genvm-modules-complete-json/w/b/wbn6hd1d29xjz910rz9haaxdj81atz1gpmk6a8910w5jj new file mode 100644 index 00000000..efcb29dd --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/w/b/wbn6hd1d29xjz910rz9haaxdj81atz1gpmk6a8910w5jj @@ -0,0 +1 @@ +91111111141111111111.11 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/w/c/wc7166zk28n709g67hkqn9e4vb38t749q18srgb226yhy b/implementation/fuzz/inputs-genvm-modules-complete-json/w/c/wc7166zk28n709g67hkqn9e4vb38t749q18srgb226yhy new file mode 100644 index 00000000..16980aa4 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/w/c/wc7166zk28n709g67hkqn9e4vb38t749q18srgb226yhy @@ -0,0 +1 @@ +661e-811 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/w/d/wd1dqfbtbahx5wngqwwzjrs59v7p10gfcndybp0d96j6j b/implementation/fuzz/inputs-genvm-modules-complete-json/w/d/wd1dqfbtbahx5wngqwwzjrs59v7p10gfcndybp0d96j6j new file mode 100644 index 00000000..4f96352b --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/w/d/wd1dqfbtbahx5wngqwwzjrs59v7p10gfcndybp0d96j6j @@ -0,0 +1,2 @@ + +"\u0000Ρ‹bbbbbbbbbbbbbOu0000Ρ‹\u00000Ρ€\u00000Ρ€\\wbbbbbbbbbbbbOu0000Ρ‹\u00000Ρ€\u00200Ρ€\\wba1 r" diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/w/e/wee7ssenrycdbatem6dtavfcxnyqs4qmaca3cnaxk1bya b/implementation/fuzz/inputs-genvm-modules-complete-json/w/e/wee7ssenrycdbatem6dtavfcxnyqs4qmaca3cnaxk1bya new file mode 100644 index 00000000..b5a8efca --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/w/e/wee7ssenrycdbatem6dtavfcxnyqs4qmaca3cnaxk1bya @@ -0,0 +1,2 @@ +222222222222222222222E+2 + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/w/f/wfr8wqbx83fgq50mww2ect3a8j08189dy7v8663mrregr b/implementation/fuzz/inputs-genvm-modules-complete-json/w/f/wfr8wqbx83fgq50mww2ect3a8j08189dy7v8663mrregr new file mode 100644 index 00000000..6916632d --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/w/f/wfr8wqbx83fgq50mww2ect3a8j08189dy7v8663mrregr @@ -0,0 +1,4 @@ +"f0000000\ro 1a+\ro 1 ( (000ΠΊΤΈΚ± ΘšΡƒΓΊo00\ro 1a+0 bbaa+000000000000000000Ρ€\\uabbubb" + + + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/w/g/wgbw3wrjhp2ywmqdg8cbgedrjy9apfx30bvm0c4h4w9gm b/implementation/fuzz/inputs-genvm-modules-complete-json/w/g/wgbw3wrjhp2ywmqdg8cbgedrjy9apfx30bvm0c4h4w9gm new file mode 100644 index 00000000..8f7f7fe6 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/w/g/wgbw3wrjhp2ywmqdg8cbgedrjy9apfx30bvm0c4h4w9gm @@ -0,0 +1,9 @@ + + + + +[ [[[[[[[[[[[[ +[[[[[ + + +[[[[[[[[[[[[[[[[[[[[[[[[=, , \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/w/g/wggdm8g9sepcbgchsj5gjfhqnn15r4xgxzv225hd94dqa b/implementation/fuzz/inputs-genvm-modules-complete-json/w/g/wggdm8g9sepcbgchsj5gjfhqnn15r4xgxzv225hd94dqa new file mode 100644 index 00000000..a3210e9f --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/w/g/wggdm8g9sepcbgchsj5gjfhqnn15r4xgxzv225hd94dqa @@ -0,0 +1,2 @@ +99999999999999999999999999 + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/w/g/wgjt61ess9kww0xvgsq35ej5new78w63pzkhfc252tchg b/implementation/fuzz/inputs-genvm-modules-complete-json/w/g/wgjt61ess9kww0xvgsq35ej5new78w63pzkhfc252tchg new file mode 100644 index 00000000..17efdd1c --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/w/g/wgjt61ess9kww0xvgsq35ej5new78w63pzkhfc252tchg @@ -0,0 +1,3 @@ +[[[[[[[[[ + +3,[[[[[[[ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/w/j/wj34mctxj9r2m4f7mp8cdmg1s776c0feyxn27h206yms6 b/implementation/fuzz/inputs-genvm-modules-complete-json/w/j/wj34mctxj9r2m4f7mp8cdmg1s776c0feyxn27h206yms6 new file mode 100644 index 00000000..71eaca19 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/w/j/wj34mctxj9r2m4f7mp8cdmg1s776c0feyxn27h206yms6 @@ -0,0 +1,5 @@ +{ + "foo": [ + + ], + "bar" "baΠ΅ Π₯本 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/w/j/wjj5qa0n4pp10ejqmt8qrzy6wvwkja36my7rhtc046q14 b/implementation/fuzz/inputs-genvm-modules-complete-json/w/j/wjj5qa0n4pp10ejqmt8qrzy6wvwkja36my7rhtc046q14 new file mode 100644 index 00000000..3b4f25ec --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/w/j/wjj5qa0n4pp10ejqmt8qrzy6wvwkja36my7rhtc046q14 @@ -0,0 +1,5 @@ +{ "Ρ€\\u 111111111": [ ], + + + + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/w/j/wjk3ttywnza76t2k3z4qvrbmyg2b9ptvr7ennmym1n59g b/implementation/fuzz/inputs-genvm-modules-complete-json/w/j/wjk3ttywnza76t2k3z4qvrbmyg2b9ptvr7ennmym1n59g new file mode 100644 index 00000000..495c4718 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/w/j/wjk3ttywnza76t2k3z4qvrbmyg2b9ptvr7ennmym1n59g @@ -0,0 +1 @@ +"UUUUwUoUUUUUUUUUUU\n]I" diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/w/k/wkgbj1ak3n7ercyw3x0f3nf884dja04kcbg7wey0kfvxc b/implementation/fuzz/inputs-genvm-modules-complete-json/w/k/wkgbj1ak3n7ercyw3x0f3nf884dja04kcbg7wey0kfvxc new file mode 100644 index 00000000..34da7763 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/w/k/wkgbj1ak3n7ercyw3x0f3nf884dja04kcbg7wey0kfvxc @@ -0,0 +1,31 @@ +3e-922 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/w/k/wkktvvynb5h7hgm8py35yf9ta0r6psqww75yg99ypm26g b/implementation/fuzz/inputs-genvm-modules-complete-json/w/k/wkktvvynb5h7hgm8py35yf9ta0r6psqww75yg99ypm26g new file mode 100644 index 00000000..0e8f8e06 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/w/k/wkktvvynb5h7hgm8py35yf9ta0r6psqww75yg99ypm26g @@ -0,0 +1,2 @@ + +"euu0000Ρ‹\u00D0Ρ‹\u00ba1 0000Ρ‹\u00D0Ρ‹\u00ba1 0D0Ρ‹\u00ba1r" diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/w/k/wkygkggtxf22b7w6d38faw092jf348v67vp78dc7esj1r b/implementation/fuzz/inputs-genvm-modules-complete-json/w/k/wkygkggtxf22b7w6d38faw092jf348v67vp78dc7esj1r new file mode 100644 index 00000000..bb40953f --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/w/k/wkygkggtxf22b7w6d38faw092jf348v67vp78dc7esj1r @@ -0,0 +1 @@ +"\nH" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/w/n/wn5e036b3prr4germa9hy89g82j908yd9tn2pg7vds984 b/implementation/fuzz/inputs-genvm-modules-complete-json/w/n/wn5e036b3prr4germa9hy89g82j908yd9tn2pg7vds984 new file mode 100644 index 00000000..32150a87 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/w/n/wn5e036b3prr4germa9hy89g82j908yd9tn2pg7vds984 @@ -0,0 +1 @@ + [1, ,  \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/w/p/wp50v8nk3akyvwhn06hwzz1b1ssz04n83wwzc1ex5p0tm b/implementation/fuzz/inputs-genvm-modules-complete-json/w/p/wp50v8nk3akyvwhn06hwzz1b1ssz04n83wwzc1ex5p0tm new file mode 100644 index 00000000..4a2bacb2 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/w/p/wp50v8nk3akyvwhn06hwzz1b1ssz04n83wwzc1ex5p0tm @@ -0,0 +1 @@ +"\f{obj ζ—₯ubb[" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/w/p/wp8ksqp4rzj5czsyq8ad6dmr0zbw8mwqpzpyt5zgvs638 b/implementation/fuzz/inputs-genvm-modules-complete-json/w/p/wp8ksqp4rzj5czsyq8ad6dmr0zbw8mwqpzpyt5zgvs638 new file mode 100644 index 00000000..7dd701ef --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/w/p/wp8ksqp4rzj5czsyq8ad6dmr0zbw8mwqpzpyt5zgvs638 @@ -0,0 +1 @@ +1e-7333 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/w/p/wphfn9cmx58aw16edmv25m9p93zvvp2wx3y9fd0e9ktey b/implementation/fuzz/inputs-genvm-modules-complete-json/w/p/wphfn9cmx58aw16edmv25m9p93zvvp2wx3y9fd0e9ktey new file mode 100644 index 00000000..44de08a7 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/w/p/wphfn9cmx58aw16edmv25m9p93zvvp2wx3y9fd0e9ktey @@ -0,0 +1,4 @@ + [ [{ + "foo" + : null , + nul \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/w/q/wq79wfbcss4wzg30xfctg28sang4r84taka353p2j7v3w b/implementation/fuzz/inputs-genvm-modules-complete-json/w/q/wq79wfbcss4wzg30xfctg28sang4r84taka353p2j7v3w new file mode 100644 index 00000000..3da4b957 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/w/q/wq79wfbcss4wzg30xfctg28sang4r84taka353p2j7v3w @@ -0,0 +1,2 @@ + +"Ρ€bbbbIbbβ€» n\/2#\\ubGqz\r" diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/w/r/wrs5t71dcjjsxgs535bvcjgn24hegspkjfzsdbajznj4c b/implementation/fuzz/inputs-genvm-modules-complete-json/w/r/wrs5t71dcjjsxgs535bvcjgn24hegspkjfzsdbajznj4c new file mode 100644 index 00000000..7b5d59f7 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/w/r/wrs5t71dcjjsxgs535bvcjgn24hegspkjfzsdbajznj4c @@ -0,0 +1 @@ +"\f\\ ζ—₯u " \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/w/s/wsfags1x4xer51704f72d0ymmk40kabm1eeezg84rf8r6 b/implementation/fuzz/inputs-genvm-modules-complete-json/w/s/wsfags1x4xer51704f72d0ymmk40kabm1eeezg84rf8r6 new file mode 100644 index 00000000..01a28d43 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/w/s/wsfags1x4xer51704f72d0ymmk40kabm1eeezg84rf8r6 @@ -0,0 +1 @@ + [1, , ] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/w/v/wv6st7rt4c9n8pqq0t14s83m0akm8cx594yv0fnc5zmg2 b/implementation/fuzz/inputs-genvm-modules-complete-json/w/v/wv6st7rt4c9n8pqq0t14s83m0akm8cx594yv0fnc5zmg2 new file mode 100644 index 00000000..b9cf704e --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/w/v/wv6st7rt4c9n8pqq0t14s83m0akm8cx594yv0fnc5zmg2 @@ -0,0 +1,2 @@ +[, + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/w/w/wwb5adm1pqs5928hf33j2sedc6j7try738exxnh6hypc0 b/implementation/fuzz/inputs-genvm-modules-complete-json/w/w/wwb5adm1pqs5928hf33j2sedc6j7try738exxnh6hypc0 new file mode 100644 index 00000000..729e63f2 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/w/w/wwb5adm1pqs5928hf33j2sedc6j7try738exxnh6hypc0 @@ -0,0 +1,2 @@ +22222222222222222222E+2 + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/w/x/wx7w4p18yyc3a01apa5qz0w31n0sbgbwj1jxvs3dwh8zc b/implementation/fuzz/inputs-genvm-modules-complete-json/w/x/wx7w4p18yyc3a01apa5qz0w31n0sbgbwj1jxvs3dwh8zc new file mode 100644 index 00000000..a8ff260e --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/w/x/wx7w4p18yyc3a01apa5qz0w31n0sbgbwj1jxvs3dwh8zc @@ -0,0 +1 @@ +"\\uΡ€UwUoUUUUUUUUUb\\\\\b\\\ba ,a ,\\\\b\\\ba\bar\\b\\\ba ,a ,\\\\b\\\ba , ,a " \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/w/x/wxd3t34dgg13219xkkbakdj9wgrshqsg2vqmj82t9hxm8 b/implementation/fuzz/inputs-genvm-modules-complete-json/w/x/wxd3t34dgg13219xkkbakdj9wgrshqsg2vqmj82t9hxm8 new file mode 100644 index 00000000..8749cde7 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/w/x/wxd3t34dgg13219xkkbakdj9wgrshqsg2vqmj82t9hxm8 @@ -0,0 +1,2 @@ +{ "o" : 9 +} \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/w/y/wyjsr0jjm2qa355th3w231761y1s3cwy2bqzd7eam3wbp b/implementation/fuzz/inputs-genvm-modules-complete-json/w/y/wyjsr0jjm2qa355th3w231761y1s3cwy2bqzd7eam3wbp new file mode 100644 index 00000000..2e1db173 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/w/y/wyjsr0jjm2qa355th3w231761y1s3cwy2bqzd7eam3wbp @@ -0,0 +1,2 @@ + +"Ρ€bbbbQbb
 T r" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/x/0/x0b9bjk8bhnjw5zvg8bmm9tbvfxjk0vp4ejy7yqn51cat b/implementation/fuzz/inputs-genvm-modules-complete-json/x/0/x0b9bjk8bhnjw5zvg8bmm9tbvfxjk0vp4ejy7yqn51cat new file mode 100644 index 00000000..d0de4b02 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/x/0/x0b9bjk8bhnjw5zvg8bmm9tbvfxjk0vp4ejy7yqn51cat @@ -0,0 +1,17 @@ +{ + "foo": [ + + 123, + -123, + -1e10, + null, + true, + false, + [1, 2, 3] + ], + "bar": [ + "ӀусскиС Π±ΡƒΠΊΨ²Ρ‹ ζ—₯本az\t\uabbb\b", + "bbbb" + + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/x/2/x2y2wxzhvb64j5v9y1wsvw25wskfh6r4r41dhcjx9vf84 b/implementation/fuzz/inputs-genvm-modules-complete-json/x/2/x2y2wxzhvb64j5v9y1wsvw25wskfh6r4r41dhcjx9vf84 new file mode 100644 index 00000000..2ab62f6f --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/x/2/x2y2wxzhvb64j5v9y1wsvw25wskfh6r4r41dhcjx9vf84 @@ -0,0 +1 @@ + false \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/x/3/x31bt75p0d02masjc7bb36dqz3w478hrp838z7vgnmkfc b/implementation/fuzz/inputs-genvm-modules-complete-json/x/3/x31bt75p0d02masjc7bb36dqz3w478hrp838z7vgnmkfc new file mode 100644 index 00000000..5763e7c7 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/x/3/x31bt75p0d02masjc7bb36dqz3w478hrp838z7vgnmkfc @@ -0,0 +1,2 @@ +{ + "" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/x/3/x3xn1mgbwk5sm1eksq5rdw36x3qs1s410zsd7fyz8tb7e b/implementation/fuzz/inputs-genvm-modules-complete-json/x/3/x3xn1mgbwk5sm1eksq5rdw36x3qs1s410zsd7fyz8tb7e new file mode 100644 index 00000000..169844fb --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/x/3/x3xn1mgbwk5sm1eksq5rdw36x3qs1s410zsd7fyz8tb7e @@ -0,0 +1 @@ +222222222222222222222222.22 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/x/7/x7z9y3rfa7r2p2as8e6cb9nd4mcze4ez61hrdc59bj774 b/implementation/fuzz/inputs-genvm-modules-complete-json/x/7/x7z9y3rfa7r2p2as8e6cb9nd4mcze4ez61hrdc59bj774 new file mode 100644 index 00000000..bede5bdc --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/x/7/x7z9y3rfa7r2p2as8e6cb9nd4mcze4ez61hrdc59bj774 @@ -0,0 +1,17 @@ +{ + "bbbbbbbbaz\tfoo": [ + "bar", + "baz\n\\z", + 123, + -123, + -1e10, + null, + true, + false, + [1, 2, 3] + ], + "bar": [ + "русск116111111111111111e-111111\uabbbbb", + "mbbrbbbbbbbbbbbaz\t\\uabbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/x/9/x9vhhbw4nq8hp3bnekcxcgkxvmz213h30h866bw592m50 b/implementation/fuzz/inputs-genvm-modules-complete-json/x/9/x9vhhbw4nq8hp3bnekcxcgkxvmz213h30h866bw592m50 new file mode 100644 index 00000000..e835a30c --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/x/9/x9vhhbw4nq8hp3bnekcxcgkxvmz213h30h866bw592m50 @@ -0,0 +1,7 @@ + [ + , + + + + + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/x/b/xbzbem843j8vaevc0551zcb60f8br7ezhwvyzfh8g9wdw b/implementation/fuzz/inputs-genvm-modules-complete-json/x/b/xbzbem843j8vaevc0551zcb60f8br7ezhwvyzfh8g9wdw new file mode 100644 index 00000000..2eda6594 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/x/b/xbzbem843j8vaevc0551zcb60f8br7ezhwvyzfh8g9wdw @@ -0,0 +1 @@ + [1,3] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/x/c/xc68061ezanp6hv7db2605w2r8v3r092yfcs6bnj4fxqy b/implementation/fuzz/inputs-genvm-modules-complete-json/x/c/xc68061ezanp6hv7db2605w2r8v3r092yfcs6bnj4fxqy new file mode 100644 index 00000000..a8ed145d --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/x/c/xc68061ezanp6hv7db2605w2r8v3r092yfcs6bnj4fxqy @@ -0,0 +1 @@ +1111111111111115111111e-6 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/x/c/xcz9d9fqq2p5tbxy7bg0kkye1zhj9dg932bfny8h7j412 b/implementation/fuzz/inputs-genvm-modules-complete-json/x/c/xcz9d9fqq2p5tbxy7bg0kkye1zhj9dg932bfny8h7j412 new file mode 100644 index 00000000..3026df79 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/x/c/xcz9d9fqq2p5tbxy7bg0kkye1zhj9dg932bfny8h7j412 @@ -0,0 +1,4 @@ + [1111111111111.1111111111E111, -1e10, + 1e+2, 111E111, 2, 111111111111e+2, 10,1e10, + 1e+2, 111E111, 2, + 1e+2, 111E111, 11111] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/x/d/xd1wspeerh2t52fm38ngbcv1a0vy24k6j2hykbtf6g7vw b/implementation/fuzz/inputs-genvm-modules-complete-json/x/d/xd1wspeerh2t52fm38ngbcv1a0vy24k6j2hykbtf6g7vw new file mode 100644 index 00000000..e7157567 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/x/d/xd1wspeerh2t52fm38ngbcv1a0vy24k6j2hykbtf6g7vw @@ -0,0 +1 @@ +t^ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/x/f/xf7nec3t7xfrezaacqkt64v40xmbapa90ra07h5kqfj12 b/implementation/fuzz/inputs-genvm-modules-complete-json/x/f/xf7nec3t7xfrezaacqkt64v40xmbapa90ra07h5kqfj12 new file mode 100644 index 00000000..4ac4f679 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/x/f/xf7nec3t7xfrezaacqkt64v40xmbapa90ra07h5kqfj12 @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/x/g/xg1nqjd3axjyy7vswtf683s63g7x4qbtynp1xjtpawrx0 b/implementation/fuzz/inputs-genvm-modules-complete-json/x/g/xg1nqjd3axjyy7vswtf683s63g7x4qbtynp1xjtpawrx0 new file mode 100644 index 0000000000000000000000000000000000000000..c2072a8bba5e9d0994416cb3ed8053a0c922e902 GIT binary patch literal 266 zcmb>CQUC&_wETP}D}`t<2P}}3Sfr$b%&khvi;1a15;HV1MkoO3$t%st0gLE@q(CCb jK*`9+2*}qpOhpzYhaC+C*z|&=pyFi=*c70O#UcR!1@JA@ literal 0 HcmV?d00001 diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/x/g/xgzc2sv4nwbamhn2p6es0n4t1r292078dwe5w9mwr1j8t b/implementation/fuzz/inputs-genvm-modules-complete-json/x/g/xgzc2sv4nwbamhn2p6es0n4t1r292078dwe5w9mwr1j8t new file mode 100644 index 00000000..3ac4a367 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/x/g/xgzc2sv4nwbamhn2p6es0n4t1r292078dwe5w9mwr1j8t @@ -0,0 +1,35 @@ + [, + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + / \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/x/h/xhmre3dg37xp3988dzxnkeaezejnbejvjt93b72g20c5j b/implementation/fuzz/inputs-genvm-modules-complete-json/x/h/xhmre3dg37xp3988dzxnkeaezejnbejvjt93b72g20c5j new file mode 100644 index 00000000..66cee0dc --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/x/h/xhmre3dg37xp3988dzxnkeaezejnbejvjt93b72g20c5j @@ -0,0 +1 @@ +{ " -@fo":22222212.2} \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/x/k/xkbr14gk3a7rr65r5tk7hfat39pevmz8why3gn0tvh57c b/implementation/fuzz/inputs-genvm-modules-complete-json/x/k/xkbr14gk3a7rr65r5tk7hfat39pevmz8why3gn0tvh57c new file mode 100644 index 00000000..d9222859 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/x/k/xkbr14gk3a7rr65r5tk7hfat39pevmz8why3gn0tvh57c @@ -0,0 +1,15 @@ +"" + + + + + + + + + + + + + + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/x/k/xkmwqaa2w39myhpa309a3b2hdr202f63jfbr343nzhmza b/implementation/fuzz/inputs-genvm-modules-complete-json/x/k/xkmwqaa2w39myhpa309a3b2hdr202f63jfbr343nzhmza new file mode 100644 index 00000000..1a87ea35 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/x/k/xkmwqaa2w39myhpa309a3b2hdr202f63jfbr343nzhmza @@ -0,0 +1 @@ +1.111e1 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/x/m/xmq5ewyq17ta0sadjn489c4dpkgn5bnrmw6be2tmw0dw8 b/implementation/fuzz/inputs-genvm-modules-complete-json/x/m/xmq5ewyq17ta0sadjn489c4dpkgn5bnrmw6be2tmw0dw8 new file mode 100644 index 00000000..0637a088 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/x/m/xmq5ewyq17ta0sadjn489c4dpkgn5bnrmw6be2tmw0dw8 @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/x/p/xp8sqt9zk6h6bt0tq2wf62xb1pck71qvn6r8d0g70mmja b/implementation/fuzz/inputs-genvm-modules-complete-json/x/p/xp8sqt9zk6h6bt0tq2wf62xb1pck71qvn6r8d0g70mmja new file mode 100644 index 00000000..d0a04647 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/x/p/xp8sqt9zk6h6bt0tq2wf62xb1pck71qvn6r8d0g70mmja @@ -0,0 +1,11 @@ + { + "":0, + "":0, + "": "": { + "": + " + + + + + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/x/p/xpvwyb9ad0rfk3yjwtfjha05etx8te3peqbwddheh8fwc b/implementation/fuzz/inputs-genvm-modules-complete-json/x/p/xpvwyb9ad0rfk3yjwtfjha05etx8te3peqbwddheh8fwc new file mode 100644 index 00000000..61f2f826 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/x/p/xpvwyb9ad0rfk3yjwtfjha05etx8te3peqbwddheh8fwc @@ -0,0 +1 @@ + [11111111111111111111111E11] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/x/q/xq3tjskn8hg3ydkemexpfa75fvm46rasg29y7zjkd7za4 b/implementation/fuzz/inputs-genvm-modules-complete-json/x/q/xq3tjskn8hg3ydkemexpfa75fvm46rasg29y7zjkd7za4 new file mode 100644 index 00000000..e5846012 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/x/q/xq3tjskn8hg3ydkemexpfa75fvm46rasg29y7zjkd7za4 @@ -0,0 +1,2 @@ +{ + "foo" 1 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/x/q/xqw45r3n70nw3wj6d638e448qf6zbrd0mt87j9698pyj0 b/implementation/fuzz/inputs-genvm-modules-complete-json/x/q/xqw45r3n70nw3wj6d638e448qf6zbrd0mt87j9698pyj0 new file mode 100644 index 00000000..ad1b70f1 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/x/q/xqw45r3n70nw3wj6d638e448qf6zbrd0mt87j9698pyj0 @@ -0,0 +1 @@ + " \fz" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/x/t/xt8jyccjejyze9n80tsqf7xsjpq5nzt1c21g8rmsa1khr b/implementation/fuzz/inputs-genvm-modules-complete-json/x/t/xt8jyccjejyze9n80tsqf7xsjpq5nzt1c21g8rmsa1khr new file mode 100644 index 00000000..90a76629 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/x/t/xt8jyccjejyze9n80tsqf7xsjpq5nzt1c21g8rmsa1khr @@ -0,0 +1 @@ +"]UUIbcγ€€," diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/x/v/xv4ne8e51c1hch84yxk993218ykvmfp3gq4c66r4x22hw b/implementation/fuzz/inputs-genvm-modules-complete-json/x/v/xv4ne8e51c1hch84yxk993218ykvmfp3gq4c66r4x22hw new file mode 100644 index 00000000..53b5d131 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/x/v/xv4ne8e51c1hch84yxk993218ykvmfp3gq4c66r4x22hw @@ -0,0 +1 @@ +"Ρ€\\ub " \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/x/w/xwhde5sh1dmy64dvf4jjs30x34d291a6v6r2d03nqnpvj b/implementation/fuzz/inputs-genvm-modules-complete-json/x/w/xwhde5sh1dmy64dvf4jjs30x34d291a6v6r2d03nqnpvj new file mode 100644 index 00000000..7c52b808 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/x/w/xwhde5sh1dmy64dvf4jjs30x34d291a6v6r2d03nqnpvj @@ -0,0 +1 @@ + " Ρƒ000 Ρƒ0000000000000000" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/y/1/y1xw7nq6z3pdayphm1vyc5rq1xsd9fadnp8m3y6ykep1y b/implementation/fuzz/inputs-genvm-modules-complete-json/y/1/y1xw7nq6z3pdayphm1vyc5rq1xsd9fadnp8m3y6ykep1y new file mode 100644 index 00000000..33405e10 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/y/1/y1xw7nq6z3pdayphm1vyc5rq1xsd9fadnp8m3y6ykep1y @@ -0,0 +1,2 @@ +true + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/y/3/y3qj9bnaksy94s17nvg4gef25kr6w1p125bq2ypdwv7ac b/implementation/fuzz/inputs-genvm-modules-complete-json/y/3/y3qj9bnaksy94s17nvg4gef25kr6w1p125bq2ypdwv7ac new file mode 100644 index 00000000..8ad6052c --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/y/3/y3qj9bnaksy94s17nvg4gef25kr6w1p125bq2ypdwv7ac @@ -0,0 +1 @@ + ": " \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/y/5/y51y1mf9z97wqwnxyrhj1qrt0tydh3cgttqwhpdxyk6by b/implementation/fuzz/inputs-genvm-modules-complete-json/y/5/y51y1mf9z97wqwnxyrhj1qrt0tydh3cgttqwhpdxyk6by new file mode 100644 index 00000000..1ffaa875 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/y/5/y51y1mf9z97wqwnxyrhj1qrt0tydh3cgttqwhpdxyk6by @@ -0,0 +1,2 @@ +{ "" : [ 3 ] +} \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/y/5/y584nfccdc4nxyk791qsyfeysmt4w73kr0t1xbffvyv76 b/implementation/fuzz/inputs-genvm-modules-complete-json/y/5/y584nfccdc4nxyk791qsyfeysmt4w73kr0t1xbffvyv76 new file mode 100644 index 00000000..899893dc --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/y/5/y584nfccdc4nxyk791qsyfeysmt4w73kr0t1xbffvyv76 @@ -0,0 +1 @@ +1.11 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/y/7/y730k36mhsd91yaj4gkennz0hng2992z3wc1d2b7xh1b6 b/implementation/fuzz/inputs-genvm-modules-complete-json/y/7/y730k36mhsd91yaj4gkennz0hng2992z3wc1d2b7xh1b6 new file mode 100644 index 00000000..da966960 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/y/7/y730k36mhsd91yaj4gkennz0hng2992z3wc1d2b7xh1b6 @@ -0,0 +1,19 @@ +{ + "foo": [ + "bar", + "baz\n\\z", + 123, + -123, + -1e10, + null, 1e20, + null, + true, + false, + [1, 2, 3] + ], + "bar": [ + "русскиС Π±ΡƒΠΊΠ²Ρ‹ ζ—₯本", + "mbbbbbbbbbbbbbbaz\t\uabbbbb", + "mbbbbbbbbbbbbbbaz\t\\uabbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/y/7/y79exx9baft7b1mvwk8w7s45arqjmgph7gepxhyjyzfyp b/implementation/fuzz/inputs-genvm-modules-complete-json/y/7/y79exx9baft7b1mvwk8w7s45arqjmgph7gepxhyjyzfyp new file mode 100644 index 00000000..e57c376f --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/y/7/y79exx9baft7b1mvwk8w7s45arqjmgph7gepxhyjyzfyp @@ -0,0 +1 @@ + "Kf\f\f:o|" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/y/7/y7jjsfaczgb0t5ef708k00m489v4b8nrkhz4y6nnxcf40 b/implementation/fuzz/inputs-genvm-modules-complete-json/y/7/y7jjsfaczgb0t5ef708k00m489v4b8nrkhz4y6nnxcf40 new file mode 100644 index 00000000..d9bc6b7e --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/y/7/y7jjsfaczgb0t5ef708k00m489v4b8nrkhz4y6nnxcf40 @@ -0,0 +1 @@ +0e555455551 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/y/7/y7sz6nb3rz8wx7y76cymb95zt6z691adqvfnpv80yvsy2 b/implementation/fuzz/inputs-genvm-modules-complete-json/y/7/y7sz6nb3rz8wx7y76cymb95zt6z691adqvfnpv80yvsy2 new file mode 100644 index 00000000..6c2bbfe6 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/y/7/y7sz6nb3rz8wx7y76cymb95zt6z691adqvfnpv80yvsy2 @@ -0,0 +1,5 @@ + [[[[[[n[ + + + + "fo \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/y/8/y8gk3gmkbgwb9fe497h121av0m23cnd6zk1rr3fyp7hjr b/implementation/fuzz/inputs-genvm-modules-complete-json/y/8/y8gk3gmkbgwb9fe497h121av0m23cnd6zk1rr3fyp7hjr new file mode 100644 index 00000000..4d9ee20c --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/y/8/y8gk3gmkbgwb9fe497h121av0m23cnd6zk1rr3fyp7hjr @@ -0,0 +1,2 @@ +{ + "foo": [[ [ "bar", [[[[[[ "bar", 33333333333333333333333333333333 [ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/y/a/ya3d6rzthp5zxm8jg5f6nh6yqs81fefzxdy874fmh6kg8 b/implementation/fuzz/inputs-genvm-modules-complete-json/y/a/ya3d6rzthp5zxm8jg5f6nh6yqs81fefzxdy874fmh6kg8 new file mode 100644 index 00000000..b48ef2f1 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/y/a/ya3d6rzthp5zxm8jg5f6nh6yqs81fefzxdy874fmh6kg8 @@ -0,0 +1,6 @@ + [{ "":{ + "o":{ "":{ "":{ + "o": [{ "":{ + "oooooooooooooooooooo":{ "":[{ "":{ + "o":{ "":{ "":{{ "":{ + "o":{{ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/y/b/ybt964nbb5yb9ps6jtk72grm9y99dv2q7xc1nbk41h5fj b/implementation/fuzz/inputs-genvm-modules-complete-json/y/b/ybt964nbb5yb9ps6jtk72grm9y99dv2q7xc1nbk41h5fj new file mode 100644 index 00000000..0011aa2e --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/y/b/ybt964nbb5yb9ps6jtk72grm9y99dv2q7xc1nbk41h5fj @@ -0,0 +1 @@ + [ { } ] diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/y/e/yex06rh07tncp10v3ck21ftg8esb1ez461dsrafy4b1cy b/implementation/fuzz/inputs-genvm-modules-complete-json/y/e/yex06rh07tncp10v3ck21ftg8esb1ez461dsrafy4b1cy new file mode 100644 index 00000000..f4578e81 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/y/e/yex06rh07tncp10v3ck21ftg8esb1ez461dsrafy4b1cy @@ -0,0 +1,2 @@ + + 22222222222222222222E-2222222222222222222222222222222222222222 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/y/h/yh3g712vt6x1b3zhcj1s1p8jxfen93gtd42fxtjktwykp b/implementation/fuzz/inputs-genvm-modules-complete-json/y/h/yh3g712vt6x1b3zhcj1s1p8jxfen93gtd42fxtjktwykp new file mode 100644 index 00000000..4df51cc9 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/y/h/yh3g712vt6x1b3zhcj1s1p8jxfen93gtd42fxtjktwykp @@ -0,0 +1 @@ + "Ρ€\thabb bb" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/y/m/ymr2htwvdcpehh55smk7n56fcmz7mmqcpfc2k83q51qkj b/implementation/fuzz/inputs-genvm-modules-complete-json/y/m/ymr2htwvdcpehh55smk7n56fcmz7mmqcpfc2k83q51qkj new file mode 100644 index 00000000..6aea5698 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/y/m/ymr2htwvdcpehh55smk7n56fcmz7mmqcpfc2k83q51qkj @@ -0,0 +1,4 @@ +[[[[[[[[ +[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ [[[[[[ + +3,[[[[[[[[ [[[[[[[[[[[[[[[[[[[[[[[ [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ [[[[[111E1 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/y/p/yp25xrkzh6cbr4afqcawsaredtfg9c5ape7seebsaax4m b/implementation/fuzz/inputs-genvm-modules-complete-json/y/p/yp25xrkzh6cbr4afqcawsaredtfg9c5ape7seebsaax4m new file mode 100644 index 00000000..26178bf1 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/y/p/yp25xrkzh6cbr4afqcawsaredtfg9c5ape7seebsaax4m @@ -0,0 +1,2 @@ +{ "o": [ ] +} \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/y/p/ypj8a1df23mmj32g64hykewsqzng41a5zkj006089msq8 b/implementation/fuzz/inputs-genvm-modules-complete-json/y/p/ypj8a1df23mmj32g64hykewsqzng41a5zkj006089msq8 new file mode 100644 index 00000000..3b7743ea --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/y/p/ypj8a1df23mmj32g64hykewsqzng41a5zkj006089msq8 @@ -0,0 +1 @@ + [1111111111111111111111111.1111111111E111,1111111111111111111111111.1111111111E111, 2, 1111111111, 2, 11111111111111E111, 2, 3] \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/y/q/yqmec8gra9kkb9kp03kfsq32en9qf0xnq8e3628mk4msa b/implementation/fuzz/inputs-genvm-modules-complete-json/y/q/yqmec8gra9kkb9kp03kfsq32en9qf0xnq8e3628mk4msa new file mode 100644 index 00000000..5d902a58 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/y/q/yqmec8gra9kkb9kp03kfsq32en9qf0xnq8e3628mk4msa @@ -0,0 +1 @@ +-0 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/y/r/yr76r68778ggn6d8n6npcppzwrkfhzk84jxv5a4ty4rgr b/implementation/fuzz/inputs-genvm-modules-complete-json/y/r/yr76r68778ggn6d8n6npcppzwrkfhzk84jxv5a4ty4rgr new file mode 100644 index 00000000..073c3d37 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/y/r/yr76r68778ggn6d8n6npcppzwrkfhzk84jxv5a4ty4rgr @@ -0,0 +1 @@ +[1111111111191111.11111e-90111 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/y/s/yswexssab5tdnfx7511f6tk9p893yhkvfe0kw4scj03dw b/implementation/fuzz/inputs-genvm-modules-complete-json/y/s/yswexssab5tdnfx7511f6tk9p893yhkvfe0kw4scj03dw new file mode 100644 index 00000000..8259b8ff --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/y/s/yswexssab5tdnfx7511f6tk9p893yhkvfe0kw4scj03dw @@ -0,0 +1,2 @@ +{ + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/y/t/yts85j4ftnhe1nvktt9j6dmch9hj2bhwcqzpjxvws2jrg b/implementation/fuzz/inputs-genvm-modules-complete-json/y/t/yts85j4ftnhe1nvktt9j6dmch9hj2bhwcqzpjxvws2jrg new file mode 100644 index 00000000..ef845566 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/y/t/yts85j4ftnhe1nvktt9j6dmch9hj2bhwcqzpjxvws2jrg @@ -0,0 +1,3 @@ +{ + "foo": [[[[[[[[[[[[[[[[[123, [ + [[[[[[[[[[[[123, [[[[[[[[[[[[[[[[[[[[[[[[[123, "bas ------[@ 1111111111 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/y/v/yvkhkdhwqsd9t5eq7ks3wf602jf3bwt8eczx2vybh6zc0 b/implementation/fuzz/inputs-genvm-modules-complete-json/y/v/yvkhkdhwqsd9t5eq7ks3wf602jf3bwt8eczx2vybh6zc0 new file mode 100644 index 00000000..af523cc9 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/y/v/yvkhkdhwqsd9t5eq7ks3wf602jf3bwt8eczx2vybh6zc0 @@ -0,0 +1,4 @@ + [{ "":{ + "o":{ "":{ + "o":{ "":{ "":{ + "o":{ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/y/w/ywmhyj7wq34fhptmx4amwpefx3nqsww494c8wyrz37eva b/implementation/fuzz/inputs-genvm-modules-complete-json/y/w/ywmhyj7wq34fhptmx4amwpefx3nqsww494c8wyrz37eva new file mode 100644 index 00000000..11a736db --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/y/w/ywmhyj7wq34fhptmx4amwpefx3nqsww494c8wyrz37eva @@ -0,0 +1 @@ +"\ti" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/y/x/yxfmh1pze8kkvezr7abqjr3tnhpqbbxp48kd2qwv0wk54 b/implementation/fuzz/inputs-genvm-modules-complete-json/y/x/yxfmh1pze8kkvezr7abqjr3tnhpqbbxp48kd2qwv0wk54 new file mode 100644 index 00000000..ee5bc3b3 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/y/x/yxfmh1pze8kkvezr7abqjr3tnhpqbbxp48kd2qwv0wk54 @@ -0,0 +1,10 @@ +{ + "foo": false, "foo":false, "foo": false, + false, + + + + + + + diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/z/0/z0fk59m6rrgjabzhxmr32c09fh750e6k03zfa20k2vrxa b/implementation/fuzz/inputs-genvm-modules-complete-json/z/0/z0fk59m6rrgjabzhxmr32c09fh750e6k03zfa20k2vrxa new file mode 100644 index 00000000..3c002ebe --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/z/0/z0fk59m6rrgjabzhxmr32c09fh750e6k03zfa20k2vrxa @@ -0,0 +1 @@ + "с\\fffffffffbffffabba\/:fffffffffffffffffffffffffffffabba\ffffffffffffffffffffffffffffabba\fffffffffffffffffffffffffffffgfffffffffffffffffffffffffffabba\/: [b| [bbbb" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/z/3/z34jsxa265sxqyb2cwx1thqtexxcar9egnjvnpcvkcrvj b/implementation/fuzz/inputs-genvm-modules-complete-json/z/3/z34jsxa265sxqyb2cwx1thqtexxcar9egnjvnpcvkcrvj new file mode 100644 index 00000000..b0853f28 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/z/3/z34jsxa265sxqyb2cwx1thqtexxcar9egnjvnpcvkcrvj @@ -0,0 +1 @@ +33333333333333333333333333 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/z/3/z39kydgt59x46cdhf6vjhy39pe44ge3rvkhvb1nkwb2zr b/implementation/fuzz/inputs-genvm-modules-complete-json/z/3/z39kydgt59x46cdhf6vjhy39pe44ge3rvkhvb1nkwb2zr new file mode 100644 index 00000000..3ee15b89 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/z/3/z39kydgt59x46cdhf6vjhy39pe44ge3rvkhvb1nkwb2zr @@ -0,0 +1 @@ + "\ubabb@" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/z/3/z3eqckb2scmz4kjf7qt1yztrsfky9qraatwe03x51h7k6 b/implementation/fuzz/inputs-genvm-modules-complete-json/z/3/z3eqckb2scmz4kjf7qt1yztrsfky9qraatwe03x51h7k6 new file mode 100644 index 00000000..8fb22b47 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/z/3/z3eqckb2scmz4kjf7qt1yztrsfky9qraatwe03x51h7k6 @@ -0,0 +1 @@ +"UUUU3UUUUUUUUTUUUUUUUoUUUUUUUUTUUUUUUUoUsUUUUUUUUUUTUUUUUUUoUUUUUUaUUUUUT," diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/z/3/z3yvw1ngyxqxp5wn2km4cd12kbm9cye0pz29wcymqgwgp b/implementation/fuzz/inputs-genvm-modules-complete-json/z/3/z3yvw1ngyxqxp5wn2km4cd12kbm9cye0pz29wcymqgwgp new file mode 100644 index 00000000..1de6c58e --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/z/3/z3yvw1ngyxqxp5wn2km4cd12kbm9cye0pz29wcymqgwgp @@ -0,0 +1 @@ +19111222222222221222222222 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/z/4/z472bjs2xxz4h1frvp4kewrt6hzem2ksne0k68c6w2478 b/implementation/fuzz/inputs-genvm-modules-complete-json/z/4/z472bjs2xxz4h1frvp4kewrt6hzem2ksne0k68c6w2478 new file mode 100644 index 00000000..216546b8 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/z/4/z472bjs2xxz4h1frvp4kewrt6hzem2ksne0k68c6w2478 @@ -0,0 +1 @@ +"[[1, B" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/z/5/z5f8bcx0bhxt4bc16ybcgjxgakyhz6081ej47wwb6yr6g b/implementation/fuzz/inputs-genvm-modules-complete-json/z/5/z5f8bcx0bhxt4bc16ybcgjxgakyhz6081ej47wwb6yr6g new file mode 100644 index 00000000..4e7542ea --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/z/5/z5f8bcx0bhxt4bc16ybcgjxgakyhz6081ej47wwb6yr6g @@ -0,0 +1 @@ + [1, , bbC \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/z/7/z70ryqvfm418005sc8r4bfm6v9ys4bbrmf0x5nnvsn1m0 b/implementation/fuzz/inputs-genvm-modules-complete-json/z/7/z70ryqvfm418005sc8r4bfm6v9ys4bbrmf0x5nnvsn1m0 new file mode 100644 index 00000000..2e287337 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/z/7/z70ryqvfm418005sc8r4bfm6v9ys4bbrmf0x5nnvsn1m0 @@ -0,0 +1,2 @@ +"\"2, t" + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/z/8/z821q1k0181fd425hst88zjpc27fcjs81z16vp3f89rm6 b/implementation/fuzz/inputs-genvm-modules-complete-json/z/8/z821q1k0181fd425hst88zjpc27fcjs81z16vp3f89rm6 new file mode 100644 index 00000000..55919c0a --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/z/8/z821q1k0181fd425hst88zjpc27fcjs81z16vp3f89rm6 @@ -0,0 +1 @@ + [[11111111111611111111111111E1 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/z/9/z9ap3tct6s8eqh3eck88e5aw9sqj3ga4yzznm2m2j0604 b/implementation/fuzz/inputs-genvm-modules-complete-json/z/9/z9ap3tct6s8eqh3eck88e5aw9sqj3ga4yzznm2m2j0604 new file mode 100644 index 00000000..8f1ae55c --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/z/9/z9ap3tct6s8eqh3eck88e5aw9sqj3ga4yzznm2m2j0604 @@ -0,0 +1,2 @@ + +"Ρ€bbbbIbbβ€» r" diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/z/a/zarvjgwmv5q3vwyd58gkw4gc501v7zkwjx00pkmgkft22 b/implementation/fuzz/inputs-genvm-modules-complete-json/z/a/zarvjgwmv5q3vwyd58gkw4gc501v7zkwjx00pkmgkft22 new file mode 100644 index 00000000..c85d3f60 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/z/a/zarvjgwmv5q3vwyd58gkw4gc501v7zkwjx00pkmgkft22 @@ -0,0 +1,18 @@ +{ + "foo": [ + "bar", + 123, + -123, + -1e10, + 1e-0, + null, + true, + false, + [1111111111111111111111111111e-111111111111111111, 2, 3] + ], + "bar": [ + "русскиС Π±ΡƒΠΊΠ²Ρ‹ ζ—₯本", + "mbbbbbbbbbbbbbbaz\t\uabbbbb", + "mbbbbbbabbbbbbbaz\t\\uabbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/z/e/zer1ffmaw8crmjm9de5g8zqtarpbwr23epvv20t5bs8x8 b/implementation/fuzz/inputs-genvm-modules-complete-json/z/e/zer1ffmaw8crmjm9de5g8zqtarpbwr23epvv20t5bs8x8 new file mode 100644 index 00000000..75a752b9 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/z/e/zer1ffmaw8crmjm9de5g8zqtarpbwr23epvv20t5bs8x8 @@ -0,0 +1 @@ +"[\\\\\\\\\\\\\\\\\\\\\\\\\\[1, B\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/z/e/zewv083f1apdzf1kg8vbg2jjfnvxgz9z00xa1h4mf65sa b/implementation/fuzz/inputs-genvm-modules-complete-json/z/e/zewv083f1apdzf1kg8vbg2jjfnvxgz9z00xa1h4mf65sa new file mode 100644 index 00000000..81724955 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/z/e/zewv083f1apdzf1kg8vbg2jjfnvxgz9z00xa1h4mf65sa @@ -0,0 +1 @@ + " \uBAb4" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/z/f/zfg0xsh5caxhzqb9z8nwpjzsvr0q8g0fjkz3y4y43kdjg b/implementation/fuzz/inputs-genvm-modules-complete-json/z/f/zfg0xsh5caxhzqb9z8nwpjzsvr0q8g0fjkz3y4y43kdjg new file mode 100644 index 00000000..a80a7670 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/z/f/zfg0xsh5caxhzqb9z8nwpjzsvr0q8g0fjkz3y4y43kdjg @@ -0,0 +1 @@ +-0 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/z/g/zgqxey2qy8j4gqm0rd91k8fbx1p23t4nwmc7gabhafhsm b/implementation/fuzz/inputs-genvm-modules-complete-json/z/g/zgqxey2qy8j4gqm0rd91k8fbx1p23t4nwmc7gabhafhsm new file mode 100644 index 00000000..5f046f0b --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/z/g/zgqxey2qy8j4gqm0rd91k8fbx1p23t4nwmc7gabhafhsm @@ -0,0 +1,2 @@ +{ "foo" Π‹ + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/z/n/zn265mn3xv8v3v6zetbedfg88vjcvffa19ebk98pc5t6e b/implementation/fuzz/inputs-genvm-modules-complete-json/z/n/zn265mn3xv8v3v6zetbedfg88vjcvffa19ebk98pc5t6e new file mode 100644 index 00000000..3966243d --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/z/n/zn265mn3xv8v3v6zetbedfg88vjcvffa19ebk98pc5t6e @@ -0,0 +1 @@ +{  \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/z/n/zntc9yjnew3s8krzbcd9m8b567mmq3d92b7e6henf2nhj b/implementation/fuzz/inputs-genvm-modules-complete-json/z/n/zntc9yjnew3s8krzbcd9m8b567mmq3d92b7e6henf2nhj new file mode 100644 index 00000000..2eff6720 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/z/n/zntc9yjnew3s8krzbcd9m8b567mmq3d92b7e6henf2nhj @@ -0,0 +1 @@ + " \uDbbb\uDDbB u" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/z/p/zp2gxjckb10wqsjcp91bmtqq9chq2qq5gpdyym87v6q1j b/implementation/fuzz/inputs-genvm-modules-complete-json/z/p/zp2gxjckb10wqsjcp91bmtqq9chq2qq5gpdyym87v6q1j new file mode 100644 index 00000000..5c1f38fd --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/z/p/zp2gxjckb10wqsjcp91bmtqq9chq2qq5gpdyym87v6q1j @@ -0,0 +1 @@ +[[[["/ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/z/p/zpm8204mejxydjsawbrhn7vje3xjgaar0cpk7zy64ws0r b/implementation/fuzz/inputs-genvm-modules-complete-json/z/p/zpm8204mejxydjsawbrhn7vje3xjgaar0cpk7zy64ws0r new file mode 100644 index 00000000..2cafaa5a --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/z/p/zpm8204mejxydjsawbrhn7vje3xjgaar0cpk7zy64ws0r @@ -0,0 +1,2 @@ +2222222222222222222222.2222222222E22 + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/z/q/zqwvef7cvxgb2tybnypx352v6tdbpmr49vx29z97z5jtr b/implementation/fuzz/inputs-genvm-modules-complete-json/z/q/zqwvef7cvxgb2tybnypx352v6tdbpmr49vx29z97z5jtr new file mode 100644 index 00000000..b3f965cb --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/z/q/zqwvef7cvxgb2tybnypx352v6tdbpmr49vx29z97z5jtr @@ -0,0 +1,3 @@ + { + "":0, + "" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/z/r/zrj953c90pegf5s0v2ehgb286hxrsgzh77cv6mx6zrfa2 b/implementation/fuzz/inputs-genvm-modules-complete-json/z/r/zrj953c90pegf5s0v2ehgb286hxrsgzh77cv6mx6zrfa2 new file mode 100644 index 00000000..6604698f --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/z/r/zrj953c90pegf5s0v2ehgb286hxrsgzh77cv6mx6zrfa2 @@ -0,0 +1 @@ +[1, \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/z/t/zt8jegx974khs3jmwa42r8h98wypp5gsv1ss4s8je0166 b/implementation/fuzz/inputs-genvm-modules-complete-json/z/t/zt8jegx974khs3jmwa42r8h98wypp5gsv1ss4s8je0166 new file mode 100644 index 00000000..2f7ef5bc --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/z/t/zt8jegx974khs3jmwa42r8h98wypp5gsv1ss4s8je0166 @@ -0,0 +1 @@ +": " \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/z/t/ztkqhjcy7hbng5ndn9f36zaxab33220jxj2d0q7rgt72j b/implementation/fuzz/inputs-genvm-modules-complete-json/z/t/ztkqhjcy7hbng5ndn9f36zaxab33220jxj2d0q7rgt72j new file mode 100644 index 00000000..0f81b3c7 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/z/t/ztkqhjcy7hbng5ndn9f36zaxab33220jxj2d0q7rgt72j @@ -0,0 +1,11 @@ +[, + + + + + + + + + + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/z/v/zvcfa3ehdtsn1djntnwmv4mn2jm17t3t659k2b3mn1ryj b/implementation/fuzz/inputs-genvm-modules-complete-json/z/v/zvcfa3ehdtsn1djntnwmv4mn2jm17t3t659k2b3mn1ryj new file mode 100644 index 00000000..02df100e --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/z/v/zvcfa3ehdtsn1djntnwmv4mn2jm17t3t659k2b3mn1ryj @@ -0,0 +1 @@ + [1, [1, [1, [1, " \uEbb \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/z/w/zwby1chwm64dgy0t105pr72a9se3x4e75xennhgf7fct2 b/implementation/fuzz/inputs-genvm-modules-complete-json/z/w/zwby1chwm64dgy0t105pr72a9se3x4e75xennhgf7fct2 new file mode 100644 index 00000000..a18c6764 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/z/w/zwby1chwm64dgy0t105pr72a9se3x4e75xennhgf7fct2 @@ -0,0 +1,14 @@ +{ + "11111111111111": 1e20, + + + + + + + + + + "11111111111111111111": 1e20, + "11111111": 1e20, +f \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/z/x/zx51d2g9vv3w2s6eh0fgy6esppgtmhary97nfcjqgtfza b/implementation/fuzz/inputs-genvm-modules-complete-json/z/x/zx51d2g9vv3w2s6eh0fgy6esppgtmhary97nfcjqgtfza new file mode 100644 index 00000000..cc3ca9b5 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/z/x/zx51d2g9vv3w2s6eh0fgy6esppgtmhary97nfcjqgtfza @@ -0,0 +1,16 @@ +{ + "bar": [ ], + "bar": [ + "baz\n\\{", + -1e10, + 1e20, + null, + true, + [1, 2, 3] + ], + "bar": [ ], + "bar": [ + "русскиС mbbbbbbbbb", + "mbbbbbbbbbbbbbbaz\t\\uabbbbb" + ] +} diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/z/y/zy2w7wxymampccdp97g9rzv76qwxk7vj05z145zenb364 b/implementation/fuzz/inputs-genvm-modules-complete-json/z/y/zy2w7wxymampccdp97g9rzv76qwxk7vj05z145zenb364 new file mode 100644 index 00000000..30d1b498 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/z/y/zy2w7wxymampccdp97g9rzv76qwxk7vj05z145zenb364 @@ -0,0 +1,2 @@ + ["baz + 01. \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-complete-json/z/z/zzn0apphqhr3g24550pxepzcyb5j2gp8px27fdjj1dt0r b/implementation/fuzz/inputs-genvm-modules-complete-json/z/z/zzn0apphqhr3g24550pxepzcyb5j2gp8px27fdjj1dt0r new file mode 100644 index 00000000..4a125f4c --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-complete-json/z/z/zzn0apphqhr3g24550pxepzcyb5j2gp8px27fdjj1dt0r @@ -0,0 +1,2 @@ + +"Ρ€bbbb\\uaaaa 111111d11 r" \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/db91d0d947f0f760465149f837c4bc02bf02c988c7eb7824c7e9496a52884908 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/0/e/0ep4zdf4ndsymnwtkeh8j0tztts0y186jq1bcw3wch8pe similarity index 100% rename from implementation/fuzz/inputs-genvm-modules-normalize-whitespace/db91d0d947f0f760465149f837c4bc02bf02c988c7eb7824c7e9496a52884908 rename to implementation/fuzz/inputs-genvm-modules-normalize-whitespace/0/e/0ep4zdf4ndsymnwtkeh8j0tztts0y186jq1bcw3wch8pe diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/0/p/0pez5en5ybbdkbh9axzqq7c4wgs8rt6mctebe7zf9kzy2 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/0/p/0pez5en5ybbdkbh9axzqq7c4wgs8rt6mctebe7zf9kzy2 new file mode 100644 index 00000000..f01735f8 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/0/p/0pez5en5ybbdkbh9axzqq7c4wgs8rt6mctebe7zf9kzy2 @@ -0,0 +1 @@ + ÍÁÀÀÀÁÍÁÁ diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/03fc8c99262f53a7d60c876faf4876a11a92c2e045c0ad0fcc738399f61fc240 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/03fc8c99262f53a7d60c876faf4876a11a92c2e045c0ad0fcc738399f61fc240 deleted file mode 100644 index 66fc93b8..00000000 --- a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/03fc8c99262f53a7d60c876faf4876a11a92c2e045c0ad0fcc738399f61fc240 +++ /dev/null @@ -1 +0,0 @@ -ÀÀÍÁÀÀÀÁÍÁÁÁ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/06e3c1413644ebd520e20235d3dfa3145613e296cb645884ffe0b885b63bd99b b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/06e3c1413644ebd520e20235d3dfa3145613e296cb645884ffe0b885b63bd99b deleted file mode 100644 index b34cdea2..00000000 --- a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/06e3c1413644ebd520e20235d3dfa3145613e296cb645884ffe0b885b63bd99b +++ /dev/null @@ -1,2 +0,0 @@ -$$$ - $ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/0a150ae9c5d282876a4e896149d6c486e771088809bf02e59aa61ca3d1c3e2e1 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/0a150ae9c5d282876a4e896149d6c486e771088809bf02e59aa61ca3d1c3e2e1 deleted file mode 100644 index 574c6e2c55a90f6da962d9d264fd3e811d7ad6a3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 74 zcmXp~P%z~(<Fc*fGEy*tu#LHFxquiV!ll4v2*kz^Ik1?C0u%tn!GM8_OOcBg0Gy8r Ax&QzG diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/0abd764f1b303911ca744e429fd729ffc1624caf063177229727e526f3dcbf1c b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/0abd764f1b303911ca744e429fd729ffc1624caf063177229727e526f3dcbf1c deleted file mode 100644 index a5952283..00000000 --- a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/0abd764f1b303911ca744e429fd729ffc1624caf063177229727e526f3dcbf1c +++ /dev/null @@ -1 +0,0 @@ -぀ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/1/4/141ka9q4c2mgw2zqpxgjpmtbrpcx3q4ywn28z3hh4ss9r b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/1/4/141ka9q4c2mgw2zqpxgjpmtbrpcx3q4ywn28z3hh4ss9r new file mode 100644 index 00000000..62aa304a --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/1/4/141ka9q4c2mgw2zqpxgjpmtbrpcx3q4ywn28z3hh4ss9r @@ -0,0 +1 @@ + ぀぀ ぀぀@=== ぀぀@=@==< ぀぀@== \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/1/s/1s28fveg5getzbynkzmb3t77fh8g40qsq7xp2ebfm6ye6 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/1/s/1s28fveg5getzbynkzmb3t77fh8g40qsq7xp2ebfm6ye6 new file mode 100644 index 00000000..7c7df0d0 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/1/s/1s28fveg5getzbynkzmb3t77fh8g40qsq7xp2ebfm6ye6 @@ -0,0 +1 @@ +β€€β€€β€ˆβ€€β€€β€€β€€β€ˆβ€€β€ˆβ€€β€€β€€β€€β€ˆβ€β€ˆβ€€ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/3caac4d461b724c627cbc28a9e96641f09c2130fb7e1fd0d9433bd63b2c12a20 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/1/t/1tk837vrcb0qn02pw3an43bzssvwsrc1e4t60fq0dknhc similarity index 94% rename from implementation/fuzz/inputs-genvm-modules-normalize-whitespace/3caac4d461b724c627cbc28a9e96641f09c2130fb7e1fd0d9433bd63b2c12a20 rename to implementation/fuzz/inputs-genvm-modules-normalize-whitespace/1/t/1tk837vrcb0qn02pw3an43bzssvwsrc1e4t60fq0dknhc index aba4d4f53b432c66b457a6a0af3049272e4d93da..1a3d9ed28613b5fbe7cc2673d95609a45153e6d4 100644 GIT binary patch literal 33 Lcmd<$!Vf9|7J31? literal 34 Mcmd<$!VlOP02fdJNB{r; diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/108efca4997c179e75c9c443def521c0bf14f754fa13984e8f6f4b25e3fe19fe b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/108efca4997c179e75c9c443def521c0bf14f754fa13984e8f6f4b25e3fe19fe deleted file mode 100644 index 0a12598f34d10e4f10bcc79c64f8fb62ac56c661..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6 Ncmd<$GUj4n0004A08#(| diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/13a6f1b73ba94d5c0ec7097fac90f1bababca6ec0095c6d7dc6fdb4680112804 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/13a6f1b73ba94d5c0ec7097fac90f1bababca6ec0095c6d7dc6fdb4680112804 deleted file mode 100644 index 56f255ffde9b6ca54538040711f04947043ba419..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 21 ZcmWe&;4<V=0AU6OCWaI)E?zS(V*nA}0o4Ei diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/18847d26806d9ac4981dd642d6aca8346cd197d647ed119cee4a6e5e95785183 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/18847d26806d9ac4981dd642d6aca8346cd197d647ed119cee4a6e5e95785183 deleted file mode 100644 index 4f079609906e781c88bbcc7bad9911807f7acb6e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 88 zcmWgV;06LNZti++ZgCDZIGcf!lT(uOabp7zIoR6T*#ZHw7&kWqvMfX)Pz^|-ku8HQ E05NYDP5=M^ diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/85870cb385a6431e482529b899cc64ac4e3a3f99424bae342e19e1ef8cdc0fb5 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/2/n/2nnyph5gamd1qcg99hg1yc9c37h5ej81ny8m647z61vj0 similarity index 100% rename from implementation/fuzz/inputs-genvm-modules-normalize-whitespace/85870cb385a6431e482529b899cc64ac4e3a3f99424bae342e19e1ef8cdc0fb5 rename to implementation/fuzz/inputs-genvm-modules-normalize-whitespace/2/n/2nnyph5gamd1qcg99hg1yc9c37h5ej81ny8m647z61vj0 diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/2afdb94524a31c4607ade0e66a33a33b94a6e6fc11da8e0ac8dc684550637684 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/2afdb94524a31c4607ade0e66a33a33b94a6e6fc11da8e0ac8dc684550637684 deleted file mode 100644 index ff8aa24d..00000000 --- a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/2afdb94524a31c4607ade0e66a33a33b94a6e6fc11da8e0ac8dc684550637684 +++ /dev/null @@ -1 +0,0 @@ -α˜€Β˜α˜€α˜€α˜€9β€€β€€β€€Β˜α˜€α˜€α˜€9β€€β€€β€€β€€β˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€9β€€β€€β€€β€€β™€β€ˆβ€€β€€β€€β€€α˜€α˜€α˜€9α˜€β€€β˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€9β€€β€€β€€β€€β™€β€ˆβ€€β€€β€€β€€α˜€α˜€α˜€9α˜€ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/2b0698696f26299c279bef4360e307c4f4b41dcf7a8fe4db7b82dde9103e1351 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/2b0698696f26299c279bef4360e307c4f4b41dcf7a8fe4db7b82dde9103e1351 deleted file mode 100644 index 995cf2c3..00000000 --- a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/2b0698696f26299c279bef4360e307c4f4b41dcf7a8fe4db7b82dde9103e1351 +++ /dev/null @@ -1 +0,0 @@ -  \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/2b4985bd1778fbb5f2aec4265651102ad3f0d53bbada124914d102b92f2be068 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/2b4985bd1778fbb5f2aec4265651102ad3f0d53bbada124914d102b92f2be068 deleted file mode 100644 index 3d684092..00000000 --- a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/2b4985bd1778fbb5f2aec4265651102ad3f0d53bbada124914d102b92f2be068 +++ /dev/null @@ -1 +0,0 @@ -⁀⁀⁀:⁀⁀:π”€Ÿ ⁀⁀E⁀⁀:π”€Ÿπ”€Ÿ:π”€Ÿπ”€Ÿ:π”€Ÿ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/2f2af5cba25cf00d90e48678f7092ae6217152bc42205385d0b5996721c6034b b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/2f2af5cba25cf00d90e48678f7092ae6217152bc42205385d0b5996721c6034b deleted file mode 100644 index 6a78c9859c51c15ef4c70d740dc4ce44676a77c6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 22 XcmaFd*zmZqLDJUN&K3k1z@#JqX8s2i diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/a77a0b19ea78584e0ea34252aa4faab2839be31b03d9f27d7cd86f503153c038 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/3/x/3xfr5w22w0a3d2c5tddyy9qwdjg55hj6ve1xmfnpct7dj similarity index 84% rename from implementation/fuzz/inputs-genvm-modules-normalize-whitespace/a77a0b19ea78584e0ea34252aa4faab2839be31b03d9f27d7cd86f503153c038 rename to implementation/fuzz/inputs-genvm-modules-normalize-whitespace/3/x/3xfr5w22w0a3d2c5tddyy9qwdjg55hj6ve1xmfnpct7dj index b583e7d9c57bf4c52aa0b2efa7961f350fbd8d74..4ab6806bef6a0b061003bf581b8df3b09d218463 100644 GIT binary patch delta 288 zcmbOven)&lFr(sRb;j_?j2y|6)flZNPMkW~o~2>(I~Kmluehd8&S&PH?7$>HIgn9G zfJ=cvflGnQ!^1-vh~2d&2Qs?|abbYT4>?mO^RTREl%KqZ#f_0^@&rcl$?utECkrt0 z0>wTuYHiMEO=M)0oP3cjg3}fV7&tjOB_~U>yal=?kxgQ<0T+<wl4azb?7&hFbxNJo zhbay7IUhAP0Fk6Cl(qs1N>0va+X6IQ4CrztCkBN|E<>RE-~i-U4-X7yq6okhdvF<Y NO;%v~Gr68y7y$0*OEdrg delta 122 zcmca3K1qB+@MJzltH~EQXF}=SoBuQNFix&zmYQtOr8qg2*?97LHkC<S!ISw})=$35 z?mPJbv;Jf&R^G`8T-uw9SQ8m>$W1O{b)S5I<LKn~%v_Ue*up1qbpwrNWSo4GTNnU8 CZYqud diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/3462c1d0fca3164ff490caa99736fa4eae1ca1a646bf880d8b5894c955033450 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/3462c1d0fca3164ff490caa99736fa4eae1ca1a646bf880d8b5894c955033450 deleted file mode 100644 index a87a78ee..00000000 --- a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/3462c1d0fca3164ff490caa99736fa4eae1ca1a646bf880d8b5894c955033450 +++ /dev/null @@ -1 +0,0 @@ -⁀⁀⁀:⁀⁀:π”€Ÿ𔀀⁀:𔀆π”€Ÿ:𔀀:π”€Ÿπ”€Ÿ:π”€Ÿ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/34e791781a9d100ab2a6d7dbef5b5eb38c4c38d887618ff5e94a8d117d4973df b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/34e791781a9d100ab2a6d7dbef5b5eb38c4c38d887618ff5e94a8d117d4973df deleted file mode 100644 index ba015c8a..00000000 --- a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/34e791781a9d100ab2a6d7dbef5b5eb38c4c38d887618ff5e94a8d117d4973df +++ /dev/null @@ -1 +0,0 @@ -β€€β€€β€ˆβ€€β€€β€€β€€β€ˆβ€€β€ˆβ€€β€€β€€β€€β€ˆβ€€β€ˆβ€€ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/4/2/42xap88t5te2mew2v7b9ekgkdy86ddj7kyg122q62qyrg b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/4/2/42xap88t5te2mew2v7b9ekgkdy86ddj7kyg122q62qyrg new file mode 100644 index 00000000..9e36d7fe --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/4/2/42xap88t5te2mew2v7b9ekgkdy86ddj7kyg122q62qyrg @@ -0,0 +1 @@ +Β…Β… \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/4/3/4372gqsr02f3mbqpv5sb1arw5zcrvjptra0e18973v30m b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/4/3/4372gqsr02f3mbqpv5sb1arw5zcrvjptra0e18973v30m new file mode 100644 index 00000000..89fae5c7 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/4/3/4372gqsr02f3mbqpv5sb1arw5zcrvjptra0e18973v30m @@ -0,0 +1 @@ + @ & \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/4/7/4730f4bwrk39hybvx8szvyvqyck9ad0aejq1zz34tgcby b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/4/7/4730f4bwrk39hybvx8szvyvqyck9ad0aejq1zz34tgcby new file mode 100644 index 00000000..1e8e450a --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/4/7/4730f4bwrk39hybvx8szvyvqyck9ad0aejq1zz34tgcby @@ -0,0 +1,2 @@ + + diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/4/b/4bnwwcdggzxf2s57jye94rmc46zrnd8551hd6gaaf0t0r b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/4/b/4bnwwcdggzxf2s57jye94rmc46zrnd8551hd6gaaf0t0r new file mode 100644 index 00000000..b5625cda --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/4/b/4bnwwcdggzxf2s57jye94rmc46zrnd8551hd6gaaf0t0r @@ -0,0 +1,32 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/4/h/4htd2gzh8zfzp34tyr6eh4jq6nyqev0nbgvtpswcybxrg b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/4/h/4htd2gzh8zfzp34tyr6eh4jq6nyqev0nbgvtpswcybxrg new file mode 100644 index 0000000000000000000000000000000000000000..a1f9aa9014042b4bc4d8d49c7e822a57dfa5b567 GIT binary patch literal 8 NcmX@Vgn^d}2mlMA0jU50 literal 0 HcmV?d00001 diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/4/r/4ryz1gbe2d7tkmrb5jab1m391vjpytdx7rgx8bne9ams6 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/4/r/4ryz1gbe2d7tkmrb5jab1m391vjpytdx7rgx8bne9ams6 new file mode 100644 index 00000000..23f7b992 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/4/r/4ryz1gbe2d7tkmrb5jab1m391vjpytdx7rgx8bne9ams6 @@ -0,0 +1 @@ +γ€Ÿ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/45eae75b464a765dc836ffdd8596874c5d0b7fa33c9685fb77ac22738dbe9edf b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/45eae75b464a765dc836ffdd8596874c5d0b7fa33c9685fb77ac22738dbe9edf deleted file mode 100644 index 86995c4a..00000000 --- a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/45eae75b464a765dc836ffdd8596874c5d0b7fa33c9685fb77ac22738dbe9edf +++ /dev/null @@ -1 +0,0 @@ -α˜€Β˜α˜€α˜€α˜€9β€€β€€β€€$Β˜α˜€α˜€α˜€9β€€β€€β€€β€€β˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€9β€€β€€β€€β€€β™€β€ˆβ€€β€€β€€β€€α˜€α˜€α˜€9α˜€β€€β˜€α˜€α˜€α˜€α˜€α˜€9β€€β€€β€€β€€β™€β€ˆβ€€β€€β€€β€€α˜€α˜€α˜€9α˜€β€€β˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€9β€€β€€β€€β€€α˜€Β˜α˜€α˜€α˜€9β€€β€€β€€$Β˜α˜€α˜€α˜€9β€€Β˜α˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€9β€€β€€β€€β€€β™€β€ˆβ€€β€€β€€β˜€α˜€α˜€Β˜α˜€α˜€α˜€9β€€β€€β€€β€€α˜€Β˜α˜€α˜€α˜€9β€€β€€β€€$Β˜α˜€α˜€α˜€9β€€Β˜α˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€9β€€β€€β€€β€€α˜€α˜€α˜€9α˜€β€€β˜€α˜€α˜€α˜€α˜€α˜€9β€€β€€β€€β€€β™€β€ˆβ€€β€€β€€β€€α˜€α˜€α˜€9α˜€β€€β˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€9β€€β€€β€€β€€α˜€Β˜α˜€α˜€α˜€9β€€β€€β€€$Β˜α˜€α˜€β€€β€€β€€β˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€9β€€β€€β€€β€€β™€β€ˆβ€€β€€β€€β€€α˜€α˜€α˜€9α˜€β€€β˜€α˜€α˜€α˜€α˜€α˜€9β€€β€€β€€β€€β™€β€ˆβ€€β€€β€€β€€α˜€α˜€α˜€9α˜€β€€β˜€α˜€α˜€α˜€β€€β€€β€€β˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€9β€€β€€β€€β€€β™€β€ˆβ€€β€€β€€β€€α˜€α˜€α˜€9α˜€β€€β˜€α˜€α˜€α˜€α˜€α˜€9β€€β€€β€€β€€β™€β€ˆβ€€β€€β€€β€€α˜€α˜€α˜€9α˜€β€€β˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€9β€€β€€β€€β€€Β˜α˜€α˜€α˜€α˜€Β˜α˜€Β˜α˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€9β€€β€€β€€β™€β€ˆβ€€β€€β€€β€€α˜€α˜€α˜€9α˜€ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/4888b4e22f1132f4617b3470c99715c2701e115f3023b1e7893d335bf807d4c9 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/4888b4e22f1132f4617b3470c99715c2701e115f3023b1e7893d335bf807d4c9 deleted file mode 100644 index 15074991..00000000 --- a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/4888b4e22f1132f4617b3470c99715c2701e115f3023b1e7893d335bf807d4c9 +++ /dev/null @@ -1 +0,0 @@ -⁀⁀:π”€Ÿ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/4a6cf03c7706d26879d5a738f98eed6e39312ebf0b61fa1f053b5cdb3c8869a0 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/4a6cf03c7706d26879d5a738f98eed6e39312ebf0b61fa1f053b5cdb3c8869a0 deleted file mode 100644 index d603bb5190b15e964b97016df1647c5b2c9ca2e1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 67 qcmWgV;06LNZti++ZgC_w11Be^B<JJC1|V{<wY9Sa0&HTo47LDy-VR*= diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/4b531e98e5057fd2eb3c96eb7973a5c90df1234758b5ee02c97755227a084e3f b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/4b531e98e5057fd2eb3c96eb7973a5c90df1234758b5ee02c97755227a084e3f deleted file mode 100644 index 2af0b4c9..00000000 --- a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/4b531e98e5057fd2eb3c96eb7973a5c90df1234758b5ee02c97755227a084e3f +++ /dev/null @@ -1 +0,0 @@ -β€€β€€β€ˆβ€€β€€β€€β€€β€ˆβ€€β€€β€€β€€β€€β€ˆβ€€β€ˆβ€€ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/5/0/50ae4phj2re52zkjrh1vj0vgh66t73e2870bhrrvey074 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/5/0/50ae4phj2re52zkjrh1vj0vgh66t73e2870bhrrvey074 new file mode 100644 index 00000000..a903cc86 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/5/0/50ae4phj2re52zkjrh1vj0vgh66t73e2870bhrrvey074 @@ -0,0 +1 @@ +:π”Ÿπ”€Ÿ:π”€Ÿπ”€Ÿ y \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/5/f/5f0c7bjwbwg1ycpwcfp5dpmmjhvm7jk2smf2eevwcsvjy b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/5/f/5f0c7bjwbwg1ycpwcfp5dpmmjhvm7jk2smf2eevwcsvjy new file mode 100644 index 00000000..4ea161d5 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/5/f/5f0c7bjwbwg1ycpwcfp5dpmmjhvm7jk2smf2eevwcsvjy @@ -0,0 +1 @@ +α˜€Β˜α˜€α˜€α˜€9β€€β€€β€€$Β˜α˜€α˜€α˜€9β€€β€€β€€β€€β˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€9β€€β€€β€€β€€β™€β€ˆβ€€β€€β€€β€€α˜€α˜€α˜€β˜€α˜€α˜€α˜€α˜€α˜€9β€€β€€β€€β€€β™€β€ˆβ€€β€€β€€β˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€9β€€β€€β€€β€€α˜€Β˜α˜€α˜€α˜€9β€€β€€β€€$Β˜α˜€α˜€α˜€9β€€Β˜α˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€9β€€β€€β€€β€€β™€β€ˆβ€€β€€β€€β˜€α˜€α˜€Β˜α˜€α˜€α˜€9β€€β€€β€€β€€α˜€Β˜α˜€α˜€α˜€9β€€β€€β€€$Β˜α˜€α˜€α˜€9β€€Β˜α˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€9β€€β€€β€€β€€α˜€α˜€α˜€9α˜€β€€β˜€α˜€α˜€α˜€α˜€α˜€9β€€β€€β€€β€€β™€β€ˆβ€€β€€β€€β€€α˜€α˜€α˜€9α˜€β€€β˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€9β€€β€€β€€β€€α˜€Β˜α˜€α˜€α˜€9β€€β€€β€€$Β˜α˜€α˜€β€€β€€β€€β˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€9β€€β€€β€€β€€β™€β€ˆβ€€β€€β€€β€€α˜€α˜€α˜€9α˜€β€€β˜€α˜€α˜€α˜€α˜€α˜€9β€€β€€β€€β€€β™€β€ˆβ€€β€€β€€β€€α˜€α˜€α˜€9α˜€β€€β˜€α˜€α˜€α˜€β€€β€€β€€β˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€9β€€β€€β€€β€€β™€β€ˆβ€€β€€β€€β€€α˜€α˜€α˜€9α˜€β€€β˜€α˜€α˜€α˜€α˜€α˜€9β€€β€€β€€β€€β™€β€ˆβ€€β€€β€€β€€α˜€α˜€α˜€9α˜€β€€β˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€9β€€β€€β€€β€€Β˜α˜€α˜€α˜€α˜€Β˜α˜€Β˜α˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€9β€€β€€β€€β™€β€ˆβ€€β€€β€€β€€α˜€α˜€α˜€9α˜€ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/50ae20ac4f0dd0a0d779ee8b2017b11348cfd8bd0f591c2abc2bbef7b6e95134 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/50ae20ac4f0dd0a0d779ee8b2017b11348cfd8bd0f591c2abc2bbef7b6e95134 deleted file mode 100644 index 64b5962e..00000000 --- a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/50ae20ac4f0dd0a0d779ee8b2017b11348cfd8bd0f591c2abc2bbef7b6e95134 +++ /dev/null @@ -1 +0,0 @@ -⁀⁀⁀:⁀⁀:π”€Ÿπ”€Ÿ:π”€Ÿ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/53a4e4cb235b274fdb21397a1141b4d6b3ee23fdde1151f799df7c3d14c86ca1 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/53a4e4cb235b274fdb21397a1141b4d6b3ee23fdde1151f799df7c3d14c86ca1 deleted file mode 100644 index d49be2f6..00000000 --- a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/53a4e4cb235b274fdb21397a1141b4d6b3ee23fdde1151f799df7c3d14c86ca1 +++ /dev/null @@ -1 +0,0 @@ -  \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/55443a6927436f5f0fde21ffd5ae2c195565e2e058ec518421272eb952dbd429 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/55443a6927436f5f0fde21ffd5ae2c195565e2e058ec518421272eb952dbd429 deleted file mode 100644 index fee7974b..00000000 --- a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/55443a6927436f5f0fde21ffd5ae2c195565e2e058ec518421272eb952dbd429 +++ /dev/null @@ -1 +0,0 @@ -α˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€9α˜€ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/59d573431e4724864b40110bb3247f783fb8fc33584238baeb1519ab70b426a1 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/59d573431e4724864b40110bb3247f783fb8fc33584238baeb1519ab70b426a1 deleted file mode 100644 index f2cb9cd7f1f27df94b43355c481b31ff077579c8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1990 zcmeHIOA5k342|NVBm;s_)ZIwIl~AmoU8&2S#=Wc7Tl9K8fOjyNPg|vR<)VuQhWR9u zyq64<x+?V#i5F_GSO6kBfr3P905u!Fg^5Pi#!E55BX9Kuuc><QN>qHL!Opx5iHZ%u zoLG=342agjj>wo{YA=ZcJh@%RM`nDnGmc_S*rpB9t9B^v9z~w#5f3k)yj%j9z1O?> z^(<-C04*G}g9C~ZnR)w5OxS@mgqm}&?*foBm!e+$IuyFM|A-VKLj$A5IE3BQz?Y;` vM}s%zM`B)w5Ff0nRC$sq15*aZlY!}{l|NA0!FMAW<xi{6O^d3lax3Ev&oJ4W diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/5c2c0103f79c00f4ee500842085ffcc424ebab149f1c85bdb6991de0b4ab86f6 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/5c2c0103f79c00f4ee500842085ffcc424ebab149f1c85bdb6991de0b4ab86f6 deleted file mode 100644 index 0f28f9e2..00000000 --- a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/5c2c0103f79c00f4ee500842085ffcc424ebab149f1c85bdb6991de0b4ab86f6 +++ /dev/null @@ -1 +0,0 @@ -  \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/5ede5b56920816525591723816c93c0d9c830264b24ca58c386075072b388372 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/5ede5b56920816525591723816c93c0d9c830264b24ca58c386075072b388372 deleted file mode 100644 index be97a30b..00000000 --- a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/5ede5b56920816525591723816c93c0d9c830264b24ca58c386075072b388372 +++ /dev/null @@ -1 +0,0 @@ -β€€ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/5f77b630b55a04c392f2caf506e2660aa0352a11c83af4f9e58b01169ecb8ac9 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/5f77b630b55a04c392f2caf506e2660aa0352a11c83af4f9e58b01169ecb8ac9 deleted file mode 100644 index ab1a2c2f778cbf151e3fddc76a299db3a5e1d405..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 67 pcmWgT=C0@FW?<q50xoWDQ4VfyaUceBpb`!YAOQ|Ag&_yj3jm!61eX8+ diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/6/n/6nzf697cve2978kqd9jby90x9ekg4m8m5nysp1jngy8jc b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/6/n/6nzf697cve2978kqd9jby90x9ekg4m8m5nysp1jngy8jc new file mode 100644 index 0000000000000000000000000000000000000000..602092b688e654f543f1eee37c6794721ab78410 GIT binary patch literal 18 Wcmd<)X5f6>*Z@QhwzhV*KpFr!$_D`e literal 0 HcmV?d00001 diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/61c8efd9f69870226747b046f077b5751015cea9dc045849d65509918973b62d b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/61c8efd9f69870226747b046f077b5751015cea9dc045849d65509918973b62d deleted file mode 100644 index 8c006e67..00000000 --- a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/61c8efd9f69870226747b046f077b5751015cea9dc045849d65509918973b62d +++ /dev/null @@ -1 +0,0 @@ -α˜€Β˜α˜€α˜€α˜€9β€€β€€β€€Β˜α˜€α˜€α˜€9β€€β€€β€€β€€β˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€9β€€β€€β€€β€€β™€β€ˆβ€€β€€β€€β€€α˜€α˜€α˜€9α˜€β€€β˜€α˜€α˜€α˜€α˜€α˜€9β€€β€€β€€β€€β™€β€ˆβ€€β€€β€€β€€α˜€α˜€α˜€9α˜€β€€β˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€9β€€β€€β€€β€€Β˜α˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€9β€€β€€β€€β™€β€ˆβ€€β€€β€€β€€α˜€α˜€α˜€9α˜€ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/620942b003b1bae02a40d38e7b03f3cc4cc9e948192cfd5e7d0cb77b31d069ef b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/620942b003b1bae02a40d38e7b03f3cc4cc9e948192cfd5e7d0cb77b31d069ef deleted file mode 100644 index 6d7d2715..00000000 --- a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/620942b003b1bae02a40d38e7b03f3cc4cc9e948192cfd5e7d0cb77b31d069ef +++ /dev/null @@ -1 +0,0 @@ -β€€β€€β€ˆβ€€β€€β€€β€ˆβ€€ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/7/4/745nee7h4x050p29a485e3t21m37hsa5y7a1wn9amb3wc b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/7/4/745nee7h4x050p29a485e3t21m37hsa5y7a1wn9amb3wc new file mode 100644 index 00000000..7409ccba --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/7/4/745nee7h4x050p29a485e3t21m37hsa5y7a1wn9amb3wc @@ -0,0 +1 @@ +⁀⁀⁀:⁀⁀:π”€Ÿ𔀍⁀:𔀆π”€Ÿ:𔀀:π”€Ÿπ§€Ÿ:π”€Ÿ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/7/c/7ccqs9fgddtc4r3s7axbxz1hr61n8rtqp6nsqc0cd5me6 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/7/c/7ccqs9fgddtc4r3s7axbxz1hr61n8rtqp6nsqc0cd5me6 new file mode 100644 index 0000000000000000000000000000000000000000..322a0216efe73dbdfb75565f4924ff4d49f074a0 GIT binary patch literal 79 mcmaDjqd}ok;?RtTGaBFs!Ud8!a28w+$iNhogh(?mFaQ87lq|*o literal 0 HcmV?d00001 diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/7/f/7fd6ksepb7j5k4q8t8e3p230agfvr357se20qa952dn2r b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/7/f/7fd6ksepb7j5k4q8t8e3p230agfvr357se20qa952dn2r new file mode 100644 index 00000000..8e5db810 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/7/f/7fd6ksepb7j5k4q8t8e3p230agfvr357se20qa952dn2r @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/70546d5fca6e8da3f4320b2bb5a2789bedcac8a5bd30820c7aac957d3c6e1684 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/70546d5fca6e8da3f4320b2bb5a2789bedcac8a5bd30820c7aac957d3c6e1684 deleted file mode 100644 index 04ead9b3..00000000 --- a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/70546d5fca6e8da3f4320b2bb5a2789bedcac8a5bd30820c7aac957d3c6e1684 +++ /dev/null @@ -1,7 +0,0 @@ -1 -1 - 2 5d - - - - diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/7923fd9b7984beaa52f2b5e97ecbf0812db63788b4eb8411bcddc4be7267e428 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/7923fd9b7984beaa52f2b5e97ecbf0812db63788b4eb8411bcddc4be7267e428 deleted file mode 100644 index 09cee908..00000000 --- a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/7923fd9b7984beaa52f2b5e97ecbf0812db63788b4eb8411bcddc4be7267e428 +++ /dev/null @@ -1 +0,0 @@ -α˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€9β€€β€€β€€β€€β€€β€ˆβ€€α˜€α˜€α˜€α˜€9α˜€ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/7b2c4f7c87a84c1e2c26e69abf2fe09932db7ef15b4c939c079cf8fb0d275cbc b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/7b2c4f7c87a84c1e2c26e69abf2fe09932db7ef15b4c939c079cf8fb0d275cbc deleted file mode 100644 index 819e9cce..00000000 --- a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/7b2c4f7c87a84c1e2c26e69abf2fe09932db7ef15b4c939c079cf8fb0d275cbc +++ /dev/null @@ -1 +0,0 @@ -⁀⁀⁀:π”€Ÿ:π”€Ÿ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/823ded55b18db8759c77a9c684b6d3af75c26871dd228082e3f3171ebda3b5a4 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/823ded55b18db8759c77a9c684b6d3af75c26871dd228082e3f3171ebda3b5a4 deleted file mode 100644 index c4110203b0509f356fc416c536125f323c1b14f3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 171 zcmY+6y$XOp41<f4m*DIsS)JrgbQ66TU&6^(x91Odg_4x!OA>?#k(<>d>swm4JwcwN xRkSCRT%^HQnMu-w86&V9e&la%`qZHnF@!$&{2FrRBi@37FSss1W|i$ga{=Jh5$FH_ diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/85ccfc35f2828100617311f9ebb138147e03034d9acbe149c2c1e93fba076dad b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/85ccfc35f2828100617311f9ebb138147e03034d9acbe149c2c1e93fba076dad deleted file mode 100644 index 76aec5bcb41977fd41829506c5ac66154bfeb5e6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 43 fcmZQz00J&9UMS$?0s=lR1qiT#P|}J(5pxCrH3|bw diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/891736f80c0acb4550ebf3a53fbde2113736faefe5e5107d3d7da299369df1b0 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/891736f80c0acb4550ebf3a53fbde2113736faefe5e5107d3d7da299369df1b0 deleted file mode 100644 index d1d6647c83a9d2205fdf82cf4a9aeb1fca681d05..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 22 WcmZSP<z?aJWl-Vf;^yXp0R{jH(E#ND diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/8fde056b9da569485292d132338d9af447d0b4e17345b30d0874b8ecccc88535 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/8fde056b9da569485292d132338d9af447d0b4e17345b30d0874b8ecccc88535 deleted file mode 100644 index 9555c91b62a1d866c6f638f622b3bfb6582160be..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 7 Ocmd<(<z+BuU;qFCmH=M> diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/9/1/91de5j750xjrn3kyfs8n3vgzt7389p5z7rnpxdw20mfrt b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/9/1/91de5j750xjrn3kyfs8n3vgzt7389p5z7rnpxdw20mfrt new file mode 100644 index 00000000..c1489732 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/9/1/91de5j750xjrn3kyfs8n3vgzt7389p5z7rnpxdw20mfrt @@ -0,0 +1,7 @@ + +2 2 + +1 + +Γ€2 + diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/9/4/94g4fe2jqv1ktz8h03heqvtybz5zekjr3kzgmg73jyzzr b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/9/4/94g4fe2jqv1ktz8h03heqvtybz5zekjr3kzgmg73jyzzr new file mode 100644 index 00000000..4cfd5012 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/9/4/94g4fe2jqv1ktz8h03heqvtybz5zekjr3kzgmg73jyzzr @@ -0,0 +1,164 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +⁀⁀Oπ”€Ÿπ”€Ÿ: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/931f79f6a9b51bb5822885e25a66a326e2beae862e1f8493d137870f9526abb0 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/9/7/974px1w6qq7hdrmawg8d8m32fccy4ft8jk32h57863qgc similarity index 93% rename from implementation/fuzz/inputs-genvm-modules-normalize-whitespace/931f79f6a9b51bb5822885e25a66a326e2beae862e1f8493d137870f9526abb0 rename to implementation/fuzz/inputs-genvm-modules-normalize-whitespace/9/7/974px1w6qq7hdrmawg8d8m32fccy4ft8jk32h57863qgc index 4308199a474eba93d32c92ad7d3e90dcd5d5e9f9..64ebffdb0e91d5b7a030c58d7d1e1ba977a1f2b6 100644 GIT binary patch delta 66 zcmZp0-R`nMkd4t~qoOktqvS@#<BTlz3=9k#74NY#a&1)n$il)P0OH<ZVvN|RsLIX) Rlmv238QG&aIgJ#I>H#>H4>14$ delta 30 dcmdn)(%`y5kZrR8*LijXvC-f+%VYy3b^xtZ3Y`D| diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/9cf2c3c978eb136c0bc952d366b028830ebe2391400b3d6658ac26f5dba36868 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/9cf2c3c978eb136c0bc952d366b028830ebe2391400b3d6658ac26f5dba36868 deleted file mode 100644 index e1ef5a72d107ff0c7926e4c806cc51e250e8ca22..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 24 Xcmd<&)Z_#LIS^6Lz`y{afq)+X6G;JY diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/8d56d269122a8bbee7db1f44f0d2bc3a834cb31b8d2538dedc5c0b6ff9a740a0 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/a/6/a68wtsxj7hgjqf2j6yrqfqcbqa85px7mms469p8j7n2bw similarity index 60% rename from implementation/fuzz/inputs-genvm-modules-normalize-whitespace/8d56d269122a8bbee7db1f44f0d2bc3a834cb31b8d2538dedc5c0b6ff9a740a0 rename to implementation/fuzz/inputs-genvm-modules-normalize-whitespace/a/6/a68wtsxj7hgjqf2j6yrqfqcbqa85px7mms469p8j7n2bw index dbd2dd8781c4ad61823ccae07ec36d8085805059..dd0e90147930c2d9ae231ccc2bcab4646882928f 100644 GIT binary patch literal 1043 zcmY#T7<#}K2!O6;;Kbr|E-YfBLPH<~6j9xng36^i&xqzWP+$QkrzGd&#)ij@levL_ Wi<`Tio11|XTk=P-L>q-iGzS3BrG*Rt literal 1057 XcmY#T7zLvtFxWysaj-dP<hlm{8_R?u diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/a/g/agnftywfhe3yc82jp0f4jad4wad9awev9gbq4px62c0gc b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/a/g/agnftywfhe3yc82jp0f4jad4wad9awev9gbq4px62c0gc new file mode 100644 index 00000000..72ab1055 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/a/g/agnftywfhe3yc82jp0f4jad4wad9awev9gbq4px62c0gc @@ -0,0 +1 @@ +Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β£ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/a/w/awkr5j8mde696580twqnhkvrkjg1b8b44pf9vvyhbyc7m b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/a/w/awkr5j8mde696580twqnhkvrkjg1b8b44pf9vvyhbyc7m new file mode 100644 index 00000000..9327a196 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/a/w/awkr5j8mde696580twqnhkvrkjg1b8b44pf9vvyhbyc7m @@ -0,0 +1 @@ +ÀÍ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/a/w/awm30caerhxjrabr6nxxp215bmmnzp576ds3n0fnd5qej b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/a/w/awm30caerhxjrabr6nxxp215bmmnzp576ds3n0fnd5qej new file mode 100644 index 0000000000000000000000000000000000000000..ec5a7e0ebfcc627ba341b67ea3be006ed81db210 GIT binary patch literal 4 LcmWe&sP_Q?0geFN literal 0 HcmV?d00001 diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/abfcf8aa5a090baf9be4f429ac9bea30c17dff4b027e19ec303d143d82cc3eb6 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/abfcf8aa5a090baf9be4f429ac9bea30c17dff4b027e19ec303d143d82cc3eb6 deleted file mode 100644 index 87b2d830..00000000 --- a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/abfcf8aa5a090baf9be4f429ac9bea30c17dff4b027e19ec303d143d82cc3eb6 +++ /dev/null @@ -1 +0,0 @@ -ÀÀÍÁÀÀÀÁÍÁÁÀÀÍÁÀÀÍÁÀÀÀÁÍÁÁÀÁÍÁÁÁÁ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/b/r/brgcy15k37c4rk914aktt5fxp4z1wdv0sjzp0va05a3s0 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/b/r/brgcy15k37c4rk914aktt5fxp4z1wdv0sjzp0va05a3s0 new file mode 100644 index 00000000..d99bfa2e --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/b/r/brgcy15k37c4rk914aktt5fxp4z1wdv0sjzp0va05a3s0 @@ -0,0 +1,2 @@ + + diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/b84925528f02d3ff5848684e0c537a889e6d301c9de358ab1959ec04e80af57f b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/b84925528f02d3ff5848684e0c537a889e6d301c9de358ab1959ec04e80af57f deleted file mode 100644 index 2501f5b0..00000000 --- a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/b84925528f02d3ff5848684e0c537a889e6d301c9de358ab1959ec04e80af57f +++ /dev/null @@ -1 +0,0 @@ -  \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/bd2ee8d25a19d78820a021ee98d8dc8de927f858011075464cb3ad4288b7ea0c b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/c/c/ccn0xhgpacrfh1aq74fkjnkd6qha0rp1z9zr22byttbgm similarity index 100% rename from implementation/fuzz/inputs-genvm-modules-normalize-whitespace/bd2ee8d25a19d78820a021ee98d8dc8de927f858011075464cb3ad4288b7ea0c rename to implementation/fuzz/inputs-genvm-modules-normalize-whitespace/c/c/ccn0xhgpacrfh1aq74fkjnkd6qha0rp1z9zr22byttbgm diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/c/s/csx0dgbx4j5a96gazak55b9ekgvw4f0cgk2zbmtjjjkve b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/c/s/csx0dgbx4j5a96gazak55b9ekgvw4f0cgk2zbmtjjjkve new file mode 100644 index 00000000..bfbf46d7 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/c/s/csx0dgbx4j5a96gazak55b9ekgvw4f0cgk2zbmtjjjkve @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/c3bcfe060468c318a0bfea760a116fd4b1e13a3905ae812b4c4434484297ca53 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/c3bcfe060468c318a0bfea760a116fd4b1e13a3905ae812b4c4434484297ca53 deleted file mode 100644 index e4032797..00000000 --- a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/c3bcfe060468c318a0bfea760a116fd4b1e13a3905ae812b4c4434484297ca53 +++ /dev/null @@ -1 +0,0 @@ -β€€β€€β€€ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/c48855b1773d9593d2bfbcd50023edf88c026e39d5a407c1b7b9155704add33d b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/c48855b1773d9593d2bfbcd50023edf88c026e39d5a407c1b7b9155704add33d deleted file mode 100644 index 4eb8eccbe3bb15db47a4237a46d1c21e8a45c94e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1195 zcmZQzU|`@f-~t0iAgQRx3kEz23MpVg1t6CPA~s@4y%Fp4QI8W50;9=;h(ID+36CJK I9FhWJ09oxlTmS$7 diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/c7df8fdd6edee85308057d4f35c608d9f2d2c8e9320b136bd312cf028400df4c b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/c7df8fdd6edee85308057d4f35c608d9f2d2c8e9320b136bd312cf028400df4c deleted file mode 100644 index e9befded0f810fa40a6b50082947c38428552220..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 274 zcmWgV;O6G8=jImYVBqBBlmru;j~g2vH%<mhaC3nrkwhCDY;EmqZ6Qja99vr?1$ecg VsD$el2e|=i)j+z#mI3HKTL9kgMSuVR diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/cd40d68396aa759f586d90a3b4887a2672716b4ef446659f26d1c030acedd63d b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/cd40d68396aa759f586d90a3b4887a2672716b4ef446659f26d1c030acedd63d deleted file mode 100644 index 0f9c86495b68ed3e42f37a8d07d280e02352e024..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 96 zcmZQz;B{c&U{DB8fB;@FVBq3PNMHa0US3WvAmHOt0D**pgi4?=Tyb$iLIESl04^>d WQdER!=276_=H_ML<z?~Z6#)R@lnz1w diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/ceb8a0e16a10fb5086e705c9cc73174b55c7a826833b26ab01a78cc5304725ea b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/ceb8a0e16a10fb5086e705c9cc73174b55c7a826833b26ab01a78cc5304725ea deleted file mode 100644 index f9eb580a15e4f158d9ff0dce2b75e7e387a1ce66..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 63 ucmWgV;06LNZti++ZgC{GB<JJC1|V{<wY9Sa0tQY_PDutJKoYZMumu24LJU;^ diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/dbfa4b3440b784d342db858d835171514a94c3b3fee05e4cf9caae20e0d28df0 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/dbfa4b3440b784d342db858d835171514a94c3b3fee05e4cf9caae20e0d28df0 deleted file mode 100644 index 53de138495fd7c8c05d87a407b9927250fd87ac7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 36 gcmZRSX8?l~E-o%JE@LheE(Qhx1_l*~D3Ap50VQhzoB#j- diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/e/4/e4692ckej8yvykkywjmdmz3y8k7062my5rk753watxfj0 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/e/4/e4692ckej8yvykkywjmdmz3y8k7062my5rk753watxfj0 new file mode 100644 index 0000000000000000000000000000000000000000..d5b6723f2b9f644fbd0f7972f72f7e531e1f4466 GIT binary patch literal 326 zcmZ`#F%H5o3`{q4O{6Xev0WEdlvM3ju~y7TUGoWsvK0Q(=OFbZ`~kZu0}`BKSx#)< zT>?SG*W;e&yA4p$OYP^$xOs97DzpOvL(e#x!rFtiHYThyb)Ew#Bv6*lIZ~dl+9C~d zy5E3DdqYS)dj_(`LMOZ?%amTyD4|-OYcYzLW1#;)r0HBcIA@Y(BNAM5rXf!7Vg(^B h79-rQ*Pt=oMTDt&HQe>_kH7B?`W1vyD3)0vBHqcdMacjF literal 0 HcmV?d00001 diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/e/7/e7yvwya7erng8g6r1xee9we5cjd7ys97ejb0yc1crfpzr b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/e/7/e7yvwya7erng8g6r1xee9we5cjd7ys97ejb0yc1crfpzr new file mode 100644 index 00000000..f0b0f494 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/e/7/e7yvwya7erng8g6r1xee9we5cjd7ys97ejb0yc1crfpzr @@ -0,0 +1,3 @@ + + +ÀÍÁ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/e6237eb52e4a8750685e1fb8741b2c0cbac5e873876430bc8f6e41b5ad0209c8 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/e6237eb52e4a8750685e1fb8741b2c0cbac5e873876430bc8f6e41b5ad0209c8 deleted file mode 100644 index 69eddd6a..00000000 --- a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/e6237eb52e4a8750685e1fb8741b2c0cbac5e873876430bc8f6e41b5ad0209c8 +++ /dev/null @@ -1 +0,0 @@ -α˜€ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/e72d0a509f5966a8861516dd92358725c1c5603183da9733bb90a125f85ab9db b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/e72d0a509f5966a8861516dd92358725c1c5603183da9733bb90a125f85ab9db deleted file mode 100644 index 285991b6385f3c409964164803309ad43f227d40..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 66 ucmZSJ;^GAZhVx4pctIEl%(#rXY%95p6pR$OxVgDO7z^O#<>cnl-~#|83I_lH diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/e8b6d77f122eb966b7a7cbb6a225ef83020029066c04ad009ec529c273de0fa8 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/e8b6d77f122eb966b7a7cbb6a225ef83020029066c04ad009ec529c273de0fa8 deleted file mode 100644 index 01a1a62a..00000000 --- a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/e8b6d77f122eb966b7a7cbb6a225ef83020029066c04ad009ec529c273de0fa8 +++ /dev/null @@ -1 +0,0 @@ -⁀ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/f/7/f717g71e7k6bcs5q565ze8z0f8xna7kpmt8j8kcp7t4rg b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/f/7/f717g71e7k6bcs5q565ze8z0f8xna7kpmt8j8kcp7t4rg new file mode 100644 index 00000000..ea1e7ab9 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/f/7/f717g71e7k6bcs5q565ze8z0f8xna7kpmt8j8kcp7t4rg @@ -0,0 +1 @@ +Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β… \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/f/j/fj28aaza5he7xhhe0pff01yxcft748ycsfxjkz1p1pzw2 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/f/j/fj28aaza5he7xhhe0pff01yxcft748ycsfxjkz1p1pzw2 new file mode 100644 index 00000000..0ed261c8 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/f/j/fj28aaza5he7xhhe0pff01yxcft748ycsfxjkz1p1pzw2 @@ -0,0 +1 @@ +βš€π”€Ÿπ”€Ÿ:π”€Ÿ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/5df9f5badb887ecd9ffbe572515a25f71356275d18023fb295c80fe8caba5c82 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/f/r/fr6sp1b7ghxs3gbj8e6je6nc4bwqra5kpkd4exvremh62 similarity index 100% rename from implementation/fuzz/inputs-genvm-modules-normalize-whitespace/5df9f5badb887ecd9ffbe572515a25f71356275d18023fb295c80fe8caba5c82 rename to implementation/fuzz/inputs-genvm-modules-normalize-whitespace/f/r/fr6sp1b7ghxs3gbj8e6je6nc4bwqra5kpkd4exvremh62 diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/f/v/fvq371s3kcysrjp95yrvxpccg93687d7c8c7ypv3xadvg b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/f/v/fvq371s3kcysrjp95yrvxpccg93687d7c8c7ypv3xadvg new file mode 100644 index 00000000..14af655f --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/f/v/fvq371s3kcysrjp95yrvxpccg93687d7c8c7ypv3xadvg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/f/w/fwqe7ds6xt6f0mvqxw2gvqmwnsgb15fas67ge0wb0mbe0 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/f/w/fwqe7ds6xt6f0mvqxw2gvqmwnsgb15fas67ge0wb0mbe0 new file mode 100644 index 00000000..940651c6 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/f/w/fwqe7ds6xt6f0mvqxw2gvqmwnsgb15fas67ge0wb0mbe0 @@ -0,0 +1 @@ +Β…Β…Β…Β…Β…Β…Β…Β… \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/f35d3bec11953f5914c9ca460ef3d4a1d5d7121d0ff0d8d6cdbc374ae9946c80 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/f35d3bec11953f5914c9ca460ef3d4a1d5d7121d0ff0d8d6cdbc374ae9946c80 deleted file mode 100644 index efa9a3eeacc3b481261cca817dfb69f36425d96a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1804 zcmd<(HDfX6vaRGYQZNcn;28z<3;{@>C)6_tfNW)8VPK%A?W1;$h5!{pU^LlMArME6 M7~l{vQcy4j0ICUCPyhe` diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/f71d8c3bc20355985702ac0223e06edd28d33945777415f169219fc560a6025b b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/f71d8c3bc20355985702ac0223e06edd28d33945777415f169219fc560a6025b deleted file mode 100644 index a7e49d3858daf8799a40dfd4ad52e20f4214bc58..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 23 acmX@Vgn{?`5(Y*f=4IyP<>CVI>KFiA2?nG9 diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/f7e3e6880c7e27d1a97f3a04f448e90e33f72f377cf4fcc582fa70c9fbeec75a b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/f7e3e6880c7e27d1a97f3a04f448e90e33f72f377cf4fcc582fa70c9fbeec75a deleted file mode 100644 index d563cdeb0ecdd0a856f8b1e957c8f5b6a8dda4d0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4343 zcmeGgyKcfj?5Z*lU)BoUb`nz6p;ATq7?7@Isf73=CPIIq9oX`1lpp9P)b~j2Mg#() zNkwBp;`1v$pY0gdt5kD{%0Ns=H0UzBV*R;H)fg}C{`HY&5nov9D<>!`qQ-b7h81w6 zYF(5d_x=RN!{PkryR3{sMJzHG6>BCJs239-H6K&%P@6cN&Ym>7;a;{DLUPw*cfI&J zMi64?Zz2|n)<VP=Wl>u1*(u+sO^uobljnl1mW&GLoRZ~7WKOcUJkpAk8b+`HGsnYB zZuKKP!9BnW67l$_0RR&L<%H-7D_78S{xC;!`4MVpXT@tm5)7NUzCugE+jcU75_n(k zO9YT|A?2Vi&}Zc?(Po)$mhZ%5)O2FZvoT&IER10SJJq=4TcJJZ)1w;jqbv9$YiWlX zrCd`zlb(p!uXpzyTR(u+NE5?W<eFXA#OgOO+W}b*D>~CQTGy?@Nc<0GS*q>sr9rVc z8|<Lr@N|6@CI!q2tQOMP4@2_iKo3!Iy-M+;@Jm)Eeq46ra~QVm$h>U`XQQ+o+3_z6 Q6tn6Yrc)T{2m9t%-{*PBvj6}9 diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/g/e/gecqrc9ajgg1n1dtv2p2v1qjxjgrh535tb5hwqkngyz00 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/g/e/gecqrc9ajgg1n1dtv2p2v1qjxjgrh535tb5hwqkngyz00 new file mode 100644 index 00000000..a0e70779 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/g/e/gecqrc9ajgg1n1dtv2p2v1qjxjgrh535tb5hwqkngyz00 @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/h/0/h013nnr4swa87d0ztyyvvrvqb28vkagw6m9xs5vb9grsm b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/h/0/h013nnr4swa87d0ztyyvvrvqb28vkagw6m9xs5vb9grsm new file mode 100644 index 00000000..fd40910d --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/h/0/h013nnr4swa87d0ztyyvvrvqb28vkagw6m9xs5vb9grsm @@ -0,0 +1,4 @@ + + + + diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/h/8/h8rtrgawv1nqn6hg52g9t04zpspzmdrakrktftpejfg4y b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/h/8/h8rtrgawv1nqn6hg52g9t04zpspzmdrakrktftpejfg4y new file mode 100644 index 00000000..842104dc --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/h/8/h8rtrgawv1nqn6hg52g9t04zpspzmdrakrktftpejfg4y @@ -0,0 +1 @@ + ş:π”€Ÿπ”€Ÿ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/f9024ec809d2d43073ab044fa702ae963fdba5c841b38a76999eb5ccead5518c b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/h/x/hxj83xa25drwh4m46gdt9naav2gzf3yg450m3gtd5xhz4 similarity index 100% rename from implementation/fuzz/inputs-genvm-modules-normalize-whitespace/f9024ec809d2d43073ab044fa702ae963fdba5c841b38a76999eb5ccead5518c rename to implementation/fuzz/inputs-genvm-modules-normalize-whitespace/h/x/hxj83xa25drwh4m46gdt9naav2gzf3yg450m3gtd5xhz4 diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/j/g/jgjsyahmzmj5v16fek5xs8aa88g7nbe8n3mwbkpy4j1ce b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/j/g/jgjsyahmzmj5v16fek5xs8aa88g7nbe8n3mwbkpy4j1ce new file mode 100644 index 00000000..dcfc9f4f --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/j/g/jgjsyahmzmj5v16fek5xs8aa88g7nbe8n3mwbkpy4j1ce @@ -0,0 +1 @@ +β€€β€ˆ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/j/j/jj3w58ycvjfwcfsv728ky0ym67zwq5ghv97tr3gvs709w b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/j/j/jj3w58ycvjfwcfsv728ky0ym67zwq5ghv97tr3gvs709w new file mode 100644 index 0000000000000000000000000000000000000000..f721ab19658948bee329cc341b97947c099fa145 GIT binary patch literal 16 Ocmd;LU}Rw6LI3~*QUH1Y literal 0 HcmV?d00001 diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/j/j/jjnrnbpw8tdkz7x7d88ksjpvma82a1eesd5kq3ma7cbma b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/j/j/jjnrnbpw8tdkz7x7d88ksjpvma82a1eesd5kq3ma7cbma new file mode 100644 index 0000000000000000000000000000000000000000..3218f4bfc231897f84b18e365ea431896ffe763f GIT binary patch literal 198 zcmaDjV_G~`kboqL%ti?3$72@<s=y(|Wyn>H&<|qS1BDbA7!-h%oGF(XmoXQYZ6%kH Vf{_B3LLO8p4#Qvu1;xi#0sxR_Lm2=7 literal 0 HcmV?d00001 diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/0a53edc743484691ae84defc2db71ac98893a8c4afc437239bbf602cadafcf01 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/j/m/jmg6zts2m7cnj5v0cwbp2tzeez12bya52tf8ff84vcx2a similarity index 89% rename from implementation/fuzz/inputs-genvm-modules-normalize-whitespace/0a53edc743484691ae84defc2db71ac98893a8c4afc437239bbf602cadafcf01 rename to implementation/fuzz/inputs-genvm-modules-normalize-whitespace/j/m/jmg6zts2m7cnj5v0cwbp2tzeez12bya52tf8ff84vcx2a index 09976b8f..a3cd1623 100644 --- a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/0a53edc743484691ae84defc2db71ac98893a8c4afc437239bbf602cadafcf01 +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/j/m/jmg6zts2m7cnj5v0cwbp2tzeez12bya52tf8ff84vcx2a @@ -2,8 +2,7 @@ 2 2 1 - -Γ€ 2 + +Γ€ 2 1 diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/07d0413d5429f6ec807e88ee6428490ca8c4e7486981a814b99a1dcd1460be16 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/j/r/jrm84mt72pk0x462yncar39r2ztrbqa2hz12d6akabk92 similarity index 69% rename from implementation/fuzz/inputs-genvm-modules-normalize-whitespace/07d0413d5429f6ec807e88ee6428490ca8c4e7486981a814b99a1dcd1460be16 rename to implementation/fuzz/inputs-genvm-modules-normalize-whitespace/j/r/jrm84mt72pk0x462yncar39r2ztrbqa2hz12d6akabk92 index dc852db2b7f636e4e28c508b28d1a03803103420..580dd6d7a03f380651f0812c2e80fa82c5dd3053 100644 GIT binary patch delta 115 zcmZ3X-J{O@a7M%AKmpZ>1}u|9m?gO-IUhGR0Fgt2?Zkt&8y75Ln!K8wiw&qs!NFnj zM(%f$6PQ>hm$92~?qKz2+FZ|^z&QB>qs8VtuGwrnj~W|*NYd)Vl!p0}A2MofPGG8M R0!r&o&f_ZIoFd}F2ml9MEO`I` delta 108 zcmeBCU!l$Xa7M%AKo*h7Jj{}l>p2TH_AX(XT*qcJ`2uU@WCN}#lOHe!O?Kv8xjCD~ tp9xJccX9yx>CHhLv)O>E%qDN(W!*fVRhkj5DuBIlau7%PW)C43MgX2~CzAjG diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/j/s/jshb3dwrfzdq3w7zdrmsf5e5ed4djp29a0a2khgaff4ky b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/j/s/jshb3dwrfzdq3w7zdrmsf5e5ed4djp29a0a2khgaff4ky new file mode 100644 index 00000000..0fddc406 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/j/s/jshb3dwrfzdq3w7zdrmsf5e5ed4djp29a0a2khgaff4ky @@ -0,0 +1 @@ +⁀⁀⁀:⁀⁀:π”€Ÿ ⁀⁀E⁀⁀:π”€Ÿ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/24047b08dbc5f81e0d744bee88f975cd94332c54c3887d0305a1b768ed23b1ab b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/k/j/kjzgqk5d7x3mfk4eh2r3erw2bcjvbb2a4tx5bmyg5h7gm similarity index 67% rename from implementation/fuzz/inputs-genvm-modules-normalize-whitespace/24047b08dbc5f81e0d744bee88f975cd94332c54c3887d0305a1b768ed23b1ab rename to implementation/fuzz/inputs-genvm-modules-normalize-whitespace/k/j/kjzgqk5d7x3mfk4eh2r3erw2bcjvbb2a4tx5bmyg5h7gm index 92b715d0..e3d16323 100644 --- a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/24047b08dbc5f81e0d744bee88f975cd94332c54c3887d0305a1b768ed23b1ab +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/k/j/kjzgqk5d7x3mfk4eh2r3erw2bcjvbb2a4tx5bmyg5h7gm @@ -126,133 +126,7 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/709af572b2d689c93a58cc6beacccd0e41429338d17234b2a1591ffbf28e0aaf b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/k/z/kz1hkv3gm079cr4xbqx5r88g7n89246m27dcckhwdrdn4 similarity index 65% rename from implementation/fuzz/inputs-genvm-modules-normalize-whitespace/709af572b2d689c93a58cc6beacccd0e41429338d17234b2a1591ffbf28e0aaf rename to implementation/fuzz/inputs-genvm-modules-normalize-whitespace/k/z/kz1hkv3gm079cr4xbqx5r88g7n89246m27dcckhwdrdn4 index 8c58ec69cf4f6f428801b8fce38bd1239ab403d3..2ea6ccbb2f3da4a37cf726524cdd2c0949cafa40 100644 GIT binary patch delta 100 zcmaFMKaEfH;fw|a5I8gg$dUwNAhMju;XW}?bz?;%)8?Pdm5iK`oDjf0F;H~#1lCSQ Q1$zQuvKLFj<VyAu0RLnk-v9sr delta 47 zcmbQn_m-de;f#ifjv|w}7;kO-#lf_A0}JD3Cgw`UNlbE+Pcd15_>-BKH%?x}RssO1 C7ZB3` diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/b39d82c804f3d1d5d8b219aab4aa30e36f7d4e00e1657d89a4dd2cefc2d47c58 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/m/k/mkvxxgdq2vw8r2zch1k1m1j8w8s48dtst8d6ckyj2hzg6 similarity index 76% rename from implementation/fuzz/inputs-genvm-modules-normalize-whitespace/b39d82c804f3d1d5d8b219aab4aa30e36f7d4e00e1657d89a4dd2cefc2d47c58 rename to implementation/fuzz/inputs-genvm-modules-normalize-whitespace/m/k/mkvxxgdq2vw8r2zch1k1m1j8w8s48dtst8d6ckyj2hzg6 index ed0bddee59266111cd2e6fee2c696d716fc633e5..2e42627a2cc4a05f88d7b4342f9b652ad439daa0 100644 GIT binary patch delta 120 zcmaE;uv%e*IpgGT#uP^H$%VXIHWx7IFivh|d_9RNX!CoPY(@z`9AKiM)aD7yyo^8{ z+nFXYnNBWXVx8Q@wVR7UKmiE2j1;&gf8-L~e1}n(c@mTUWOdf^$+G+(7|ka02?PTG DJ%A*- delta 61 zcmV-D0K)&PAmJdeHvy60Ad@!%WRp+~A+yg0O9PW|4tJAi1-!Gc3_k(0Py#FgvyTXX T0+X-|3bPptz61e4lh_YawJ;S- diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/m/z/mzfpjfw6n0r4khj55ptpnx9cayycmwj6rb0hc1q6m8rb4 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/m/z/mzfpjfw6n0r4khj55ptpnx9cayycmwj6rb0hc1q6m8rb4 new file mode 100644 index 00000000..991aa1a5 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/m/z/mzfpjfw6n0r4khj55ptpnx9cayycmwj6rb0hc1q6m8rb4 @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/add963d8792a3271636e0d20bb97a73de0ea4220567f4fba49a271018360f15f b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/n/e/neab4np5n178b82cn37w6wwwew2s3hbz2xgtawd732qm4 similarity index 75% rename from implementation/fuzz/inputs-genvm-modules-normalize-whitespace/add963d8792a3271636e0d20bb97a73de0ea4220567f4fba49a271018360f15f rename to implementation/fuzz/inputs-genvm-modules-normalize-whitespace/n/e/neab4np5n178b82cn37w6wwwew2s3hbz2xgtawd732qm4 index 1901ba93..a02e2345 100644 --- a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/add963d8792a3271636e0d20bb97a73de0ea4220567f4fba49a271018360f15f +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/n/e/neab4np5n178b82cn37w6wwwew2s3hbz2xgtawd732qm4 @@ -10,3 +10,7 @@ + + + + diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/n/r/nr89fnaja49pea92v7ynqbrmtv9fmeqp91kpvb5cxkqj6 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/n/r/nr89fnaja49pea92v7ynqbrmtv9fmeqp91kpvb5cxkqj6 new file mode 100644 index 00000000..b28b04f6 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/n/r/nr89fnaja49pea92v7ynqbrmtv9fmeqp91kpvb5cxkqj6 @@ -0,0 +1,3 @@ + + + diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/p/1/p1zpp6xkygnwpdwe92026bywfvmxgwjmms4xp1p031gne b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/p/1/p1zpp6xkygnwpdwe92026bywfvmxgwjmms4xp1p031gne new file mode 100644 index 00000000..0facbfe4 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/p/1/p1zpp6xkygnwpdwe92026bywfvmxgwjmms4xp1p031gne @@ -0,0 +1 @@ + ぀぀@=== ぀぀@== \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/p/8/p8awpn2bs92fdxdgba32kgfqzy20eqysnv60tc2bdtw00 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/p/8/p8awpn2bs92fdxdgba32kgfqzy20eqysnv60tc2bdtw00 new file mode 100644 index 00000000..79ea433d --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/p/8/p8awpn2bs92fdxdgba32kgfqzy20eqysnv60tc2bdtw00 @@ -0,0 +1,12 @@ +β€€Í +3 +=y +2 2 + +1 + +32 5 +6y +ÁÁ +=y + diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/p/e/pe5n6cfvewh2w64bd0za74k56dm5w120w067p8e7s8zqe b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/p/e/pe5n6cfvewh2w64bd0za74k56dm5w120w067p8e7s8zqe new file mode 100644 index 00000000..a63a7165 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/p/e/pe5n6cfvewh2w64bd0za74k56dm5w120w067p8e7s8zqe @@ -0,0 +1 @@ +2 5 \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/167391bcddba8328537beeeb9c29c8ae7c1f4a23113ce87e30aa365fdec40971 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/p/x/pxbseqw4f9cwxkxm8qjv999ydvxydehy9xfrw5x0pynjg similarity index 84% rename from implementation/fuzz/inputs-genvm-modules-normalize-whitespace/167391bcddba8328537beeeb9c29c8ae7c1f4a23113ce87e30aa365fdec40971 rename to implementation/fuzz/inputs-genvm-modules-normalize-whitespace/p/x/pxbseqw4f9cwxkxm8qjv999ydvxydehy9xfrw5x0pynjg index b3b39908dc175af6b93cf60fc9332b8d090ccb2b..0b07e9066f4cba2b3c4d709877d42d5e09e81cc1 100644 GIT binary patch delta 146 zcmdnMbA(&%;fw|aD9~r&QR4vu1_osa#mB>9q@cjSz{5DvtCfWd2sZC#Qe>RW#uO+f zY4u@B!+g$1jSWBq;Q$$vbD8Ql^Dr|qGV*PH$jr;A0@SU*Wyob@YFo(#G|K}FxPT%G UKyiC8%`o{Pa|<K)<Xfy^0KTju<p2Nx delta 120 zcmX@Yy@AK;;fw|aJV2j?M~w#v7#Nfx6ax=~k%9sP0}ms*u8C@`lk=I}H$HG=oEYl| dVorSUVR9R@;p7X<UxE0><O__>lQ*&o0{~hA9PI!A diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/q/d/qda1v0f123z425e6e676d6jxpx5depn9zy8mrncp32ky2 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/q/d/qda1v0f123z425e6e676d6jxpx5depn9zy8mrncp32ky2 new file mode 100644 index 00000000..5474ceaa --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/q/d/qda1v0f123z425e6e676d6jxpx5depn9zy8mrncp32ky2 @@ -0,0 +1 @@ +:π”€Ÿ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/q/d/qdq0byn41gzhq0wqwfcs77e4kcmftqh4r11237dqpg8j4 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/q/d/qdq0byn41gzhq0wqwfcs77e4kcmftqh4r11237dqpg8j4 new file mode 100644 index 00000000..c7a0c8d7 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/q/d/qdq0byn41gzhq0wqwfcs77e4kcmftqh4r11237dqpg8j4 @@ -0,0 +1 @@ +ÀÀÍÁÀÀÀÁÍÁÁÀÀÍÁÀ܀ÍÁÀÀÀÁÍÁÁÀÁÍÁÁÁÁ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/q/j/qjrs2n5jd5yenchgmwk4zny08by0d7qtqq6paysg185h6 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/q/j/qjrs2n5jd5yenchgmwk4zny08by0d7qtqq6paysg185h6 new file mode 100644 index 00000000..297d4117 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/q/j/qjrs2n5jd5yenchgmwk4zny08by0d7qtqq6paysg185h6 @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/q/w/qwk9094vv8hx6rmr13g2ncxp0wjj2pbvd73t02498vdq8 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/q/w/qwk9094vv8hx6rmr13g2ncxp0wjj2pbvd73t02498vdq8 new file mode 100644 index 0000000000000000000000000000000000000000..0f541b4504c730797028e7145db4d7ad2958e638 GIT binary patch literal 4 LcmZQjP*4B>0K@=c literal 0 HcmV?d00001 diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/r/5/r5jvp155d7b5sa8zbgqzc2x6176bgs69qz9g0hx99r79e b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/r/5/r5jvp155d7b5sa8zbgqzc2x6176bgs69qz9g0hx99r79e new file mode 100644 index 00000000..d782e2eb --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/r/5/r5jvp155d7b5sa8zbgqzc2x6176bgs69qz9g0hx99r79e @@ -0,0 +1,18 @@ +E + + + + + + + + + + + + + + + + + diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/r/7/r7bm9nh1znjv0nh4qcy272s9brgsz64pz4dq52sx0ajec b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/r/7/r7bm9nh1znjv0nh4qcy272s9brgsz64pz4dq52sx0ajec new file mode 100644 index 00000000..e5e0b204 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/r/7/r7bm9nh1znjv0nh4qcy272s9brgsz64pz4dq52sx0ajec @@ -0,0 +1 @@ +⁀E⁀⁀ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/r/m/rm7j76kpffjnykfqxcgx0925xen98vn3t09atfgxapbmp b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/r/m/rm7j76kpffjnykfqxcgx0925xen98vn3t09atfgxapbmp new file mode 100644 index 00000000..12a6f48e --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/r/m/rm7j76kpffjnykfqxcgx0925xen98vn3t09atfgxapbmp @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/r/r/rrg0pbmdjev80ps4d0js812gzqa86q9jpa7rj5dn29rrp b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/r/r/rrg0pbmdjev80ps4d0js812gzqa86q9jpa7rj5dn29rrp new file mode 100644 index 0000000000000000000000000000000000000000..e58e603162b579e576db80e66819a4bd8b0d6e85 GIT binary patch literal 17 QcmZQjFogpyQwD~502xyP@c;k- literal 0 HcmV?d00001 diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/r/z/rzpzfk0mrj3zpzt3637hkx9zr4578xrbznyq6e2a7w91g b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/r/z/rzpzfk0mrj3zpzt3637hkx9zr4578xrbznyq6e2a7w91g new file mode 100644 index 00000000..597faa26 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/r/z/rzpzfk0mrj3zpzt3637hkx9zr4578xrbznyq6e2a7w91g @@ -0,0 +1 @@ +β€ˆβ€€α˜€ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/s/j/sjnxj30614zv737b6tjpmtnce0992v52bzadh2rhxzjf0 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/s/j/sjnxj30614zv737b6tjpmtnce0992v52bzadh2rhxzjf0 new file mode 100644 index 00000000..85830459 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/s/j/sjnxj30614zv737b6tjpmtnce0992v52bzadh2rhxzjf0 @@ -0,0 +1 @@ +Β…Β…Β…Β… \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/s/s/ssx9arzd57h63sh653erphygbrm9xnq5t4e4yhs2yq5aj b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/s/s/ssx9arzd57h63sh653erphygbrm9xnq5t4e4yhs2yq5aj new file mode 100644 index 00000000..dd9d674b --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/s/s/ssx9arzd57h63sh653erphygbrm9xnq5t4e4yhs2yq5aj @@ -0,0 +1 @@ +Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β…Β… \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/s/w/swzsf6kzdrj1gay4k3mpkhwen689j0z2191zpt6qw1bm0 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/s/w/swzsf6kzdrj1gay4k3mpkhwen689j0z2191zpt6qw1bm0 new file mode 100644 index 00000000..1f039545 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/s/w/swzsf6kzdrj1gay4k3mpkhwen689j0z2191zpt6qw1bm0 @@ -0,0 +1,26 @@ + =y + 2 5 +6 +Γ€ 2 + +1 +    2 + +1 + + + + + + + +  2 5 +6 +Γ€ 2 + +1 + +β€€β€€=m + 2 5 +6 +Γ€ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/s/y/sybedhhm3tshmxs05mtw2vsqenrw8wv4cvw7wae800jk0 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/s/y/sybedhhm3tshmxs05mtw2vsqenrw8wv4cvw7wae800jk0 new file mode 100644 index 00000000..5459a8c9 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/s/y/sybedhhm3tshmxs05mtw2vsqenrw8wv4cvw7wae800jk0 @@ -0,0 +1 @@ +α˜€Β˜α˜€α˜€α˜€9β€€β€€β€€Β˜α˜€α˜€α˜€9β€€β€€β€€β˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€β€€   ♀    ‖α˜€α˜€α˜€9α˜€β€€β˜€α˜žα˜€α˜€Β˜α˜€α˜€α˜€α˜€Β˜α˜€α˜€α˜€9β€€β”€β€€β€€β™€β€ˆβ€€β€€β€€β€€α˜€α˜€α˜€ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/w/9/w9frp6nrpx9teh9vx8q52bwe8pz3gmzv2320qv3mttqtw b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/w/9/w9frp6nrpx9teh9vx8q52bwe8pz3gmzv2320qv3mttqtw new file mode 100644 index 00000000..4b11cd50 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/w/9/w9frp6nrpx9teh9vx8q52bwe8pz3gmzv2320qv3mttqtw @@ -0,0 +1 @@ +β€€β€ˆβ€€β€€ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/w/a/wa80m1ba2zgfd9y9bqvh13wfm83kqbqpsjwhv4mjmqebj b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/w/a/wa80m1ba2zgfd9y9bqvh13wfm83kqbqpsjwhv4mjmqebj new file mode 100644 index 00000000..4dcc5e79 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/w/a/wa80m1ba2zgfd9y9bqvh13wfm83kqbqpsjwhv4mjmqebj @@ -0,0 +1 @@ +β€€β€€β€ˆβ€€β€€β€€β€€β€ˆ \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/x/4/x4gh60rdd803axv8pw9kh27xe1enmx1nhfhmjrags7jy8 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/x/4/x4gh60rdd803axv8pw9kh27xe1enmx1nhfhmjrags7jy8 new file mode 100644 index 0000000000000000000000000000000000000000..de0c3e4baf4aee4dfdd83c9c24c602c5d9b0f43a GIT binary patch literal 212 zcmd<(;O6G8=jImYVBqBBlmru;j~g2vH%<mhaC3nrkwhCDY;EmqZ6Qh^92;9(Bn5c2 Yp{Rr!Wos*r;ReDEz;2Q)15n%+07eEh@&Et; literal 0 HcmV?d00001 diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/y/b/yb68cax2dvjwqrnpxdq5srdezdcfwjfsfx6bsfcf9sk78 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/y/b/yb68cax2dvjwqrnpxdq5srdezdcfwjfsfx6bsfcf9sk78 new file mode 100644 index 00000000..dfc6b101 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/y/b/yb68cax2dvjwqrnpxdq5srdezdcfwjfsfx6bsfcf9sk78 @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/y/q/yqzk58tjef594ycch1z76qwvtyms5e4rvj2989her6rey b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/y/q/yqzk58tjef594ycch1z76qwvtyms5e4rvj2989her6rey new file mode 100644 index 0000000000000000000000000000000000000000..edb5ce3e9fb7142333f8d957791dd7d08b20c4a8 GIT binary patch literal 6 NcmZ?Z;^E<80006c0B-;Q literal 0 HcmV?d00001 diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/z/9/z941c9vqggbmfxm8shz7vrn24bvpvajg0wpdeym43vbym b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/z/9/z941c9vqggbmfxm8shz7vrn24bvpvajg0wpdeym43vbym new file mode 100644 index 0000000000000000000000000000000000000000..ae581cf0d8bca5b9ef3b72adaa61b7774b975c9f GIT binary patch literal 344 zcmaJ-!4ZHU48tF^`sO^x;9ZZM96*@C5n03nqHw7BaU^My;GHE}i6N9yvkKk?e;}hg zoCRP){$VyVSR#h)0#?E=t`Nz$+<LtShseX`;}TE>i7qh*Rv<wyk{Lx_+tPHmI- T-Yn}<`~+L>VaS-YBacHTzvW*y literal 0 HcmV?d00001 diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/z/g/zg4trx14yq823m5wamqwvdrz61sh6vkhc4y3fps0m02w6 b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/z/g/zg4trx14yq823m5wamqwvdrz61sh6vkhc4y3fps0m02w6 new file mode 100644 index 00000000..4f63c411 --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/z/g/zg4trx14yq823m5wamqwvdrz61sh6vkhc4y3fps0m02w6 @@ -0,0 +1 @@ +Β…Β…Β… \ No newline at end of file diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/z/x/zx6bt1mp5mztw0q4nzhjezfq249br5779q1v51s6hk85e b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/z/x/zx6bt1mp5mztw0q4nzhjezfq249br5779q1v51s6hk85e new file mode 100644 index 00000000..aba2382c --- /dev/null +++ b/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/z/x/zx6bt1mp5mztw0q4nzhjezfq249br5779q1v51s6hk85e @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/implementation/src/lib.rs b/implementation/src/lib.rs index 387588f3..ec5f273c 100644 --- a/implementation/src/lib.rs +++ b/implementation/src/lib.rs @@ -7,13 +7,35 @@ pub mod web; pub use scripting::filters; +/// code point of the trailing `\uXXXX` escape, if the string ends in one +fn trailing_unicode_escape(s: &str) -> Option<u32> { + let start = s.len().checked_sub(6)?; + let b = s.as_bytes(); + + if b[start] != b'\\' || b[start + 1] != b'u' { + return None; + } + + let hex = &s[start + 2..]; + if !hex.bytes().all(|c| c.is_ascii_hexdigit()) { + return None; + } + + let slashes = s[..start].bytes().rev().take_while(|&c| c == b'\\').count(); + if slashes % 2 == 1 { + return None; + } + + u32::from_str_radix(hex, 16).ok() +} + pub fn complete_json(partial: &str) -> String { let mut result = partial.trim_end().to_string(); - let mut stack = Vec::new(); + // each frame is (closing char, whether the current object member already has a colon) + let mut stack: Vec<(char, bool)> = Vec::new(); let mut in_string = false; let mut escape_next = false; - let mut has_colon = false; for ch in partial.chars() { if escape_next { @@ -24,8 +46,8 @@ pub fn complete_json(partial: &str) -> String { match ch { '\\' if in_string => escape_next = true, '"' => in_string = !in_string, - '{' if !in_string => stack.push('}'), - '[' if !in_string => stack.push(']'), + '{' if !in_string => stack.push(('}', false)), + '[' if !in_string => stack.push((']', false)), '}' if !in_string => { stack.pop(); } @@ -33,10 +55,14 @@ pub fn complete_json(partial: &str) -> String { stack.pop(); } ':' if !in_string => { - has_colon = true; + if let Some(frame) = stack.last_mut() { + frame.1 = true; + } } - ',' if !in_string && stack.last() == Some(&'}') => { - has_colon = false; + ',' if !in_string => { + if let Some(frame @ ('}', _)) = stack.last_mut() { + frame.1 = false; + } } _ => {} } @@ -79,6 +105,19 @@ pub fn complete_json(partial: &str) -> String { } } + // a surrogate pair cut in half leaves a lone surrogate, which JSON rejects + while let Some(cp) = trailing_unicode_escape(&result) { + let paired = (0xdc00..0xe000).contains(&cp) + && trailing_unicode_escape(&result[..result.len() - 6]) + .is_some_and(|hi| (0xd800..0xdc00).contains(&hi)); + + if paired || !(0xd800..0xe000).contains(&cp) { + break; + } + + result.truncate(result.len() - 6); + } + result.push('"'); } @@ -89,15 +128,16 @@ pub fn complete_json(partial: &str) -> String { // nothing } else if result.ends_with(':') { result.push_str("null"); - } else if result.ends_with('-') || result.ends_with('.') || result.ends_with('+') { - result.push('0'); - } else if (result.ends_with('e') || result.ends_with('E')) - && result.chars().rev().nth(1).unwrap_or('a').is_ascii_digit() + } else if result.ends_with('-') + || result.ends_with('.') + || result.ends_with('+') + || ((result.ends_with('e') || result.ends_with('E')) + && result.chars().rev().nth(1).unwrap_or('a').is_ascii_digit()) { result.push('0'); } else if result.ends_with(',') { result.pop(); - } else if stack.last() == Some(&'}') && !has_colon && !result.ends_with('{') { + } else if stack.last() == Some(&('}', false)) && !result.ends_with('{') { result.push_str(":null"); } else { const CONSTS: &[&str] = &["true", "false", "null"]; @@ -115,7 +155,7 @@ pub fn complete_json(partial: &str) -> String { } } - while let Some(closing) = stack.pop() { + while let Some((closing, _)) = stack.pop() { result.push(closing); } diff --git a/implementation/src/llm/handler.rs b/implementation/src/llm/handler.rs index cb346e39..30b01c4a 100644 --- a/implementation/src/llm/handler.rs +++ b/implementation/src/llm/handler.rs @@ -190,17 +190,15 @@ impl Inner { consumed_gen: primitive_types::U256::MAX, }); } - if let Some(mlua_err) = err.downcast_ref::<mlua::Error>() { - if let mlua::Error::ExternalError(ext) = mlua_err { - if ext - .downcast_ref::<crate::common::BudgetExhausted>() - .is_some() - { - return Ok(llm_iface::PromptAnswer { - data: llm_iface::PromptAnswerData::Text(String::new()), - consumed_gen: primitive_types::U256::MAX, - }); - } + if let Some(mlua::Error::ExternalError(ext)) = err.downcast_ref::<mlua::Error>() { + if ext + .downcast_ref::<crate::common::BudgetExhausted>() + .is_some() + { + return Ok(llm_iface::PromptAnswer { + data: llm_iface::PromptAnswerData::Text(String::new()), + consumed_gen: primitive_types::U256::MAX, + }); } } Err(err) diff --git a/implementation/src/manager/handlers.rs b/implementation/src/manager/handlers.rs index 79721ab0..902f53db 100644 --- a/implementation/src/manager/handlers.rs +++ b/implementation/src/manager/handlers.rs @@ -21,6 +21,8 @@ pub async fn handle_status(ctx: sync::DArc<AppContext>) -> Result<Json<serde_jso "permits": { "current": ctx.run_ctx.get_current_permits(), "max": ctx.run_ctx.get_max_permits().await, + "per_sync_run": ctx.run_ctx.permits_sync(), + "per_nondet_run": ctx.run_ctx.permits_nondet(), }, "executions": ctx.run_ctx.status_executions(), }))) @@ -347,7 +349,9 @@ fn describe_vm_error(error: &str) -> Option<&'static str> { Some( r##"The contract was detected to be a plain text contract, however it contains non-UTF8 bytes and hence cannot be parsed. Is deployed runner a valid contract?"##, ) - } else if is_sub_error(error, "invalid_contract absent_runner_comment") { + } else if is_sub_error(error, "invalid_contract runner absent") + || is_sub_error(error, "invalid_contract absent_runner_comment") + { Some( r##"The contract was detected to be a plain text contract, however it does not start with a runner comment (such as `# { "Depends": "py-genlayer:..." }`), hence it is impossible to run. Have you forgotten to add it or is there other content before it?"##, ) diff --git a/implementation/src/manager/mod.rs b/implementation/src/manager/mod.rs index c8f262e5..bc0a2dec 100644 --- a/implementation/src/manager/mod.rs +++ b/implementation/src/manager/mod.rs @@ -29,13 +29,124 @@ pub struct Config { pub base: genvm_common::BaseConfig, pub manifest_path: String, - pub permits: Option<usize>, + /// Absent or null keeps every default. + pub permits: Option<PermitsConfig>, #[serde(default = "default_execution_retention")] pub execution_retention: run::ManagerDuration, #[serde(default = "default_max_message_bytes")] pub max_message_bytes: usize, } +/// Concurrency budget, denominated in gigabytes of RAM. +#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] +pub struct PermitsConfig { + /// Total pool; null derives it from free memory. + #[serde(default)] + pub total: Option<usize>, + #[serde(default = "default_permits_per_gib")] + pub per_gib: PermitsPerGig, + #[serde(default = "default_permits_sync")] + pub sync: usize, + #[serde(default = "default_permits_nondet")] + pub nondet: usize, +} + +impl Default for PermitsConfig { + fn default() -> Self { + Self { + total: None, + per_gib: default_permits_per_gib(), + sync: default_permits_sync(), + nondet: default_permits_nondet(), + } + } +} + +/// Permits handed out per gibibyte of free memory. +/// +/// A ratio rather than a float: `1/4` is exactly one permit per four gibibytes, +/// and a config saying so must not depend on how a decimal rounds. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct PermitsPerGig(num_rational::Ratio<u64>); + +impl PermitsPerGig { + /// Floors, so a pool is never wider than the memory backing it. + pub fn apply(self, gigabytes: usize) -> usize { + let scaled = gigabytes as u128 * *self.0.numer() as u128 / *self.0.denom() as u128; + scaled.min(usize::MAX as u128) as usize + } +} + +impl std::str::FromStr for PermitsPerGig { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result<Self> { + let (numer, denom) = match s.split_once('/') { + Some((numer, denom)) => (numer.trim(), denom.trim()), + None => (s.trim(), "1"), + }; + let numer: u64 = numer + .parse() + .with_context(|| format!("invalid permits per_gib numerator in {s:?}"))?; + let denom: u64 = denom + .parse() + .with_context(|| format!("invalid permits per_gib denominator in {s:?}"))?; + anyhow::ensure!(numer > 0, "permits per_gib must be positive, got {s:?}"); + anyhow::ensure!(denom > 0, "permits per_gib must not divide by zero"); + + Ok(Self(num_rational::Ratio::new(numer, denom))) + } +} + +impl std::fmt::Display for PermitsPerGig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}/{}", self.0.numer(), self.0.denom()) + } +} + +impl serde::Serialize for PermitsPerGig { + fn serialize<S: serde::Serializer>( + &self, + serializer: S, + ) -> std::result::Result<S::Ok, S::Error> { + serializer.collect_str(self) + } +} + +impl<'de> serde::Deserialize<'de> for PermitsPerGig { + fn deserialize<D: serde::Deserializer<'de>>( + deserializer: D, + ) -> std::result::Result<Self, D::Error> { + /// `1` and `"1/4"` are both natural spellings in yaml + #[derive(serde::Deserialize)] + #[serde(untagged)] + enum Repr { + Integer(u64), + Str(String), + } + + use std::str::FromStr as _; + + match Repr::deserialize(deserializer)? { + Repr::Integer(n) => Self::from_str(&n.to_string()), + Repr::Str(s) => Self::from_str(&s), + } + .map_err(serde::de::Error::custom) + } +} + +fn default_permits_per_gib() -> PermitsPerGig { + PermitsPerGig(num_rational::Ratio::from_integer(1)) +} + +fn default_permits_sync() -> usize { + 4 +} + +fn default_permits_nondet() -> usize { + 8 +} + fn default_execution_retention() -> run::ManagerDuration { run::ManagerDuration::try_from("5m").expect("default execution_retention is valid") } @@ -129,6 +240,13 @@ fn internal_error(error: anyhow::Error) -> Response { error_response(StatusCode::INTERNAL_SERVER_ERROR, format!("{:#}", error)) } +/// A malformed body, query or header is the caller's fault: it answers 400 and +/// stays out of the error log, which is for this process' own failures. +fn bad_request(error: anyhow::Error) -> Response { + log_debug!(err:ah = &error; "bad request"); + error_response(StatusCode::BAD_REQUEST, format!("{:#}", error)) +} + fn unwrap_all_anyhow<R: IntoResponse>(result: anyhow::Result<R>) -> Response { match result { Ok(response) => response.into_response(), @@ -189,7 +307,7 @@ fn create_router(app_ctx: sync::DArc<AppContext>) -> Router { body: Bytes| async move { let body = match parse_json_body(body) { Ok(body) => body, - Err(error) => return internal_error(error), + Err(error) => return bad_request(error), }; unwrap_all_anyhow(handlers::handle_module_start(ctx, body).await) }, @@ -202,7 +320,7 @@ fn create_router(app_ctx: sync::DArc<AppContext>) -> Router { body: Bytes| async move { let body = match parse_json_body(body) { Ok(body) => body, - Err(error) => return internal_error(error), + Err(error) => return bad_request(error), }; unwrap_all_anyhow(handlers::handle_module_stop(ctx, body).await) }, @@ -215,7 +333,7 @@ fn create_router(app_ctx: sync::DArc<AppContext>) -> Router { body: Bytes| async move { let body = match parse_json_body(body) { Ok(body) => body, - Err(error) => return internal_error(error), + Err(error) => return bad_request(error), }; unwrap_all_anyhow(handlers::handle_module_restart(ctx, body).await) }, @@ -237,7 +355,7 @@ fn create_router(app_ctx: sync::DArc<AppContext>) -> Router { contract_code: Bytes| async move { let deployment_timestamp = match deployment_timestamp(&headers) { Ok(deployment_timestamp) => deployment_timestamp, - Err(error) => return internal_error(error), + Err(error) => return bad_request(error), }; unwrap_all_anyhow( handlers::handle_contract_detect_version( @@ -257,7 +375,7 @@ fn create_router(app_ctx: sync::DArc<AppContext>) -> Router { body: Bytes| async move { let body = match parse_json_body(body) { Ok(body) => body, - Err(error) => return internal_error(error), + Err(error) => return bad_request(error), }; unwrap_all_anyhow(handlers::handle_set_log_level(ctx, body).await) }, @@ -279,7 +397,7 @@ fn create_router(app_ctx: sync::DArc<AppContext>) -> Router { body: Bytes| async move { let body = match parse_json_body(body) { Ok(body) => body, - Err(error) => return internal_error(error), + Err(error) => return bad_request(error), }; unwrap_all_anyhow(handlers::handle_set_permits(ctx, body).await) }, @@ -296,7 +414,7 @@ fn create_router(app_ctx: sync::DArc<AppContext>) -> Router { }; let query = match parse_query(query) { Ok(query) => query, - Err(error) => return internal_error(error), + Err(error) => return bad_request(error), }; unwrap_all_anyhow(handlers::handle_genvm_shutdown(ctx, genvm_id, query).await) }, @@ -310,7 +428,7 @@ fn create_router(app_ctx: sync::DArc<AppContext>) -> Router { }; let query = match parse_query(query) { Ok(query) => query, - Err(error) => return internal_error(error), + Err(error) => return bad_request(error), }; unwrap_all_anyhow( handlers::handle_genvm_status(ctx, genvm_id, query).await, @@ -329,7 +447,7 @@ fn create_router(app_ctx: sync::DArc<AppContext>) -> Router { }; let query = match parse_query(query) { Ok(query) => query, - Err(error) => return internal_error(error), + Err(error) => return bad_request(error), }; unwrap_all_anyhow(handlers::handle_genvm_artifact(ctx, genvm_id, query).await) }, @@ -342,7 +460,7 @@ fn create_router(app_ctx: sync::DArc<AppContext>) -> Router { body: Bytes| async move { let body = match parse_json_body(body) { Ok(body) => body, - Err(error) => return internal_error(error), + Err(error) => return bad_request(error), }; unwrap_all_anyhow(handlers::handle_llm_check(ctx, body).await) }, @@ -355,7 +473,7 @@ fn create_router(app_ctx: sync::DArc<AppContext>) -> Router { query: std::result::Result<Query<handlers::DescribeVmErrorRequest>, QueryRejection>| async move { let query = match parse_query(query) { Ok(query) => query, - Err(error) => return internal_error(error), + Err(error) => return bad_request(error), }; unwrap_all_anyhow(handlers::handle_describe_vm_error(ctx, query).await) }, diff --git a/implementation/src/manager/run.rs b/implementation/src/manager/run.rs index c67ed736..beea74b0 100644 --- a/implementation/src/manager/run.rs +++ b/implementation/src/manager/run.rs @@ -23,6 +23,25 @@ use crate::common::{LogSink, LogSinkElement, LogSinkInner, LoggerWithId, GENVM_B const ARTIFACT_CHUNK_CAP: usize = 256 * 1024; +/// How many delegated (cross-major) hops one chain of contract calls may make. +/// +/// Deliberately far below any line's `VM_RECURSION`: a hop costs a whole +/// executor process, and the manager -- not any single line -- is the only +/// party that sees the chain as a whole. +const CROSS_MAJOR_RECURSION: u32 = 6; + +/// The recursion budget a delegated callee is given: whatever is left of the +/// caller's, but never more than [`CROSS_MAJOR_RECURSION`]. +/// +/// Clamping rather than counting keeps this stateless -- the manager reads one +/// number off the envelope and writes one back -- and it can only ever tighten +/// the bound the caller already had. The subtree below a delegated call is +/// bounded by the same number, which is the point: a chain that has already +/// crossed a boundary does not get to recurse deeply in-process either. +fn cross_major_recursion(remaining: u32) -> u32 { + remaining.min(CROSS_MAJOR_RECURSION) +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct ManagerDuration(std::time::Duration); @@ -437,6 +456,9 @@ pub struct Ctx { next_genvm_id: std::sync::atomic::AtomicU64, pub permits: Arc<tokio::sync::Semaphore>, max_permits: tokio::sync::Mutex<PermitsData>, + /// Permits a single run costs, by kind. One permit is one gigabyte of RAM. + permits_sync: usize, + permits_nondet: usize, executors_path: std::path::PathBuf, } @@ -485,6 +507,28 @@ impl Ctx { self.permits.available_permits() } + pub fn permits_sync(&self) -> usize { + self.permits_sync + } + + pub fn permits_nondet(&self) -> usize { + self.permits_nondet + } + + /// Below this the most expensive run could never be admitted. + pub fn min_permits(&self) -> usize { + self.permits_sync.max(self.permits_nondet) + } + + fn permits_for(&self, req: &Request) -> u32 { + let count = if req.needs_modules() { + self.permits_nondet + } else { + self.permits_sync + }; + count as u32 + } + pub async fn set_permits(&self, permits: usize) -> usize { let mut permits_lock = self.max_permits.lock().await; @@ -494,8 +538,9 @@ impl Ctx { // actually this causes drop of previous one, so we can enter more genvms than we have permits, but it's ok for now // especially since this method is expected to be called before starting any genvms at all - if permits < 2 { - log_warn!(permits = permits; "cannot set permits below 2"); + let min = self.min_permits(); + if permits < min { + log_warn!(permits = permits, min = min; "cannot set permits below the most expensive run"); return permits_lock.max; } @@ -662,14 +707,14 @@ impl Ctx { } #[cfg(target_os = "macos")] - fn get_default_permits() -> usize { + fn detect_free_gigabytes() -> usize { log_warn!("automatic permits detection is not supported on macOS, using default value"); - 8 + 32 } #[cfg(not(target_os = "macos"))] - fn get_default_permits() -> usize { + fn detect_free_gigabytes() -> usize { let mut sys = sysinfo::System::new_all(); sys.refresh_memory(); sys.refresh_all(); @@ -685,19 +730,41 @@ impl Ctx { "memory status" ); - let total_mem_kb = free_memory / 1024 + free_swap / 1024; - let total_mem_gb = total_mem_kb / 1024 / 1024; - - (total_mem_gb / 4).max(2) as usize + ((free_memory + free_swap) / (1024 * 1024 * 1024)) as usize } pub fn new(config: &crate::manager::Config) -> anyhow::Result<Self> { - let permits = if let Some(p) = config.permits { - p + let config_permits = config.permits.clone().unwrap_or_default(); + let permits_sync = config_permits.sync; + let permits_nondet = config_permits.nondet; + // the upper bound keeps the cast in `permits_for` honest + anyhow::ensure!( + (1..=u32::MAX as usize).contains(&permits_sync) + && (1..=u32::MAX as usize).contains(&permits_nondet), + "per-run permits must be positive and fit in u32" + ); + let min_permits = permits_sync.max(permits_nondet); + + let permits = match config_permits.total { + Some(p) => p, + None => config_permits.per_gib.apply(Self::detect_free_gigabytes()), + }; + let permits = if permits < min_permits { + log_warn!( + permits = permits, min = min_permits; + "permits are below the most expensive run, raising" + ); + min_permits } else { - Self::get_default_permits() + permits }; - log_info!(permits = permits; "estimated concurrent GenVM permits"); + log_info!( + permits = permits, + permits_sync = permits_sync, + permits_nondet = permits_nondet, + per_gib = config_permits.per_gib; + "estimated concurrent GenVM permits" + ); let mut exe_path = std::env::current_exe()?; exe_path.pop(); @@ -716,6 +783,8 @@ impl Ctx { num_throttled: 0, throttled: None, }), + permits_sync, + permits_nondet, executors_path: exe_path, }) @@ -1329,6 +1398,7 @@ fn nested_reply_from_consumed_result( 1 => genvm_modules_interfaces::ResultCode::UserError, 2 => genvm_modules_interfaces::ResultCode::VmError, 3 => genvm_modules_interfaces::ResultCode::InternalError, + 4 => genvm_modules_interfaces::ResultCode::FatalVmError, value => anyhow::bail!("nested executor returned unknown result code {value}"), }; let reported: genvm_modules_interfaces::ReportedResult = calldata::decode_obj(encoded)?; @@ -2045,7 +2115,8 @@ impl Ctx { execution_data.leader_nondet_results = None; execution_data.record_actions.clear(); execution_data.message_fee_allocation.clear(); - execution_data.remaining_recursion = Some(envelope.remaining_recursion); + execution_data.remaining_recursion = + Some(cross_major_recursion(envelope.remaining_recursion)); execution_data.nested = Some(genvm_modules_interfaces::NestedExecutionData { memory_limit: envelope.memory_limit, stack: envelope.stack, @@ -2084,7 +2155,7 @@ async fn supervise_genvm( modules_lock: Box<dyn std::any::Any + Send + Sync>, ) { let ctx = full_ctx.gep(|x| &x.run_ctx); - let permit_count = if req.needs_modules() { 2 } else { 1 }; + let permit_count = ctx.permits_for(&req); let permit_future = acquire_run_permits(ctx.permits.clone(), permit_count); tokio::pin!(permit_future); diff --git a/implementation/src/manager/run_test.rs b/implementation/src/manager/run_test.rs index 9744ad02..1c8a2be8 100644 --- a/implementation/src/manager/run_test.rs +++ b/implementation/src/manager/run_test.rs @@ -13,6 +13,8 @@ fn test_ctx(retention: &str, permits: usize) -> Ctx { num_throttled: 0, throttled: None, }), + permits_sync: 1, + permits_nondet: 2, executors_path: std::path::PathBuf::new(), } } @@ -443,6 +445,18 @@ async fn cancel_while_queued_consumes_no_permit() { assert_eq!(permits.available_permits(), 1); } +#[tokio::test] +async fn permits_cannot_drop_below_the_most_expensive_run() { + let ctx = test_ctx("5m", 8); + assert_eq!(ctx.min_permits(), 2); + + assert_eq!(ctx.set_permits(1).await, 8); + assert_eq!(ctx.get_current_permits(), 8); + + assert_eq!(ctx.set_permits(2).await, 2); + assert_eq!(ctx.get_current_permits(), 2); +} + #[tokio::test] async fn nested_run_does_not_consume_parent_permit() { let ctx = test_ctx("5m", 1); @@ -830,3 +844,36 @@ fn message_schema_matches_the_rust_struct() { assert_eq!(documented_properties("MessageData"), accepted); } + +#[test] +fn a_delegated_chain_is_bounded_more_tightly_than_plain_recursion() { + // The first hop clamps to the cross-major cap, and every hop after it + // spends one, so a chain of delegated calls dies well before the VM + // recursion budget it was minted from would. + let mut remaining = 1_000; + let mut hops = 0; + loop { + remaining = cross_major_recursion(remaining); + if remaining == 0 { + break; + } + hops += 1; + // What an executor spends on the sub-VM it spawns for the next hop. + remaining -= 1; + } + + assert_eq!(hops, CROSS_MAJOR_RECURSION as usize); +} + +#[test] +fn a_delegated_chain_never_widens_the_budget_it_was_given() { + for remaining in [ + 0, + 1, + CROSS_MAJOR_RECURSION - 1, + CROSS_MAJOR_RECURSION, + 1_000, + ] { + assert!(cross_major_recursion(remaining) <= remaining, "{remaining}"); + } +} diff --git a/implementation/src/manager/socket.rs b/implementation/src/manager/socket.rs index e0dd4dab..67b66612 100644 --- a/implementation/src/manager/socket.rs +++ b/implementation/src/manager/socket.rs @@ -48,16 +48,8 @@ struct Writer { } impl Writer { - /// Non-blocking, for the paths that run inside the read loop's error - /// handling and must not park. Everything that can await uses the async - /// twin instead, so a stalled peer applies backpressure rather than - /// costing frames. - fn send_control(&self, frame: Frame) -> Result<()> { - self.control - .try_send(frame) - .context("connection writer queue is full") - } - + /// Parks when the queue is full, so a slow reader is throttled rather than + /// disconnected. async fn send_control_async(&self, frame: Frame) -> Result<()> { self.control .send(frame) @@ -72,8 +64,9 @@ impl Writer { .context("connection writer queue is closed") } - fn error(&self, request_id: u64, code: Errors, message: impl Into<String>) -> Result<()> { - self.send_control(error_frame(request_id, code, message)) + async fn error(&self, request_id: u64, code: Errors, message: impl Into<String>) -> Result<()> { + self.send_control_async(error_frame(request_id, code, message)) + .await } } @@ -390,7 +383,9 @@ where let body = match message.context("reading websocket message")? { Message::Binary(body) => body, Message::Text(_) => { - writer.error(0, Errors::MalformedFrame, "text messages are not valid")?; + writer + .error(0, Errors::MalformedFrame, "text messages are not valid") + .await?; continue; } Message::Close(_) => return Ok(None), @@ -399,7 +394,9 @@ where }; if body.len() < HEADER_LEN { - writer.error(0, Errors::MalformedFrame, "message is shorter than header")?; + writer + .error(0, Errors::MalformedFrame, "message is shorter than header") + .await?; continue; } @@ -409,11 +406,13 @@ where let method = match Methods::try_from(method_id) { Ok(method) => method, Err(()) => { - writer.error( - request_id, - Errors::UnknownMethod, - format!("unknown method id {method_id}"), - )?; + writer + .error( + request_id, + Errors::UnknownMethod, + format!("unknown method id {method_id}"), + ) + .await?; continue; } }; @@ -453,11 +452,13 @@ where async fn dispatch(&mut self, method: Methods, request_id: u64, payload: &[u8]) -> Result<()> { if request_id == 0 { - self.writer.error( - 0, - Errors::BadRequestId, - "client requests must use request_id != 0", - )?; + self.writer + .error( + 0, + Errors::BadRequestId, + "client requests must use request_id != 0", + ) + .await?; return Ok(()); } @@ -468,11 +469,13 @@ where Methods::Ack => self.handle_ack(request_id, payload).await, Methods::GetArtifact => self.handle_get_artifact(request_id, payload).await, Methods::Error | Methods::Hello | Methods::Event => { - self.writer.error( - request_id, - Errors::UnknownMethod, - format!("method {} is not a client request", method.value()), - )?; + self.writer + .error( + request_id, + Errors::UnknownMethod, + format!("method {} is not a client request", method.value()), + ) + .await?; Ok(()) } } @@ -483,7 +486,8 @@ where Ok(req) => req, Err(e) => { self.writer - .error(request_id, Errors::MalformedFrame, e.to_string())?; + .error(request_id, Errors::MalformedFrame, e.to_string()) + .await?; return Ok(()); } }; @@ -493,11 +497,13 @@ where .get(1) .is_some_and(|data| !data.is_empty()) { - self.writer.error( - request_id, - Errors::MalformedFrame, - "host_hello_data for manager-owned host index 1 is not allowed", - )?; + self.writer + .error( + request_id, + Errors::MalformedFrame, + "host_hello_data for manager-owned host index 1 is not allowed", + ) + .await?; return Ok(()); } @@ -505,11 +511,13 @@ where match super::modules::Ctx::get_module_locks(self.ctx.gep(|x| &x.mod_ctx)).await { Some(lock) => Some(lock), None => { - self.writer.error( - request_id, - Errors::Internal, - "modules are required but not running", - )?; + self.writer + .error( + request_id, + Errors::Internal, + "modules are required but not running", + ) + .await?; return Ok(()); } } @@ -524,7 +532,8 @@ where Ok(id) => id, Err(e) => { self.writer - .error(request_id, Errors::Internal, format!("{:#}", e))?; + .error(request_id, Errors::Internal, format!("{:#}", e)) + .await?; return Ok(()); } }; @@ -532,24 +541,28 @@ where Ok(result) => result, Err(e) => { self.writer - .error(request_id, Errors::UnknownId, format!("{:#}", e))?; + .error(request_id, Errors::UnknownId, format!("{:#}", e)) + .await?; return Ok(()); } }; - self.writer.send_control(Frame::new( - Methods::Run, - request_id, - &RunResponse { genvm_id }, - ))?; + self.writer + .send_control_async(Frame::new( + Methods::Run, + request_id, + &RunResponse { genvm_id }, + )) + .await?; if let run::Snapshot::Event(event) = snapshot { - self.writer.send_control(event_frame(event))?; + self.writer.send_control_async(event_frame(event)).await?; } // Subscribing last keeps the frames ordered: the receiver was taken // above, so nothing is missed, but a terminal landing meanwhile cannot // produce an event for a genvm_id the client has not been told yet. if let Err(e) = self.subscribe(genvm_id, Ok(rx)) { self.writer - .error(request_id, Errors::UnknownId, format!("{:#}", e))?; + .error(request_id, Errors::UnknownId, format!("{:#}", e)) + .await?; return Ok(()); } Ok(()) @@ -560,7 +573,8 @@ where Ok(req) => req, Err(e) => { self.writer - .error(request_id, Errors::MalformedFrame, e.to_string())?; + .error(request_id, Errors::MalformedFrame, e.to_string()) + .await?; return Ok(()); } }; @@ -570,23 +584,30 @@ where Ok(result) => result, Err(e) if e.to_string() == "boot_id_mismatch" => { self.writer - .error(request_id, Errors::BootIdMismatch, "boot_id_mismatch")?; + .error(request_id, Errors::BootIdMismatch, "boot_id_mismatch") + .await?; return Ok(()); } Err(e) => { self.writer - .error(request_id, Errors::UnknownId, format!("{:#}", e))?; + .error(request_id, Errors::UnknownId, format!("{:#}", e)) + .await?; return Ok(()); } }; + self.writer + .send_control_async(Frame::new( + Methods::Attach, + request_id, + &AttachResponse { + snapshot: snapshot_payload(snapshot), + }, + )) + .await?; + // Subscribing last, as in `handle_run`: the receiver was taken above so + // nothing is missed, and the client cannot see an event that predates + // the snapshot it is answered with. self.subscribe(req.genvm_id, Ok(rx))?; - self.writer.send_control(Frame::new( - Methods::Attach, - request_id, - &AttachResponse { - snapshot: snapshot_payload(snapshot), - }, - ))?; Ok(()) } @@ -595,19 +616,22 @@ where Ok(req) => req, Err(e) => { self.writer - .error(request_id, Errors::MalformedFrame, e.to_string())?; + .error(request_id, Errors::MalformedFrame, e.to_string()) + .await?; return Ok(()); } }; match self.ctx.run_ctx.cancel(req.genvm_id).await { - Ok(()) => self.writer.send_control(Frame::new( - Methods::Cancel, - request_id, - &EmptyResponse {}, - ))?, - Err(e) => self - .writer - .error(request_id, Errors::UnknownId, format!("{:#}", e))?, + Ok(()) => { + self.writer + .send_control_async(Frame::new(Methods::Cancel, request_id, &EmptyResponse {})) + .await? + } + Err(e) => { + self.writer + .error(request_id, Errors::UnknownId, format!("{:#}", e)) + .await? + } } Ok(()) } @@ -617,7 +641,8 @@ where Ok(req) => req, Err(e) => { self.writer - .error(request_id, Errors::MalformedFrame, e.to_string())?; + .error(request_id, Errors::MalformedFrame, e.to_string()) + .await?; return Ok(()); } }; @@ -626,19 +651,19 @@ where if let Some(handle) = self.subscriptions.remove(&req.genvm_id) { handle.abort(); } - self.writer.send_control(Frame::new( - Methods::Ack, - request_id, - &EmptyResponse {}, - ))?; + self.writer + .send_control_async(Frame::new(Methods::Ack, request_id, &EmptyResponse {})) + .await?; } run::AckOutcome::NotFinished => { self.writer - .error(request_id, Errors::NotFinished, "run has not finished")? + .error(request_id, Errors::NotFinished, "run has not finished") + .await? } run::AckOutcome::Unknown => { self.writer - .error(request_id, Errors::UnknownId, "unknown_id")? + .error(request_id, Errors::UnknownId, "unknown_id") + .await? } } Ok(()) @@ -650,7 +675,8 @@ where Ok(req) => req, Err(e) => { self.writer - .error(request_id, Errors::MalformedFrame, e.to_string())?; + .error(request_id, Errors::MalformedFrame, e.to_string()) + .await?; return Ok(()); } }; @@ -674,11 +700,14 @@ where } Err(e) if e.to_string() == "unknown_id" => { self.writer - .error(request_id, Errors::UnknownId, "unknown_id")? + .error(request_id, Errors::UnknownId, "unknown_id") + .await? + } + Err(e) => { + self.writer + .error(request_id, Errors::MalformedFrame, format!("{:#}", e)) + .await? } - Err(e) => self - .writer - .error(request_id, Errors::MalformedFrame, format!("{:#}", e))?, } Ok(()) } @@ -733,9 +762,10 @@ pub async fn handle_connection(socket: WebSocket, ctx: sync::DArc<AppContext>) { artifact: artifact_tx, }; - let writer_task = tokio::spawn(writer_loop(writer_stream, control_rx, artifact_rx)); + let mut writer_task = tokio::spawn(writer_loop(writer_stream, control_rx, artifact_rx)); if writer - .send_control(hello_frame(ctx.run_ctx.boot_id())) + .send_control_async(hello_frame(ctx.run_ctx.boot_id())) + .await .is_err() { writer_task.abort(); @@ -762,7 +792,18 @@ pub async fn handle_connection(socket: WebSocket, ctx: sync::DArc<AppContext>) { _ = cancel.chan.closed() => {} } - match writer_task.await { + // The same peer defeats the select above if it stops reading: `sink.send` + // parks once the socket buffers fill, so the writer never drains its queue + // and never returns. + let writer_result = tokio::select! { + result = &mut writer_task => result, + _ = cancel.chan.closed() => { + writer_task.abort(); + return; + } + }; + + match writer_result { Ok(Ok(())) => {} Ok(Err(e)) => log_debug!(error:ah = e; "manager websocket writer stopped"), Err(e) if e.is_cancelled() => {} diff --git a/implementation/src/manager/socket_test.rs b/implementation/src/manager/socket_test.rs index 452f16c8..97ebf2a3 100644 --- a/implementation/src/manager/socket_test.rs +++ b/implementation/src/manager/socket_test.rs @@ -137,7 +137,8 @@ async fn full_control_queue_does_not_kill_the_subscription() { // until the peer reads. A stalled client's connection looks exactly like // this from the manager's side. writer - .send_control(test_frame(Methods::Event, 99, "occupier")) + .control + .try_send(test_frame(Methods::Event, 99, "occupier")) .unwrap(); let forwarder = tokio::spawn(forward_events(writer.clone(), events_rx)); diff --git a/implementation/src/scripting/ctx/dflt.rs b/implementation/src/scripting/ctx/dflt.rs index d3942f4e..bd0a90cd 100644 --- a/implementation/src/scripting/ctx/dflt.rs +++ b/implementation/src/scripting/ctx/dflt.rs @@ -17,8 +17,7 @@ use super::CtxPart; static START: std::sync::LazyLock<std::time::Instant> = std::sync::LazyLock::new(std::time::Instant::now); -/// Process-relative monotonic milliseconds: comparable only within one process, -/// not a wall clock. Exists because the Lua sandbox loads no os library. +/// Process-relative monotonic milliseconds fn monotonic_ms() -> mlua::Integer { let elapsed = START.elapsed().as_millis(); elapsed.min(mlua::Integer::MAX as u128) as mlua::Integer diff --git a/implementation/tests/dflt_requests.rs b/implementation/tests/dflt_requests.rs index 721ad908..eda571f0 100644 --- a/implementation/tests/dflt_requests.rs +++ b/implementation/tests/dflt_requests.rs @@ -1,3 +1,5 @@ +// genvm-tool-test-tags: feature-web-request-body, feature-web-request-status, needs-web + use std::collections::BTreeMap; use std::sync::Arc; diff --git a/implementation/tests/manager_permits.rs b/implementation/tests/manager_permits.rs new file mode 100644 index 00000000..c8dfebe0 --- /dev/null +++ b/implementation/tests/manager_permits.rs @@ -0,0 +1,66 @@ +use genvm_modules::manager::{PermitsConfig, PermitsPerGig}; + +fn per_gib(yaml: &str) -> PermitsPerGig { + parse(&format!("per_gib: {yaml}")).per_gib +} + +fn parse(yaml: &str) -> PermitsConfig { + serde_yaml::from_str(yaml).unwrap() +} + +fn parse_err(yaml: &str) -> String { + serde_yaml::from_str::<PermitsConfig>(&format!("per_gib: {yaml}")) + .unwrap_err() + .to_string() +} + +#[test] +fn a_ratio_is_exact() { + assert_eq!(per_gib("1/4").apply(16), 4); + assert_eq!(per_gib("1/4").apply(15), 3); + assert_eq!(per_gib("2/3").apply(10), 6); +} + +#[test] +fn an_integer_is_a_whole_ratio() { + assert_eq!(per_gib("1").apply(7), 7); + assert_eq!(per_gib("3").apply(7), 21); + assert_eq!(per_gib("'3/1'").to_string(), "3/1"); +} + +#[test] +fn defaults_hand_out_one_permit_per_gibibyte() { + let config = parse("{}"); + assert_eq!(config.per_gib.apply(12), 12); + assert_eq!(config.total, None); + assert_eq!(config.sync, 4); + assert_eq!(config.nondet, 8); +} + +#[test] +fn a_ratio_is_normalized_when_printed() { + assert_eq!(per_gib("2/8").to_string(), "1/4"); +} + +#[test] +fn scaling_saturates_instead_of_wrapping() { + let huge = format!("{}", u64::MAX); + assert_eq!(per_gib(&huge).apply(usize::MAX), usize::MAX); +} + +#[test] +fn a_zero_or_malformed_ratio_is_refused() { + for (input, expected) in [ + ("0", "must be positive"), + ("1/0", "divide by zero"), + ("'1/'", "denominator"), + ("'-1'", "numerator"), + ("'1/2/3'", "denominator"), + ] { + let err = parse_err(input); + assert!( + err.contains(expected), + "{input}: expected {expected:?}, got {err}" + ); + } +} diff --git a/implementation/tests/merge.rs b/implementation/tests/merge.rs index 6aa9cec3..1ce611e2 100644 --- a/implementation/tests/merge.rs +++ b/implementation/tests/merge.rs @@ -1,3 +1,5 @@ +// genvm-tool-test-tags: feature-prompt + use genvm_modules::llm::merge::{merge_extra, merge_values, MergeStrategy}; use serde_json::json; use std::collections::BTreeMap; diff --git a/implementation/tests/overloaded.rs b/implementation/tests/overloaded.rs index 5bdeb774..fe431994 100644 --- a/implementation/tests/overloaded.rs +++ b/implementation/tests/overloaded.rs @@ -1,3 +1,5 @@ +// genvm-tool-test-tags: feature-prompt-text, needs-llm, needs-web + use genvm_common::logger; use genvm_common::*; use genvm_modules_interfaces::llm::{self as llm_iface}; diff --git a/implementation/tests/policy_dispatch.rs b/implementation/tests/policy_dispatch.rs index 28ad1cb9..c2d7f061 100644 --- a/implementation/tests/policy_dispatch.rs +++ b/implementation/tests/policy_dispatch.rs @@ -1,3 +1,5 @@ +// genvm-tool-test-tags: feature-prompt-text, feature-prompt-json + // End-to-end tests for the LLM dispatch script after backend selection and the // retry state machine were delegated to the `llm_policy` engine. Everything here // runs offline against fake local TCP backends; no API keys, no network. diff --git a/implementation/tests/providers.rs b/implementation/tests/providers.rs index c41ad70a..680c8801 100644 --- a/implementation/tests/providers.rs +++ b/implementation/tests/providers.rs @@ -1,5 +1,7 @@ #![allow(non_upper_case_globals, dead_code)] +// genvm-tool-test-tags: feature-prompt-text, feature-prompt-json, needs-llm, needs-web + use std::collections::BTreeMap; use std::collections::HashMap; diff --git a/implementation/tests/request_body_limit.rs b/implementation/tests/request_body_limit.rs index 6cfa568f..e3fb32e8 100644 --- a/implementation/tests/request_body_limit.rs +++ b/implementation/tests/request_body_limit.rs @@ -1,3 +1,5 @@ +// genvm-tool-test-tags: feature-web-request-body, needs-web + use std::collections::BTreeMap; use genvm_common::*; diff --git a/implementation/tests/request_localhost.rs b/implementation/tests/request_localhost.rs index 105bf453..bded29d6 100644 --- a/implementation/tests/request_localhost.rs +++ b/implementation/tests/request_localhost.rs @@ -1,3 +1,5 @@ +// genvm-tool-test-tags: feature-web-request, needs-web + use genvm_common::*; use genvm_modules::common; use genvm_modules::scripting::{self, send_request_get_lua_compatible_response_bytes, Metrics}; diff --git a/implementation/tests/signing_server.rs b/implementation/tests/signing_server.rs index bbe57d7d..e0dfc9a1 100644 --- a/implementation/tests/signing_server.rs +++ b/implementation/tests/signing_server.rs @@ -1,3 +1,5 @@ +// genvm-tool-test-tags: feature-web-request-body, feature-web-request-signed, needs-web + use std::collections::BTreeMap; use std::sync::Arc; diff --git a/implementation/tests/timeout.rs b/implementation/tests/timeout.rs index 45bd15a1..3953f561 100644 --- a/implementation/tests/timeout.rs +++ b/implementation/tests/timeout.rs @@ -1,3 +1,5 @@ +// genvm-tool-test-tags: feature-prompt-text, needs-time + use genvm_common::logger; use genvm_common::*; use std::collections::BTreeMap; diff --git a/implementation/tests/web_render_webdriver.rs b/implementation/tests/web_render_webdriver.rs index ae80f1a8..24b5e23e 100644 --- a/implementation/tests/web_render_webdriver.rs +++ b/implementation/tests/web_render_webdriver.rs @@ -4,6 +4,8 @@ //! would otherwise drop. That connection must therefore go through the unfiltered //! client -- while contract-driven `Request` traffic stays filtered. +//! genvm-tool-test-tags: feature-web-render, feature-web-request, needs-web + use std::collections::BTreeMap; use std::sync::Arc; diff --git a/install/bin/genvm-post-install b/install/bin/genvm-post-install index 3c4de8e7..948894b1 100755 --- a/install/bin/genvm-post-install +++ b/install/bin/genvm-post-install @@ -47,7 +47,8 @@ genvm_root_dir = Path(__file__).parent.parent def requirements_for(base: Path) -> Path: - """The requirements to install into the venv. + """ + The requirements to install into the venv. lief is a heavy binary wheel whose only job is rewriting the ELF interpreter, so `--use-patchelf` drops it β€” patchelf does that instead. The venv is keyed diff --git a/install/config/genvm-manager.yaml b/install/config/genvm-manager.yaml index a33cdd78..930a47c3 100644 --- a/install/config/genvm-manager.yaml +++ b/install/config/genvm-manager.yaml @@ -7,6 +7,10 @@ log_disable: tracing*,polling*,tungstenite*,tokio_tungstenite* log_level: info manifest_path: ${exeDir}/../data/manifest.yaml -permits: null # if null will be auto-discovered; full VM takes 2, sync VM takes 1 +permits: # 1 permit is 1 GiB of RAM + total: null # if null, free memory times `per_gib` + per_gib: 1 # permits per gibibyte of free memory, an integer or a ratio (1/4 = one permit per 4 GiB) + sync: 4 # permits a sync run costs + nondet: 8 # permits a nondet run costs, also the minimum for `total` execution_retention: 5m # finished executions are retained until ack or this TTL max_message_bytes: 67108864 # 64 MiB manager websocket message cap diff --git a/install/lib/python/post-install/__main__.py b/install/lib/python/post-install/__main__.py index 3d492862..5caf5f32 100644 --- a/install/lib/python/post-install/__main__.py +++ b/install/lib/python/post-install/__main__.py @@ -324,7 +324,8 @@ def patch_executable(path: Path): def detect_executable_platform(path: Path) -> BinaryOS | None: - """Sniff a binary's header to identify its target OS: ELF -> linux, Mach-O + """ + Sniff a binary's header to identify its target OS: ELF -> linux, Mach-O (any variant/byte order) -> macos. Returns None for anything unrecognized (wrapper scripts, truncated/empty files, unknown formats).""" try: @@ -340,7 +341,8 @@ def detect_executable_platform(path: Path) -> BinaryOS | None: def check_executable_platform(path: Path): - """Refuse a wrong-OS binary before we trust it. A bad release once shipped a + """ + Refuse a wrong-OS binary before we trust it. A bad release once shipped a macOS (Mach-O) executor inside a linux tarball; the missing-only download guard accepted it and it surfaced as an opaque `Exec format error` at runtime. Here we error on a clear OS mismatch instead. Unrecognized formats @@ -435,7 +437,7 @@ def _download_template(descr: str, templates: list[str], vars: dict[str, str]) - def download_runners_from_json( - file: str | Path, runners_dir: Path, verify_hash: bool = True + file: str | Path, runners_dir: Path, extension: str, verify_hash: bool = True ): # `verify_hash` is disabled for the v0.2.x legacy line, whose registry hashes # use Nix base32 rather than the Crockford scheme `runner_check_bytes` @@ -454,7 +456,7 @@ def download_runners_from_json( for name, hashes in all_runners.items(): for hash in hashes: - cur_dst = runners_dir.joinpath(name, hash[:2], hash[2:] + '.tar') + cur_dst = runners_dir.joinpath(name, hash[:2], hash[2:] + '.' + extension) if cur_dst.exists(): if not verify_hash: @@ -476,6 +478,7 @@ def download_runners_from_json( 'hash': hash, 'hash_0_2': hash[:2], 'hash_2_': hash[2:], + 'ext': extension, }, ) if verify_hash and not runner_check_bytes(data, hash): @@ -486,7 +489,8 @@ def download_runners_from_json( def download_executor(executor_version: str): - """Fetch an executor line from its own release and unpack it at the install root. + """ + Fetch an executor line from its own release and unpack it at the install root. Since the repo split the manager release carries no executors: each line is released separately, under its own executor-version. The tarball is laid out @@ -578,7 +582,8 @@ def process_executor_version(executor_version: str): # root (executor/<version>/legacy-runners, see its genvm.yaml); every # other line shares the manager-root runners/ dir. v0.2.x also uses a # Nix-base32 registry hash this installer can't reproduce, so hash - # verification is left to that executor's `check` command. + # verification is left to that executor's `check` command, and it packages + # its runners as ustar rather than the zip every other line uses. is_legacy = (major, minor) == (0, 2) if is_legacy: runners_dir = executor_root_dir.joinpath('legacy-runners') @@ -587,6 +592,7 @@ def process_executor_version(executor_version: str): download_runners_from_json( executor_root_dir.joinpath('data', 'all.json'), runners_dir, + extension='tar' if is_legacy else 'zip', verify_hash=not is_legacy, ) diff --git a/libs/iconv.nix b/libs/iconv.nix index c0651d19..f1b5b7a5 100644 --- a/libs/iconv.nix +++ b/libs/iconv.nix @@ -8,7 +8,7 @@ let iconv-src = deps."libiconv-1.18"; in pkgs.stdenvNoCC.mkDerivation { - name = "libiconv"; + name = "genvm-libiconv"; src = iconv-src; diff --git a/libs/libc.nix b/libs/libc.nix index 6a0468c5..40953087 100644 --- a/libs/libc.nix +++ b/libs/libc.nix @@ -14,7 +14,7 @@ let aarch64-compiler-rt = deps."compiler-rt-aarch64-21.1.5-1"; in pkgs.stdenvNoCC.mkDerivation { - name = "libc-${name-target}"; + name = "genvm-libc-${name-target}"; src = musl-src; diff --git a/libs/liblua.nix b/libs/liblua.nix index 3af8ff98..1846b8e6 100644 --- a/libs/liblua.nix +++ b/libs/liblua.nix @@ -14,7 +14,7 @@ let installNameFlag = if isMacos then "-Wl,-install_name,@rpath/liblua.dylib" else ""; in pkgs.stdenvNoCC.mkDerivation { - name = "liblua-${name-target}"; + name = "genvm-liblua-${name-target}"; inherit version; diff --git a/libs/lsqlite3.nix b/libs/lsqlite3.nix index 5db3e106..43162e42 100644 --- a/libs/lsqlite3.nix +++ b/libs/lsqlite3.nix @@ -11,7 +11,7 @@ let isMacos = name-target == "arm64-macos"; in pkgs.stdenvNoCC.mkDerivation { - name = "lsqlite3-${name-target}"; + name = "genvm-lsqlite3-${name-target}"; src = lsqlite3-src; diff --git a/ruff.toml b/ruff.toml index c2ceb589..8fe74f41 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,6 +1,6 @@ indent-width = 2 -exclude = ["executors/**"] +exclude = ["executors/**", "libs/unhardcoded-engine"] [format] indent-style = "tab" @@ -13,4 +13,15 @@ skip-magic-trailing-comma = false [lint] ignore = ["E741"] -extend-select = ["I"] +[lint.per-file-ignores] +# contract assets: the v0.2 sdk is used through a star import +"tests/system/**/assets/**" = ["F403", "F405"] + +extend-select = [ + "E9", + "F", + "ASYNC", + "RUF006", + "D213", # docstring summary on next line + "I", # sort imports +] diff --git a/runners/default.nix b/runners/default.nix index df37cd8d..c5e9312f 100644 --- a/runners/default.nix +++ b/runners/default.nix @@ -25,7 +25,4 @@ in list forExecutor ; - - # We always build all of them: { uid -> derivation } over the full list. - universal = import ./views/universal.nix args list; } diff --git a/support/ci/pipelines/tests.py b/support/ci/pipelines/tests.py index 10654dcb..7219e213 100644 --- a/support/ci/pipelines/tests.py +++ b/support/ci/pipelines/tests.py @@ -209,7 +209,9 @@ def group(gh_step: str, name: str, stage: Stage, body: GithubStep) -> None: tool_step=Stage( data=ToolTest( tests_name_filter_for_or='^executors/', - tests_tags_filter_for_or='rust', + # integration cases tag their Rust contracts `rust` too, and those need + # a built manager -- which this cell deliberately does not build + tests_tags_filter_for_or='rust & !integration', ), ), ) @@ -297,7 +299,7 @@ def _cell( QUEUE_CELLS = [ - _cell('other', TEST_OTHER), + _cell('other', TEST_OTHER, fuzz_host=True), _cell('rust-executors', TEST_RUST_EXECUTORS, fuzz_host=True, disk_reclaim=True), _cell('rust-manager', TEST_RUST_MANAGER, fuzz_host=True, disk_reclaim=True), _cell( diff --git a/support/ci/tools/open_executor_prs.py b/support/ci/tools/open_executor_prs.py index 68b7fe7c..85da2307 100644 --- a/support/ci/tools/open_executor_prs.py +++ b/support/ci/tools/open_executor_prs.py @@ -173,7 +173,7 @@ def open_executor_prs(ctx: gh_common.Ctx) -> None: continue url = existing_pr(ctx, head, base) or open_pr(ctx, head, base) print(f'{line}: executor PR {url}') - lines.append(f'executor: {url}') + lines.append(f'executor: {url} ({line})') if lines: upsert_comment(ctx, lines) else: diff --git a/support/manifest-base.yaml b/support/manifest-base.yaml index 25b07e31..654bad6f 100644 --- a/support/manifest-base.yaml +++ b/support/manifest-base.yaml @@ -3,7 +3,7 @@ # is derived from the active executor submodules; everything here is # build-independent and lives under version control. runners_download_urls: - - "https://storage.googleapis.com/gh-af/genvm_runners/{name}/{hash}.tar" + - "https://storage.googleapis.com/gh-af/genvm_runners/{name}/{hash}.{ext}" # Each executor line is released on its own, in its own repo, tagged with its # executor-version β€” the manager bundle does not carry them. `{version}` is that # tag and `{platform}` is the nix-style token (amd64-linux, arm64-macos, ...). diff --git a/support/runner-script.py b/support/runner-script.py index 39bfc1b9..20628778 100755 --- a/support/runner-script.py +++ b/support/runner-script.py @@ -52,8 +52,7 @@ def run_verify_file(args): path = Path(args.file) expected_hash: str | None = args.expected_hash if expected_hash is None: - expected_hash = path.name - expected_hash = expected_hash.removesuffix('.tar') + expected_hash = path.stem if not all([x in HASH_VALID_CHARS for x in expected_hash]): print(f'invalid hash {expected_hash}', file=sys.stderr) @@ -93,12 +92,12 @@ def _load_registry(file: str) -> dict[str, list[str]]: return ret -def _object_gcs_path(name: str, hash: str) -> str: - return f'genvm_runners/{name}/{hash}.tar' +def _object_gcs_path(name: str, hash: str, ext: str) -> str: + return f'genvm_runners/{name}/{hash}.{ext}' -def _download_single(name: str, hash: str) -> bytes: - url = f'https://storage.googleapis.com/gh-af/{_object_gcs_path(name, hash)}' +def _download_single(name: str, hash: str, ext: str) -> bytes: + url = f'https://storage.googleapis.com/gh-af/{_object_gcs_path(name, hash, ext)}' with urllib.request.urlopen(url) as f: return f.read() @@ -141,7 +140,7 @@ def run_download(args): for name, hashes in registry.items(): for hash in hashes: try: - cur_dst = dst.joinpath(name, hash[:2], hash[2:] + '.tar') + cur_dst = dst.joinpath(name, hash[:2], hash[2:] + '.' + args.extension) if cur_dst.exists(): data = cur_dst.read_bytes() @@ -154,7 +153,7 @@ def run_download(args): else: print(f'info: not found {cur_dst}') - data = _download_single(name, hash) + data = _download_single(name, hash, args.extension) if not check_bytes(data, hash): raise ValueError('hash mismatch') @@ -175,11 +174,11 @@ def run_download(args): print(json.dumps({'downloaded': successful}, sort_keys=True)) -def _upload_single(name: str, hash: str, contents: bytes, *, token: str): +def _upload_single(name: str, hash: str, contents: bytes, *, ext: str, token: str): if not check_bytes(contents, hash): raise ValueError('hash mismatch') - object_name = urllib.parse.quote_plus(_object_gcs_path(name, hash)) + object_name = urllib.parse.quote_plus(_object_gcs_path(name, hash, ext)) upload_url = f'https://storage.googleapis.com/upload/storage/v1/b/gh-af/o?uploadType=media&name={object_name}' @@ -249,8 +248,10 @@ def run_upload(args): for hash in hashes: try: print(f'trying {name}:{hash} ...') - data = root.joinpath(name, hash[:2], hash[2:] + '.tar').read_bytes() - _upload_single(name, hash, data, token=token) + data = root.joinpath( + name, hash[:2], hash[2:] + '.' + args.extension + ).read_bytes() + _upload_single(name, hash, data, ext=args.extension, token=token) except Exception as e: print(f'warn: failed to upload {name}:{hash}, {e}', file=sys.stderr) @@ -273,7 +274,8 @@ def run_merge(args): def _expand_targets(targets: set[str]) -> set[str]: - """Add the per-platform alias of every per-line executor target. + """ + Add the per-platform alias of every per-line executor target. Every executor line builds against the same dependency set, so the registry keys them by platform (`executor-amd64-linux`), while the build targets name @@ -493,6 +495,11 @@ def run_mirror_to_gcs(args): verify_file_parser.set_defaults(func=run_verify_file) download_parser = subparsers.add_parser('download') + download_parser.add_argument( + '--extension', + default='zip', + help='runner archive extension; the frozen v0.2.x line uses `tar`', + ) download_parser.add_argument('--dest', default='.') download_parser.add_argument('--allow-partial', default=False, action='store_true') download_parser.add_argument('--nix-preload', default=False, action='store_true') @@ -500,6 +507,11 @@ def run_mirror_to_gcs(args): download_parser.set_defaults(func=run_download) upload_file_parser = subparsers.add_parser('upload') + upload_file_parser.add_argument( + '--extension', + default='zip', + help='runner archive extension; the frozen v0.2.x line uses `tar`', + ) upload_file_parser.add_argument('--root', default='.') upload_file_parser.add_argument('--registry', required=True, metavar='FILE') upload_file_parser.set_defaults(func=run_upload) diff --git a/support/scripts/check-commit-message.py b/support/scripts/check-commit-message.py index e82dc297..3d8ffa39 100755 --- a/support/scripts/check-commit-message.py +++ b/support/scripts/check-commit-message.py @@ -112,7 +112,8 @@ class CommitPart: def parse_commit(message: str) -> list[CommitPart]: - """Split a commit message into parts to validate independently. + """ + Split a commit message into parts to validate independently. A squash carries a bullet per collapsed commit, each its own subject and body; the umbrella takes what precedes the first bullet. @@ -211,7 +212,8 @@ def validate_part(part: CommitPart) -> tuple[list[str], list[str]]: def _check_control_chars(message: str, errors: list[str]) -> None: - """Reject control characters: a message is text, newlines and tabs. + """ + Reject control characters: a message is text, newlines and tabs. Anything else is invisible where the message is written and meaningful to something downstream that reads it. diff --git a/support/scripts/check-source-text.py b/support/scripts/check-source-text.py index 0046e8a5..86bf8414 100644 --- a/support/scripts/check-source-text.py +++ b/support/scripts/check-source-text.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 -"""Pre-commit guard: keep source text plain and citation-free. +""" +Pre-commit guard: keep source text plain and citation-free. Comments should read on their own terms and stay ASCII. Rejected anywhere in a scanned file: architecture-decision and issue-ticket references -- they diff --git a/support/scripts/ci-changes.py b/support/scripts/ci-changes.py index 402f588d..cb92f2ca 100755 --- a/support/scripts/ci-changes.py +++ b/support/scripts/ci-changes.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 -"""Report what each repo changed in this CI run, as JSON. +""" +Report what each repo changed in this CI run, as JSON. Emits one entry per repo: the manager keyed as ``.``, every executor submodule by its path, each holding ``base_commit``, ``branch_commit`` and ``has_changes``. @@ -60,7 +61,8 @@ def endpoints(base: str, head: str, path: str) -> tuple[str | None, str | None]: def fetch_pr_refs(base_ref: str, pr_number: str) -> tuple[str, str]: - """Materialize the base tip and the PR head, and return (merge_base, head). + """ + Materialize the base tip and the PR head, and return (merge_base, head). A `pull_request` checkout leaves HEAD on the merge commit, and the base branch is usually not fetched at all, so both sides are fetched by explicit @@ -97,7 +99,8 @@ def gh(*args: str) -> str: def base_ref_of(pr_number: str) -> str: - """The PR's base branch, read from the API. + """ + The PR's base branch, read from the API. `GITHUB_BASE_REF` is only set on `pull_request` events, but these checks also run on the panel's `workflow_dispatch`, where the PR number is known and the diff --git a/support/scripts/get-all-git.py b/support/scripts/get-all-git.py index c2293673..e90e272e 100755 --- a/support/scripts/get-all-git.py +++ b/support/scripts/get-all-git.py @@ -108,7 +108,8 @@ def _read_gitmodules_entries(root: Path) -> dict[str, dict[str, str]]: def _gitlink_hash(root: Path, path: str) -> str: - """The commit ``path`` is pinned to, read from the index like git does. + """ + The commit ``path`` is pinned to, read from the index like git does. Staged and committed agree except right after a gitlink bump, where the index is the one the next commit will carry. @@ -214,7 +215,8 @@ def has_commit(repo: Path, commit: str) -> bool: def open_cache(root: Path, url: str, env: dict[str, str]) -> Path: - """Create (if needed) and configure the bare cache repo for ``url``. + """ + Create (if needed) and configure the bare cache repo for ``url``. Every submodule sharing this URL is checked out of this one repository, so its objects are fetched once and its refs are what keeps them alive. @@ -265,7 +267,8 @@ def open_cache(root: Path, url: str, env: dict[str, str]) -> Path: def fetch_pins( root: Path, cache: Path, submodules: list[SubmoduleGitInfo], env: dict[str, str] ) -> None: - """Make sure every pinned commit is in ``cache``, and keep it reachable. + """ + Make sure every pinned commit is in ``cache``, and keep it reachable. An unreferenced commit is only kept alive by the worktree HEADs pointing at it, and those checkouts do not exist yet β€” hence a ref per pin. @@ -338,7 +341,8 @@ def has_staged_changes(path: Path) -> bool: def clear_staging(staging: Path, name: str) -> None: - """Remove ``staging``, provided it is a leftover of an interrupted conversion. + """ + Remove ``staging``, provided it is a leftover of an interrupted conversion. Which holds one `--no-checkout` worktree, so at most a `.git` file. Anything else under that name is somebody's data. @@ -356,7 +360,8 @@ def clear_staging(staging: Path, name: str) -> None: def adopt_as_worktree( root: Path, cache: Path, dest: Path, head: str, env: dict[str, str] ) -> None: - """Give the populated directory ``dest`` the `.git` of a worktree at ``head``. + """ + Give the populated directory ``dest`` the `.git` of a worktree at ``head``. `git worktree add` cannot be pointed at a directory that has files in it, so the worktree is created empty beside it and only its `.git` file moves. No @@ -406,7 +411,8 @@ def adopt_as_worktree( def convert_from_dangling( root: Path, cache: Path, submodule: SubmoduleGitInfo, env: dict[str, str] ) -> None: - """Re-attach a checkout whose `.git` names a gitdir that is gone. + """ + Re-attach a checkout whose `.git` names a gitdir that is gone. The files are all there is to go on, so the checkout is re-attached at the gitlink. Files that the old checkout had at some other commit stay as they @@ -422,7 +428,8 @@ def convert_to_worktree( gitdir: Path | None, env: dict[str, str], ) -> None: - """Re-point an existing checkout at ``cache``, keeping every file in place. + """ + Re-point an existing checkout at ``cache``, keeping every file in place. Deleting and re-adding the directory would take the ignored content with it β€” the materialized third-party trees above all β€” so only the `.git` file is diff --git a/support/scripts/md-local-links.py b/support/scripts/md-local-links.py index f1c75760..82ec3d95 100644 --- a/support/scripts/md-local-links.py +++ b/support/scripts/md-local-links.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 -"""Pre-commit guard: verify local file-path links in Markdown resolve. +""" +Pre-commit guard: verify local file-path links in Markdown resolve. Validates that **local** link targets (and reference definitions) point at existing files; ignores external links (http(s)/mailto/other schemes), @@ -55,7 +56,8 @@ def _rel(p: Path, root: Path) -> str: def broken_links(md: Path, root: Path) -> list[str]: - """Return human-readable messages for each broken local link in `md`. + """ + Return human-readable messages for each broken local link in `md`. Targets resolve relative to the markdown file's directory; a leading `/` is repo-root-relative (`root`). Paths in messages are shown relative to `root`. diff --git a/support/tools/genvm-tool/default.nix b/support/tools/genvm-tool/default.nix index c0cc1d19..cd479fe8 100644 --- a/support/tools/genvm-tool/default.nix +++ b/support/tools/genvm-tool/default.nix @@ -8,7 +8,14 @@ python.pkgs.buildPythonApplication { pname = "genvm-tool"; version = "0.1.0"; - src = ./.; + src = pkgs.lib.cleanSourceWith { + src = ./.; + filter = + path: type: + pkgs.lib.cleanSourceFilter path type + && !pkgs.lib.hasSuffix ".nix" path + && baseNameOf path != "flake.lock"; + }; pyproject = true; build-system = [ python.pkgs.setuptools ]; @@ -27,9 +34,11 @@ python.pkgs.buildPythonApplication { pkgs.git pkgs.installShellFiles pkgs.pre-commit - python.pkgs.pytestCheckHook ]; + doCheck = false; + dontUsePytestCheck = true; + # Generate the shell completions and man page from the freshly built binary # (shtab / argparse-manpage both walk the whole argparse command tree) and drop # them in the standard share/ locations. @@ -42,8 +51,6 @@ python.pkgs.buildPythonApplication { installManPage genvm-tool.1 ''; - pytestFlags = [ "unit_tests" ]; - meta = { description = "GenVM monorepo git helper (build, tests, hooks, ls)"; mainProgram = "genvm-tool"; diff --git a/support/tools/genvm-tool/flake.lock b/support/tools/genvm-tool/flake.lock new file mode 100644 index 00000000..db686c56 --- /dev/null +++ b/support/tools/genvm-tool/flake.lock @@ -0,0 +1,61 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1767313136, + "narHash": "sha256-16KkgfdYqjaeRGBaYsNrhPRRENs0qzkQVUooNHtoy2w=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "ac62194c3917d5f474c1a844b6fd6da2db95077d", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-25.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/support/tools/genvm-tool/flake.nix b/support/tools/genvm-tool/flake.nix new file mode 100644 index 00000000..47cde49f --- /dev/null +++ b/support/tools/genvm-tool/flake.nix @@ -0,0 +1,65 @@ +{ + description = "genvm-tool unit-test Python environment (pinned, independent flake)"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.05"; + flake-utils.url = "github:numtide/flake-utils"; + }; + + outputs = + { + self, + nixpkgs, + flake-utils, + }: + flake-utils.lib.eachDefaultSystem ( + system: + let + pkgs = import nixpkgs { inherit system; }; + python = pkgs.python312; + # The tool itself, only to borrow its dependency list: duplicating the + # one in `default.nix` here is what would drift. + tool = pkgs.callPackage ./default.nix { inherit python; }; + # Instruments a fuzz target for AFL; not in nixpkgs. The same derivation + # is in executors/v0.3.x/support/nix/py-test/flake.nix, which cannot be + # shared from here: it lives in another repository. + pythonAfl = python.pkgs.buildPythonPackage rec { + pname = "python-afl"; + version = "0.7.3"; + pyproject = true; + + src = pkgs.fetchPypi { + inherit pname version; + hash = "sha256-s3MZbXRyZiMLvoQu2qrWWWJTY4V8jnZVBddl0GE/lUQ="; + }; + + build-system = with python.pkgs; [ + cython + setuptools + wheel + ]; + pythonImportsCheck = [ "afl" ]; + }; + # `genvm_tool` is NOT installed into this env. The pytest plugin puts the + # project root on PYTHONPATH so the tests import the working tree, the + # same way genlayer-py-std's tests import `src/`. + pythonEnv = python.withPackages ( + ps: + [ ps.pytest ] + ++ tool.propagatedBuildInputs + ++ pkgs.lib.optionals (system == "x86_64-linux") [ pythonAfl ] + ); + in + { + # AFL++ carries `afl-fuzz` and friends, which the fuzz cases exec from + # this env's bin/ (the python-afl wrappers are not used) + packages.default = pkgs.buildEnv { + name = "genvm-tool-test"; + paths = [ + pythonEnv + ] + ++ pkgs.lib.optionals (system == "x86_64-linux") [ pkgs.aflplusplus ]; + }; + } + ); +} diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/0/a/0a77zeewq0aqkcykz2bb1xwhdr99xh4w8sajwhz3k7t4c b/support/tools/genvm-tool/fuzz/inputs/scheduling/0/a/0a77zeewq0aqkcykz2bb1xwhdr99xh4w8sajwhz3k7t4c new file mode 100644 index 0000000000000000000000000000000000000000..0dcb79eb2f480feb52d401838539ad27d678d133 GIT binary patch literal 26 XcmcDqfdEDZo8<cR6e!QeCW8wARAdHO literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/1/j/1j7zdpm0keqes0emct1z75fvxqefmg86rabh7f58b9j1t b/support/tools/genvm-tool/fuzz/inputs/scheduling/1/j/1j7zdpm0keqes0emct1z75fvxqefmg86rabh7f58b9j1t new file mode 100644 index 0000000000000000000000000000000000000000..e496ac8e62cf2318e70c1af97051844e945f7070 GIT binary patch literal 76 zcmdnIIU_V>^XB85H%CROMMeGp9|Z!*$w|p64h#%}4OzKaAP}0>kp6<9ej!95Pz6{a T!+)=2pvc0_n<M`>{Eq|xW4<MR literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/1/m/1m3m9gb15nqrbsf0s0yxnwkxw8bvjchzt3g3292nfh89r b/support/tools/genvm-tool/fuzz/inputs/scheduling/1/m/1m3m9gb15nqrbsf0s0yxnwkxw8bvjchzt3g3292nfh89r new file mode 100644 index 0000000000000000000000000000000000000000..95244c9cf52bf56cf8cea68e26bdaf82ea5dfc3c GIT binary patch literal 22 YcmZP&VDKmk0fDTbkjRh%5GyDJ07B&kq5uE@ literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/1/q/1qak1b713h3b43wtnpxr3k4szgy779kkpt5fmmgk3k7r8 b/support/tools/genvm-tool/fuzz/inputs/scheduling/1/q/1qak1b713h3b43wtnpxr3k4szgy779kkpt5fmmgk3k7r8 new file mode 100644 index 0000000000000000000000000000000000000000..a766be3ca8e98df4b830636b21b362b17c1db905 GIT binary patch literal 68 zcmdnIIU_V>^XB85H%CROMMeGp9|Z!*$w|p64p~`Qp;-;-FBs|<LKFaHzzP`tdnE%! N7H-}g`G3OyNC1|sBT@hW literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/1/r/1rm6sfd3ce16mneqnza9a6k44q1q0b630bjyjsr0bay2t b/support/tools/genvm-tool/fuzz/inputs/scheduling/1/r/1rm6sfd3ce16mneqnza9a6k44q1q0b630bjyjsr0bay2t new file mode 100644 index 00000000..50c8a687 --- /dev/null +++ b/support/tools/genvm-tool/fuzz/inputs/scheduling/1/r/1rm6sfd3ce16mneqnza9a6k44q1q0b630bjyjsr0bay2t @@ -0,0 +1 @@ +ccccccccccccfh diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/2/8/2889c6vjcr10k5mct810px4hwtvkjjmfap9g4v7vqw49m b/support/tools/genvm-tool/fuzz/inputs/scheduling/2/8/2889c6vjcr10k5mct810px4hwtvkjjmfap9g4v7vqw49m new file mode 100644 index 00000000..eedf6cf4 --- /dev/null +++ b/support/tools/genvm-tool/fuzz/inputs/scheduling/2/8/2889c6vjcr10k5mct810px4hwtvkjjmfap9g4v7vqw49m @@ -0,0 +1 @@ +----΅ \ No newline at end of file diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/2/z/2z4a5kfma2ck2ptfr4vgf7x93035nc52ysw31vcmwz8kw b/support/tools/genvm-tool/fuzz/inputs/scheduling/2/z/2z4a5kfma2ck2ptfr4vgf7x93035nc52ysw31vcmwz8kw new file mode 100644 index 0000000000000000000000000000000000000000..da64e2bc4cfc9ca78a0221f3e520a58d0757fe15 GIT binary patch literal 87 zcmd-q=`JZLdHM1ugTTw(yTQztpI?Fi=PMAo^8ESp8#%epKYaM`0}vyD{R}BDfm(Lo I!UVfz0fx#zga7~l literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/2/z/2ztshe2mqj9vscyc77m0xhsh0mv1syb797gssvxk2qgrg b/support/tools/genvm-tool/fuzz/inputs/scheduling/2/z/2ztshe2mqj9vscyc77m0xhsh0mv1syb797gssvxk2qgrg new file mode 100644 index 0000000000000000000000000000000000000000..f13a52956100da21d24ceb1c3849462a36e577b2 GIT binary patch literal 4 LcmYdh5a0p;0lENw literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/3/x/3xjn8db9k106dpsxg3p4safeddkwr8vyb46fzb42ycz6m b/support/tools/genvm-tool/fuzz/inputs/scheduling/3/x/3xjn8db9k106dpsxg3p4safeddkwr8vyb46fzb42ycz6m new file mode 100644 index 0000000000000000000000000000000000000000..38ddecf8f6e0f2636649fbd8467f858b4cb761a4 GIT binary patch literal 30 VcmZ3u5Ct$Wey9WqfasJAE&vOb51s%3 literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/4/e/4ebgh2n7349b26mbas03fnk883hbd8bfxkk3ta9gfzem8 b/support/tools/genvm-tool/fuzz/inputs/scheduling/4/e/4ebgh2n7349b26mbas03fnk883hbd8bfxkk3ta9gfzem8 new file mode 100644 index 00000000..4ad446b1 --- /dev/null +++ b/support/tools/genvm-tool/fuzz/inputs/scheduling/4/e/4ebgh2n7349b26mbas03fnk883hbd8bfxkk3ta9gfzem8 @@ -0,0 +1,2 @@ +c`€h +cccWcccccccccccccccccccccccccccccccccccccccccc \ No newline at end of file diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/4/y/4ysd4fq9jcf95wvg4fpk73h0ftkyg9dy285arxvgqft2j b/support/tools/genvm-tool/fuzz/inputs/scheduling/4/y/4ysd4fq9jcf95wvg4fpk73h0ftkyg9dy285arxvgqft2j new file mode 100644 index 0000000000000000000000000000000000000000..37e31bc2a83300a80b6009e88447ac60bd537ac0 GIT binary patch literal 62 zcmdnIIU_VVDLLgA0|P@tL;4Gb`h}a1Z{8dg73CQ9|38pI2q`c`L4=~Bypn;67H-}Q E0HeAd%m4rY literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/5/n/5n4jjf4c6xgejs3ntw83jbpakfa8y26hrka9d9ykpkt7y b/support/tools/genvm-tool/fuzz/inputs/scheduling/5/n/5n4jjf4c6xgejs3ntw83jbpakfa8y26hrka9d9ykpkt7y new file mode 100644 index 0000000000000000000000000000000000000000..587c7db4b28013f1bfa76b8f0323ab46468d8d66 GIT binary patch literal 19 Zcmc}@P1(Hp__WPHu#ka4@I^`n7XW1U2_XOg literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/5/n/5nmfkg45vmckx2qj1gpm5tbke0cqedmvsgky5pa5fdhdj b/support/tools/genvm-tool/fuzz/inputs/scheduling/5/n/5nmfkg45vmckx2qj1gpm5tbke0cqedmvsgky5pa5fdhdj new file mode 100644 index 0000000000000000000000000000000000000000..37fa078065e48f14dc983a18a44eff841f5e8ac7 GIT binary patch literal 67 rcmYdx&qz*TU|>i|FE5{8US1vr1poj4ZvfFC0AgbZLZm=s8V~~j_Wm91 literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/5/q/5qh9w8qmh1wz818jmnwnrd7aqq6f7y39hx1fqn6gret3a b/support/tools/genvm-tool/fuzz/inputs/scheduling/5/q/5qh9w8qmh1wz818jmnwnrd7aqq6f7y39hx1fqn6gret3a new file mode 100644 index 0000000000000000000000000000000000000000..e003a5b661bdedbd7ff2d6cf5b412ff6fc900ce1 GIT binary patch literal 61 zcmYc*i8OIEtqKHE<wmhVW+2+dpxDDQ%S|Ia#4yj-yfDf*Il`jMn}Go=Ug%~D6bmv= I4uACu09B|H+yDRo literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/6/d/6dge9wd637rp52z6bazc1ph9jbacfknb3mpkcwff8y8be b/support/tools/genvm-tool/fuzz/inputs/scheduling/6/d/6dge9wd637rp52z6bazc1ph9jbacfknb3mpkcwff8y8be new file mode 100644 index 00000000..fa7af8bf --- /dev/null +++ b/support/tools/genvm-tool/fuzz/inputs/scheduling/6/d/6dge9wd637rp52z6bazc1ph9jbacfknb3mpkcwff8y8be @@ -0,0 +1 @@ +z \ No newline at end of file diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/7/a/7a5y9xan3xzj1vam3946gqbepe41p9471t8f5gw0gfn1a b/support/tools/genvm-tool/fuzz/inputs/scheduling/7/a/7a5y9xan3xzj1vam3946gqbepe41p9471t8f5gw0gfn1a new file mode 100644 index 00000000..f9169df0 --- /dev/null +++ b/support/tools/genvm-tool/fuzz/inputs/scheduling/7/a/7a5y9xan3xzj1vam3946gqbepe41p9471t8f5gw0gfn1a @@ -0,0 +1 @@ +Mα \ No newline at end of file diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/7/d/7dbdpj58bhnvv9zcqaxg1gyf6ty707k25avczqr13n414 b/support/tools/genvm-tool/fuzz/inputs/scheduling/7/d/7dbdpj58bhnvv9zcqaxg1gyf6ty707k25avczqr13n414 new file mode 100644 index 0000000000000000000000000000000000000000..63f92d3c22c10258f9981a9ebdf62f7d17a0341e GIT binary patch literal 3 KcmXSBU<3dGT>y0e literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/7/m/7mne39ergtbmmtvn8dwf5rts4c4k5aaac1t6rf7rypn1y b/support/tools/genvm-tool/fuzz/inputs/scheduling/7/m/7mne39ergtbmmtvn8dwf5rts4c4k5aaac1t6rf7rypn1y new file mode 100644 index 00000000..2373feab --- /dev/null +++ b/support/tools/genvm-tool/fuzz/inputs/scheduling/7/m/7mne39ergtbmmtvn8dwf5rts4c4k5aaac1t6rf7rypn1y @@ -0,0 +1 @@ +23•••••••3••3 \ No newline at end of file diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/8/h/8h406nwh64gfav2jxv96hbmhrb1t03yqf5e61s7jf7tfm b/support/tools/genvm-tool/fuzz/inputs/scheduling/8/h/8h406nwh64gfav2jxv96hbmhrb1t03yqf5e61s7jf7tfm new file mode 100644 index 00000000..f6a5679b --- /dev/null +++ b/support/tools/genvm-tool/fuzz/inputs/scheduling/8/h/8h406nwh64gfav2jxv96hbmhrb1t03yqf5e61s7jf7tfm @@ -0,0 +1 @@ +ΛccGuxhΛccd κκ \ No newline at end of file diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/8/v/8vfbamrjk4e8cm6nc8qmp2jx8dkqqhfs13wkb1hb5rq90 b/support/tools/genvm-tool/fuzz/inputs/scheduling/8/v/8vfbamrjk4e8cm6nc8qmp2jx8dkqqhfs13wkb1hb5rq90 new file mode 100644 index 0000000000000000000000000000000000000000..859fd8586464ba2caf2664d0c33bfcbec9efaa5e GIT binary patch literal 44 kcmZq3XvpZuXl6))Qpw3&PVyiCXUNOf|7Vjo^zzC8074uL%K!iX literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/9/v/9vv23ykykwfjgdmnsfxxtrqzfjx5r0b30frarwecm3rd8 b/support/tools/genvm-tool/fuzz/inputs/scheduling/9/v/9vv23ykykwfjgdmnsfxxtrqzfjx5r0b30frarwecm3rd8 new file mode 100644 index 0000000000000000000000000000000000000000..91f09c04ddc93b159f85a27aeeb3faf537f79b11 GIT binary patch literal 19 acmYch6ij3N|DTmX;J;u7mms&1AQu2S6a?@9 literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/9/w/9wmhqg28z4ergsgh57jkqfyx2p66pdby1r1zs5wth779p b/support/tools/genvm-tool/fuzz/inputs/scheduling/9/w/9wmhqg28z4ergsgh57jkqfyx2p66pdby1r1zs5wth779p new file mode 100644 index 00000000..9a18f1fd --- /dev/null +++ b/support/tools/genvm-tool/fuzz/inputs/scheduling/9/w/9wmhqg28z4ergsgh57jkqfyx2p66pdby1r1zs5wth779p @@ -0,0 +1 @@ +hŸc$h*—ό€c \ No newline at end of file diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/a/0/a03g00cnvfa0gs77n0y1ww0tdn8908qnja4bxw9a7gpmw b/support/tools/genvm-tool/fuzz/inputs/scheduling/a/0/a03g00cnvfa0gs77n0y1ww0tdn8908qnja4bxw9a7gpmw new file mode 100644 index 0000000000000000000000000000000000000000..3f8316e7e24fa12749cced18a54fc2a3b4680161 GIT binary patch literal 20 bcmazvEO#;J@vzKtGfj6g%wXt9$>0J2K|Tfu literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/a/4/a4r7021ggw1rmd6qz9p3spjn266k328eh5vr8pg0pm2gm b/support/tools/genvm-tool/fuzz/inputs/scheduling/a/4/a4r7021ggw1rmd6qz9p3spjn266k328eh5vr8pg0pm2gm new file mode 100644 index 0000000000000000000000000000000000000000..5d97b58f2a14e70a8c64bea47c2c1abc92444850 GIT binary patch literal 22 ZcmYdhcn${7faD9t=g&_+e=hJo1pvBS5F7vi literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/a/7/a77dmacngtkjb9y93610yv1zh6a9rtd1s6dt3vjz00zcg b/support/tools/genvm-tool/fuzz/inputs/scheduling/a/7/a77dmacngtkjb9y93610yv1zh6a9rtd1s6dt3vjz00zcg new file mode 100644 index 0000000000000000000000000000000000000000..349de638cecdd46412195d7ae4289ec6bedf5329 GIT binary patch literal 21 bcmYdx&qzMwKK*|^14Bk~GLYk*k--H3U&#mm literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/a/y/ayj9117xzpdr731k488aawap9fa25kbg0yez0yxncp8r6 b/support/tools/genvm-tool/fuzz/inputs/scheduling/a/y/ayj9117xzpdr731k488aawap9fa25kbg0yez0yxncp8r6 new file mode 100644 index 0000000000000000000000000000000000000000..eae688e594cd97c1addaa4b6798c1db1d418344b GIT binary patch literal 50 ncmYe!4opeONbxggaOz5KU`S3rfC_+;hae)!Tnr2h4oN8hZG;%p literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/a/z/azat8a0r7d706fa7tnee39qz1rbdc92ekgbxtgejyntrc b/support/tools/genvm-tool/fuzz/inputs/scheduling/a/z/azat8a0r7d706fa7tnee39qz1rbdc92ekgbxtgejyntrc new file mode 100644 index 00000000..dd825a3f --- /dev/null +++ b/support/tools/genvm-tool/fuzz/inputs/scheduling/a/z/azat8a0r7d706fa7tnee39qz1rbdc92ekgbxtgejyntrc @@ -0,0 +1,3 @@ +cc€ +dc€ +dh8h8 \ No newline at end of file diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/b/h/bh4wrvwgtw8a0xw2836gdfej6qxws1f66ej3hxtqf480m b/support/tools/genvm-tool/fuzz/inputs/scheduling/b/h/bh4wrvwgtw8a0xw2836gdfej6qxws1f66ej3hxtqf480m new file mode 100644 index 00000000..20cf60aa --- /dev/null +++ b/support/tools/genvm-tool/fuzz/inputs/scheduling/b/h/bh4wrvwgtw8a0xw2836gdfej6qxws1f66ej3hxtqf480m @@ -0,0 +1 @@ +»»»»»»Ό»ύ diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/c/a/ca97ypke6xjfbx2n9jpex04wy2rd0nh950vrteqpx6v4c b/support/tools/genvm-tool/fuzz/inputs/scheduling/c/a/ca97ypke6xjfbx2n9jpex04wy2rd0nh950vrteqpx6v4c new file mode 100644 index 00000000..30c800a5 --- /dev/null +++ b/support/tools/genvm-tool/fuzz/inputs/scheduling/c/a/ca97ypke6xjfbx2n9jpex04wy2rd0nh950vrteqpx6v4c @@ -0,0 +1,3 @@ + +h +ΔΔΔΔΔδ \ No newline at end of file diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/d/1/d1ftb59xsy5bkg0vacd8wrqrhm8c5s10zd3xcpc7edtn4 b/support/tools/genvm-tool/fuzz/inputs/scheduling/d/1/d1ftb59xsy5bkg0vacd8wrqrhm8c5s10zd3xcpc7edtn4 new file mode 100644 index 00000000..56d0e0f0 --- /dev/null +++ b/support/tools/genvm-tool/fuzz/inputs/scheduling/d/1/d1ftb59xsy5bkg0vacd8wrqrhm8c5s10zd3xcpc7edtn4 @@ -0,0 +1 @@ +c diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/d/j/djj76h73qn7c9yh8r6xfzehdtvsbn0hkxvz5gz1w8cw6r b/support/tools/genvm-tool/fuzz/inputs/scheduling/d/j/djj76h73qn7c9yh8r6xfzehdtvsbn0hkxvz5gz1w8cw6r new file mode 100644 index 00000000..0b3721bb --- /dev/null +++ b/support/tools/genvm-tool/fuzz/inputs/scheduling/d/j/djj76h73qn7c9yh8r6xfzehdtvsbn0hkxvz5gz1w8cw6r @@ -0,0 +1 @@ +cdhΡ \ No newline at end of file diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/d/s/ds11mdgj72234c89gzj6k0cvw7vpb5v2n3c9327rmch3a b/support/tools/genvm-tool/fuzz/inputs/scheduling/d/s/ds11mdgj72234c89gzj6k0cvw7vpb5v2n3c9327rmch3a new file mode 100644 index 0000000000000000000000000000000000000000..ed29f75af059798e75435a581485fb93b3ca9dc4 GIT binary patch literal 5 McmezMi-A!d00{5`6#xJL literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/d/t/dtjpxmbt1f6axs8fh7kz3mb4xhstcm3erhpx9nnsmb5c8 b/support/tools/genvm-tool/fuzz/inputs/scheduling/d/t/dtjpxmbt1f6axs8fh7kz3mb4xhstcm3erhpx9nnsmb5c8 new file mode 100644 index 00000000..04fe2cda --- /dev/null +++ b/support/tools/genvm-tool/fuzz/inputs/scheduling/d/t/dtjpxmbt1f6axs8fh7kz3mb4xhstcm3erhpx9nnsmb5c8 @@ -0,0 +1 @@ +6tYf9A5zPΡmN7bC3x4fA5zQ1mN7b \ No newline at end of file diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/d/t/dtt19bz3ejma3sk5xyk9bhnz9yna4w7qwc5xvqt12n1nr b/support/tools/genvm-tool/fuzz/inputs/scheduling/d/t/dtt19bz3ejma3sk5xyk9bhnz9yna4w7qwc5xvqt12n1nr new file mode 100644 index 0000000000000000000000000000000000000000..89c283db41e78749772a94f0489ff82b690f8bcb GIT binary patch literal 30 TcmZP&Fa!f52!%|3Yp4eRQuPMV literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/e/5/e56xem18j5vnd0db4gmheccjjrzp6114yjzv0rwpvkarr b/support/tools/genvm-tool/fuzz/inputs/scheduling/e/5/e56xem18j5vnd0db4gmheccjjrzp6114yjzv0rwpvkarr new file mode 100644 index 00000000..416e0b93 --- /dev/null +++ b/support/tools/genvm-tool/fuzz/inputs/scheduling/e/5/e56xem18j5vnd0db4gmheccjjrzp6114yjzv0rwpvkarr @@ -0,0 +1 @@ +c}____________o_____________h diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/e/k/ek756vzwq4xcp9g7ncsqmc1n0bpa2391kkjwa70anx5km b/support/tools/genvm-tool/fuzz/inputs/scheduling/e/k/ek756vzwq4xcp9g7ncsqmc1n0bpa2391kkjwa70anx5km new file mode 100644 index 00000000..0c8aeb1b --- /dev/null +++ b/support/tools/genvm-tool/fuzz/inputs/scheduling/e/k/ek756vzwq4xcp9g7ncsqmc1n0bpa2391kkjwa70anx5km @@ -0,0 +1 @@ +czQ1mN7bC3xV8kL2pR6tY4wD0sdh diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/e/w/ewzaq332qw2ef9kjhgknt5jgzqvq9959t18a9netpe7n2 b/support/tools/genvm-tool/fuzz/inputs/scheduling/e/w/ewzaq332qw2ef9kjhgknt5jgzqvq9959t18a9netpe7n2 new file mode 100644 index 0000000000000000000000000000000000000000..cda97f7a1db6a2b924c8a828469623ec39edf905 GIT binary patch literal 79 xcmd-q=`JZLdHM2M!^@YuCBe*>$YAAp5V*j3{`^BAfHOcG1lZ4z0@MuzvH<*HI{N?s literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/f/2/f2977sr15mekm241hw6307vdxw6jfpvtzcv96e0kpb30t b/support/tools/genvm-tool/fuzz/inputs/scheduling/f/2/f2977sr15mekm241hw6307vdxw6jfpvtzcv96e0kpb30t new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/support/tools/genvm-tool/fuzz/inputs/scheduling/f/2/f2977sr15mekm241hw6307vdxw6jfpvtzcv96e0kpb30t @@ -0,0 +1 @@ + diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/f/r/frwjgmhyf89bdtnk33sfphqeqh99a7bgnjm672d7491sp b/support/tools/genvm-tool/fuzz/inputs/scheduling/f/r/frwjgmhyf89bdtnk33sfphqeqh99a7bgnjm672d7491sp new file mode 100644 index 0000000000000000000000000000000000000000..7833cd80dacf035b2b66c37e1ded5cf0c25059f1 GIT binary patch literal 21 bcmYdx&qzMwp7Fn)fgvL~8OU+Z$lwA1T?hxz literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/f/s/fs19mwm00yyna4dgd0vfdd2xsz9tta24qqnb3a5j2fpb4 b/support/tools/genvm-tool/fuzz/inputs/scheduling/f/s/fs19mwm00yyna4dgd0vfdd2xsz9tta24qqnb3a5j2fpb4 new file mode 100644 index 0000000000000000000000000000000000000000..5d0bffa059a165e1b6de398305e232f824922f1f GIT binary patch literal 13 UcmYdx&qx+v$Vm3ha4+Ek02^Qf%K!iX literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/f/y/fypk3np5dvfnrfhm0mp84qmkpbweyfyqtenpb9tybvv4r b/support/tools/genvm-tool/fuzz/inputs/scheduling/f/y/fypk3np5dvfnrfhm0mp84qmkpbweyfyqtenpb9tybvv4r new file mode 100644 index 0000000000000000000000000000000000000000..18704e0022863042da14ea4e5514b181067dc921 GIT binary patch literal 25 gcmYe!4opeO@bU9yaPCTOs83E#R;~!k;POiW0CLj^qyPW_ literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/g/k/gk362a9qzh27m009584enyxjpcgrydm3gpvh2pfgragvp b/support/tools/genvm-tool/fuzz/inputs/scheduling/g/k/gk362a9qzh27m009584enyxjpcgrydm3gpvh2pfgragvp new file mode 100644 index 0000000000000000000000000000000000000000..4170489786e1321cb80b0c5f8dcc807ff45b9c89 GIT binary patch literal 19 YcmYdOoaM+M5NMn0=fwa7-fpJG04)jw{{R30 literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/g/s/gs7tthhttxra0bw9pr03wswx1zca0tasa30ban582d1ry b/support/tools/genvm-tool/fuzz/inputs/scheduling/g/s/gs7tthhttxra0bw9pr03wswx1zca0tasa30ban582d1ry new file mode 100644 index 00000000..36175568 --- /dev/null +++ b/support/tools/genvm-tool/fuzz/inputs/scheduling/g/s/gs7tthhttxra0bw9pr03wswx1zca0tasa30ban582d1ry @@ -0,0 +1 @@ +WWWdch0"00ŠŠŠŠŠŠŠŠŠŠŠŠŠŠŠŠchŸdh diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/h/0/h0rj8gfaf7y5xn1bd2f55dxj285rfmrv8ja0zypgzdjc2 b/support/tools/genvm-tool/fuzz/inputs/scheduling/h/0/h0rj8gfaf7y5xn1bd2f55dxj285rfmrv8ja0zypgzdjc2 new file mode 100644 index 0000000000000000000000000000000000000000..3ad4a98b5dd31fa7888c984c29dba201e2bb0bfe GIT binary patch literal 46 zcmXTRO0G&tDXB_vNJ&X%V32%~kdT;=kPx4d%`iLTN`?Rf14G8=41xcQ4h;Vj5&({} B5ZC|! literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/h/2/h2vx2w3me566fak1gsq86nytqw7x2egsgqntxh4m811rp b/support/tools/genvm-tool/fuzz/inputs/scheduling/h/2/h2vx2w3me566fak1gsq86nytqw7x2egsgqntxh4m811rp new file mode 100644 index 00000000..5593c0ac --- /dev/null +++ b/support/tools/genvm-tool/fuzz/inputs/scheduling/h/2/h2vx2w3me566fak1gsq86nytqw7x2egsgqntxh4m811rp @@ -0,0 +1 @@ +bdhYYYYYYYYYYYYYYY diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/h/8/h8s06mw10v4v4hs3dd98gfqnpszmh7s2bfp3bgstpx1sc b/support/tools/genvm-tool/fuzz/inputs/scheduling/h/8/h8s06mw10v4v4hs3dd98gfqnpszmh7s2bfp3bgstpx1sc new file mode 100644 index 0000000000000000000000000000000000000000..219bc57046475954d326defa07159b8fee2e3901 GIT binary patch literal 42 wcmYexVsJ<kb2Lp3G|crgDhM(wi8Lv9Q7rbb1WKek8Acg1Mi|>0<{EJU0RO%VQ~&?~ literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/h/k/hke62nwahseh6j9eb81qsbggf2k585jf9t01jf5s49cwt b/support/tools/genvm-tool/fuzz/inputs/scheduling/h/k/hke62nwahseh6j9eb81qsbggf2k585jf9t01jf5s49cwt new file mode 100644 index 0000000000000000000000000000000000000000..a59f6c4d05db244e33c1fcca67a7ad57feedf4b1 GIT binary patch literal 21 bcmZR`jEegI|9?YN7-ME8kR6qhlEDQ4X}t)~ literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/j/g/jgvxrb5teh8rxxejpttpqzwhthnpctfrnxma8fzf07zqt b/support/tools/genvm-tool/fuzz/inputs/scheduling/j/g/jgvxrb5teh8rxxejpttpqzwhthnpctfrnxma8fzf07zqt new file mode 100644 index 00000000..dad5f9ed --- /dev/null +++ b/support/tools/genvm-tool/fuzz/inputs/scheduling/j/g/jgvxrb5teh8rxxejpttpqzwhthnpctfrnxma8fzf07zqt @@ -0,0 +1 @@ + t‹ttt»»» \ No newline at end of file diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/j/q/jq7d47bpcygzb55b6jsjwh3xb7nq5p5swwg52dv0je7qt b/support/tools/genvm-tool/fuzz/inputs/scheduling/j/q/jq7d47bpcygzb55b6jsjwh3xb7nq5p5swwg52dv0je7qt new file mode 100644 index 00000000..033187ab --- /dev/null +++ b/support/tools/genvm-tool/fuzz/inputs/scheduling/j/q/jq7d47bpcygzb55b6jsjwh3xb7nq5p5swwg52dv0je7qt @@ -0,0 +1,3 @@ +chε +œd +hcdhdr diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/j/t/jt773qwy1mj50y89pkcasgkkm326mykrzx9yvx94ts4kw b/support/tools/genvm-tool/fuzz/inputs/scheduling/j/t/jt773qwy1mj50y89pkcasgkkm326mykrzx9yvx94ts4kw new file mode 100644 index 0000000000000000000000000000000000000000..d6c91e947cf9b161f884f02305f29a2e76fb7c68 GIT binary patch literal 10 PcmZSJU}$h)0AdaR2U-D< literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/j/t/jtccnqnb3bh8m1dtcd9wh20qxbyhggqhk559d4xafecb8 b/support/tools/genvm-tool/fuzz/inputs/scheduling/j/t/jtccnqnb3bh8m1dtcd9wh20qxbyhggqhk559d4xafecb8 new file mode 100644 index 0000000000000000000000000000000000000000..825fb8b6be40d175bbc51761683d632640757435 GIT binary patch literal 64 xcmezW5(pR=7#u+K%Jb)eB-bkt<NSXh023g_{{;*w3l=PxxB#RKtPmpg8UQR+GR6P^ literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/k/3/k3n2b2f1cvyd4nnep751pzhdh0fn48ga066nn3d2jpgs4 b/support/tools/genvm-tool/fuzz/inputs/scheduling/k/3/k3n2b2f1cvyd4nnep751pzhdh0fn48ga066nn3d2jpgs4 new file mode 100644 index 00000000..ccfc151e --- /dev/null +++ b/support/tools/genvm-tool/fuzz/inputs/scheduling/k/3/k3n2b2f1cvyd4nnep751pzhdh0fn48ga066nn3d2jpgs4 @@ -0,0 +1 @@ +fS=9A5zQ1mN7 \ No newline at end of file diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/k/j/kjy0wbyzj59320tt8pbm99abg80w2v0spd0v1g1q2jxaa b/support/tools/genvm-tool/fuzz/inputs/scheduling/k/j/kjy0wbyzj59320tt8pbm99abg80w2v0spd0v1g1q2jxaa new file mode 100644 index 00000000..5b00cfa0 --- /dev/null +++ b/support/tools/genvm-tool/fuzz/inputs/scheduling/k/j/kjy0wbyzj59320tt8pbm99abg80w2v0spd0v1g1q2jxaa @@ -0,0 +1 @@ +ccccccccccccccccdd diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/k/p/kpwwz2fcjyav4ksb0rcj53w0ansjt4gb9yh1ynhapkz0j b/support/tools/genvm-tool/fuzz/inputs/scheduling/k/p/kpwwz2fcjyav4ksb0rcj53w0ansjt4gb9yh1ynhapkz0j new file mode 100644 index 00000000..14fe11e8 --- /dev/null +++ b/support/tools/genvm-tool/fuzz/inputs/scheduling/k/p/kpwwz2fcjyav4ksb0rcj53w0ansjt4gb9yh1ynhapkz0j @@ -0,0 +1,4 @@ +chhcdh +%%%% +cd +dh diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/k/q/kq8w2s5d2rq6ncbzmryxtzcc62jz09j2vzb1cs24t8mcg b/support/tools/genvm-tool/fuzz/inputs/scheduling/k/q/kq8w2s5d2rq6ncbzmryxtzcc62jz09j2vzb1cs24t8mcg new file mode 100644 index 00000000..319fec19 --- /dev/null +++ b/support/tools/genvm-tool/fuzz/inputs/scheduling/k/q/kq8w2s5d2rq6ncbzmryxtzcc62jz09j2vzb1cs24t8mcg @@ -0,0 +1 @@ +œ›—υ \ No newline at end of file diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/k/x/kxhe3xvpb0s9d1f7ffrnkwrg2h61dqqkgrn5sap51xkqm b/support/tools/genvm-tool/fuzz/inputs/scheduling/k/x/kxhe3xvpb0s9d1f7ffrnkwrg2h61dqqkgrn5sap51xkqm new file mode 100644 index 00000000..695b58f6 --- /dev/null +++ b/support/tools/genvm-tool/fuzz/inputs/scheduling/k/x/kxhe3xvpb0s9d1f7ffrnkwrg2h61dqqkgrn5sap51xkqm @@ -0,0 +1 @@ +ΧΧΧςΧΧΧ \ No newline at end of file diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/n/v/nv0apj6jkdsnskpkhbnpxhq75zq69msynk5mab9mgv54w b/support/tools/genvm-tool/fuzz/inputs/scheduling/n/v/nv0apj6jkdsnskpkhbnpxhq75zq69msynk5mab9mgv54w new file mode 100644 index 0000000000000000000000000000000000000000..d0d12442e87a8b2726a16d6b774bc0cd4093f6a8 GIT binary patch literal 21 bcmYdx&qzMwp7Fn)fgvL~8OU+ZX!s8RUM&dS literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/p/s/ps065bgt8wk77gp5pacydweetf153h40chbhpb5advvj8 b/support/tools/genvm-tool/fuzz/inputs/scheduling/p/s/ps065bgt8wk77gp5pacydweetf153h40chbhpb5advvj8 new file mode 100644 index 00000000..5ef81b63 --- /dev/null +++ b/support/tools/genvm-tool/fuzz/inputs/scheduling/p/s/ps065bgt8wk77gp5pacydweetf153h40chbhpb5advvj8 @@ -0,0 +1 @@ +cc \ No newline at end of file diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/q/1/q1ggb0ky8p285276zx7gnbyfd8ygy3t6r7vrv2x9y7rjt b/support/tools/genvm-tool/fuzz/inputs/scheduling/q/1/q1ggb0ky8p285276zx7gnbyfd8ygy3t6r7vrv2x9y7rjt new file mode 100644 index 0000000000000000000000000000000000000000..87db8bdc4df1d09cd2faa33641ac5e32c143dcf7 GIT binary patch literal 46 zcmZ=WO?NWP^EG!eG)|5Xa4@Y3G|crgPjWV{2(!rcQ3J{pMj1CSFfe!<aT##|0560L A3IG5A literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/q/q/qqkmwgxfve8zjkwyag90ha2k573f1sfja9mcxttfmqkw0 b/support/tools/genvm-tool/fuzz/inputs/scheduling/q/q/qqkmwgxfve8zjkwyag90ha2k573f1sfja9mcxttfmqkw0 new file mode 100644 index 0000000000000000000000000000000000000000..8ad6e5c48ac02d4bb00ead36fcbeb6e65901f6c9 GIT binary patch literal 10 OcmYeEox;da03rYqOajyZ literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/q/r/qr29xg001ksft36c10dn085qrsfwqv5q0414rnrpwtzgm b/support/tools/genvm-tool/fuzz/inputs/scheduling/q/r/qr29xg001ksft36c10dn085qrsfwqv5q0414rnrpwtzgm new file mode 100644 index 00000000..dc11fac6 --- /dev/null +++ b/support/tools/genvm-tool/fuzz/inputs/scheduling/q/r/qr29xg001ksft36c10dn085qrsfwqv5q0414rnrpwtzgm @@ -0,0 +1 @@ +c??????????????? \ No newline at end of file diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/r/2/r284nd9vqrwfx5hb6ve95gbakraz1e9xmqse6m6vyme7e b/support/tools/genvm-tool/fuzz/inputs/scheduling/r/2/r284nd9vqrwfx5hb6ve95gbakraz1e9xmqse6m6vyme7e new file mode 100644 index 0000000000000000000000000000000000000000..364ede9ae53a6a6f1d679e7e5769f52987092f80 GIT binary patch literal 12 TcmXRemW}3@m1b!8|6djW7qbLG literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/r/5/r5zhh7c9j6r51vggk6ab360krd6x252xxgh3tdfp6rksa b/support/tools/genvm-tool/fuzz/inputs/scheduling/r/5/r5zhh7c9j6r51vggk6ab360krd6x252xxgh3tdfp6rksa new file mode 100644 index 0000000000000000000000000000000000000000..05e0c796b4c63951a87a77d0c788337763f4f27f GIT binary patch literal 20 bcmd1EW|*CEB}0ILfg$5_hQNPD2ZsLuNfHM& literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/r/e/rekdzpff11mgb0mtkjc166frrkre6smknejpqcpt7f22p b/support/tools/genvm-tool/fuzz/inputs/scheduling/r/e/rekdzpff11mgb0mtkjc166frrkre6smknejpqcpt7f22p new file mode 100644 index 0000000000000000000000000000000000000000..d1ea454ac1d55ac614d3533f1d3aa43305106873 GIT binary patch literal 27 icmYdOoaM+M5GW96o9pM5<ZM*tZS=>_JUPOm!VLgyv<PDW literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/r/t/rt4ykf5cbnv0n2r107km5yh0qqmx2vsjjp1qwybpjgy5t b/support/tools/genvm-tool/fuzz/inputs/scheduling/r/t/rt4ykf5cbnv0n2r107km5yh0qqmx2vsjjp1qwybpjgy5t new file mode 100644 index 0000000000000000000000000000000000000000..29924f1059133de272c3cfcb33921eda289fead9 GIT binary patch literal 35 jcmYdx&qz)NVt0nR3@!!%29Kf;5XcG&i3}+Kv4T<ny$uS( literal 0 HcmV?d00001 diff --git a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/0c8b1f460e1ccc4f95050847930e5b35a31848eeee570692c202ed78f3fa739e b/support/tools/genvm-tool/fuzz/inputs/scheduling/s/k/sk5exxw1y9681n735pv748t072w0qz7s9x23907cdjj68 similarity index 50% rename from implementation/fuzz/inputs-genvm-modules-normalize-whitespace/0c8b1f460e1ccc4f95050847930e5b35a31848eeee570692c202ed78f3fa739e rename to support/tools/genvm-tool/fuzz/inputs/scheduling/s/k/sk5exxw1y9681n735pv748t072w0qz7s9x23907cdjj68 index 4bc7fcde..f02ee1d8 100644 --- a/implementation/fuzz/inputs-genvm-modules-normalize-whitespace/0c8b1f460e1ccc4f95050847930e5b35a31848eeee570692c202ed78f3fa739e +++ b/support/tools/genvm-tool/fuzz/inputs/scheduling/s/k/sk5exxw1y9681n735pv748t072w0qz7s9x23907cdjj68 @@ -1 +1 @@ -β€€β€€ \ No newline at end of file +cwwwwwwwwwwZZZZZ \ No newline at end of file diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/t/1/t1hj52m3a64ws7xxvx7qps9xsnhehfay1k6j88tr3ypke b/support/tools/genvm-tool/fuzz/inputs/scheduling/t/1/t1hj52m3a64ws7xxvx7qps9xsnhehfay1k6j88tr3ypke new file mode 100644 index 00000000..dc98b2a1 --- /dev/null +++ b/support/tools/genvm-tool/fuzz/inputs/scheduling/t/1/t1hj52m3a64ws7xxvx7qps9xsnhehfay1k6j88tr3ypke @@ -0,0 +1 @@ +ssssssdS \ No newline at end of file diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/t/r/trx1968q1gq2y6dbw2nbykrrendnqp1stprbxvd7mt558 b/support/tools/genvm-tool/fuzz/inputs/scheduling/t/r/trx1968q1gq2y6dbw2nbykrrendnqp1stprbxvd7mt558 new file mode 100644 index 0000000000000000000000000000000000000000..8ea76f159845d60258bb02ffba33af8ea9545d17 GIT binary patch literal 28 dcmYdFf&mAH|H&yS8Be+9q+}#HB&TF>0RW(g3fTYv literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/t/t/ttdsymeqpec56fp0mrr72150b33memf6nex59dhnbkqfg b/support/tools/genvm-tool/fuzz/inputs/scheduling/t/t/ttdsymeqpec56fp0mrr72150b33memf6nex59dhnbkqfg new file mode 100644 index 0000000000000000000000000000000000000000..7056e77a2ffee575297954c43d14261501f92539 GIT binary patch literal 14 ScmezM3k=wP>n0~LG5`Qn;RwqB literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/t/z/tzhred7xrw335kmfkv4px3sppvkjx8kff9ykcv0nfr1gg b/support/tools/genvm-tool/fuzz/inputs/scheduling/t/z/tzhred7xrw335kmfkv4px3sppvkjx8kff9ykcv0nfr1gg new file mode 100644 index 0000000000000000000000000000000000000000..43f54c2f336e45338465c233c03c205bb304c1ad GIT binary patch literal 12 RcmezWzkz{~m64GN2ml;E0*?Ry literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/t/z/tzm8wzrej02f2qhyj478hsx7jzj55ge236qr4hz2zajdt b/support/tools/genvm-tool/fuzz/inputs/scheduling/t/z/tzm8wzrej02f2qhyj478hsx7jzj55ge236qr4hz2zajdt new file mode 100644 index 0000000000000000000000000000000000000000..413a4e9453c96400a838c4f6691897bcef593f8b GIT binary patch literal 6 NcmZR0`Tzg_dH@ay1ZDsL literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/v/9/v9bn5x7sxq15qm7yckys35008w0b0d4mp59gcp0th6hta b/support/tools/genvm-tool/fuzz/inputs/scheduling/v/9/v9bn5x7sxq15qm7yckys35008w0b0d4mp59gcp0th6hta new file mode 100644 index 0000000000000000000000000000000000000000..00ddcc4f32285e0c41057d7d44e581844c48feca GIT binary patch literal 36 QcmezW9|{B*U@Qz80A!6J?*IS* literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/v/q/vq8mbh2tkh0ce4nq474f49zrg0egdva3bxyhs2m0jhv7m b/support/tools/genvm-tool/fuzz/inputs/scheduling/v/q/vq8mbh2tkh0ce4nq474f49zrg0egdva3bxyhs2m0jhv7m new file mode 100644 index 0000000000000000000000000000000000000000..d4c79f400afda03755a06ebe9f926c91d308651b GIT binary patch literal 51 wcmZ3*`uxi(AZB1-NMT4x@w)K7oq>xH#7pry4`z!pg1}3L3@*lJPk^ky0N!E{egFUf literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/w/9/w9b8068b46gp8ajd9xzv810kws9j84q5eb2rj5pgmn9t2 b/support/tools/genvm-tool/fuzz/inputs/scheduling/w/9/w9b8068b46gp8ajd9xzv810kws9j84q5eb2rj5pgmn9t2 new file mode 100644 index 00000000..16063911 --- /dev/null +++ b/support/tools/genvm-tool/fuzz/inputs/scheduling/w/9/w9b8068b46gp8ajd9xzv810kws9j84q5eb2rj5pgmn9t2 @@ -0,0 +1 @@ +cdh* \ No newline at end of file diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/w/9/w9exq2ds6v5amcy0sv3q7s2c8thhpb4tafjsmawb7f132 b/support/tools/genvm-tool/fuzz/inputs/scheduling/w/9/w9exq2ds6v5amcy0sv3q7s2c8thhpb4tafjsmawb7f132 new file mode 100644 index 00000000..cf99539d --- /dev/null +++ b/support/tools/genvm-tool/fuzz/inputs/scheduling/w/9/w9exq2ds6v5amcy0sv3q7s2c8thhpb4tafjsmawb7f132 @@ -0,0 +1 @@ +gcbgcb diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/w/b/wbjmbw1dr6vxhxvmavvsa3zq0h4mm209am9qteesgyekg b/support/tools/genvm-tool/fuzz/inputs/scheduling/w/b/wbjmbw1dr6vxhxvmavvsa3zq0h4mm209am9qteesgyekg new file mode 100644 index 00000000..25038d7b --- /dev/null +++ b/support/tools/genvm-tool/fuzz/inputs/scheduling/w/b/wbjmbw1dr6vxhxvmavvsa3zq0h4mm209am9qteesgyekg @@ -0,0 +1 @@ +cd \ No newline at end of file diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/w/h/whza2rpj7esftmqkevjy248v1p0a13rawsz3cz6jg172a b/support/tools/genvm-tool/fuzz/inputs/scheduling/w/h/whza2rpj7esftmqkevjy248v1p0a13rawsz3cz6jg172a new file mode 100644 index 00000000..c67b7b40 --- /dev/null +++ b/support/tools/genvm-tool/fuzz/inputs/scheduling/w/h/whza2rpj7esftmqkevjy248v1p0a13rawsz3cz6jg172a @@ -0,0 +1 @@ +c2 \ No newline at end of file diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/w/j/wj4fkpxc9em4mhhhymes72frtqxkjckgjbvwn5cdnzy4e b/support/tools/genvm-tool/fuzz/inputs/scheduling/w/j/wj4fkpxc9em4mhhhymes72frtqxkjckgjbvwn5cdnzy4e new file mode 100644 index 00000000..d17f3443 --- /dev/null +++ b/support/tools/genvm-tool/fuzz/inputs/scheduling/w/j/wj4fkpxc9em4mhhhymes72frtqxkjckgjbvwn5cdnzy4e @@ -0,0 +1,3 @@ +`h +d›—h +Cdh diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/x/2/x25dmm6a6rbny7cwfsbxz7p2ccdjg3r42cfxqjaqj0vea b/support/tools/genvm-tool/fuzz/inputs/scheduling/x/2/x25dmm6a6rbny7cwfsbxz7p2ccdjg3r42cfxqjaqj0vea new file mode 100644 index 00000000..e9efb02b --- /dev/null +++ b/support/tools/genvm-tool/fuzz/inputs/scheduling/x/2/x25dmm6a6rbny7cwfsbxz7p2ccdjg3r42cfxqjaqj0vea @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/x/3/x36pbrm7ks6yytqpe4eye2n8zbhh15n3fnkng3p5m9a9w b/support/tools/genvm-tool/fuzz/inputs/scheduling/x/3/x36pbrm7ks6yytqpe4eye2n8zbhh15n3fnkng3p5m9a9w new file mode 100644 index 0000000000000000000000000000000000000000..90522c992da10637ea7fdcef4289cfc5f5d09b86 GIT binary patch literal 25 gcmYe!4opeONbxggaPCTOU`S3*=BfzG;7UpX0BtD<A^-pY literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/x/f/xfnb5rmw43mb4w66amzywjvy5hrdy2zj8s9wy15g2vfsc b/support/tools/genvm-tool/fuzz/inputs/scheduling/x/f/xfnb5rmw43mb4w66amzywjvy5hrdy2zj8s9wy15g2vfsc new file mode 100644 index 00000000..9bc92f3d --- /dev/null +++ b/support/tools/genvm-tool/fuzz/inputs/scheduling/x/f/xfnb5rmw43mb4w66amzywjvy5hrdy2zj8s9wy15g2vfsc @@ -0,0 +1 @@ +‚wh \ No newline at end of file diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/x/h/xhebxpwxedbn894hgq2x212xe7nc1ky9brgnqpt9emw6y b/support/tools/genvm-tool/fuzz/inputs/scheduling/x/h/xhebxpwxedbn894hgq2x212xe7nc1ky9brgnqpt9emw6y new file mode 100644 index 0000000000000000000000000000000000000000..fd19bd007259fc1c310427f2dd89c25eb48bfa2d GIT binary patch literal 24 gcmZSZtxw4~fBw8e_=WTDFFP>!*8e|$<^1`}0GYrI{Qv*} literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/x/x/xxthw0f8kdytdk9ww8p3pb17seayx23p73ye02px68sra b/support/tools/genvm-tool/fuzz/inputs/scheduling/x/x/xxthw0f8kdytdk9ww8p3pb17seayx23p73ye02px68sra new file mode 100644 index 00000000..40c042a5 --- /dev/null +++ b/support/tools/genvm-tool/fuzz/inputs/scheduling/x/x/xxthw0f8kdytdk9ww8p3pb17seayx23p73ye02px68sra @@ -0,0 +1 @@ +ch diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/x/y/xyechygnfyvjc98mxcqa33pxqdtny51d3avkgy5rqrmbc b/support/tools/genvm-tool/fuzz/inputs/scheduling/x/y/xyechygnfyvjc98mxcqa33pxqdtny51d3avkgy5rqrmbc new file mode 100644 index 0000000000000000000000000000000000000000..86c36258438f0b8164ecf2fd81024a516cdfc880 GIT binary patch literal 53 ncmezW5(pR=7#u+K%Jb(z6qs@TKM<V%zhJ?Fi3?B#|Gx$RL-Q)A literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/y/1/y1x97drzaeq8r3a3tsb44s1xhd9rq5w34jceqcf0cnzk2 b/support/tools/genvm-tool/fuzz/inputs/scheduling/y/1/y1x97drzaeq8r3a3tsb44s1xhd9rq5w34jceqcf0cnzk2 new file mode 100644 index 0000000000000000000000000000000000000000..5e7c196d9d586b9bed53565209128837d7fd665d GIT binary patch literal 64 ncmezW5(pR=7#u+K%Jb(z6qs@TKM;Tk5aa&>1_*^J|Nk`rey%mj literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/y/3/y3sn7080jpjd06ttd23q11f1gvf7e8x2vksq2987e74x6 b/support/tools/genvm-tool/fuzz/inputs/scheduling/y/3/y3sn7080jpjd06ttd23q11f1gvf7e8x2vksq2987e74x6 new file mode 100644 index 0000000000000000000000000000000000000000..28ca77f7b82db0fa75236c8a1f4903a3b1bc7886 GIT binary patch literal 8 PcmYdhWcZz&@jn>=4Ko9c literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/y/b/yb13jgagqpa7t8wm0p1cm563cqzs3t7bnma3b1c15ks70 b/support/tools/genvm-tool/fuzz/inputs/scheduling/y/b/yb13jgagqpa7t8wm0p1cm563cqzs3t7bnma3b1c15ks70 new file mode 100644 index 00000000..b538e360 --- /dev/null +++ b/support/tools/genvm-tool/fuzz/inputs/scheduling/y/b/yb13jgagqpa7t8wm0p1cm563cqzs3t7bnma3b1c15ks70 @@ -0,0 +1 @@ +cttttttt»»»»»»»»»»»»»»»»»»»»»»tttttht diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/y/e/yep2zs7xpq5936mtf5cachrha72j0j1512g1c8hbgvmnm b/support/tools/genvm-tool/fuzz/inputs/scheduling/y/e/yep2zs7xpq5936mtf5cachrha72j0j1512g1c8hbgvmnm new file mode 100644 index 00000000..c194d9bc --- /dev/null +++ b/support/tools/genvm-tool/fuzz/inputs/scheduling/y/e/yep2zs7xpq5936mtf5cachrha72j0j1512g1c8hbgvmnm @@ -0,0 +1,2 @@ +ΞΞΞΞΞΞάΞΞΞιΞΞΞΞΞΞΞΞ + diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/y/m/ym3qhn9hdmjz2f11dxm70nzrwmb4akc4rh536xmx8mkvj b/support/tools/genvm-tool/fuzz/inputs/scheduling/y/m/ym3qhn9hdmjz2f11dxm70nzrwmb4akc4rh536xmx8mkvj new file mode 100644 index 0000000000000000000000000000000000000000..e6efd8edf27098e218b07de52545f337caf07d03 GIT binary patch literal 4 LcmYdhVB!J*0iFPG literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/z/1/z1tpx2ffwwzkhmysrgp72696z3nwm1b9c0w0rjqs21ge0 b/support/tools/genvm-tool/fuzz/inputs/scheduling/z/1/z1tpx2ffwwzkhmysrgp72696z3nwm1b9c0w0rjqs21ge0 new file mode 100644 index 00000000..0395fed0 --- /dev/null +++ b/support/tools/genvm-tool/fuzz/inputs/scheduling/z/1/z1tpx2ffwwzkhmysrgp72696z3nwm1b9c0w0rjqs21ge0 @@ -0,0 +1 @@ +cd555555 \ No newline at end of file diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/z/a/za6qeb27pkvczyyz2b75c59yaa5c9sghmgrekcr80ea28 b/support/tools/genvm-tool/fuzz/inputs/scheduling/z/a/za6qeb27pkvczyyz2b75c59yaa5c9sghmgrekcr80ea28 new file mode 100644 index 0000000000000000000000000000000000000000..760fa2c36c4771911b543de349555e5757f49df5 GIT binary patch literal 25 gcmYe!4opeONbxggaPCTOU`kF-=BfzG;7UpX0Bt`ABme*a literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/z/e/zef9y9cs4pzfg1bwv73k271wp240w8gnpgdbs1532n3mp b/support/tools/genvm-tool/fuzz/inputs/scheduling/z/e/zef9y9cs4pzfg1bwv73k271wp240w8gnpgdbs1532n3mp new file mode 100644 index 00000000..70e179a8 --- /dev/null +++ b/support/tools/genvm-tool/fuzz/inputs/scheduling/z/e/zef9y9cs4pzfg1bwv73k271wp240w8gnpgdbs1532n3mp @@ -0,0 +1,2 @@ +žžžžžžh +h \ No newline at end of file diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/z/j/zj9tvrqmg8a20yn6zjg040fznngbfzqc6wd8hxqjza3de b/support/tools/genvm-tool/fuzz/inputs/scheduling/z/j/zj9tvrqmg8a20yn6zjg040fznngbfzqc6wd8hxqjza3de new file mode 100644 index 00000000..e6c0fb33 --- /dev/null +++ b/support/tools/genvm-tool/fuzz/inputs/scheduling/z/j/zj9tvrqmg8a20yn6zjg040fznngbfzqc6wd8hxqjza3de @@ -0,0 +1 @@ +1G11511111111111tY4w0sH9π \ No newline at end of file diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/z/m/zm2bj14d4d6jx6t9dwevjz9qjy9hvkm6yq0zsvhq0traa b/support/tools/genvm-tool/fuzz/inputs/scheduling/z/m/zm2bj14d4d6jx6t9dwevjz9qjy9hvkm6yq0zsvhq0traa new file mode 100644 index 0000000000000000000000000000000000000000..64d860c4ae67afc7a674efd6082fe0dc681b41fe GIT binary patch literal 13 RcmezWKO-3k8W?~C7XUFA1*QN1 literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/z/t/ztgvvbrx1tcewc1h1932qkz4qm9yjdbs7g3072ackqg06 b/support/tools/genvm-tool/fuzz/inputs/scheduling/z/t/ztgvvbrx1tcewc1h1932qkz4qm9yjdbs7g3072ackqg06 new file mode 100644 index 00000000..a1d4813f --- /dev/null +++ b/support/tools/genvm-tool/fuzz/inputs/scheduling/z/t/ztgvvbrx1tcewc1h1932qkz4qm9yjdbs7g3072ackqg06 @@ -0,0 +1 @@ +odh diff --git a/support/tools/genvm-tool/fuzz/inputs/scheduling/z/t/ztkrh5101zazqjygn9mnf19aa05ybkq9hyh1w1as4pbjc b/support/tools/genvm-tool/fuzz/inputs/scheduling/z/t/ztkrh5101zazqjygn9mnf19aa05ybkq9hyh1w1as4pbjc new file mode 100644 index 0000000000000000000000000000000000000000..ce6bfaae95f4f1f34b4b49487ea0c4e1cd9d73f0 GIT binary patch literal 86 zcmd-q=`JbxU;qDq!^@YucbAluynOib<x2q|hJcmlAz<%+pb`l4ArS26P64Uky;~N@ XIe+bZMl(Z7ax#~byuAE*&hzI1mF7B? literal 0 HcmV?d00001 diff --git a/support/tools/genvm-tool/fuzz/src/scheduling.py b/support/tools/genvm-tool/fuzz/src/scheduling.py new file mode 100755 index 00000000..fff32a8f --- /dev/null +++ b/support/tools/genvm-tool/fuzz/src/scheduling.py @@ -0,0 +1,346 @@ +#!/usr/bin/env python3 + +import sys +from argparse import Namespace +from collections.abc import Iterable +from dataclasses import dataclass + +import genvm_tool.tests +from genvm_tool.tests.stage import scheduling +from genvm_tool.tests.stage.collection import Env, Semaphore, Service + + +MAX_SERVICES = 8 +MAX_SEMAPHORES = 4 +MAX_CASES = 12 + + +class _Service(genvm_tool.tests.exec.service.Service): + async def start(self): + raise AssertionError('fuzz scheduling must not start services') + + +@dataclass +class Reader: + data: bytes + offset: int = 0 + + def byte(self) -> int: + if self.offset >= len(self.data): + return 0 + result = self.data[self.offset] + self.offset += 1 + return result + + def mask(self, width: int) -> int: + result = 0 + for shift in range(0, width, 8): + result |= self.byte() << shift + return result + + +def _service_closure(service: Service) -> set[Service]: + result: set[Service] = set() + pending = [service] + while pending: + current = pending.pop() + if current in result: + continue + result.add(current) + pending.extend(current.depends_on or []) + return result + + +def _closure_of(services: Iterable[Service]) -> set[Service]: + result: set[Service] = set() + for service in services: + result.update(_service_closure(service)) + return result + + +def _case_services(case: genvm_tool.tests.test.Case) -> set[Service]: + return _closure_of(case.description.needed_services) + + +def _conflict_free(services: set[Service]) -> bool: + claimed: set[Semaphore] = set() + for service in services: + if claimed & service.semaphores: + return False + claimed.update(service.semaphores) + return True + + +def _make_services(reader: Reader) -> list[Service]: + semaphore_count = reader.byte() % (MAX_SEMAPHORES + 1) + semaphores = [Semaphore(f'mutex-{index}') for index in range(semaphore_count)] + services: list[Service] = [] + + for index in range(reader.byte() % (MAX_SERVICES + 1)): + semaphore_mask = reader.mask(semaphore_count) + service = Service( + name=f'service-{index}', + manager=_Service(), + semaphores=frozenset( + semaphore + for bit, semaphore in enumerate(semaphores) + if semaphore_mask & (1 << bit) + ), + order=index, + depends_on=[], + ) + # A service whose closure holds two owners of one semaphore could never be + # up, since starting the second one stops the first + dependency_mask = reader.mask(index) + closure = {service} + for bit, dependency in enumerate(services): + if not dependency_mask & (1 << bit): + continue + candidate = closure | _service_closure(dependency) + if _conflict_free(candidate): + service.depends_on.append(dependency) + closure = candidate + services.append(service) + + return services + + +def _cover_case(service: Service, index: int) -> genvm_tool.tests.test.Case: + return genvm_tool.tests.test.StepsCase( + description=genvm_tool.tests.test.Description( + name=f'cover-{index}', + needed_services=frozenset({service}), + ), + steps=[], + ) + + +def _service_timeline(services: list[Service]) -> list[Service]: + """ + The order the scheduler starts ``services`` in, when every one of them is + needed and no test dependency constrains their lifetimes. + + Taken from a probe schedule rather than recomputed here: the timeline is the + scheduler's own policy, and a harness reimplementing it would only be + checking its copy against the original. What the invariants below check is + the meaning of a schedule, not this order. + """ + probe = Env( + cases=[_cover_case(service, index) for index, service in enumerate(services)], + args=Namespace(), + ) + return [ + action.service + for action in scheduling.run(None, probe).actions + if isinstance(action, scheduling.StartService) + ] + + +class _Timeline: + """Which services can be up together, given the order they are started in.""" + + def __init__(self, services: list[Service]): + self.order = _service_timeline(services) + self.position = {service: index for index, service in enumerate(self.order)} + + def _survives(self, service: Service, until: int) -> bool: + # Starting a service stops every owner of a semaphore it takes, and + # whatever depends on those transitively + closure = _service_closure(service) + for other in self.order[self.position[service] + 1 : until + 1]: + if any( + member.semaphores & other.semaphores + for member in closure + if member is not other + ): + return False + return True + + def phase(self, services: set[Service]) -> int: + """When the last of ``services`` starts, or -1 for a case needing none.""" + return max((self.position[service] for service in services), default=-1) + + def can_be_up_together(self, services: set[Service]) -> bool: + until = self.phase(services) + return all(self._survives(service, until) for service in services) + + def can_precede(self, before: set[Service], after: set[Service]) -> bool: + """ + Whether a case on ``before`` may be a dependency of one on ``after``. + + Their shared semaphores are handed over in one direction only, and the + dependency has to have run by the time the dependent's services are up + """ + if self.phase(before) > self.phase(after): + return False + return all( + self.position[left] < self.position[right] + for left in before + for right in after + if left is not right and left.semaphores & right.semaphores + ) + + +def _make_input(data: bytes) -> Env: + reader = Reader(data) + services = _make_services(reader) + timeline = _Timeline(services) + + # Every service is covered, so the timeline the cases are built against is the + # one the scheduler ends up using. A service no case needs is not an input the + # scheduler ever sees: it reaches it through the cases alone + cases = [_cover_case(service, index) for index, service in enumerate(services)] + case_services: list[set[Service]] = [_case_services(case) for case in cases] + case_dependencies: list[set[int]] = [set() for _ in cases] + + for index in range(reader.byte() % (MAX_CASES + 1)): + service_mask = reader.mask(len(services)) + needed: set[Service] = set() + for bit, service in enumerate(services): + if not service_mask & (1 << bit): + continue + candidate = _closure_of(needed | {service}) + if timeline.can_be_up_together(candidate): + needed.add(service) + required = _closure_of(needed) + + dependency_mask = reader.mask(len(cases)) + dependencies: set[int] = set() + for bit in range(len(cases)): + if not dependency_mask & (1 << bit): + continue + candidate_dependencies = {bit} | case_dependencies[bit] + if all( + timeline.can_precede(case_services[dependency], required) + for dependency in candidate_dependencies + ): + dependencies.update(candidate_dependencies) + + case_services.append(required) + case_dependencies.append(dependencies) + cases.append( + genvm_tool.tests.test.StepsCase( + description=genvm_tool.tests.test.Description( + name=f'case-{index}', + needed_services=frozenset(needed), + console_pool=bool(reader.byte() & 1), + depends_on=frozenset( + cases[dependency].description.name for dependency in dependencies + ), + ), + steps=[], + ) + ) + + # Collection order is whatever the suites happened to register, and the + # schedule must not depend on it + for index in range(len(cases) - 1, 0, -1): + other = reader.byte() % (index + 1) + cases[index], cases[other] = cases[other], cases[index] + + return Env(cases=cases, args=Namespace()) + + +def _test_dependencies( + case: genvm_tool.tests.test.Case, + cases_by_name: dict[str, genvm_tool.tests.test.Case], +) -> set[str]: + result: set[str] = set() + pending = list(case.description.depends_on) + while pending: + name = pending.pop() + if name in result: + continue + result.add(name) + pending.extend(cases_by_name[name].description.depends_on) + return result + + +def _check_invariants( + cases: list[genvm_tool.tests.test.Case], + actions: list[scheduling.Action], +) -> None: + cases_by_name = {case.description.name: case for case in cases} + needed_services = {service for case in cases for service in _case_services(case)} + active_services: set[Service] = set() + started_services: set[Service] = set() + stopped_services: set[Service] = set() + started_cases: set[str] = set() + running_batches: dict[int, list[genvm_tool.tests.test.Case]] = {} + seen_batches: set[int] = set() + + for action in actions: + if isinstance(action, scheduling.StartService): + service = action.service + assert service in needed_services + assert service not in started_services + assert set(service.depends_on or []).issubset(active_services) + assert all( + not service.semaphores & active.semaphores for active in active_services + ) + started_services.add(service) + active_services.add(service) + elif isinstance(action, scheduling.StopService): + service = action.service + assert service in active_services + assert all( + service not in _service_closure(active) - {active} for active in active_services + ) + assert all( + service not in _case_services(case) + for batch in running_batches.values() + for case in batch + ) + active_services.remove(service) + stopped_services.add(service) + elif isinstance(action, scheduling.StartCases): + assert action.id not in seen_batches + assert action.cases + assert len({case.description.name for case in action.cases}) == len(action.cases) + previously_started = set(started_cases) + for case in action.cases: + assert cases_by_name.get(case.description.name) is case + assert case.description.name not in started_cases + assert _case_services(case).issubset(active_services) + if case.description.console_pool: + # It holds the whole case pool, so nothing it waits for may + # still be waiting for the runner to start it + assert len(action.cases) == 1 + assert _test_dependencies(case, cases_by_name).issubset(previously_started) + started_cases.add(case.description.name) + seen_batches.add(action.id) + running_batches[action.id] = action.cases + elif isinstance(action, scheduling.AwaitAllCases): + assert action.id in running_batches + batch = running_batches.pop(action.id) + for case in batch: + # The runner blocks here, so a dependency that has not started by + # now never will + assert _test_dependencies(case, cases_by_name).issubset(started_cases) + else: + raise AssertionError(f'unknown scheduling action: {action!r}') + + assert started_cases == set(cases_by_name) + assert started_services == needed_services + assert stopped_services == needed_services + assert not active_services + assert not running_batches + + +def fuzz(data: bytes) -> None: + collection_env = _make_input(data) + schedule = scheduling.run(None, collection_env) + _check_invariants(collection_env.cases, schedule.actions) + + +def main() -> None: + import afl + + stream = sys.stdin.buffer + while afl.loop(1000): + stream.seek(0) + fuzz(stream.read()) + + +if __name__ == '__main__': + main() diff --git a/support/tools/genvm-tool/genvm_tool/__init__.py b/support/tools/genvm-tool/genvm_tool/__init__.py index 84e96444..d5838fef 100644 --- a/support/tools/genvm-tool/genvm_tool/__init__.py +++ b/support/tools/genvm-tool/genvm_tool/__init__.py @@ -4,7 +4,9 @@ 'common', 'formatter', 'gvm32', + 'io', 'manifest', + 'misc', ) @@ -12,5 +14,7 @@ common, formatter, gvm32, + io, manifest, + misc, ) diff --git a/support/tools/genvm-tool/genvm_tool/__main__.py b/support/tools/genvm-tool/genvm_tool/__main__.py index 9cd56b92..b0e68bc0 100644 --- a/support/tools/genvm-tool/genvm_tool/__main__.py +++ b/support/tools/genvm-tool/genvm_tool/__main__.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 -"""Command-line interface for genvm-tool. +""" +Command-line interface for genvm-tool. Top-level commands (`configure`, `test`) sit alongside command *groups* (`git` with `git ls` / `git list-repo`). Each group is a sub-package exposing @@ -24,13 +25,21 @@ cmd_codegen, cmd_configure, cmd_docs, + cmd_fuzz_corpus, cmd_test, common, formatter, git, ) -TOPLEVEL = [cmd_configure, cmd_test, cmd_build_manifest, cmd_codegen, cmd_docs] +TOPLEVEL = [ + cmd_configure, + cmd_test, + cmd_build_manifest, + cmd_codegen, + cmd_docs, + cmd_fuzz_corpus, +] GROUPS = [git] # One-line tagline for the man page NAME line and completion header; the fuller @@ -67,7 +76,8 @@ def _rst_to_plain(text: str) -> str: def _long_help(mod): - """The module docstring as a subparser `description` (falls back to HELP). + """ + The module docstring as a subparser `description` (falls back to HELP). `inspect.getdoc` dedents and strips; `_rst_to_plain` drops the RST markup so the rich per-command docstrings feed both the subcommand's `-h` and the man @@ -88,7 +98,8 @@ def _add_leaf(subparsers, mod) -> None: class _PrintManpageAction(argparse.Action): - """`--print-manpage`: emit a roff man page for the whole tree, then exit. + """ + `--print-manpage`: emit a roff man page for the whole tree, then exit. The manager's `default.nix` pipes this into `installManPage` at build time, mirroring how shtab's `--print-completion` feeds `installShellCompletion`. diff --git a/support/tools/genvm-tool/genvm_tool/cmd_build_manifest.py b/support/tools/genvm-tool/genvm_tool/cmd_build_manifest.py index 0058c0fe..c6d5e164 100644 --- a/support/tools/genvm-tool/genvm_tool/cmd_build_manifest.py +++ b/support/tools/genvm-tool/genvm_tool/cmd_build_manifest.py @@ -1,4 +1,5 @@ -"""`genvm-tool build-manifest` β€” generate the manager's `data/manifest.yaml`. +""" +`genvm-tool build-manifest` β€” generate the manager's `data/manifest.yaml`. Writes the runtime manifest (executor versions from the active submodules plus the static base fields) to `--output`. Used by release packaging; dev builds get diff --git a/support/tools/genvm-tool/genvm_tool/cmd_codegen.py b/support/tools/genvm-tool/genvm_tool/cmd_codegen.py index 7bbe5923..505bb122 100644 --- a/support/tools/genvm-tool/genvm_tool/cmd_codegen.py +++ b/support/tools/genvm-tool/genvm_tool/cmd_codegen.py @@ -1,4 +1,5 @@ -"""`genvm-tool codegen` β€” generate language bindings from genvm data JSON. +""" +`genvm-tool codegen` β€” generate language bindings from genvm data JSON. Renders a data JSON file (`-i`) to a `--lang` backend (rust/python/rst/go) at `-o`. The build runs this for the rust/python/rst outputs; `--lang go` is a diff --git a/support/tools/genvm-tool/genvm_tool/cmd_configure.py b/support/tools/genvm-tool/genvm_tool/cmd_configure.py index 25bcbce0..ad1285ec 100644 --- a/support/tools/genvm-tool/genvm_tool/cmd_configure.py +++ b/support/tools/genvm-tool/genvm_tool/cmd_configure.py @@ -1,4 +1,5 @@ -"""`genvm-tool configure` β€” generate the build graph. +""" +`genvm-tool configure` β€” generate the build graph. Writes `build/build.ninja` and `build/info.json`, from which `ninja -C build` drives the cargo builds, `genvm-tool codegen` outputs, the runner build, and the @@ -26,7 +27,8 @@ class ExecVersion(NamedTuple): - """One active executor line resolved to its on-disk locations. + """ + One active executor line resolved to its on-disk locations. - ``key``: the top-level version line (e.g. ``v0.3``) as listed in ``.genvm-monorepo-root``. @@ -42,7 +44,8 @@ class ExecVersion(NamedTuple): def _load_versions(monorepo_cfg, source_dir: Path) -> tuple[str, list[ExecVersion]]: - """Resolve every active executor line from ``.genvm-monorepo-root``. + """ + Resolve every active executor line from ``.genvm-monorepo-root``. Each active line is mounted at ``executors/<line>.x`` and its ``manifest.json`` pins the concrete ``executor-version`` that becomes the @@ -59,7 +62,8 @@ def _load_versions(monorepo_cfg, source_dir: Path) -> tuple[str, list[ExecVersio def build_independent_info(monorepo_cfg, source_dir: Path) -> dict: - """The parts of ``info.json`` derivable from source alone β€” no build needed. + """ + The parts of ``info.json`` derivable from source alone β€” no build needed. ``executor_versions`` maps each active line (e.g. ``v0.3``) to its built ``out/executor/<real>`` directory name; ``primary_executor_version`` is the @@ -79,7 +83,8 @@ def build_independent_info(monorepo_cfg, source_dir: Path) -> dict: def base_info( monorepo_cfg, source_dir: Path, build_dir: Path, rust_target_dir: Path ) -> dict: - """Every ``info.json`` key that does not depend on having built anything. + """ + Every ``info.json`` key that does not depend on having built anything. Two commands write ``info.json``: ``configure``, and the test harness when it finds none (so CI can run tests without configuring first, see the @@ -102,7 +107,8 @@ def base_info( def rust_target_dirs_info( monorepo_cfg, source_dir: Path, rust_target_dir: Path ) -> dict: - """``rust_target_dirs``: line checkout (``executors/v0.3.x``) β†’ its cargo target dir. + """ + ``rust_target_dirs``: line checkout (``executors/v0.3.x``) β†’ its cargo target dir. Keyed by mount so a consumer only has to ask which one contains its crate; crates under none of them use ``rust_target_dir`` itself. Why lines cannot @@ -138,7 +144,8 @@ def detect_rust_target() -> str: def _line_configurator(ctx: common.Context, source_dir: Path, exec_rel: str): - """Resolve a line's `configure(line)` hook from its `.genvm-tool.py`. + """ + Resolve a line's `configure(line)` hook from its `.genvm-tool.py`. Falls back to the plugin's default (committed-registry-or-nix) when the line carries no hook, so a line without one still configures sensibly. @@ -315,9 +322,12 @@ def main(ctx: common.Context, args) -> int: ) public_abi_py = source_dir / 'tests/runner/origin/public_abi.py' constants_rst = source_dir / 'docs/website/src/spec/appendix/constants.rst' - # Pending public-abi constants (ADR-012): documented in their own appendix - # page so the spec can reference them, without feeding the runner-hashed - # `public_abi.py`. Sourced from the primary line's pending data file. + internal_constants_rst = ( + source_dir / 'docs/website/src/spec/appendix/internal-constants.rst' + ) + # Public-ABI constants staged for a future release: documented in their own + # appendix page so the spec can reference them, without feeding the + # runner-hashed `public_abi.py`. Empty while nothing is staged. constants_pending_rst = ( source_dir / 'docs/website/src/spec/appendix/constants-pending.rst' ) @@ -329,6 +339,11 @@ def main(ctx: common.Context, args) -> int: n.codegen(manager_socket_consts_rst, 'rst', shared_data / 'manager-api.json') n.codegen(public_abi_py, 'python', p_data / 'public-abi.json') n.codegen(constants_rst, 'rst', p_data / 'public-abi.json') + n.codegen( + internal_constants_rst, + 'rst', + p_data / 'internal-constants.json', + ) n.codegen( constants_pending_rst, 'rst', @@ -343,6 +358,7 @@ def main(ctx: common.Context, args) -> int: manager_socket_consts_rst, public_abi_py, constants_rst, + internal_constants_rst, constants_pending_rst, ): codegen_phony.add_dependency(out) @@ -352,6 +368,7 @@ def main(ctx: common.Context, args) -> int: # CI mode forbids touching Cargo.lock during the build. locked = ['--locked'] if args.ci else [] + # Lint edges (clippy) never create `$out`, so ninja re-runs them every time. n.rule( 'cargo', command=[ @@ -368,12 +385,6 @@ def main(ctx: common.Context, args) -> int: ninja.RawStr('$target_dir'), *locked, ninja.RawStr('$extra_args'), - ninja.AND, - 'cd', - str(build_dir), - ninja.AND, - 'touch', - ninja.VAR_OUT, ], description='Running cargo $subcommand', pool='console', diff --git a/support/tools/genvm-tool/genvm_tool/cmd_docs.py b/support/tools/genvm-tool/genvm_tool/cmd_docs.py index 1cd05782..6b034e7f 100644 --- a/support/tools/genvm-tool/genvm_tool/cmd_docs.py +++ b/support/tools/genvm-tool/genvm_tool/cmd_docs.py @@ -1,4 +1,5 @@ -"""`genvm-tool docs` β€” list the contributor documentation. +""" +`genvm-tool docs` β€” list the contributor documentation. Walks each quadrant of `docs/contributing/` (tutorial, how-to, explanation), reads every page's one-line description from that quadrant's `README.md` index, and prints diff --git a/support/tools/genvm-tool/genvm_tool/cmd_fuzz_corpus.py b/support/tools/genvm-tool/genvm_tool/cmd_fuzz_corpus.py new file mode 100644 index 00000000..17e21b5f --- /dev/null +++ b/support/tools/genvm-tool/genvm_tool/cmd_fuzz_corpus.py @@ -0,0 +1,429 @@ +""" +`genvm-tool fuzz-corpus` β€” seed the fuzz corpora from a finished test run. + +The integration suite encodes real calldata, reads real contract code and +collects real leader results on every run, then leaves them in the artifacts +dir. This walks that dir, maps each blob to the fuzz targets that consume its +format, and writes the new ones into the committed `fuzz/inputs-<target>` +corpora, named by content hash exactly as `test run --fuzz-update-corpus` does. + +Meant to be run by hand after a suite run, and followed by a fuzz run with +`--fuzz-update-corpus`: that one `cmin`s the union back down before the corpus +is committed. +""" + +import dataclasses +import json +import os +import pickle +import re +import shutil +import subprocess +import tempfile +from collections.abc import Iterator +from pathlib import Path + +from . import common, misc + +NAME = 'fuzz-corpus' +HELP = 'seed the fuzz corpora from a finished test run' + +# `arbitrary::Unstructured::int_in_range` eats the first byte and takes it modulo +# the number of variants, so a leading zero selects the variant that hands the +# rest of the input to the parser verbatim. A target taking `|data: &[u8]|` +# parses the whole input and needs no such byte. +_RAW = b'\x00' + + +@dataclasses.dataclass(frozen=True) +class Target: + """A fuzz target that accepts bytes a run produced, and how to frame them.""" + + crate: str + name: str + prefix: bytes = b'' + + +# What each kind of harvested blob is a seed for. Only the targets that take a +# format are listed: a `-structured` target and its op-sequence kin take a +# serialized value instead, which a run does not produce. Seed those by driving +# their mutator library, as `docs/contributing/howto/testing/fuzzing.md` describes. +# +# Each `-raw` target is paired with the un-split name the older lines still use, +# where the payload is still one `Arbitrary` variant behind a selector byte. A +# name no line has resolves to no corpus dir and is skipped. +TARGETS: dict[str, tuple[Target, ...]] = { + 'calldata': ( + Target('executor/crates/calldata', 'genvm-common-decode'), + Target('executor', 'entry-data-raw'), + Target('executor', 'entry-data', _RAW), + ), + 'code': ( + Target('executor', 'runners-parse-raw'), + Target('executor', 'runners-parse', _RAW), + ), + 'leader-result': ( + Target('executor', 'leader-result-raw'), + Target('executor', 'leader-result', _RAW), + ), +} + +_LINE_DIR = re.compile(r'v(\d+\.\d+)\.x') +# Fed to `afl-showmap`, matching what the fuzz cases use. +_EXEC_TIMEOUT_MS = '5000' + + +@dataclasses.dataclass(frozen=True) +class Blob: + """One recovered payload: `kind` says which format it is in.""" + + line: str + kind: str + data: bytes + origin: str + + +@dataclasses.dataclass +class Corpus: + """The seeds one target gained, against what its corpus already holds.""" + + line: str + target: Target + inputs: Path + known: set[str] + seeds: dict[str, bytes] = dataclasses.field(default_factory=dict) + + +def configure(parser): + parser.add_argument( + '--artifacts-dir', + help='run to harvest, defaults to the configured artifacts dir', + ) + parser.add_argument( + '--max-size', + type=int, + default=64 * 1024, + help='drop a seed above this many bytes (default: 65536)', + ) + parser.add_argument( + '-n', + '--dry-run', + action='store_true', + help='report what would be added without writing it', + ) + parser.add_argument( + '--no-verify', + action='store_true', + help='keep every new seed, without checking with afl-showmap that it ' + 'reaches code the corpus does not', + ) + + +def main(ctx: common.Context, args) -> int: + config = common.monorepo_config(ctx.root) + if args.artifacts_dir: + artifacts = Path(args.artifacts_dir) + else: + artifacts = ctx.root / config.get('artifacts_dir', 'build/test-artifacts') + cases = artifacts / 'cases' + if not cases.is_dir(): + raise common.ToolError(f'no case artifacts under {cases}: run the suite first') + + # The blobs are recovered with the repo's own encoders, which live on the + # test runner's import path rather than in this package. + common.add_extra_python_paths(ctx.root) + + plan = _plan(ctx, args, _blobs(ctx, cases)) + verify = not args.no_verify and _have_showmap(ctx) + + total = 0 + for corpus in sorted(plan.values(), key=lambda c: c.inputs): + seeds = _verified(ctx, corpus) if verify else corpus.seeds + if not seeds: + continue + if not args.dry_run: + for name, data in seeds.items(): + path = misc.fuzz_input_path(corpus.inputs, name) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + total += len(seeds) + ctx.printer.put( + 'seeded', + corpus=str(corpus.inputs.relative_to(ctx.root)), + added=len(seeds), + had=len(corpus.known), + ) + + ctx.printer.put('fuzz-corpus', added=total, dry_run=bool(args.dry_run)) + return 0 + + +# --- harvesting ------------------------------------------------------------- + + +def _case_line(path: Path, cases: Path) -> str | None: + """The executor line an artifact belongs to, or None when it names none.""" + parts = path.relative_to(cases).parts + if len(parts) < 2 or parts[0] != 'executors': + return None + match = _LINE_DIR.fullmatch(parts[1]) + return match.group(1) if match else None + + +def _blobs(ctx: common.Context, cases: Path) -> Iterator[Blob]: + """Every payload a finished run left behind, in no particular order.""" + import genvm_tool_plugins.runners as runners + from gvm_extra.case_inputs import encode_calldata + + unresolved = '' + for config_path in sorted(cases.rglob('config.json')): + line = _case_line(config_path, cases) + if line is None: + continue + conf = json.loads(config_path.read_text()) + if 'calldata' not in conf: # not an integration case + continue + + try: + calldata = encode_calldata(conf) + except Exception as exc: + print(f'{config_path}: calldata does not evaluate: {exc}') + else: + yield Blob(line, 'calldata', calldata, str(config_path)) + + code = _code_path(conf, config_path.parent) + if code is None: + continue + data = code.read_bytes() + try: + data = runners.substitute(data, ctx.root) + except (KeyError, FileNotFoundError) as exc: + # A uid token only differs from the built uid by its bytes, so the + # seed is still worth keeping; the run itself would have failed. + unresolved = str(exc) + yield Blob(line, 'code', data, str(code)) + + if unresolved: + ctx.logger.warning('kept runner uid tokens verbatim', reason=unresolved) + + for pickle_path in sorted(cases.rglob('*_leader_nondet.pickle')): + line = _case_line(pickle_path, cases) + if line is None: + continue + # Written by the leader step of the same suite, as `code || payload` + # frames -- which is what a validator's parser is handed. + for index, result in enumerate(pickle.loads(pickle_path.read_bytes())): + yield Blob(line, 'leader-result', bytes(result), f'{pickle_path}#{index}') + + +def _code_path(conf: dict, case_dir: Path) -> Path | None: + """The contract code as the executor saw it, or None if the case ships none.""" + code = conf.get('code') + if code is None: + return None + path = Path(code) + if path.suffix == '.wat': + # `.wat` is a source the run compiles next to the config; the parser only + # ever sees the module. + path = case_dir / path.with_suffix('.wasm').name + return path if path.is_file() else None + + +# --- planning --------------------------------------------------------------- + + +def _plan(ctx: common.Context, args, blobs: Iterator[Blob]) -> dict[Path, Corpus]: + """Group the seeds worth adding by corpus, printing every rejection.""" + plan: dict[Path, Corpus] = {} + for blob in blobs: + for target in TARGETS[blob.kind]: + inputs = _inputs_dir(ctx.root, blob.line, target) + if inputs is None: # the line has no such target + continue + + data = target.prefix + blob.data + if len(data) > args.max_size: + print(f'{blob.origin}: {len(data)} bytes, over --max-size') + continue + host_path = _host_path(ctx.root, data) + if host_path is not None: + print(f'{blob.origin}: embeds a host path ({host_path})') + continue + + corpus = plan.get(inputs) + if corpus is None: + known = _known(inputs) | { + misc.fuzz_input_name(data) for data in _curated(inputs).values() + } + corpus = Corpus(line=blob.line, target=target, inputs=inputs, known=known) + plan[inputs] = corpus + + name = misc.fuzz_input_name(data) + if name not in corpus.known and name not in corpus.seeds: + corpus.seeds[name] = data + return plan + + +def _known(inputs: Path) -> set[str]: + return {entry.name for entry in misc.fuzz_corpus_entries(inputs)} + + +def _curated(inputs: Path) -> dict[str, bytes]: + """ + The hand-written seeds beside a corpus, by their free-form file names. + + They are fed to the fuzzer as seeds and never rewritten, so a harvested blob + one of them already holds is not worth adding. + """ + curated = inputs.parent / f'{inputs.name}-curated' + if not curated.is_dir(): + return {} + return { + entry.name: entry.read_bytes() + for entry in sorted(curated.iterdir()) + if entry.is_file() and not entry.name.startswith('.') + } + + +def _inputs_dir(root: Path, line: str, target: Target) -> Path | None: + """A line's corpus dir for `target`, or None when that line lacks the target.""" + fuzz = root / 'executors' / f'v{line}.x' / target.crate / 'fuzz' + if not fuzz.joinpath(f'{target.name}.rs').is_file(): + return None + return fuzz / f'inputs-{target.name}' + + +def _host_path(root: Path, data: bytes) -> str | None: + """ + The first path of the harvesting machine `data` carries, if any. + + Corpus entries are committed, and a zip or a wasm built here embeds the + directory it was built in; that turns a seed into a record of somebody's + home dir. + """ + for candidate in (str(root), str(Path.home()), '/nix/store'): + if candidate.encode() in data: + return candidate + return None + + +# --- verification ----------------------------------------------------------- + + +def _have_showmap(ctx: common.Context) -> bool: + if shutil.which('cargo-afl'): + return True + ctx.logger.warning('cargo-afl not found, keeping every seed unverified') + return False + + +def _verified(ctx: common.Context, corpus: Corpus) -> dict[str, bytes]: + """ + The seeds that reach an edge the corpus does not already cover. + + A harvested blob can be rejected by the target for a reason of its own -- + the wrong line's format, a shape the parser drops at the first byte -- and + then it only makes the committed corpus bigger. + """ + binary = _fuzz_binary(ctx.root, corpus.line, corpus.target.name) + if binary is None: + ctx.logger.warning( + 'fuzz target is not built, keeping its seeds unverified', + target=corpus.target.name, + line=corpus.line, + ) + return corpus.seeds + + with tempfile.TemporaryDirectory() as tmp_name: + tmp = Path(tmp_name) + staged = tmp / 'seeds' + staged.mkdir() + for name, data in corpus.seeds.items(): + staged.joinpath(name).write_bytes(data) + + # What the committed seeds already reach: a harvested blob is only worth + # adding on top of both dirs, the curated one included + baseline = tmp / 'baseline' + baseline.mkdir() + for name, data in _curated(corpus.inputs).items(): + # Curated names are free-form, so they are not guaranteed to differ + # from the corpus' content hashes + baseline.joinpath(f'curated-{name}').write_bytes(data) + # Flattened: `showmap` keys its output by file name, which is unique anyway + for entry in misc.fuzz_corpus_entries(corpus.inputs): + shutil.copyfile(entry, baseline / entry.name) + + covered: set[int] = set() + if any(baseline.iterdir()): + for edges in _showmap(binary, baseline, tmp / 'corpus').values(): + covered |= edges + reached = _showmap(binary, staged, tmp / 'new') + + kept: dict[str, bytes] = {} + # Smallest first, so a seed is judged against the cheapest cover of its edges. + for name, data in sorted(corpus.seeds.items(), key=lambda item: len(item[1])): + edges = reached.get(name) + if not edges: + print(f'{corpus.inputs}/{name}: the target did not run it') + continue + if edges <= covered: + print(f'{corpus.inputs}/{name}: reaches nothing new') + continue + covered |= edges + kept[name] = data + return kept + + +def _showmap(binary: Path, inputs: Path, out: Path) -> dict[str, set[int]]: + """Edge ids each file in `inputs` reaches, batched through one forkserver.""" + env = dict(os.environ) + # The targets are persistent, so a run reseeds itself from the entropy the + # shim fakes; without it two runs of one input disagree on their edges. + preload = binary.parent.parent / 'libgenvm_fuzz_preload.so' + if preload.is_file(): + env['AFL_PRELOAD'] = str(preload) + + result = subprocess.run( + [ + 'cargo-afl', + 'afl', + 'showmap', + '-i', + str(inputs), + '-o', + str(out), + '-t', + _EXEC_TIMEOUT_MS, + '--', + str(binary), + ], + env=env, + capture_output=True, + text=True, + ) + if not out.is_dir(): + raise common.ToolError( + f'afl-showmap failed for {binary.name}: {result.stderr.strip()}' + ) + + edges: dict[str, set[int]] = {} + for path in out.iterdir(): + edges[path.name] = { + int(line.split(':', 1)[0]) for line in path.read_text().splitlines() if line + } + return edges + + +def _fuzz_binary(root: Path, line: str, name: str) -> Path | None: + """Where `afl build` puts a target, as `configure` recorded the target dirs.""" + info_path = root / 'build' / 'info.json' + if not info_path.is_file(): + return None + info = json.loads(info_path.read_text()) + target_dir = info.get('rust_target_dirs', {}).get(f'executors/v{line}.x') + if target_dir is None: + return None + binary = ( + Path(target_dir) / info['rust_target'] / 'debug' / 'examples' / f'fuzz-{name}' + ) + return binary if binary.is_file() else None diff --git a/support/tools/genvm-tool/genvm_tool/cmd_test.py b/support/tools/genvm-tool/genvm_tool/cmd_test.py index f8f781b2..b221ab73 100644 --- a/support/tools/genvm-tool/genvm_tool/cmd_test.py +++ b/support/tools/genvm-tool/genvm_tool/cmd_test.py @@ -1,4 +1,5 @@ -"""`genvm-tool test` β€” run the test suite. +""" +`genvm-tool test` β€” run the test suite. Everything after `test` is forwarded to the test runner, which has its own `run` and `show plan|test|services|tags` subcommands plus filter flags. Run diff --git a/support/tools/genvm-tool/genvm_tool/codegen/__init__.py b/support/tools/genvm-tool/genvm_tool/codegen/__init__.py index 05aac43d..0578fe8e 100644 --- a/support/tools/genvm-tool/genvm_tool/codegen/__init__.py +++ b/support/tools/genvm-tool/genvm_tool/codegen/__init__.py @@ -1,4 +1,5 @@ -"""`genvm-tool codegen` language backends. +""" +`genvm-tool codegen` language backends. A single front-end (:mod:`genvm_tool.codegen.model`) parses the codegen data JSON into a typed model + shared string-trie; each language module renders that model. diff --git a/support/tools/genvm-tool/genvm_tool/codegen/go.py b/support/tools/genvm-tool/genvm_tool/codegen/go.py index 818e5a8d..2561fde7 100644 --- a/support/tools/genvm-tool/genvm_tool/codegen/go.py +++ b/support/tools/genvm-tool/genvm_tool/codegen/go.py @@ -1,4 +1,5 @@ -"""Go backend for ``genvm-tool codegen``. +""" +Go backend for ``genvm-tool codegen``. Ports the standalone go generator (trie builders emit flat ``func``s rather than structs). The package name is configurable via ``--go-package`` since the @@ -73,8 +74,17 @@ def render(defs: list[Definition], *, go_package: str = 'genvm', **_opts) -> str buf.append(f'package {go_package}\n') needs_fmt = any(isinstance(d, StrTrie) and _has_param(d.root) for d in defs) - if needs_fmt: - buf.append('\nimport "fmt"\n') + needs_strings = any(isinstance(d, StrTrie) and d.suffix is not None for d in defs) + imports = [ + n for n, needed in (('fmt', needs_fmt), ('strings', needs_strings)) if needed + ] + if len(imports) == 1: + buf.append(f'\nimport "{imports[0]}"\n') + elif imports: + buf.append('\nimport (\n') + for name in imports: + buf.append(f'\t"{name}"\n') + buf.append(')\n') for d in defs: if isinstance(d, Enum): @@ -99,6 +109,14 @@ def render(defs: list[Definition], *, go_package: str = 'genvm', **_opts) -> str root_camel = to_camel(d.name) buf.append(f'\ntype {root_camel} string\n\n') _trie_inner(d.root, root_camel, buf) + for name, _parts in d.suffix.leaves if d.suffix else []: + # Mirrors the rust/python guard: a value carries at most one detail. + buf.append( + f'func (v {root_camel}) {to_camel(name)}() {root_camel} {{ ' + f'if strings.Contains(string(v), " # ") {{ ' + f'panic("a value carries at most one detail") }}; ' + f'return v + {_dump(f" # {name}")} }}\n' + ) return ''.join(buf) diff --git a/support/tools/genvm-tool/genvm_tool/codegen/model.py b/support/tools/genvm-tool/genvm_tool/codegen/model.py index a7912d99..e9f319c0 100644 --- a/support/tools/genvm-tool/genvm_tool/codegen/model.py +++ b/support/tools/genvm-tool/genvm_tool/codegen/model.py @@ -1,4 +1,5 @@ -"""Shared front-end for the ``genvm-tool codegen`` language backends. +""" +Shared front-end for the ``genvm-tool codegen`` language backends. Parses the codegen data JSON (a list of typed definitions) into a small typed model and provides the shared string-trie machinery every backend walks. The @@ -45,7 +46,8 @@ class Consts: @dataclasses.dataclass class TrieNode: - """One node of an unfolded string trie. + """ + One node of an unfolded string trie. ``suffix`` is the CamelCase concatenation of the path heads from the root to this node (``''`` at the root); backends use it to name the generated @@ -76,13 +78,18 @@ class StrTrie: root: TrieNode entries: list # unfolded entries, kept for flat path enumeration (rst) docs: dict[str, str] + # Details reachable from any built value, spelled ``<path> # <detail>``. A + # root-level ``#`` entry declares them; they are not paths, so they stay out + # of ``root``/``entries`` and out of the generated validity check. + suffix: TrieNode | None = None Definition = Enum | Const | Consts | StrTrie def _unfold(entry): - """Normalize a trie entry to ``{'head', 'tail'}`` form (recursively). + """ + Normalize a trie entry to ``{'head', 'tail'}`` form (recursively). A bare string ``"x"`` becomes ``{'head': 'x', 'tail': []}``; a dict whose ``tail`` is a list has each child unfolded; everything else (``$param`` tails) @@ -167,6 +174,29 @@ def enumerate_paths(entries, prefix: str = ''): return result +DETAIL_HEAD = '#' +"""Head of the root entry declaring a trie's detail suffixes.""" + + +def _split_details(name: str, entries: list) -> tuple[list, TrieNode | None]: + """Carve the ``#`` entry out of a trie's root, returning ``(paths, details)``.""" + found = [e for e in entries if e is not None and e['head'] == DETAIL_HEAD] + if not found: + return entries, None + if len(found) > 1: + raise ValueError(f'{name}: more than one {DETAIL_HEAD!r} entry') + tail = found[0]['tail'] + if not isinstance(tail, list): + raise ValueError(f'{name}: {DETAIL_HEAD!r} must list details') + for detail in tail: + # A detail is appended to an already-built value, so it has no place to + # hang a subtree off and no argument to render. + if detail is None or detail['tail'] != []: + raise ValueError(f'{name}: a detail may not be nested or take a parameter') + paths = [e for e in entries if e is None or e['head'] != DETAIL_HEAD] + return paths, _build(tail, [], '', False) + + def parse(data) -> list[Definition]: defs: list[Definition] = [] for t in data: @@ -178,13 +208,14 @@ def parse(data) -> list[Definition]: elif kind == 'consts': defs.append(Consts(t['name'], t['repr'], t['values'])) elif kind == 'str_trie': - entries = [_unfold(e) for e in t['values']] + entries, details = _split_details(t['name'], [_unfold(e) for e in t['values']]) defs.append( StrTrie( t['name'], _build(entries, [], '', False), entries, t.get('docs', {}), + details, ) ) else: diff --git a/support/tools/genvm-tool/genvm_tool/codegen/python.py b/support/tools/genvm-tool/genvm_tool/codegen/python.py index 6c07c3a5..122ad849 100644 --- a/support/tools/genvm-tool/genvm_tool/codegen/python.py +++ b/support/tools/genvm-tool/genvm_tool/codegen/python.py @@ -1,4 +1,5 @@ -"""Python backend for ``genvm-tool codegen`` (ports the old ``py.rb`` template). +""" +Python backend for ``genvm-tool codegen`` (ports the old ``py.rb`` template). Output is byte-for-byte identical to the previous ruby generator (tab-indented, single-quoted), so regeneration produces no diff. @@ -83,6 +84,11 @@ def _trie(t: StrTrie, buf: list[str]) -> None: buf.append('\t@staticmethod\n') buf.append(f"\tdef {head.lower()}() -> '_{root_name}{child.suffix}':\n") buf.append(f'\t\treturn _{root_name}{child.suffix}()\n') + if t.suffix is not None: + for name, _parts in t.suffix.leaves: + buf.append(f"\tdef {name.lower()}(self) -> '{root_name}':\n") + buf.append("\t\tassert ' # ' not in self.value\n") + buf.append(f"\t\treturn {root_name}(f'{{self.value}} # {name}')\n") buf.append('\n') diff --git a/support/tools/genvm-tool/genvm_tool/codegen/rst.py b/support/tools/genvm-tool/genvm_tool/codegen/rst.py index e21e48a2..6b495b71 100644 --- a/support/tools/genvm-tool/genvm_tool/codegen/rst.py +++ b/support/tools/genvm-tool/genvm_tool/codegen/rst.py @@ -1,4 +1,5 @@ -"""reStructuredText backend for ``genvm-tool codegen`` (ports ``rst.rb``). +""" +reStructuredText backend for ``genvm-tool codegen`` (ports ``rst.rb``). Generates the spec appendix listing every constant/enum/trie path with stable ``.. _gvm-def-*`` cross-reference anchors. @@ -66,6 +67,16 @@ def render(defs: list[Definition], rst_anchor_ns: str = '', **_opts) -> str: buf.append(f'Param: {param}\n\n') if path in d.docs: buf.append(d.docs[path].rstrip() + '\n\n') + if d.suffix is not None: + for name, _parts in d.suffix.leaves: + path = f'{model.DETAIL_HEAD} {name}' + buf.append( + f'.. _gvm-def-{ns}str-trie-detail-{_dash(d.name)}-{_dash(name)}:\n\n' + ) + buf.append(f'``{path}``\n') + buf.append('~' * (len(path) + 4) + '\n\n') + if path in d.docs: + buf.append(d.docs[path].rstrip() + '\n\n') # Sections are emitted blank-line-separated, so the last one leaves a trailing # blank line that `end-of-file-fixer` would strip β€” permanent codegen drift. return ''.join(buf).rstrip('\n') + '\n' diff --git a/support/tools/genvm-tool/genvm_tool/codegen/rust.py b/support/tools/genvm-tool/genvm_tool/codegen/rust.py index 68e52f03..649e4064 100644 --- a/support/tools/genvm-tool/genvm_tool/codegen/rust.py +++ b/support/tools/genvm-tool/genvm_tool/codegen/rust.py @@ -1,4 +1,5 @@ -"""Rust backend for ``genvm-tool codegen`` (ports the old ``rs.rb`` template). +""" +Rust backend for ``genvm-tool codegen`` (ports the old ``rs.rb`` template). Output follows the previous ruby generator, plus the later `prefix_` and `is_valid_` additions: the build regenerates these files in-tree and a non-empty @@ -207,6 +208,21 @@ def _trie(t: StrTrie, buf: list[str]) -> None: ) buf.append('}\n\n') + if t.suffix is not None: + buf.append('#[rustfmt::skip]\n') + buf.append(f'impl {root_name} {{\n') + for name, _parts in t.suffix.leaves: + # A value carries at most one detail: the reader splits on the first + # separator, so a second one would be silently dropped. Checked in + # release too, so the invariant does not depend on the build -- these + # are error paths, and the scan is over a short fixed string. + buf.append( + f' pub fn {name.lower()}(self) -> Self {{ ' + f'assert!(!self.0.contains(" # "), "a value carries at most one detail"); ' + f'Self(Cow::Owned(format!("{{}} # {name}", self.0))) }}\n' + ) + buf.append('}\n\n') + _emit_is_valid(t, root_name, buf) @@ -267,7 +283,7 @@ def render(defs: list[Definition], **_opts) -> str: has_serialize = has_deserialize or has_str_trie buf: list[str] = [] buf.append('// This file is auto-generated. Do not edit!\n\n') - buf.append('#![allow(dead_code, clippy::redundant_static_lifetimes)]\n') + buf.append('#![allow(dead_code, clippy::all)]\n') buf.append('\n') if has_deserialize: buf.append('use serde::{Deserialize, Serialize};\n\n') diff --git a/support/tools/genvm-tool/genvm_tool/common.py b/support/tools/genvm-tool/genvm_tool/common.py index 35c1b1c7..9b81b6d0 100644 --- a/support/tools/genvm-tool/genvm_tool/common.py +++ b/support/tools/genvm-tool/genvm_tool/common.py @@ -1,4 +1,5 @@ -"""Shared primitives: repo discovery and git queries. +""" +Shared primitives: repo discovery and git queries. A "repo" is the manager root plus every executor submodule (`executors/<v>.x/`, discovered from `.genvm-monorepo-root`). @@ -9,6 +10,7 @@ import os import re import subprocess +import sys import types from dataclasses import dataclass from pathlib import Path @@ -125,6 +127,21 @@ def active_versions_from_config(config: dict) -> list[str]: return sorted(vers, key=_ver_key) +def add_extra_python_paths(root: Path) -> None: + """ + Put `extra_python_paths` on `sys.path` (the repo's importable test plugins). + + They are libraries of the repo rather than of this package, so anything that + reaches for one imports it lazily, after this call. + """ + for entry in monorepo_config(root).get('extra_python_paths', []): + path = Path(entry) + if not path.is_absolute(): + path = root / path + if str(path) not in sys.path: + sys.path.append(str(path)) + + def executor_rel(version: str) -> str: """Manager-relative checkout/gitlink path for an executor version line.""" return f'executors/v{_bare(version)}.x' @@ -153,13 +170,15 @@ def prefix(self, rel_path: str) -> str: @property def line(self) -> str | None: - """The version-line tag (e.g. `v0.3`) for an executor submodule, or None + """ + The version-line tag (e.g. `v0.3`) for an executor submodule, or None for the manager.""" m = re.fullmatch(r'executors/(v\d+\.\d+)\.x', self.name) return m.group(1) if m else None def feature_branch(self, name: str) -> str: - """Physical branch name for the logical feature branch `name` in this repo. + """ + Physical branch name for the logical feature branch `name` in this repo. The manager is its own repo, so it carries `name` verbatim. Every executor submodule, however, shares ONE remote (`genvm-executor`): a bare `name` diff --git a/support/tools/genvm-tool/genvm_tool/formatter.py b/support/tools/genvm-tool/genvm_tool/formatter.py index 34319c24..aad44ba6 100644 --- a/support/tools/genvm-tool/genvm_tool/formatter.py +++ b/support/tools/genvm-tool/genvm_tool/formatter.py @@ -1,4 +1,5 @@ -"""Structured text/json output. +""" +Structured text/json output. A `Formatter` is both a leveled logger (`logger.info('msg', key=val)`) and a `Sink` (`printer.put('topic', **kv)`). Output is rendered either as indented @@ -8,6 +9,7 @@ import abc import collections.abc +import dataclasses import enum import io import json @@ -232,6 +234,10 @@ def accepts(self, level: Level) -> bool: def _do_dump(self, f: io.TextIOBase, ind, data): if isinstance(data, (set, frozenset, BaseException)): data = _log_unwrap(data) + if dataclasses.is_dataclass(data): + data = { + field.name: getattr(data, field.name) for field in dataclasses.fields(data) + } if isinstance(data, collections.abc.Mapping): for k, v in data.items(): diff --git a/support/tools/genvm-tool/genvm_tool/git/check_for_push.py b/support/tools/genvm-tool/genvm_tool/git/check_for_push.py index 53695a9d..4bd9ad12 100644 --- a/support/tools/genvm-tool/genvm_tool/git/check_for_push.py +++ b/support/tools/genvm-tool/genvm_tool/git/check_for_push.py @@ -1,4 +1,5 @@ -"""`genvm-tool git check-for-push` β€” pre-push status across the sub-repos. +""" +`genvm-tool git check-for-push` β€” pre-push status across the sub-repos. Read-only survey of the manager and every executor submodule. For each repo it reports whether it is dirty, on a branch, committed into the manager, and in sync @@ -53,7 +54,8 @@ def _origin_url(repo: common.Repo) -> str: def _worktree_state( repo: common.Repo, ignore_submodules: bool ) -> tuple[int, bool, bool]: - """Porcelain summary: (changed-line count, has-staged, has-unstaged/untracked). + """ + Porcelain summary: (changed-line count, has-staged, has-unstaged/untracked). The manager passes ``--ignore-submodules=dirty`` so a submodule's own dirty tree is left to that submodule's row, while a *moved gitlink* (new commits) @@ -84,7 +86,8 @@ def _committed_in_manager( def _launch_origin_queries(targets: list[tuple[common.Repo, str]]): - """Start one parallel `ls-remote` per distinct origin URL. + """ + Start one parallel `ls-remote` per distinct origin URL. `targets` is (repo, branch) for the repos we want a live remote answer for. Repos that share an origin URL (the executor submodules) are batched into a @@ -120,7 +123,8 @@ def _launch_origin_queries(targets: list[tuple[common.Repo, str]]): def _collect_origin_queries(launched) -> dict[str, tuple[str | None, bool]]: - """Wait for the launched `ls-remote` processes and map each repo to its remote + """ + Wait for the launched `ls-remote` processes and map each repo to its remote branch hash: `{repo.name: (hash | None, reachable)}` (None hash = branch absent). """ procs, result = launched diff --git a/support/tools/genvm-tool/genvm_tool/git/create_branches.py b/support/tools/genvm-tool/genvm_tool/git/create_branches.py index ba1c0f63..e90ae20e 100644 --- a/support/tools/genvm-tool/genvm_tool/git/create_branches.py +++ b/support/tools/genvm-tool/genvm_tool/git/create_branches.py @@ -1,4 +1,5 @@ -"""`genvm-tool git create-branches` β€” branch the sub-repos that have new content. +""" +`genvm-tool git create-branches` β€” branch the sub-repos that have new content. Surveys the manager and every checked-out executor submodule, keeps the ones with uncommitted or unpushed work, lets you tick which of those to branch, and creates @@ -35,7 +36,8 @@ def _git(repo: common.Repo, *args: str) -> subprocess.CompletedProcess: def _new_content(repo: common.Repo) -> str | None: - """Describe a repo's pending work, or None if it has nothing to branch. + """ + Describe a repo's pending work, or None if it has nothing to branch. "New content" is a dirty working tree (staged / unstaged / untracked) or commits ahead of the upstream branch β€” anything a fresh branch would capture. @@ -80,7 +82,8 @@ def _current_branch(repo: common.Repo) -> str: def _default_name(repo: common.Repo) -> str: - """The manager's current branch as a prompt default, unless it is a release/ + """ + The manager's current branch as a prompt default, unless it is a release/ branch-model branch (main / version / dev) or detached β€” then no default. """ cur = _current_branch(repo) diff --git a/support/tools/genvm-tool/genvm_tool/git/verify_pushed_executors.py b/support/tools/genvm-tool/genvm_tool/git/verify_pushed_executors.py index ba689a79..b1d707b1 100644 --- a/support/tools/genvm-tool/genvm_tool/git/verify_pushed_executors.py +++ b/support/tools/genvm-tool/genvm_tool/git/verify_pushed_executors.py @@ -1,4 +1,5 @@ -"""Verify that an outgoing manager commit pins published executor commits. +""" +Verify that an outgoing manager commit pins published executor commits. The manager ref defaults to ``PRE_COMMIT_TO_REF`` in a pre-push hook and to ``HEAD`` for manual runs. The active executor set and gitlinks are read from @@ -157,7 +158,8 @@ def _targets_from_commit( def _published(target: _Target, remote: str) -> tuple[bool | None, str]: - """Whether ``remote`` already holds the gitlink commit. + """ + Whether ``remote`` already holds the gitlink commit. ``--negotiate-only`` runs the fetch negotiation and stops before any transfer, printing the commits both sides have. Our own sha comes back only diff --git a/support/tools/genvm-tool/genvm_tool/io.py b/support/tools/genvm-tool/genvm_tool/io.py new file mode 100644 index 00000000..3375d8e2 --- /dev/null +++ b/support/tools/genvm-tool/genvm_tool/io.py @@ -0,0 +1,82 @@ +""" +Filesystem helpers that do not block the event loop. + +Python has no real asynchronous file IO, so every helper here is the +corresponding :mod:`pathlib` call handed to a worker thread. +""" + +__all__ = ( + 'AsyncFile', + 'read_file_bytes', + 'read_file_text', + 'write_file_bytes', + 'write_file_text', + 'open_file', + 'path_exists', + 'make_dir', +) + +import asyncio +import typing +from pathlib import Path + + +async def read_file_bytes(path: Path) -> bytes: + return await asyncio.to_thread(path.read_bytes) + + +async def read_file_text(path: Path) -> str: + return await asyncio.to_thread(path.read_text) + + +async def write_file_bytes(path: Path, data: bytes) -> None: + await asyncio.to_thread(path.write_bytes, data) + + +async def write_file_text(path: Path, data: str) -> None: + await asyncio.to_thread(path.write_text, data) + + +class AsyncFile[T: (str, bytes)]: + """ + Open file whose `open` and `close` do not block the event loop. + + `raw` is the underlying file object, for an fd (subprocess redirection) or a + write path too hot to pay a thread hop per call. + """ + + def __init__(self, raw: typing.IO[T]): + self.raw = raw + + async def __aenter__(self) -> typing.Self: + return self + + async def __aexit__(self, *_args) -> None: + await self.close() + + async def close(self) -> None: + await asyncio.to_thread(self.raw.close) + + +@typing.overload +async def open_file( + path: Path, mode: typing.Literal['r', 'w', 'a'] +) -> AsyncFile[str]: ... + + +@typing.overload +async def open_file( + path: Path, mode: typing.Literal['rb', 'wb', 'ab'] +) -> AsyncFile[bytes]: ... + + +async def open_file(path: Path, mode: str) -> AsyncFile: + return AsyncFile(await asyncio.to_thread(open, path, mode)) + + +async def path_exists(path: Path) -> bool: + return await asyncio.to_thread(path.exists) + + +async def make_dir(path: Path) -> None: + await asyncio.to_thread(path.mkdir, parents=True, exist_ok=True) diff --git a/support/tools/genvm-tool/genvm_tool/manifest.py b/support/tools/genvm-tool/genvm_tool/manifest.py index 4b36039b..e6696d82 100644 --- a/support/tools/genvm-tool/genvm_tool/manifest.py +++ b/support/tools/genvm-tool/genvm_tool/manifest.py @@ -1,4 +1,5 @@ -"""Assemble the manager's runtime manifest (`data/manifest.yaml`). +""" +Assemble the manager's runtime manifest (`data/manifest.yaml`). The manager reads this YAML at startup: `executor_versions` maps each active executor version to its `available_after` timestamp, alongside the runner @@ -33,7 +34,8 @@ def _executor_manifest(root: Path, bare_version: str) -> dict: def build(root: Path): - """Return the manifest document: `executor_versions` followed by the static + """ + Return the manifest document: `executor_versions` followed by the static base fields. Keeps ruamel's round-trip type so comments/quoting in the base file survive the dump.""" executor_versions: dict = {} diff --git a/support/tools/genvm-tool/genvm_tool/misc.py b/support/tools/genvm-tool/genvm_tool/misc.py new file mode 100644 index 00000000..cd6620e3 --- /dev/null +++ b/support/tools/genvm-tool/genvm_tool/misc.py @@ -0,0 +1,71 @@ +import hashlib +from pathlib import Path + +from . import gvm32 + + +def fuzz_input_name(data: bytes) -> str: + """ + The file name a fuzzing corpus entry is stored under. + + Named by content, in the encoding the rest of GenVM names hashes in. The + digest is truncated to 224 bits: it only has to tell two corpus entries + apart. + + :param data: the corpus entry + :return: its name + """ + return gvm32.encode(hashlib.sha3_224(data).digest()) + + +def fuzz_input_path(inputs_dir: Path, name: str) -> Path: + """ + Where a corpus entry of that name lives. + + Sharded by the first two characters of the name, so a corpus of thousands of + entries is a tree of small directories rather than one unreadable listing. + + :param inputs_dir: the corpus + :param name: as `fuzz_input_name` gives it + :return: the file to write + """ + return inputs_dir / name[0] / name[1] / name + + +def fuzz_corpus_entries(inputs_dir: Path) -> list[Path]: + """ + Every entry of a corpus, sharded or not. + + Entries written before sharding sit directly in the corpus dir and are still + read from there; nothing rewrites them until a corpus update replaces the + whole dir. + + :param inputs_dir: the corpus + :return: its entries, in path order + """ + if not inputs_dir.is_dir(): + return [] + return sorted( + entry + for entry in inputs_dir.rglob('*') + if entry.is_file() and not entry.name.startswith('.') and entry.name != 'README.txt' + ) + + +def parse_duration(s: str) -> float: + """ + Parse a duration string into seconds. + + :param s: The duration string to parse. + :return: The duration in seconds. + """ + if s.endswith('ms'): + return float(s[:-2]) / 1000.0 + elif s.endswith('s'): + return float(s[:-1]) + elif s.endswith('m'): + return float(s[:-1]) * 60.0 + elif s.endswith('h'): + return float(s[:-1]) * 3600.0 + else: + raise ValueError(f'Invalid duration string: {s} , expected 30s | 2m | 1h') diff --git a/support/tools/genvm-tool/genvm_tool/tests/__init__.py b/support/tools/genvm-tool/genvm_tool/tests/__init__.py index 3c372951..acf5214e 100644 --- a/support/tools/genvm-tool/genvm_tool/tests/__init__.py +++ b/support/tools/genvm-tool/genvm_tool/tests/__init__.py @@ -7,6 +7,7 @@ 'const', 'exec', 'test', + 'tags', 'stage', 'util', 'formatter', @@ -23,6 +24,7 @@ from genvm_tool import common, formatter from genvm_tool.formatter import Formatter, Sink +from . import tags from .util.watchdog import Watchdog _UNSAFE_NAME_RE = re.compile(r'[^A-Za-z0-9._-]+') @@ -67,6 +69,8 @@ class SharedContext: _git_files: list[Path] | None = None _interrupted: threading.Event = field(default_factory=threading.Event) _config: dict[str, Any] | None = None + _tags_registry: tags.Registry | None = None + _tags_registry_loaded: bool = False def bump_metric[T](self, name: str, initial: T, delta: T) -> None: """Increment a named metric by ``delta``, locked: collectors are threads.""" @@ -75,7 +79,8 @@ def bump_metric[T](self, name: str, initial: T, delta: T) -> None: @property def config(self) -> dict[str, Any]: - """Runner config (``artifacts_dir`` / ``extra_python_paths``). + """ + Runner config (``artifacts_dir`` / ``extra_python_paths``). Read from the monorepo marker ``.genvm-monorepo-root``, which doubles as the top-level runner config. @@ -91,6 +96,20 @@ def config(self) -> dict[str, Any]: self._config = conf return conf + @property + def tags_registry(self) -> tags.Registry | None: + if self._tags_registry_loaded: + return self._tags_registry + + registry_path = self.config.get('tags_registry') + if registry_path is not None: + path = Path(registry_path) + if not path.is_absolute(): + path = self.root_dir / path + self._tags_registry = tags.load(path) + self._tags_registry_loaded = True + return self._tags_registry + @property def artifacts_dir(self) -> Path: """Get the artifacts directory from config, or default to root_dir/build/test-artifacts.""" @@ -128,6 +147,10 @@ def git_files(self) -> list[Path]: capture_output=True, text=True, env=common.git_env(), + # The paths below are resolved against the root, so they have to be + # listed from there: run from a subdirectory, git would print them + # relative to that instead + cwd=self.root_dir, ) r = [ diff --git a/support/tools/genvm-tool/genvm_tool/tests/cli.py b/support/tools/genvm-tool/genvm_tool/tests/cli.py index 4d83c6a5..f729d170 100755 --- a/support/tools/genvm-tool/genvm_tool/tests/cli.py +++ b/support/tools/genvm-tool/genvm_tool/tests/cli.py @@ -13,7 +13,7 @@ import genvm_tool.tests import genvm_tool.tests.stage -from genvm_tool import formatter +from genvm_tool import common, formatter from .stage.pipeline import ( CollectionStage, @@ -131,12 +131,15 @@ def _show_execution_plan( immediate_await_batches.add(action.id) def _case_info(case: genvm_tool.tests.test.Case) -> str | dict: + info: dict[str, typing.Any] = {} if case.description.depends_on: - return { - 'name': case.description.name, - 'depends_on': sorted(case.description.depends_on), - } - return case.description.name + info['depends_on'] = sorted(case.description.depends_on) + if case.description.permits > 1 and not case.description.console_pool: + # Otherwise a case holding a third of the pool looks like any other one + info['permits'] = case.description.permits + if not info: + return case.description.name + return {'name': case.description.name, **info} plan_items = [] for action in actions: @@ -334,6 +337,8 @@ async def workflow_services( info: dict[str, typing.Any] = {'name': svc.name} if svc.depends_on: info['depends_on'] = [dep.name for dep in svc.depends_on] + if svc.semaphores: + info['semaphores'] = sorted(semaphore.name for semaphore in svc.semaphores) services_info.append(info) shared_context.printer.put( @@ -376,7 +381,8 @@ def run( root: Path, suite: typing.Callable, ) -> None: - """Run the test runner under genvm-tool's shared ``logger`` / ``printer``. + """ + Run the test runner under genvm-tool's shared ``logger`` / ``printer``. ``argv`` is everything after ``genvm-tool test`` β€” the runner's own ``run`` / ``show ...`` subcommands plus suite-provided filter flags. The @@ -400,14 +406,7 @@ def run( suite=suite, ) - for p in shared_context.config.get('extra_python_paths', []): - extra_path = Path(p) - if not extra_path.is_absolute(): - extra_path = shared_context.root_dir / extra_path - extra_path_str = str(extra_path) - if extra_path_str not in sys.path: - logger.debug('adding extra python path', path=extra_path) - sys.path.append(extra_path_str) + common.add_extra_python_paths(shared_context.root_dir) parser_result = create_parser() diff --git a/support/tools/genvm-tool/genvm_tool/tests/exec/command.py b/support/tools/genvm-tool/genvm_tool/tests/exec/command.py index a13a138b..6ec73e10 100644 --- a/support/tools/genvm-tool/genvm_tool/tests/exec/command.py +++ b/support/tools/genvm-tool/genvm_tool/tests/exec/command.py @@ -1,10 +1,13 @@ import asyncio import collections.abc +import contextlib import enum +import functools import io import os import re import shlex +import signal import sys import time from dataclasses import dataclass @@ -14,6 +17,47 @@ _ANSI_ESCAPE_RE = re.compile(rb'\x1b\[[0-9;]*[A-Za-z]') +# A command gets its own session, so a signal aimed at ours never reaches it: it +# has to be killed by pid, and it outlives us if we die before doing that +_KILL_GRACE = 5.0 + + +def _kill(pid: int, *, group: bool) -> None: + """ + Kill ``pid``, politely first. + + Runs in the watchdog, which is not the parent and can reap nothing, so the + exit is waited for by looking rather than by `wait`. + """ + send = os.killpg if group else os.kill + for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGKILL): + try: + send(pid, sig) + except OSError: + return + deadline = time.monotonic() + _KILL_GRACE + while time.monotonic() < deadline: + time.sleep(0.1) + try: + send(pid, 0) + except OSError: + return + + +def kill_session_cleanup(pid: int) -> functools.partial: + """ + The picklable cleanup the watchdog runs for a command left behind. + + The whole group goes, not the session leader alone: what leaks is the fleet + the command spawned. + """ + return functools.partial(_kill, pid, group=True) + + +def kill_process_cleanup(pid: int) -> functools.partial: + """Same, for a process sharing our own group, which must survive this.""" + return functools.partial(_kill, pid, group=False) + @dataclass class Result: @@ -144,7 +188,19 @@ async def run(self, ctx: SharedContext, *, mode: RunMode) -> Result: os.close(stderr_writer) ctx.logger.debug('process started', pid=process.pid, id=uid) - res = await process.wait() + # Registered for as long as it runs: a hard death of the tool leaves the + # watchdog to kill it, and a cancellation is handled right here + token = ctx.watchdog.register(kill_session_cleanup(process.pid)) + try: + res = await process.wait() + except asyncio.CancelledError: + # Ask it to stop and leave the cleanup registered: whatever survives + # this is killed when the watchdog is stopped, or when it notices we + # are gone + with contextlib.suppress(OSError): + os.killpg(process.pid, signal.SIGINT) + raise + ctx.watchdog.unregister(token) end = time.monotonic() ctx.logger.debug('process ended', pid=process.pid, exit_code=res, id=uid) diff --git a/support/tools/genvm-tool/genvm_tool/tests/exec/process.py b/support/tools/genvm-tool/genvm_tool/tests/exec/process.py new file mode 100644 index 00000000..3b41b772 --- /dev/null +++ b/support/tools/genvm-tool/genvm_tool/tests/exec/process.py @@ -0,0 +1,46 @@ +import asyncio + + +async def stop( + process: asyncio.subprocess.Process, + *, + terminate_timeout: float = 5.0, + kill_timeout: float = 5.0, +) -> None: + """ + End a process without ever waiting on it indefinitely. + + A reap that never lands leaves a zombie, which is the lesser evil: the + process has been killed either way, and a teardown that cannot finish wedges + the whole run. + """ + try: + process.terminate() + except ProcessLookupError: + return + try: + await asyncio.wait_for(process.wait(), timeout=terminate_timeout) + return + except asyncio.TimeoutError: + pass + process.kill() + try: + await asyncio.wait_for(process.wait(), timeout=kill_timeout) + except asyncio.TimeoutError: + pass + + +async def drain(tasks: list[asyncio.Task], *, timeout: float = 5.0) -> None: + """ + End tasks reading a process' pipes, without waiting for EOF. + + A reader only sees EOF once every process holding the write end is gone, and + children the process spawned inherit it, so EOF may never come. Whatever is + still unread when the timeout expires is dropped. + """ + _, pending = await asyncio.wait(tasks, timeout=timeout) + for task in pending: + task.cancel() + # every task, not just the cancelled ones: an exception nobody retrieves + # resurfaces as a warning at collection time + await asyncio.gather(*tasks, return_exceptions=True) diff --git a/support/tools/genvm-tool/genvm_tool/tests/exec/service.py b/support/tools/genvm-tool/genvm_tool/tests/exec/service.py index cda1081c..138bef82 100644 --- a/support/tools/genvm-tool/genvm_tool/tests/exec/service.py +++ b/support/tools/genvm-tool/genvm_tool/tests/exec/service.py @@ -4,19 +4,63 @@ class Handle(metaclass=abc.ABCMeta): + # Budgets for a plain service; a subclass whose probe is slower to answer + # (see :class:`ContainerHandle`) raises them. + STARTUP_TIMEOUT = 60.0 + INTERRUPT_TIMEOUT = 30.0 + @abc.abstractmethod async def healthy(self) -> bool: ... @abc.abstractmethod async def interrupt(self) -> None: ... + async def death_reason(self) -> str | None: + """ + Why the service can no longer become healthy, or None while it still might. + """ + return None + + async def shutdown(self) -> None: + """ + :meth:`interrupt` that cannot outlive its budget. + """ + try: + await asyncio.wait_for(self.interrupt(), timeout=self.INTERRUPT_TIMEOUT) + except asyncio.TimeoutError: + raise RuntimeError( + f'Service did not stop within {self.INTERRUPT_TIMEOUT:g}s' + ) from None + async def await_startup(self) -> typing.Self: - for i in range(10): - if await self.healthy(): - return self - await asyncio.sleep(1) - await self.interrupt() - raise RuntimeError('Service failed to become healthy') + # the deadline covers the probes themselves, not just the gaps between them + try: + async with asyncio.timeout(self.STARTUP_TIMEOUT): + while True: + reason = await self.death_reason() + if reason is not None: + break + if await self.healthy(): + return self + await asyncio.sleep(1) + except TimeoutError: + reason = f'it was not healthy within {self.STARTUP_TIMEOUT:g}s' + teardown = await self._shutdown_after_failure() + if teardown is not None: + reason = f'{reason}; teardown failed too: {teardown}' + raise RuntimeError(f'Service failed to start: {reason}') + + async def _shutdown_after_failure(self) -> str | None: + """ + Tear down a service that never started; reports why teardown itself failed. + """ + # reported rather than raised: a container left behind must not hide why + # the service never came up + try: + await self.shutdown() + except Exception as e: + return str(e) + return None class Service(metaclass=abc.ABCMeta): diff --git a/support/tools/genvm-tool/genvm_tool/tests/exec/step.py b/support/tools/genvm-tool/genvm_tool/tests/exec/step.py index e40970d3..9d6720be 100644 --- a/support/tools/genvm-tool/genvm_tool/tests/exec/step.py +++ b/support/tools/genvm-tool/genvm_tool/tests/exec/step.py @@ -27,6 +27,14 @@ def to_str(self) -> str: return f'export {self.key}={shlex.quote(str(self.value))}' +@dataclass +class UnsetEnv: + key: str + + def to_str(self) -> str: + return f'unset {self.key}' + + @dataclass class Run: args: collections.abc.Sequence[str | Path] @@ -63,13 +71,14 @@ async def run(self, previous_results: list[typing.Any]) -> typing.Any: return await self.func(previous_results) -type Step = SetCwd | SetEnv | Run | MkDir | Python +type Step = SetCwd | SetEnv | UnsetEnv | Run | MkDir | Python def optimize_steps(steps: list[Step]) -> list[Step]: has_effect = [False] * len(steps) last_cwd_idx: int | None = None - last_env: dict[str, tuple[int, str | Path]] = {} + # `None` is "unset", which only matters after the key was set + last_env: dict[str, tuple[int, str | Path | None]] = {} for i, step in enumerate(steps): if isinstance(step, SetCwd): if last_cwd_idx is None or steps[last_cwd_idx].path != step.path: @@ -77,6 +86,9 @@ def optimize_steps(steps: list[Step]) -> list[Step]: elif isinstance(step, SetEnv): if step.key not in last_env or last_env[step.key][1] != step.value: last_env[step.key] = (i, step.value) + elif isinstance(step, UnsetEnv): + if last_env.get(step.key, (0, None))[1] is not None: + last_env[step.key] = (i, None) elif isinstance(step, Run): has_effect[i] = True # run always has effect if last_cwd_idx is not None: @@ -107,6 +119,8 @@ async def run_steps(ctx: SharedContext, steps: list[Step]) -> list[typing.Any]: cwd = s.path elif isinstance(s, SetEnv): env[s.key] = str(s.value) + elif isinstance(s, UnsetEnv): + env.pop(s.key, None) elif isinstance(s, Run): cmd = command.Command(s.args, cwd, env) results.append( diff --git a/support/tools/genvm-tool/genvm_tool/tests/stage/collection.py b/support/tools/genvm-tool/genvm_tool/tests/stage/collection.py index 537228df..e23bb33f 100644 --- a/support/tools/genvm-tool/genvm_tool/tests/stage/collection.py +++ b/support/tools/genvm-tool/genvm_tool/tests/stage/collection.py @@ -6,18 +6,32 @@ from typing import NamedTuple import genvm_tool.tests -from genvm_tool.tests import SharedContext, const +from genvm_tool.tests import SharedContext, const, tags from .configuration import Env as ConfigurationEnv from .configuration import eval_module +@dataclass(frozen=False, eq=False) +class Semaphore: + name: str + + def __hash__(self): + return id(self) + + def __eq__(self, other): + return self is other + + @dataclass(frozen=False, eq=False) class Service: name: str manager: genvm_tool.tests.exec.service.Service depends_on: list['Service'] | None = None + semaphores: frozenset[Semaphore] = frozenset() meta: dict[str, typing.Any] = field(default_factory=dict) + handle: genvm_tool.tests.exec.service.Handle | None = None + order: int = 0 def __hash__(self): return id(self) @@ -30,23 +44,46 @@ class Context: shared: SharedContext configuration: ConfigurationEnv _all_services: list[Service] + _all_semaphores: list[Semaphore] _all_cases: list[genvm_tool.tests.test.Case] + _tag_offenders: dict[str, set[str]] + _tag_declaration_errors: set[tuple[str, str]] + + def new_semaphore(self, name: str) -> Semaphore: + semaphore = Semaphore(name=name) + self._all_semaphores.append(semaphore) + return semaphore def new_service( self, name: str, manager: genvm_tool.tests.exec.service.Service, depends_on: list['Service'] | None = None, + semaphores: typing.Iterable[Semaphore] = (), ) -> Service: - svc = Service(name=name, manager=manager, depends_on=depends_on) + svc = Service( + name=name, + manager=manager, + depends_on=depends_on, + semaphores=frozenset(semaphores), + order=len(self._all_services), + ) + self._all_services.append(svc) return svc def add_case(self, case: genvm_tool.tests.test.Case): assert isinstance(case, genvm_tool.tests.test.Case) self._all_cases.append(case) + def add_tag_offender(self, name: str, invalid: frozenset[str]) -> None: + self._tag_offenders.setdefault(name, set()).update(invalid) + + def add_tag_declaration_error(self, name: str, error: str) -> None: + self._tag_declaration_errors.add((name, error)) + def collect_dir(self, relative: str, **kwargs: typing.Any) -> None: - """Evaluate ``<relative>/test.py`` and invoke its ``collect(ctx, **kwargs)``. + """ + Evaluate ``<relative>/test.py`` and invoke its ``collect(ctx, **kwargs)``. This is how a test suite registers its cases: the ``test.py`` next to the tests owns the discovery + case-building, keeping it out of the importable @@ -67,11 +104,42 @@ def run(shared: SharedContext, configuration: ConfigurationEnv) -> Env: ctx.shared = shared ctx.configuration = configuration ctx._all_services = [] + ctx._all_semaphores = [] ctx._all_cases = [] + ctx._tag_offenders = {} + ctx._tag_declaration_errors = set() for collector in configuration.collectors: collector(ctx) + registry = shared.tags_registry + for case in ctx._all_cases: + expanded = tags.expand(case.description.tags, registry=registry) + case.description = case.description._replace(tags=expanded) + if registry is not None and (invalid := tags.unknown(registry, expanded)): + ctx.add_tag_offender(case.description.name, invalid) + + if ctx._tag_offenders or ctx._tag_declaration_errors: + errors = [ + (name, f'unknown test tags: {", ".join(sorted(invalid))}') + for name, invalid in ctx._tag_offenders.items() + ] + errors.extend(ctx._tag_declaration_errors) + errors.sort() + shown = errors[:20] + lines = [f'{name}: {error}' for name, error in shown] + if remaining := len(errors) - len(shown): + lines.append(f'... and {remaining} more errors') + registry_path = shared.config.get('tags_registry') + registry_detail = ( + f'; valid tags are loaded from {registry_path}' + if registry_path is not None + else '' + ) + raise ValueError( + f'Invalid test tag declarations{registry_detail}:\n' + '\n'.join(lines) + ) + return Env( cases=ctx._all_cases, args=configuration.args, diff --git a/support/tools/genvm-tool/genvm_tool/tests/stage/configuration.py b/support/tools/genvm-tool/genvm_tool/tests/stage/configuration.py index 3de2bdfa..93e5b06a 100644 --- a/support/tools/genvm-tool/genvm_tool/tests/stage/configuration.py +++ b/support/tools/genvm-tool/genvm_tool/tests/stage/configuration.py @@ -15,7 +15,8 @@ def eval_module(file: Path, root_dir: Path) -> types.ModuleType: - """Compile and exec ``file`` as a throwaway module, returning it. + """ + Compile and exec ``file`` as a throwaway module, returning it. The module is registered in ``sys.modules`` (under a dotted name derived from its path relative to ``root_dir``) so its own relative imports resolve. Used diff --git a/support/tools/genvm-tool/genvm_tool/tests/stage/execution.py b/support/tools/genvm-tool/genvm_tool/tests/stage/execution.py index e007d35a..5b0d3989 100644 --- a/support/tools/genvm-tool/genvm_tool/tests/stage/execution.py +++ b/support/tools/genvm-tool/genvm_tool/tests/stage/execution.py @@ -158,12 +158,19 @@ async def wait(self): await self._event.wait() +# Spawning a service is not yet covered by a Handle's own budget, and the work +# behind it (`docker run`, say) can wedge before there is anything to time out. +SPAWN_TIMEOUT = 300.0 + + async def _start_service(ctx: _ExecutionContext, service: Service) -> None: """Start a service and track its handle.""" ctx.shared.logger.info('Starting service', service_name=service.name) try: - handle = await service.manager.start() + async with asyncio.timeout(SPAWN_TIMEOUT): + handle = await service.manager.start() await handle.await_startup() + service.handle = handle ctx.running_services[service.name] = handle ctx.shared.logger.info('Service started', service_name=service.name) except Exception as e: @@ -178,10 +185,11 @@ async def _start_service(ctx: _ExecutionContext, service: Service) -> None: async def _stop_service(ctx: _ExecutionContext, service: Service) -> None: """Stop a running service.""" handle = ctx.running_services.pop(service.name, None) + service.handle = None if handle is not None: ctx.shared.logger.info('Stopping service', service_name=service.name) try: - await handle.interrupt() + await handle.shutdown() ctx.shared.logger.info('Service stopped', service_name=service.name) except Exception as e: ctx.shared.logger.error( @@ -199,7 +207,7 @@ async def _stop_all_services(ctx: _ExecutionContext) -> None: if handle is not None: ctx.shared.logger.info('Stopping service (cleanup)', service_name=name) try: - await handle.interrupt() + await handle.shutdown() except Exception as e: ctx.shared.logger.error( 'Failed to stop service during cleanup', @@ -250,9 +258,12 @@ async def _run_case( ) return - permits = 1 - if case.description.console_pool: - permits = ctx.semaphore.max_value + permits = min( + ctx.semaphore.max_value, + ctx.semaphore.max_value + if case.description.console_pool + else max(1, case.description.permits), + ) await ctx.semaphore.acquire(permits) try: if ctx.should_stop.is_set(): diff --git a/support/tools/genvm-tool/genvm_tool/tests/stage/filter.py b/support/tools/genvm-tool/genvm_tool/tests/stage/filter.py index 6076edd2..a47f46af 100644 --- a/support/tools/genvm-tool/genvm_tool/tests/stage/filter.py +++ b/support/tools/genvm-tool/genvm_tool/tests/stage/filter.py @@ -35,6 +35,10 @@ def add_args(parser: argparse.ArgumentParser) -> None: ) +def _is_tag_char(char: str) -> bool: + return char.isalnum() or char in '-_' + + def _tokenize_expr(expr: str) -> list[str]: tokens = [] idx = 0 @@ -47,7 +51,7 @@ def _tokenize_expr(expr: str) -> list[str]: idx += 1 continue word_end = idx - while word_end < len(expr) and expr[word_end].isalpha(): + while word_end < len(expr) and _is_tag_char(expr[word_end]): word_end += 1 if word_end == idx: raise ValueError(f'Unexpected character at position {idx}: {expr[idx]}') @@ -72,7 +76,7 @@ def _parse_tags_expr( inner = _parse_tags_expr(toks, 0) return lambda tags: not inner(tags) tag = toks.pop() - if not tag[0].isalpha(): + if not _is_tag_char(tag[0]): raise ValueError(f'Unexpected token in tags expression: {tag}') if tag == 'true': return lambda tags: True diff --git a/support/tools/genvm-tool/genvm_tool/tests/stage/scheduling.py b/support/tools/genvm-tool/genvm_tool/tests/stage/scheduling.py index 2657fcfa..0641ee2d 100644 --- a/support/tools/genvm-tool/genvm_tool/tests/stage/scheduling.py +++ b/support/tools/genvm-tool/genvm_tool/tests/stage/scheduling.py @@ -35,36 +35,94 @@ class Env(typing.NamedTuple): args: SimpleNamespace -def _topo_sort_services(services: set[Service]) -> list[Service]: - """ - Topological sort of services via DFS. - Returns services in dependency order: dependencies come before dependents. - """ - result: list[Service] = [] - visited: set[str] = set() - in_stack: set[str] = set() - - def visit(svc: Service) -> None: - if svc.name in visited: - return - if svc.name in in_stack: - raise ValueError(f'Circular dependency detected involving {svc.name}') - - in_stack.add(svc.name) - - # Visit dependencies first - if svc.depends_on: - for dep in svc.depends_on: - if dep in services: - visit(dep) - - in_stack.remove(svc.name) - visited.add(svc.name) - result.append(svc) +def _service_closure(service: Service) -> set[Service]: + result: set[Service] = set() + pending = [service] + while pending: + current = pending.pop() + if current in result: + continue + result.add(current) + pending.extend(current.depends_on or []) + return result - for svc in services: - visit(svc) +def _topo_sort_services( + services: set[Service], + precedence: set[tuple[Service, Service]], +) -> list[Service]: + """ + The order the services are started in. + + Dependencies come before dependents, and the owners of one semaphore come one + after another, with the whole dependent subtree of the previous owner started + before the next one takes it. Which owner goes first is a free choice, and + two semaphores whose owners depend on each other can only be ordered jointly, + so the choices are searched rather than fixed one semaphore at a time. Among + the services that may go next the one declared first wins, so an + unconstrained suite keeps its declaration order. + """ + closures = {service: _service_closure(service) & services for service in services} + predecessors: dict[Service, set[Service]] = { + service: { + dependency for dependency in service.depends_on or [] if dependency in services + } + for service in services + } + for before, after in precedence: + if before in services and after in services: + predecessors[after].add(before) + + def can_start( + candidate: Service, + started: frozenset[Service], + remaining: tuple[Service, ...], + ) -> bool: + if not predecessors[candidate].issubset(started): + return False + # Taking a semaphore stops whoever owns it and everything depending on + # that owner, so all of it has to have been started already + return not any( + holder in started and holder.semaphores & candidate.semaphores + for other in remaining + if other is not candidate + for holder in closures[other] + ) + + # One dead end is enough to rule out every way of reaching the same set of + # started services, which keeps the search out of the combinatorial case + dead_ends: set[frozenset[Service]] = set() + + def search( + started: frozenset[Service], + remaining: tuple[Service, ...], + ) -> list[Service] | None: + if not remaining: + return [] + if started in dead_ends: + return None + for candidate in remaining: + if not can_start(candidate, started, remaining): + continue + tail = search( + started | {candidate}, + tuple(item for item in remaining if item is not candidate), + ) + if tail is not None: + return [candidate, *tail] + dead_ends.add(started) + return None + + result = search( + frozenset(), + tuple(sorted(services, key=lambda service: service.order)), + ) + if result is None: + names = ', '.join(sorted(service.name for service in services)) + raise ValueError( + 'services cannot be started in any order, their dependencies and ' + f'semaphores contradict each other: {names}' + ) return result @@ -103,40 +161,30 @@ def visit(name: str) -> None: return cases_by_name -def _get_effective_services( +def _case_services(case: genvm_tool.tests.test.Case) -> set[Service]: + services: set[Service] = set() + for svc in case.description.needed_services: + services.update(_service_closure(svc)) + return services + + +def _test_dependency_names( case: genvm_tool.tests.test.Case, cases_by_name: dict[str, genvm_tool.tests.test.Case], - _memo: dict[str, frozenset[str]] | None = None, + memo: dict[str, frozenset[str]], ) -> frozenset[str]: - """ - Compute effective service names for a test case: its own services plus - services of all transitive test dependencies. This ensures a dependent - test is never placed in an earlier batch than its dependency. - """ - if _memo is None: - _memo = {} + if case.description.name in memo: + return memo[case.description.name] - name = case.description.name - if name in _memo: - return _memo[name] - - names: set[str] = set() - # Own services - for svc in case.description.needed_services: - names.add(svc.name) - if svc.depends_on: - for dep in svc.depends_on: - names.add(dep.name) - - # Transitive test dependency services - for dep_name in case.description.depends_on: - dep_case = cases_by_name.get(dep_name) - if dep_case is not None: - names.update(_get_effective_services(dep_case, cases_by_name, _memo)) - - result = frozenset(names) - _memo[name] = result - return result + result: set[str] = set() + for name in case.description.depends_on: + dependency = cases_by_name.get(name) + if dependency is None: + continue + result.add(name) + result.update(_test_dependency_names(dependency, cases_by_name, memo)) + memo[case.description.name] = frozenset(result) + return memo[case.description.name] def run(shared: SharedContext, collection_env: CollectionEnv) -> Env: @@ -149,117 +197,194 @@ def run(shared: SharedContext, collection_env: CollectionEnv) -> Env: # Validate test dependencies and build name lookup cases_by_name = _validate_test_dependencies(collection_env.cases) - # Collect all services needed by all cases (using effective services) + # Collect all services needed by the selected cases all_needed_services: set[Service] = set() for case in collection_env.cases: - # Collect direct services - for svc in case.description.needed_services: - all_needed_services.add(svc) - if svc.depends_on: - for dep in svc.depends_on: - all_needed_services.add(dep) - # Also collect services from test dependencies (transitively) - for dep_name in case.description.depends_on: + all_needed_services.update(_case_services(case)) + + # A dependency between tests using different holders of one semaphore orders + # those service lifetimes. The dependency runs first, then its holder can be + # stopped and the dependent's holder started + service_precedence: set[tuple[Service, Service]] = set() + test_dependencies_memo: dict[str, frozenset[str]] = {} + for case in collection_env.cases: + for dep_name in _test_dependency_names( + case, + cases_by_name, + test_dependencies_memo, + ): dep_case = cases_by_name.get(dep_name) - if dep_case is not None: - for svc in dep_case.description.needed_services: - all_needed_services.add(svc) - if svc.depends_on: - for dep in svc.depends_on: - all_needed_services.add(dep) + if dep_case is None: + continue + for before in _case_services(dep_case): + for after in _case_services(case): + if before is not after and before.semaphores & after.semaphores: + service_precedence.add((before, after)) # Topo sort: dependencies first, dependents last - topo_sorted_services = _topo_sort_services(all_needed_services) - - # Use effective services for batch placement - effective_services_memo: dict[str, frozenset[str]] = {} - - def get_effective_service_names(case: genvm_tool.tests.test.Case) -> frozenset[str]: - return _get_effective_services(case, cases_by_name, effective_services_memo) + service_names = [service.name for service in all_needed_services] + if len(service_names) != len(set(service_names)): + duplicates = sorted( + {name for name in service_names if service_names.count(name) > 1} + ) + raise ValueError(f'duplicate service names: {", ".join(duplicates)}') + topo_sorted_services = _topo_sort_services( + all_needed_services, + service_precedence, + ) + services_by_name = {service.name: service for service in topo_sorted_services} - # Separate cases by whether they have effective services - cases_without_services: list[genvm_tool.tests.test.Case] = [] - cases_with_services: list[genvm_tool.tests.test.Case] = [] + def get_required_service_names(case: genvm_tool.tests.test.Case) -> frozenset[str]: + return frozenset(service.name for service in _case_services(case)) for case in collection_env.cases: - if len(get_effective_service_names(case)) > 0: - cases_with_services.append(case) - else: - cases_without_services.append(case) - - # Schedule cases without services first (they can run immediately) - parallel_batch_no_services: list[genvm_tool.tests.test.Case] = [] - for case in cases_without_services: - if case.description.console_pool: - # Console pool: run alone, await immediately - actions.append(StartCases(id=next_id, cases=[case])) - actions.append(AwaitAllCases(id=next_id)) - next_id += 1 - else: - parallel_batch_no_services.append(case) - - if parallel_batch_no_services: - batch_id = next_id - next_id += 1 - actions.append(StartCases(id=batch_id, cases=parallel_batch_no_services)) - running_batches[batch_id] = set() # No service dependencies - - # Now schedule cases with services - # Start services in topo order, start tests as soon as their services are ready + services = [services_by_name[name] for name in get_required_service_names(case)] + for i, left in enumerate(services): + for right in services[i + 1 :]: + shared_semaphores = left.semaphores & right.semaphores + if shared_semaphores: + names = ', '.join(sorted(item.name for item in shared_semaphores)) + raise ValueError( + f'{case.description.name} requires mutually exclusive services ' + f'{left.name} and {right.name} (semaphores: {names})' + ) + + # Start services in topo order and cases as soon as their services are ready active_services: set[str] = set() - remaining_cases = list(cases_with_services) + remaining_cases = list(collection_env.cases) + scheduled_cases: set[str] = set() + + def await_batches_for(service_names: set[str]) -> None: + for batch_id, batch_services in list(running_batches.items()): + if batch_services & service_names: + actions.append(AwaitAllCases(id=batch_id)) + del running_batches[batch_id] + + def stop_services(service_names: set[str]) -> None: + await_batches_for(service_names) + for service in reversed(topo_sorted_services): + if service.name in service_names and service.name in active_services: + actions.append(StopService(service=service)) + active_services.remove(service.name) + + def services_ready(case: genvm_tool.tests.test.Case) -> bool: + return get_required_service_names(case).issubset(active_services) + + def dependencies_ready( + case: genvm_tool.tests.test.Case, + also_starting: frozenset[str], + ) -> bool: + return _test_dependency_names( + case, + cases_by_name, + test_dependencies_memo, + ).issubset(scheduled_cases | also_starting) + + def schedule_ready_cases() -> None: + nonlocal next_id, remaining_cases + + while True: + # A batch is awaited as a whole, so a case waiting for a dependency + # that only starts after that await would block the runner forever. + # Cases of one batch run concurrently, hence may depend on each other + parallel_batch = [ + case + for case in remaining_cases + if not case.description.console_pool and services_ready(case) + ] + while True: + starting = frozenset(case.description.name for case in parallel_batch) + kept = [case for case in parallel_batch if dependencies_ready(case, starting)] + if len(kept) == len(parallel_batch): + break + parallel_batch = kept + + if parallel_batch: + batch_id = next_id + next_id += 1 + batch_services: set[str] = set() + for case in parallel_batch: + batch_services.update(get_required_service_names(case)) + scheduled_cases.add(case.description.name) + started = {case.description.name for case in parallel_batch} + remaining_cases = [ + case for case in remaining_cases if case.description.name not in started + ] + actions.append(StartCases(id=batch_id, cases=parallel_batch)) + running_batches[batch_id] = batch_services + + # A console case holds the whole case pool, so it runs alone. Start it + # only once every case it depends on has started, otherwise it would + # wait for a later service that cannot start while it holds the pool + ready_console = next( + ( + case + for case in remaining_cases + if case.description.console_pool + and services_ready(case) + and dependencies_ready(case, frozenset()) + ), + None, + ) + if ready_console is not None: + remaining_cases.remove(ready_console) + scheduled_cases.add(ready_console.description.name) + actions.append(StartCases(id=next_id, cases=[ready_console])) + actions.append(AwaitAllCases(id=next_id)) + next_id += 1 + + # A console case that finished may unblock cases that depend on it + if not parallel_batch and ready_console is None: + return + + # Cases without services can start before the first service + schedule_ready_cases() for svc in topo_sorted_services: + conflicting = { + name + for name in active_services + if services_by_name[name].semaphores & svc.semaphores + } + if conflicting: + # A dependent cannot outlive a service being displaced by a mutually + # exclusive alternative + while True: + dependents = { + name + for name in active_services + if any( + dep.name in conflicting for dep in services_by_name[name].depends_on or [] + ) + } + if dependents.issubset(conflicting): + break + conflicting.update(dependents) + stop_services(conflicting) + + missing_dependencies = { + dep.name for dep in svc.depends_on or [] if dep.name not in active_services + } + if missing_dependencies: + raise ValueError( + f'{svc.name} cannot start after mutually exclusive dependencies stopped: ' + + ', '.join(sorted(missing_dependencies)) + ) + # Start this service actions.append(StartService(service=svc)) active_services.add(svc.name) - # Find cases that can now run (all their effective services are active) - ready_cases: list[genvm_tool.tests.test.Case] = [] - still_waiting: list[genvm_tool.tests.test.Case] = [] - - for case in remaining_cases: - required_services = get_effective_service_names(case) - if required_services.issubset(active_services): - ready_cases.append(case) - else: - still_waiting.append(case) - - remaining_cases = still_waiting - - # Schedule ready cases - parallel_batch: list[genvm_tool.tests.test.Case] = [] - for case in ready_cases: - if case.description.console_pool: - # Console pool: run alone, await immediately - actions.append(StartCases(id=next_id, cases=[case])) - actions.append(AwaitAllCases(id=next_id)) - next_id += 1 - else: - parallel_batch.append(case) - - if parallel_batch: - batch_id = next_id - next_id += 1 - # Track which services this batch depends on - batch_services: set[str] = set() - for case in parallel_batch: - batch_services.update(get_effective_service_names(case)) - actions.append(StartCases(id=batch_id, cases=parallel_batch)) - running_batches[batch_id] = batch_services + schedule_ready_cases() + + if remaining_cases: + names = ', '.join(case.description.name for case in remaining_cases) + raise ValueError( + f'cases could not be scheduled before their services stopped: {names}' + ) # Ending: stop services in reverse topo order (dependents first, dependencies last) - for svc in reversed(topo_sorted_services): - # Await all batches that depend on this service - batches_to_await = [ - batch_id for batch_id, services in running_batches.items() if svc.name in services - ] - for batch_id in batches_to_await: - actions.append(AwaitAllCases(id=batch_id)) - del running_batches[batch_id] - - # Stop this service - actions.append(StopService(service=svc)) + stop_services(set(active_services)) # Await any remaining batches (e.g., those without service dependencies) for batch_id in list(running_batches.keys()): diff --git a/support/tools/genvm-tool/genvm_tool/tests/tags.py b/support/tools/genvm-tool/genvm_tool/tests/tags.py new file mode 100644 index 00000000..f457339f --- /dev/null +++ b/support/tools/genvm-tool/genvm_tool/tests/tags.py @@ -0,0 +1,68 @@ +"""Tag registry: expansion of hierarchical feature tags and validation.""" + +import json +from dataclasses import dataclass +from pathlib import Path + +FEATURE_PREFIX = 'feature-' + + +@dataclass(frozen=True) +class Registry: + general: dict[str, str] + features: dict[str, str] + + @property + def known(self) -> frozenset[str]: + return frozenset(self.general) | frozenset( + FEATURE_PREFIX + name for name in self.features + ) + + +def load(path: Path) -> Registry: + try: + raw = json.loads(path.read_text()) + except json.JSONDecodeError as error: + raise ValueError(f'{path}: {error}') from error + if not isinstance(raw, dict): + raise ValueError(f'{path}: tag registry must be an object') + + values: dict[str, dict[str, str]] = {} + for section in ('general', 'features'): + value = raw.get(section) + if not isinstance(value, dict) or not all( + isinstance(name, str) and isinstance(description, str) + for name, description in value.items() + ): + raise ValueError(f'{path}: {section} must map tag names to descriptions') + values[section] = value + + return Registry(general=values['general'], features=values['features']) + + +def expand(tags: frozenset[str], *, registry: Registry | None) -> frozenset[str]: + """ + Add every declared ancestor of a hierarchical feature tag. + + ``feature-web-request-body`` also emits ``feature-web-request`` and + ``feature-web``, so selecting a root selects the whole subtree while matching + stays exact. Ancestors are declared dash-prefixes in the registry, preserving + compound path segments such as ``runner-map-vm``. Without a registry, tags are + returned unchanged. + """ + expanded = set(tags) + if registry is None: + return frozenset(expanded) + for tag in tags: + if not tag.startswith(FEATURE_PREFIX): + continue + expanded.update( + FEATURE_PREFIX + name + for name in registry.features + if tag.startswith(FEATURE_PREFIX + name + '-') + ) + return frozenset(expanded) + + +def unknown(registry: Registry, tags: frozenset[str]) -> frozenset[str]: + return tags - registry.known diff --git a/support/tools/genvm-tool/genvm_tool/tests/test.py b/support/tools/genvm-tool/genvm_tool/tests/test.py index 2a74cb88..639ae204 100644 --- a/support/tools/genvm-tool/genvm_tool/tests/test.py +++ b/support/tools/genvm-tool/genvm_tool/tests/test.py @@ -17,6 +17,9 @@ class Description(typing.NamedTuple): tags: frozenset[str] = frozenset() console_pool: bool = False depends_on: frozenset[str] = frozenset() + # How much of the case pool this case occupies. Above one for a case that is + # itself a fleet of processes, so the pool bounds threads and not cases + permits: int = 1 def with_tags(self, new_tags: typing.Iterable[str]) -> 'Description': return self._replace(tags=self.tags.union(new_tags)) diff --git a/support/tools/genvm-tool/genvm_tool/tests/util/watchdog.py b/support/tools/genvm-tool/genvm_tool/tests/util/watchdog.py index a6796753..63348c67 100644 --- a/support/tools/genvm-tool/genvm_tool/tests/util/watchdog.py +++ b/support/tools/genvm-tool/genvm_tool/tests/util/watchdog.py @@ -1,6 +1,9 @@ +import os +import selectors import subprocess import sys import threading +import time import typing from pathlib import Path @@ -8,7 +11,8 @@ class Watchdog: - """Owns the lifecycle of registered cleanups (e.g. ``docker stop``). + """ + Owns the lifecycle of registered cleanups (e.g. ``docker stop``). Cleanups are picklable zero-argument callables shipped to a watchdog subprocess. The subprocess is the single executor: it runs a cleanup when @@ -22,6 +26,10 @@ class Watchdog: cleanly). Otherwise a broken watchdog would silently leak every container. """ + # Has to outlast the slowest cleanup the watchdog runs for us: `docker stop` + # gives a container 10s to exit and is itself capped at 30s. + ACK_TIMEOUT = 60.0 + def __init__(self, process: subprocess.Popen): self._process = process self._lock = threading.Lock() @@ -30,6 +38,7 @@ def __init__(self, process: subprocess.Popen): # Cleared as soon as the watchdog misses anything we asked of it: from # then on its exit status says nothing about what it actually ran. self._healthy = True + self._ack_buffer = b'' @staticmethod def start() -> 'Watchdog': @@ -75,7 +84,8 @@ def unregister(self, token: int) -> None: self._send({'action': 'remove', 'token': token}) def kill(self, token: int) -> None: - """Run a registered cleanup now (via the watchdog) and forget it. + """ + Run a registered cleanup now (via the watchdog) and forget it. Blocks until the watchdog acknowledges completion, so callers may rely on the resource (e.g. a container's port) being released on return. Safe to @@ -93,23 +103,44 @@ def kill(self, token: int) -> None: _run_cleanup(cleanup) def _await_ack(self, token: int) -> bool: - """Wait for this kill's ack, skipping stale ones. ``False`` on EOF.""" + """ + Wait for this kill's ack, skipping stale ones. ``False`` on EOF or timeout. + + The wait is bounded because this runs on a thread holding :attr:`_lock`: + a watchdog that stops answering would otherwise block every later + :meth:`register` and :meth:`kill`, and hang the interpreter at exit. + """ assert self._process.stdout is not None - while True: - try: - line = self._process.stdout.readline() - except (OSError, ValueError): - return False - if not line: - return False - try: - if int(line) == token: - return True - except ValueError: - return False + deadline = time.monotonic() + self.ACK_TIMEOUT + with selectors.DefaultSelector() as selector: + selector.register(self._process.stdout, selectors.EVENT_READ) + while True: + # Lines are cut from our own buffer rather than read through the + # stream's: a line the stream has already buffered is one the + # selector can no longer see, and we would wait out the deadline + # with the answer in hand. The remainder outlives the call, so + # back-to-back kills do not drop each other's ack. + while b'\n' in self._ack_buffer: + line, _, self._ack_buffer = self._ack_buffer.partition(b'\n') + try: + if int(line) == token: + return True + except ValueError: + return False + remaining = deadline - time.monotonic() + if remaining <= 0 or not selector.select(remaining): + return False + try: + chunk = os.read(self._process.stdout.fileno(), 4096) + except (OSError, ValueError): + return False + if not chunk: + return False + self._ack_buffer += chunk def stop(self) -> None: - """Stop the watchdog, running any cleanups still registered. + """ + Stop the watchdog, running any cleanups still registered. Closing stdin signals the subprocess to exit: it tears down every dangling cleanup first, so we wait for it to finish. It exits 0 only after they all @@ -128,7 +159,12 @@ def stop(self) -> None: code = self._process.wait(timeout=60) except subprocess.TimeoutExpired: self._process.kill() - code = self._process.wait() + try: + code = self._process.wait(timeout=10) + except subprocess.TimeoutExpired: + # Killed and still not reaped. Ending the run matters more than + # the exit status; below, a non-zero one runs the cleanups here. + code = -1 with self._lock: cleanups, self._cleanups = self._cleanups, {} healthy = self._healthy diff --git a/support/tools/genvm-tool/unit_tests/test_codegen_detail_suffix.py b/support/tools/genvm-tool/unit_tests/test_codegen_detail_suffix.py new file mode 100644 index 00000000..b0951ea0 --- /dev/null +++ b/support/tools/genvm-tool/unit_tests/test_codegen_detail_suffix.py @@ -0,0 +1,94 @@ +"""A ``#`` trie entry becomes a detail suffix on the built value, not a path.""" + +import pytest +from genvm_tool.codegen import go, model, python, rst, rust + +DATA = [ + { + 'type': 'str_trie', + 'name': 'vm_error', + 'values': [ + 'timeout', + {'head': 'out_of', 'tail': ['memory']}, + {'head': '#', 'tail': ['internal', 'external']}, + ], + } +] + + +@pytest.fixture +def defs(): + return model.parse(DATA) + + +def test_suffix_is_kept_out_of_the_path_trie(defs): + (trie,) = defs + assert [head for head, _ in trie.root.methods] == ['out_of'] + assert [path for path, _ in model.enumerate_paths(trie.entries)] == [ + 'timeout', + 'out_of memory', + ] + assert [name for name, _ in trie.suffix.leaves] == ['internal', 'external'] + + +def test_rust_emits_detail_methods_on_the_value(defs): + out = rust.render(defs) + for name in ('internal', 'external'): + assert ( + f'pub fn {name}(self) -> Self ' + '{ assert!(!self.0.contains(" # "), "a value carries at most one detail"); ' + f'Self(Cow::Owned(format!("{{}} # {name}", self.0))) }}' in out + ) + + +def test_every_backend_refuses_to_stack_two_details(defs): + assert out_has_guard(rust.render(defs)) + assert out_has_guard(python.render(defs)) + assert out_has_guard(go.render(defs)) + assert 'strings' in go.render(defs) + + +def out_has_guard(out: str) -> bool: + return out.count(' # ') >= 2 and 'at most one detail' in out or "' # ' not in" in out + + +def test_a_detail_with_a_non_list_tail_is_rejected(): + data = [ + { + 'type': 'str_trie', + 'name': 'vm_error', + 'values': [{'head': '#', 'tail': [{'head': 'internal', 'tail': '$str'}]}], + } + ] + with pytest.raises(ValueError, match='nested'): + model.parse(data) + + +def test_rust_is_valid_ignores_details(defs): + out = rust.render(defs) + assert '"out_of memory"' in out + assert '# internal' not in out.split('pub fn is_valid_')[1] + + +def test_python_emits_detail_methods_on_the_value(defs): + out = python.render(defs) + assert "\tdef internal(self) -> 'VmError':\n" in out + assert "\t\treturn VmError(f'{self.value} # internal')\n" in out + + +def test_rst_documents_details_under_the_trie(defs): + out = rst.render(defs) + assert '``# internal``' in out + assert '``# external``' in out + + +def test_a_nested_detail_is_rejected(): + data = [ + { + 'type': 'str_trie', + 'name': 'vm_error', + 'values': [{'head': '#', 'tail': [{'head': 'a', 'tail': ['b']}]}], + } + ] + with pytest.raises(ValueError, match='nested'): + model.parse(data) diff --git a/support/tools/genvm-tool/unit_tests/test_fuzz_corpus.py b/support/tools/genvm-tool/unit_tests/test_fuzz_corpus.py new file mode 100644 index 00000000..90944777 --- /dev/null +++ b/support/tools/genvm-tool/unit_tests/test_fuzz_corpus.py @@ -0,0 +1,150 @@ +"""What a harvested blob becomes: a framed seed, in the corpus of its own line.""" + +import types + +import pytest +from genvm_tool import cmd_fuzz_corpus +from genvm_tool.misc import fuzz_input_name, fuzz_input_path + + +@pytest.fixture +def root(tmp_path): + """A repo with the split targets on v0.3 only, and calldata on both lines.""" + for line, targets in ( + ('0.3', ('executor/fuzz/runners-parse-raw', 'executor/fuzz/entry-data-raw')), + ('0.2', ()), + ): + for target in targets: + path = tmp_path / 'executors' / f'v{line}.x' / f'{target}.rs' + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text('fn main() {}') + decode = ( + tmp_path + / 'executors' + / f'v{line}.x' + / 'executor/crates/calldata/fuzz/genvm-common-decode.rs' + ) + decode.parent.mkdir(parents=True, exist_ok=True) + decode.write_text('fn main() {}') + return tmp_path + + +def _plan(root, *blobs, max_size=1024): + ctx = types.SimpleNamespace(root=root) + args = types.SimpleNamespace(max_size=max_size) + return cmd_fuzz_corpus._plan(ctx, args, iter(blobs)) + + +def _blob(kind, data, line='0.3'): + return cmd_fuzz_corpus.Blob(line=line, kind=kind, data=data, origin='<test>') + + +def test_code_reaches_a_raw_target_verbatim(root): + plan = _plan(root, _blob('code', b'PK\x03\x04zip')) + + (corpus,) = plan.values() + assert ( + corpus.inputs == root / 'executors/v0.3.x/executor/fuzz/inputs-runners-parse-raw' + ) + ((name, data),) = corpus.seeds.items() + assert data == b'PK\x03\x04zip' + assert name == fuzz_input_name(data) + + +def test_a_line_that_has_not_split_still_gets_the_selector_byte(root): + """The un-split target reads its payload through `Arbitrary`, which eats a byte.""" + unsplit = root / 'executors/v0.2.x/executor/fuzz/runners-parse.rs' + unsplit.parent.mkdir(parents=True, exist_ok=True) + unsplit.write_text('fn main() {}') + + plan = _plan(root, _blob('code', b'PK\x03\x04zip', line='0.2')) + + (corpus,) = plan.values() + assert corpus.inputs == root / 'executors/v0.2.x/executor/fuzz/inputs-runners-parse' + ((_, data),) = corpus.seeds.items() + assert data == b'\x00PK\x03\x04zip' + + +def test_calldata_seeds_every_target_that_parses_it(root): + plan = _plan(root, _blob('calldata', b'\x00')) + + assert {corpus.inputs.name for corpus in plan.values()} == { + 'inputs-genvm-common-decode', + 'inputs-entry-data-raw', + } + assert all( + data == b'\x00' for corpus in plan.values() for data in corpus.seeds.values() + ) + + +def test_a_line_only_gets_the_targets_it_has(root): + plan = _plan(root, _blob('code', b'wasm', line='0.2')) + + assert plan == {} + + +def test_an_entry_the_corpus_already_holds_is_not_re_added(root): + inputs = root / 'executors/v0.3.x/executor/fuzz/inputs-entry-data-raw' + path = fuzz_input_path(inputs, fuzz_input_name(b'seen')) + path.parent.mkdir(parents=True) + path.write_bytes(b'seen') + + plan = _plan(root, _blob('calldata', b'seen'), _blob('calldata', b'seen')) + + assert plan[inputs].seeds == {} + assert plan[inputs].known == {fuzz_input_name(b'seen')} + + +def test_an_entry_left_unsharded_by_an_older_run_still_counts(root): + inputs = root / 'executors/v0.3.x/executor/fuzz/inputs-entry-data-raw' + inputs.mkdir() + inputs.joinpath(fuzz_input_name(b'seen')).write_bytes(b'seen') + + plan = _plan(root, _blob('calldata', b'seen')) + + assert plan[inputs].seeds == {} + + +def test_a_blob_a_curated_seed_already_holds_is_not_added(root): + """Curated names are free-form, so the blob is matched by content.""" + curated = root / 'executors/v0.3.x/executor/fuzz/inputs-entry-data-raw-curated' + curated.mkdir() + curated.joinpath('empty-calldata').write_bytes(b'seen') + + plan = _plan(root, _blob('calldata', b'seen')) + + inputs = root / 'executors/v0.3.x/executor/fuzz/inputs-entry-data-raw' + assert plan[inputs].seeds == {} + + +def test_an_oversized_blob_is_dropped(root, capsys): + plan = _plan(root, _blob('code', b'x' * 100), max_size=10) + + assert plan == {} + assert 'over --max-size' in capsys.readouterr().out + + +def test_a_blob_carrying_a_host_path_is_dropped(root, capsys): + plan = _plan(root, _blob('code', b'\x00asm' + str(root).encode())) + + assert plan == {} + assert str(root) in capsys.readouterr().out + + +def test_case_line_names_the_executor_the_artifact_came_from(tmp_path): + cases = tmp_path / 'cases' + assert ( + cmd_fuzz_corpus._case_line(cases / 'executors/v0.3.x/tests/x/config.json', cases) + == '0.3' + ) + assert cmd_fuzz_corpus._case_line(cases / 'tests/system/x/config.json', cases) is None + + +def test_a_wat_case_yields_the_module_the_run_compiled(tmp_path): + case_dir = tmp_path / 'case' + case_dir.mkdir() + case_dir.joinpath('contract.wasm').write_bytes(b'\0asm') + + conf = {'code': str(tmp_path / 'src' / 'contract.wat')} + assert cmd_fuzz_corpus._code_path(conf, case_dir) == case_dir / 'contract.wasm' + assert cmd_fuzz_corpus._code_path({}, case_dir) is None diff --git a/support/tools/genvm-tool/unit_tests/test_fuzz_corpus_layout.py b/support/tools/genvm-tool/unit_tests/test_fuzz_corpus_layout.py new file mode 100644 index 00000000..29537fd0 --- /dev/null +++ b/support/tools/genvm-tool/unit_tests/test_fuzz_corpus_layout.py @@ -0,0 +1,27 @@ +"""Where a corpus entry is written, and what counts as one when reading back.""" + +from genvm_tool.misc import fuzz_corpus_entries, fuzz_input_name, fuzz_input_path + + +def test_an_entry_is_sharded_by_the_first_two_characters_of_its_name(tmp_path): + name = fuzz_input_name(b'payload') + + assert fuzz_input_path(tmp_path, name) == tmp_path / name[0] / name[1] / name + + +def test_reading_a_corpus_takes_sharded_and_unsharded_entries(tmp_path): + sharded = fuzz_input_path(tmp_path, fuzz_input_name(b'new')) + sharded.parent.mkdir(parents=True) + sharded.write_bytes(b'new') + tmp_path.joinpath(fuzz_input_name(b'old')).write_bytes(b'old') + tmp_path.joinpath('.hidden').write_bytes(b'') + tmp_path.joinpath('README.txt').write_bytes(b'') + + assert {entry.name for entry in fuzz_corpus_entries(tmp_path)} == { + fuzz_input_name(b'new'), + fuzz_input_name(b'old'), + } + + +def test_a_corpus_that_does_not_exist_yet_is_empty(tmp_path): + assert fuzz_corpus_entries(tmp_path / 'inputs-nothing') == [] diff --git a/support/tools/genvm-tool/unit_tests/test_process_teardown.py b/support/tools/genvm-tool/unit_tests/test_process_teardown.py new file mode 100644 index 00000000..d6500e41 --- /dev/null +++ b/support/tools/genvm-tool/unit_tests/test_process_teardown.py @@ -0,0 +1,90 @@ +"""Ending a service must not depend on a reap landing or a pipe reaching EOF.""" + +import asyncio + +from genvm_tool.tests.exec import process + + +class _Process: + """A process that ignores every signal and is never reaped.""" + + returncode = None + + def __init__(self): + self.terminated = False + self.killed = False + + def terminate(self): + self.terminated = True + + def kill(self): + self.killed = True + + async def wait(self): + await asyncio.Event().wait() + + +class _ExitingProcess(_Process): + async def wait(self): + self.returncode = 0 + return 0 + + +def test_a_process_that_exits_is_not_killed(): + proc = _ExitingProcess() + asyncio.run(process.stop(proc)) + assert proc.terminated + assert not proc.killed + + +def test_an_unreapable_process_does_not_wedge_teardown(): + proc = _Process() + + async def go(): + await asyncio.wait_for( + process.stop(proc, terminate_timeout=0.1, kill_timeout=0.1), timeout=5 + ) + + asyncio.run(go()) + assert proc.killed + + +def test_a_vanished_process_is_not_an_error(): + class _Gone(_Process): + def terminate(self): + raise ProcessLookupError + + asyncio.run(process.stop(_Gone())) + + +def test_readers_that_never_see_eof_are_cancelled(): + async def go(): + tasks = [asyncio.create_task(asyncio.Event().wait()) for _ in range(2)] + await asyncio.wait_for(process.drain(tasks, timeout=0.1), timeout=5) + return tasks + + tasks = asyncio.run(go()) + assert all(task.cancelled() for task in tasks) + + +def test_a_reader_that_finished_is_left_alone(): + async def go(): + tasks = [asyncio.create_task(asyncio.sleep(0))] + await process.drain(tasks, timeout=5) + return tasks + + tasks = asyncio.run(go()) + assert all(task.done() and not task.cancelled() for task in tasks) + + +def test_a_reader_that_raised_does_not_warn(): + async def boom(): + raise RuntimeError('log write failed') + + async def go(): + tasks = [asyncio.create_task(boom())] + await process.drain(tasks, timeout=5) + # retrieved by drain, so nothing resurfaces at collection time + assert tasks[0].exception() is not None + + asyncio.run(go()) diff --git a/support/tools/genvm-tool/unit_tests/test_service_semaphores.py b/support/tools/genvm-tool/unit_tests/test_service_semaphores.py new file mode 100644 index 00000000..f5d5a1f1 --- /dev/null +++ b/support/tools/genvm-tool/unit_tests/test_service_semaphores.py @@ -0,0 +1,356 @@ +from argparse import Namespace + +import genvm_tool.tests +from genvm_tool.tests.stage import scheduling +from genvm_tool.tests.stage.collection import Env, Semaphore, Service + + +class _Service(genvm_tool.tests.exec.service.Service): + async def start(self): + raise NotImplementedError + + +def _case( + name: str, + service: Service | None, + *, + depends_on: frozenset[str] = frozenset(), + console_pool: bool = False, +): + return genvm_tool.tests.test.StepsCase( + description=genvm_tool.tests.test.Description( + name=name, + needed_services=frozenset() if service is None else frozenset({service}), + depends_on=depends_on, + console_pool=console_pool, + ), + steps=[], + ) + + +def _service_action_index(actions, action_type, service): + return next( + index + for index, action in enumerate(actions) + if isinstance(action, action_type) and action.service is service + ) + + +def test_mutually_exclusive_services_do_not_overlap(): + semaphore = Semaphore('manager-listener') + first = Service('first', _Service(), semaphores=frozenset({semaphore}), order=0) + second = Service('second', _Service(), semaphores=frozenset({semaphore}), order=1) + + env = scheduling.run( + None, + Env( + cases=[_case('first-case', first), _case('second-case', second)], args=Namespace() + ), + ) + + actions = env.actions + stop_first = _service_action_index(actions, scheduling.StopService, first) + start_second = _service_action_index(actions, scheduling.StartService, second) + assert stop_first < start_second + assert isinstance(actions[stop_first - 1], scheduling.AwaitAllCases) + + +def test_dependents_finish_before_conflicting_service_starts(): + semaphore = Semaphore('manager-listener') + manager = Service('manager', _Service(), semaphores=frozenset({semaphore}), order=0) + replacement = Service( + 'replacement', _Service(), semaphores=frozenset({semaphore}), order=1 + ) + webdriver = Service('webdriver', _Service(), order=2) + modules = Service('modules', _Service(), depends_on=[manager, webdriver], order=3) + + env = scheduling.run( + None, + Env( + cases=[_case('integration', modules), _case('replacement-case', replacement)], + args=Namespace(), + ), + ) + + actions = env.actions + start_modules = _service_action_index(actions, scheduling.StartService, modules) + stop_modules = _service_action_index(actions, scheduling.StopService, modules) + stop_manager = _service_action_index(actions, scheduling.StopService, manager) + start_replacement = _service_action_index( + actions, scheduling.StartService, replacement + ) + stop_webdriver = _service_action_index(actions, scheduling.StopService, webdriver) + assert start_modules < stop_manager + assert stop_modules < stop_manager + assert stop_manager < start_replacement + assert start_replacement < stop_webdriver + + +def test_case_cannot_require_mutually_exclusive_services(): + semaphore = Semaphore('manager-listener') + first = Service('first', _Service(), semaphores=frozenset({semaphore}), order=0) + second = Service('second', _Service(), semaphores=frozenset({semaphore}), order=1) + case = genvm_tool.tests.test.StepsCase( + description=genvm_tool.tests.test.Description( + name='impossible', + needed_services=frozenset({first, second}), + ), + steps=[], + ) + + try: + scheduling.run(None, Env(cases=[case], args=Namespace())) + except ValueError as exc: + assert 'mutually exclusive services' in str(exc) + else: + raise AssertionError('mutually exclusive service requirement was accepted') + + +def test_test_dependency_crosses_semaphore_transition(): + semaphore = Semaphore('manager-listener') + first = Service('first', _Service(), semaphores=frozenset({semaphore}), order=0) + second = Service('second', _Service(), semaphores=frozenset({semaphore}), order=1) + first_case = _case('first-case', first) + second_case = _case( + 'second-case', + second, + depends_on=frozenset({'first-case'}), + ) + + env = scheduling.run( + None, + Env(cases=[first_case, second_case], args=Namespace()), + ) + actions = env.actions + stop_first = _service_action_index(actions, scheduling.StopService, first) + start_second = _service_action_index(actions, scheduling.StartService, second) + assert isinstance(actions[stop_first - 1], scheduling.AwaitAllCases) + assert stop_first < start_second + + +def test_test_dependency_can_reverse_service_registration_order(): + semaphore = Semaphore('manager-listener') + dependent_service = Service( + 'dependent', + _Service(), + semaphores=frozenset({semaphore}), + order=0, + ) + prerequisite_service = Service( + 'prerequisite', + _Service(), + semaphores=frozenset({semaphore}), + order=1, + ) + prerequisite = _case('prerequisite-case', prerequisite_service) + dependent = _case( + 'dependent-case', + dependent_service, + depends_on=frozenset({'prerequisite-case'}), + ) + + env = scheduling.run( + None, + Env(cases=[dependent, prerequisite], args=Namespace()), + ) + actions = env.actions + stop_prerequisite = _service_action_index( + actions, + scheduling.StopService, + prerequisite_service, + ) + start_dependent = _service_action_index( + actions, + scheduling.StartService, + dependent_service, + ) + assert stop_prerequisite < start_dependent + + +def test_console_dependency_is_started_before_scheduler_waits(): + service = Service('service', _Service(), order=0) + prerequisite = _case('prerequisite', service, console_pool=True) + dependent = _case( + 'dependent', + service, + depends_on=frozenset({'prerequisite'}), + console_pool=True, + ) + + actions = scheduling.run( + None, + Env(cases=[dependent, prerequisite], args=Namespace()), + ).actions + # Waiting for a case whose dependency has not started yet never ends + batches: dict[int, list] = {} + started: set[str] = set() + for action in actions: + if isinstance(action, scheduling.StartCases): + batches[action.id] = action.cases + started.update(case.description.name for case in action.cases) + elif isinstance(action, scheduling.AwaitAllCases): + for case in batches[action.id]: + assert case.description.depends_on.issubset(started) + assert started == {'prerequisite', 'dependent'} + + +def test_console_case_waits_until_later_service_dependency_is_scheduled(): + console_service = Service('console-service', _Service(), order=0) + prerequisite_service = Service('prerequisite-service', _Service(), order=1) + prerequisite = _case('prerequisite', prerequisite_service) + dependent = _case( + 'dependent', + console_service, + depends_on=frozenset({'prerequisite'}), + console_pool=True, + ) + + actions = scheduling.run( + None, + Env(cases=[dependent, prerequisite], args=Namespace()), + ).actions + start_prerequisite_service = _service_action_index( + actions, + scheduling.StartService, + prerequisite_service, + ) + start_dependent = next( + index + for index, action in enumerate(actions) + if isinstance(action, scheduling.StartCases) and action.cases == [dependent] + ) + assert start_prerequisite_service < start_dependent + + +def test_no_service_console_case_waits_for_service_dependency(): + service = Service('service', _Service(), order=0) + prerequisite = _case('prerequisite', service) + dependent = _case( + 'dependent', + None, + depends_on=frozenset({'prerequisite'}), + console_pool=True, + ) + + actions = scheduling.run( + None, + Env(cases=[dependent, prerequisite], args=Namespace()), + ).actions + start_service = _service_action_index(actions, scheduling.StartService, service) + start_dependent = next( + index + for index, action in enumerate(actions) + if isinstance(action, scheduling.StartCases) and action.cases == [dependent] + ) + assert start_service < start_dependent + + +def test_case_cannot_depend_on_a_case_of_a_later_service(): + # `replacement` displaces `first`, so `early-case` is over before `last-case` + # can even start; waiting for it would hang the runner + semaphore = Semaphore('manager-listener') + first = Service('first', _Service(), semaphores=frozenset({semaphore}), order=0) + replacement = Service( + 'replacement', _Service(), semaphores=frozenset({semaphore}), order=1 + ) + unrelated = Service('unrelated', _Service(), order=2) + early = _case( + 'early-case', + first, + depends_on=frozenset({'last-case'}), + ) + + try: + scheduling.run( + None, + Env( + cases=[ + early, + _case('replacement-case', replacement), + _case('last-case', unrelated), + ], + args=Namespace(), + ), + ) + except ValueError as exc: + assert 'could not be scheduled' in str(exc) + else: + raise AssertionError('a case waiting for a case that runs later was accepted') + + +def test_semaphore_handover_order_is_chosen_across_semaphores(): + # `both` conflicts with either other service, so it has to run on its own; + # ordering the owners of one semaphore at a time claims that is impossible + left = Semaphore('left') + right = Semaphore('right') + first = Service('first', _Service(), semaphores=frozenset({right}), order=0) + both = Service('both', _Service(), semaphores=frozenset({left, right}), order=1) + dependent = Service( + 'dependent', + _Service(), + semaphores=frozenset({left}), + depends_on=[first], + order=2, + ) + + actions = scheduling.run( + None, + Env( + cases=[ + _case('first-case', first), + _case('both-case', both), + _case('dependent-case', dependent), + ], + args=Namespace(), + ), + ).actions + start_first = _service_action_index(actions, scheduling.StartService, first) + start_dependent = _service_action_index(actions, scheduling.StartService, dependent) + stop_dependent = _service_action_index(actions, scheduling.StopService, dependent) + start_both = _service_action_index(actions, scheduling.StartService, both) + assert start_first < start_dependent + assert stop_dependent < start_both + + +def test_crossed_semaphore_dependencies_are_orderable(): + # Each semaphore's owners can be handed over in one order only, and the two + # orders are opposite: the second `left` owner depends on the second `right` + # one and vice versa + left = Semaphore('left') + right = Semaphore('right') + first_left = Service('first-left', _Service(), semaphores=frozenset({left}), order=0) + first_right = Service( + 'first-right', _Service(), semaphores=frozenset({right}), order=1 + ) + second_left = Service( + 'second-left', + _Service(), + semaphores=frozenset({left}), + depends_on=[first_right], + order=2, + ) + second_right = Service( + 'second-right', + _Service(), + semaphores=frozenset({right}), + depends_on=[first_left], + order=3, + ) + + services = [first_left, first_right, second_left, second_right] + actions = scheduling.run( + None, + Env( + cases=[_case(f'{service.name}-case', service) for service in services], + args=Namespace(), + ), + ).actions + started = [ + action.service for action in actions if isinstance(action, scheduling.StartService) + ] + assert set(started) == set(services) + for service in services: + start = _service_action_index(actions, scheduling.StartService, service) + for dependency in service.depends_on or []: + stop = _service_action_index(actions, scheduling.StopService, dependency) + assert start < stop diff --git a/support/tools/genvm-tool/unit_tests/test_service_startup.py b/support/tools/genvm-tool/unit_tests/test_service_startup.py new file mode 100644 index 00000000..4d999b09 --- /dev/null +++ b/support/tools/genvm-tool/unit_tests/test_service_startup.py @@ -0,0 +1,79 @@ +"""A service that never comes up must not wedge the run.""" + +import asyncio + +import pytest +from genvm_tool.tests.exec import service + + +class _Handle(service.Handle): + STARTUP_TIMEOUT = 0.3 + INTERRUPT_TIMEOUT = 0.2 + + def __init__(self, *, healthy_after: int = 10**9, reason: str | None = None): + self.probes = 0 + self.interrupted = False + self._healthy_after = healthy_after + self._reason = reason + + async def healthy(self) -> bool: + self.probes += 1 + return self.probes >= self._healthy_after + + async def death_reason(self) -> str | None: + return self._reason + + async def interrupt(self) -> None: + self.interrupted = True + + +class _WedgedHandle(_Handle): + async def interrupt(self) -> None: + await asyncio.Event().wait() + + +def test_a_healthy_service_is_not_torn_down(): + handle = _Handle(healthy_after=1) + assert asyncio.run(handle.await_startup()) is handle + assert not handle.interrupted + + +def test_startup_gives_up_on_its_own_deadline(): + handle = _Handle() + with pytest.raises(RuntimeError, match='not healthy within'): + asyncio.run(handle.await_startup()) + assert handle.interrupted + + +def test_a_dead_service_is_not_waited_out(): + handle = _Handle(reason='exited with code 1') + with pytest.raises(RuntimeError, match='exited with code 1'): + asyncio.run(handle.await_startup()) + assert handle.probes == 0 + + +def test_shutdown_outlives_a_hanging_interrupt(): + async def go(): + with pytest.raises(RuntimeError, match='did not stop'): + await asyncio.wait_for(_WedgedHandle().shutdown(), timeout=5) + + asyncio.run(go()) + + +def test_a_hanging_interrupt_does_not_mask_the_startup_failure(): + async def go(): + with pytest.raises(RuntimeError, match='not healthy within.*teardown failed too'): + await asyncio.wait_for(_WedgedHandle().await_startup(), timeout=5) + + asyncio.run(go()) + + +def test_a_wedged_probe_cannot_outlive_the_deadline(): + class _WedgedProbe(_Handle): + async def healthy(self) -> bool: + await asyncio.Event().wait() + + handle = _WedgedProbe() + with pytest.raises(RuntimeError, match='not healthy within'): + asyncio.run(asyncio.wait_for(handle.await_startup(), timeout=5)) + assert handle.interrupted diff --git a/support/tools/genvm-tool/unit_tests/test_watchdog_ack.py b/support/tools/genvm-tool/unit_tests/test_watchdog_ack.py new file mode 100644 index 00000000..3f96e68d --- /dev/null +++ b/support/tools/genvm-tool/unit_tests/test_watchdog_ack.py @@ -0,0 +1,55 @@ +"""A watchdog that stops answering must not block the thread that asked it.""" + +import contextlib +import os +import time +import types + +import pytest +from genvm_tool.tests.util import watchdog + + +@pytest.fixture +def pipe(): + read_fd, write_fd = os.pipe() + reader = os.fdopen(read_fd, 'rb') + try: + yield reader, write_fd + finally: + reader.close() + # a test may have closed the write end itself, to signal EOF + with contextlib.suppress(OSError): + os.close(write_fd) + + +def _dog(reader, *, timeout: float = 5.0) -> watchdog.Watchdog: + dog = watchdog.Watchdog(types.SimpleNamespace(stdout=reader)) + dog.ACK_TIMEOUT = timeout + return dog + + +def test_a_silent_watchdog_times_out(pipe): + reader, _write_fd = pipe + dog = _dog(reader, timeout=0.2) + start = time.monotonic() + # the write end stays open, so this can only end on the deadline + assert dog._await_ack(1) is False + assert time.monotonic() - start < 5 + + +def test_the_matching_ack_is_accepted(pipe): + reader, write_fd = pipe + os.write(write_fd, b'7\n') + assert _dog(reader)._await_ack(7) is True + + +def test_stale_acks_are_skipped(pipe): + reader, write_fd = pipe + os.write(write_fd, b'3\n7\n') + assert _dog(reader)._await_ack(7) is True + + +def test_a_closed_pipe_is_not_mistaken_for_an_ack(pipe): + reader, write_fd = pipe + os.close(write_fd) + assert _dog(reader)._await_ack(1) is False diff --git a/tests/presets/python-fuzz.txt b/tests/presets/python-fuzz.txt new file mode 100644 index 00000000..4efb4837 --- /dev/null +++ b/tests/presets/python-fuzz.txt @@ -0,0 +1 @@ +python & needs-fuzz diff --git a/tests/presets/python.txt b/tests/presets/python.txt index fdc793e7..b215238b 100644 --- a/tests/presets/python.txt +++ b/tests/presets/python.txt @@ -1 +1 @@ -python +python & !needs-fuzz diff --git a/tests/runner/genvm_tool_plugins/afl.py b/tests/runner/genvm_tool_plugins/afl.py new file mode 100644 index 00000000..10ded7ae --- /dev/null +++ b/tests/runner/genvm_tool_plugins/afl.py @@ -0,0 +1,594 @@ +import asyncio +import atexit +import collections.abc +import contextlib +import functools +import math +import os +import shutil +import signal +import tempfile +import typing +from pathlib import Path + +import genvm_tool.tests + +Command = collections.abc.Sequence[str | Path] +Wrapper = collections.abc.Callable[[Command], Command] + +# Secondaries outlive their step only if it never finishes β€” a cancelled case, +# `--fail-fast`, Ctrl-C. `run_steps` has no cleanup hook, so the fleet is bounded +# by its own `-V` and, when the tool itself goes away, by this +_live: set[asyncio.subprocess.Process] = set() + + +@atexit.register +def _stop_live_fuzzers() -> None: + for process in list(_live): + if process.returncode is None: + with contextlib.suppress(OSError): + process.send_signal(signal.SIGINT) + _live.clear() + + +# What a fuzzing run wants and a minimization run must not have: `cmin` measures +# one input per run through `afl-showmap`, and these either steer `afl-fuzz` +# alone or (`AFL_FUZZER_LOOPCOUNT`) make the target loop instead of returning +FUZZ_ONLY_ENV = ( + 'AFL_AUTORESUME', + 'AFL_BENCH_UNTIL_CRASH', + 'AFL_FUZZER_LOOPCOUNT', + 'AFL_NO_FASTRESUME', + 'AFL_TRY_AFFINITY', +) + + +def register(ctx: genvm_tool.tests.stage.configuration.Context) -> None: + ctx.run_parser.add_argument( + '--fuzz-timeout', + type=str, + default='30s', + help='Timeout for each fuzzing run (e.g. 30s, 5m, 1h)', + ) + ctx.run_parser.add_argument( + '--fuzz-jobs', + type=int, + default=0, + help='Fuzzer processes per target, defaults to the cores left over by --fuzz-concurrent', + ) + ctx.run_parser.add_argument( + '--fuzz-concurrent', + type=int, + default=0, + help='Fuzz targets to run side by side, defaults to the core count minus two. ' + 'Above one there is no AFL console, only the logs', + ) + ctx.run_parser.add_argument( + '--fuzz-update-corpus', + default=False, + action='store_true', + help='Whether to update the fuzzing corpus', + ) + + +def _budget() -> int: + """Cores this machine is willing to hand to fuzzing.""" + return max(1, (os.cpu_count() or 1) - 2) + + +@functools.cache +def concurrent(ctx: genvm_tool.tests.stage.collection.Context) -> int: + """ + How many fuzz targets may run at once. + + Different targets are worth more than instances of one: a fleet only shares + its corpus every `AFL_SYNC_TIME`, so below that it is one search duplicated, + while another target explores code the first one never reaches. + """ + # `test show` builds cases without the `run` parser, so the flags may be absent + requested = getattr(ctx.configuration.args, 'fuzz_concurrent', 0) + source = '--fuzz-concurrent' + if requested <= 0: + requested = _budget() + source = f'os.cpu_count() - 2 = {requested}' + ctx.shared.logger.info('Concurrent fuzz targets', count=requested, source=source) + return requested + + +@functools.cache +def jobs(ctx: genvm_tool.tests.stage.collection.Context) -> int: + requested = getattr(ctx.configuration.args, 'fuzz_jobs', 0) + source = '--fuzz-jobs' + if requested <= 0: + requested = max(1, min(8, _budget() // concurrent(ctx))) + source = f'(os.cpu_count() - 2) // {concurrent(ctx)}' + ctx.shared.logger.info('Fuzz jobs', count=requested, source=source) + return requested + + +def pool_usage(ctx: genvm_tool.tests.stage.collection.Context) -> dict[str, typing.Any]: + """ + How a fuzz case takes the case pool. + + A lone target owns the console and therefore the whole pool; several of them + share it, each holding as many permits as it starts fuzzer processes. + """ + solo = concurrent(ctx) == 1 + return {'console_pool': solo, 'permits': 1 if solo else jobs(ctx)} + + +def timeout(ctx: genvm_tool.tests.stage.collection.Context) -> int: + value = str(getattr(ctx.configuration.args, 'fuzz_timeout', '30s')) + return max(1, math.ceil(genvm_tool.misc.parse_duration(value))) + + +def environment(base: dict[str, str]) -> dict[str, str]: + """``base`` plus the AFL knobs every target shares.""" + env = dict(base) + env.update( + { + # The output dir is reused so a rerun continues the search; without + # this AFL refuses to start on it. NB it then reads its own queue and + # ignores `-i`, so a corpus update only takes effect after the output + # dir is removed + 'AFL_AUTORESUME': '1', + # Stop *successfully* on the first saved crash, so a finding is + # reported by `fleet_steps` rather than by the exit code + 'AFL_BENCH_UNTIL_CRASH': '1', + 'AFL_FUZZER_LOOPCOUNT': '20', # without it no coverage will be written! + 'AFL_NO_CFG_FUZZING': '1', + # The fast-resume cache was observed to fail the next run with `Bad + # file descriptor` after a fleet was interrupted + 'AFL_NO_FASTRESUME': '1', + # Persistent mode carries state across iterations, so calibration + # reports most of the corpus as unstable; the warning is pure noise + 'AFL_NO_WARN_INSTABILITY': '1', + # A fleet wants a core each, and AFL aborts rather than share one; on + # a busy machine an unbound instance is still better than a failed case + 'AFL_TRY_AFFINITY': '1', + } + ) + return env + + +def output_dir( + shared: genvm_tool.tests.SharedContext, project_dir: Path, name: str +) -> Path: + """ + Where one target's AFL output lives. + + Keyed by project, not by target name alone: the executor lines share fuzz + target stems, and one output dir per stem would mix their corpora β€” which + `cmin` then writes back into the committed inputs. + """ + return shared.artifacts_dir / 'fuzz' / project_dir.relative_to(shared.root_dir) / name + + +def current_umask() -> int: + mask = os.umask(0) + os.umask(mask) + return mask + + +def saved_findings(path: Path) -> list[Path]: + """Inputs AFL saved in one of its finding dirs, minus the README it drops there.""" + if not path.is_dir(): + return [] + return sorted( + entry + for entry in path.iterdir() + if entry.is_file() and entry.name != 'README.txt' and not entry.name.startswith('.') + ) + + +def curated_dir(inputs_dir: Path) -> Path: + """ + Where a target's hand-written seeds live, beside its generated corpus. + + Not in it: a corpus update replaces that dir with a `cmin` of the queue, and a + seed someone wrote to pin a format down has to survive the run that decides it + is redundant. Names here are free-form, they are for humans to read. + """ + return inputs_dir.parent / f'{inputs_dir.name}-curated' + + +def curated_inputs(inputs_dir: Path) -> dict[str, bytes]: + """Curated seeds of a target, keyed by the name the corpus would give them.""" + result: dict[str, bytes] = {} + for entry in saved_findings(curated_dir(inputs_dir)): + data = entry.read_bytes() + result[genvm_tool.misc.fuzz_input_name(data)] = data + return result + + +def add_to_corpus( + inputs_dir: Path, findings: collections.abc.Iterable[Path] +) -> list[str]: + """ + Commit findings as generated corpus entries, under the names a corpus uses. + + A crash is only replayed by a later run if some dir handed to `-i` holds it, + and the output dir it was saved in is wiped by the next run's crash reset. + """ + known = set(curated_inputs(inputs_dir)) | { + entry.name for entry in genvm_tool.misc.fuzz_corpus_entries(inputs_dir) + } + added: list[str] = [] + for entry in findings: + data = entry.read_bytes() + name = genvm_tool.misc.fuzz_input_name(data) + if name in known: + continue + path = genvm_tool.misc.fuzz_input_path(inputs_dir, name) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + known.add(name) + added.append(name) + return added + + +def fleet_steps( + ctx: genvm_tool.tests.stage.collection.Context, + *, + cwd: Path, + env: dict[str, str], + inputs_dir: Path, + out_dir: Path, + launcher: Command, + target: Command, + extra_args: Command = (), + wrap: Wrapper = lambda argv: argv, + prepare_command: Command | None = None, + status_command: Command | None = None, + replay: str | None = None, +) -> list[genvm_tool.tests.exec.step.Step]: + """ + Steps running one AFL fleet over ``target`` and reporting what it saved. + + One `-M` plus N-1 `-S` instances share ``out_dir`` and sync their corpora + through it, so the whole fleet is one case with one verdict. ``wrap`` gets + the last word on the argv, for a launcher that has to be reached through a + shell (the Python env, say). + """ + instances = ['main'] + [f'sec{i}' for i in range(1, jobs(ctx))] + secondaries: list[tuple[asyncio.subprocess.Process, typing.IO[bytes]]] = [] + watchdog_tokens: dict[int, int] = {} + # AFL takes one `-i`, so the two committed dirs are staged into one + seeds_dir = out_dir.parent / f'{out_dir.name}-seeds' + # Targets sharing the terminal cannot each draw a full-screen UI, so with more + # than one in flight nobody does and `main` is read from its log like the rest + solo = concurrent(ctx) == 1 + env = dict(env) if solo else {**env, 'AFL_NO_UI': '1'} + + def command(role: str, instance: str) -> Command: + return wrap( + [ + *launcher, + # CmpLog needs a second instrumented binary and buys little on + # these targets, see the fuzzing explanation page + '-c', + '-', + role, + instance, + '-i', + seeds_dir, + '-o', + out_dir, + '-V', + str(timeout(ctx)), + *extra_args, + *target, + ] + ) + + def crash_dirs() -> list[Path]: + # Globbed, not derived from `instances`: a run with fewer jobs than the + # last one still has to account for what the wider fleet left behind + return sorted(out_dir.glob('*/crashes')) + + async def stage_seeds(_) -> genvm_tool.tests.test.Result | None: + # Rebuilt every run: a curated seed added since the last one has to reach + # `-i`, and one removed from either dir must not linger + shutil.rmtree(seeds_dir, ignore_errors=True) + seeds_dir.mkdir(parents=True) + # Flattened: the corpus is sharded by name, and AFL is happier with one dir + for entry in genvm_tool.misc.fuzz_corpus_entries(inputs_dir): + shutil.copyfile(entry, seeds_dir / entry.name) + for name, data in curated_inputs(inputs_dir).items(): + seeds_dir.joinpath(name).write_bytes(data) + + staged = saved_findings(seeds_dir) + # A result either way: this is the first step, so the error check after it + # has nothing else to read + return genvm_tool.tests.test.Result( + passed=bool(staged), + context={'error': f'no seeds in {inputs_dir} or {curated_dir(inputs_dir)}'} + if not staged + else {'seeds': len(staged)}, + elapsed_seconds=0, + ) + + async def drop_previous_crashes(_): + # The verdict below describes this run. AFL keeps what earlier ones saved, + # and the output dir is deliberately reused so fuzzing resumes rather than + # restarts + for path in crash_dirs(): + shutil.rmtree(path, ignore_errors=True) + + async def stop(process: asyncio.subprocess.Process, log: typing.IO[bytes]) -> None: + try: + if process.returncode is None: + # AFL writes its queue out on SIGINT, so the next run resumes + process.send_signal(signal.SIGINT) + else: + # It was gone before the fleet was stopped, so it never fuzzed for + # its whole `-V`; the case still passes on `main` alone + ctx.shared.logger.warning( + 'a fuzz secondary exited early', + returncode=process.returncode, + log=str(getattr(log, 'name', '')), + ) + try: + await asyncio.wait_for(process.wait(), timeout=30) + except TimeoutError: + process.kill() + await process.wait() + finally: + log.close() + _live.discard(process) + token = watchdog_tokens.pop(process.pid, None) + if token is not None: + ctx.shared.watchdog.unregister(token) + + async def start_secondaries(_): + # The console, when there is one, belongs to `main`; the others write to a + # log next to their queue instead. They exit on their own `-V`; the stop + # step is for the run that ends early, on a crash or a failure of `main` + try: + for instance in instances[1:]: + log = (out_dir / f'{instance}.log').open('wb') + try: + process = await asyncio.create_subprocess_exec( + *command('-S', instance), + cwd=cwd, + env=env, + stdin=asyncio.subprocess.DEVNULL, + stdout=log, + stderr=asyncio.subprocess.STDOUT, + ) + except BaseException: + log.close() + raise + secondaries.append((process, log)) + _live.add(process) + # `_live` only covers an orderly exit; a fleet outliving a killed + # tool is what the watchdog is for + watchdog_tokens[process.pid] = ctx.shared.watchdog.register( + genvm_tool.tests.exec.command.kill_process_cleanup(process.pid) + ) + except BaseException: + # A half-started fleet would otherwise hold its cores and its output + # dir for the rest of the suite + for process, log in secondaries: + await stop(process, log) + secondaries.clear() + raise + + async def stop_secondaries(previous_results: list[typing.Any]) -> typing.Any: + for process, log in secondaries: + try: + await stop(process, log) + except Exception as error: + # One instance that will not die must not strand the others + ctx.shared.logger.warning('could not stop a fuzz secondary', error=error) + secondaries.clear() + # `CommandToResultStep` reads the step before it, which must stay the + # command result of `main` + return previous_results[-1] + + async def save_main_log(previous_results: list[typing.Any]) -> typing.Any: + result = previous_results[-1] + assert isinstance(result, genvm_tool.tests.exec.command.Result) + # Without the console `main`'s output is only in the case log, next to + # everything else; the fleet's own logs are what one reads afterwards + (out_dir / 'main.log').write_text(result.stdout + result.stderr) + return result + + async def report_findings( + previous_results: list[typing.Any], + ) -> genvm_tool.tests.test.Result: + result = previous_results[-1] + assert isinstance(result, genvm_tool.tests.test.Result) + + crashes = [crash for path in crash_dirs() for crash in saved_findings(path)] + context = dict(result.context) + if crashes: + context['crashes'] = [str(crash) for crash in crashes] + if replay is not None: + context['replay'] = replay + added = add_to_corpus(inputs_dir, crashes) + if added: + context['corpus'] = f'{len(added)} crash(es) added to {inputs_dir}' + if status_command is not None and len(instances) > 1: + # `-M`'s console shows that instance alone, so the fleet is invisible + # without asking for the aggregate + try: + process = await asyncio.create_subprocess_exec( + *status_command, + cwd=cwd, + env=env, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.DEVNULL, + ) + summary, _ = await process.communicate() + context['fleet'] = ( + summary.decode(errors='replace') + if process.returncode == 0 + else f'{status_command[0]} exited with {process.returncode}' + ) + except OSError as error: + # A status tool that cannot run must not turn a clean run into a + # failure + context['fleet'] = f'{status_command[0]}: {error}' + + return genvm_tool.tests.test.Result( + passed=result.passed and not crashes, + context=context, + elapsed_seconds=result.elapsed_seconds, + ) + + return [ + genvm_tool.tests.exec.step.SetCwd(path=cwd), + *[genvm_tool.tests.exec.step.SetEnv(key=k, value=v) for k, v in env.items()], + genvm_tool.tests.exec.step.MkDir(path=out_dir), + genvm_tool.tests.exec.step.PythonFunction(stage_seeds), + genvm_tool.tests.test.ResultStopIfErrorStep(), + genvm_tool.tests.exec.step.PythonFunction(drop_previous_crashes), + *( + [] + if prepare_command is None + else [ + genvm_tool.tests.exec.step.Run( + args=prepare_command, + mode=genvm_tool.tests.exec.command.RunMode.INTERACTIVE, + ), + genvm_tool.tests.test.CommandToResultStep(), + genvm_tool.tests.test.ResultStopIfErrorStep(), + ] + ), + genvm_tool.tests.exec.step.PythonFunction(start_secondaries), + genvm_tool.tests.exec.step.Run( + # A lone instance runs as `-S`: `-M` would add the deterministic + # stage, which a short budget cannot afford + args=command('-M' if len(instances) > 1 else '-S', 'main'), + mode=genvm_tool.tests.exec.command.RunMode.INTERACTIVE_TTY + if solo + else genvm_tool.tests.exec.command.RunMode.SILENT, + ), + *([] if solo else [genvm_tool.tests.exec.step.PythonFunction(save_main_log)]), + genvm_tool.tests.exec.step.PythonFunction(stop_secondaries), + genvm_tool.tests.test.CommandToResultStep(), + # Findings are reported before the abort check, so a run that crashed and + # then failed still names the crash + genvm_tool.tests.exec.step.PythonFunction(report_findings), + genvm_tool.tests.test.ResultStopIfErrorStep(), + ] + + +def corpus_update_steps( + ctx: genvm_tool.tests.stage.collection.Context, + *, + inputs_dir: Path, + out_dir: Path, + cmin_command: collections.abc.Callable[[Path, Path], Command], + unset_env: collections.abc.Sequence[str] = (), +) -> list[genvm_tool.tests.exec.step.Step]: + """ + Steps replacing the committed corpus with a `cmin` of what the fleet found. + + ``cmin_command`` is given the input and the output dir, in that order. These + steps go after `fleet_steps`, whose cwd and environment they reuse. + """ + if not getattr(ctx.configuration.args, 'fuzz_update_corpus', False): + return [] + + opt_dir = out_dir.parent / f'{out_dir.name}-opt' + # `cmin` walks its input dir recursively and skips only dotfiles, so it would + # otherwise mistake crashes and instance logs for corpus entries + queue_dir = out_dir.parent / f'{out_dir.name}-queue' + + async def prepare(_) -> genvm_tool.tests.test.Result | None: + for path in (opt_dir, queue_dir): + shutil.rmtree(path, ignore_errors=True) + path.mkdir(parents=True) + + for entry in sorted(out_dir.glob('*/queue/*')): + if not entry.is_file() or entry.name.startswith('.'): + continue + data = entry.read_bytes() + queue_dir.joinpath(genvm_tool.misc.fuzz_input_name(data)).write_bytes(data) + + if not any(queue_dir.iterdir()): + return genvm_tool.tests.test.Result( + passed=False, + context={'error': f'no queue entries under {out_dir}'}, + elapsed_seconds=0, + ) + return None + + async def replace(_) -> genvm_tool.tests.test.Result: + minimized = saved_findings(opt_dir) + # A curated seed is fed to every run, so `cmin` keeps it whenever it is + # worth keeping; written back it would be committed twice + curated = set(curated_inputs(inputs_dir)) + kept: dict[str, bytes] = {} + for entry in minimized: + data = entry.read_bytes() + name = genvm_tool.misc.fuzz_input_name(data) + if name not in curated: + kept[name] = data + + if not minimized: + # The old corpus is the only copy of the seeds someone wrote by hand, + # so an empty minimization is a failure, not a reason to delete it + return genvm_tool.tests.test.Result( + passed=False, + context={'error': f'cmin produced no inputs in {opt_dir}'}, + elapsed_seconds=0, + ) + + inputs_dir.parent.mkdir(parents=True, exist_ok=True) + staged = Path( + tempfile.mkdtemp(prefix=f'.{inputs_dir.name}.new-', dir=inputs_dir.parent) + ) + # `mkdtemp` is 0700, and this directory ends up as the committed corpus + staged.chmod(0o777 & ~current_umask()) + backup: Path | None = None + try: + for name, data in kept.items(): + path = genvm_tool.misc.fuzz_input_path(staged, name) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + if inputs_dir.exists(): + backup = Path( + tempfile.mkdtemp(prefix=f'.{inputs_dir.name}.old-', dir=inputs_dir.parent) + ) + backup.rmdir() + inputs_dir.rename(backup) + try: + staged.rename(inputs_dir) + except BaseException as error: + if backup is not None: + try: + backup.rename(inputs_dir) + except OSError: + raise RuntimeError( + f'corpus swap failed and the old corpus is left in {backup}' + ) from error + raise + if backup is not None: + shutil.rmtree(backup, ignore_errors=True) + finally: + shutil.rmtree(staged, ignore_errors=True) + + context = {'corpus': f'{len(kept)} inputs written to {inputs_dir}'} + if len(kept) != len(minimized): + context['curated'] = ( + f'{len(minimized) - len(kept)} kept by {curated_dir(inputs_dir)}' + ) + return genvm_tool.tests.test.Result(passed=True, context=context, elapsed_seconds=0) + + return [ + genvm_tool.tests.exec.step.PythonFunction(prepare), + genvm_tool.tests.test.ResultStopIfErrorStep(), + *[genvm_tool.tests.exec.step.UnsetEnv(key=key) for key in FUZZ_ONLY_ENV], + *[genvm_tool.tests.exec.step.UnsetEnv(key=key) for key in unset_env], + genvm_tool.tests.exec.step.Run( + args=cmin_command(queue_dir, opt_dir), + mode=genvm_tool.tests.exec.command.RunMode.INTERACTIVE, + ), + genvm_tool.tests.test.CommandToResultStep(), + genvm_tool.tests.test.ResultStopIfErrorStep(), + genvm_tool.tests.exec.step.PythonFunction(replace), + genvm_tool.tests.test.ResultStopIfErrorStep(), + ] diff --git a/tests/runner/genvm_tool_plugins/cargo.py b/tests/runner/genvm_tool_plugins/cargo.py index 528fcf89..35884bf1 100644 --- a/tests/runner/genvm_tool_plugins/cargo.py +++ b/tests/runner/genvm_tool_plugins/cargo.py @@ -1,13 +1,15 @@ import json import os +import re import shutil import subprocess import sys from pathlib import Path import genvm_tool.tests +import genvm_tool_plugins.afl as afl import tomllib -from genvm_tool.tests.test import CONST_PASSED +from genvm_tool.tests import tags as test_tags local_ctx = genvm_tool.tests.stage.configuration.current_context() @@ -75,16 +77,6 @@ def _get_default_env() -> dict[str, str]: else: env['LLVM_PROFILE_FILE'] = '/dev/null' - env['AFL_FUZZER_LOOPCOUNT'] = '20' # without it no coverage will be written! - env['AFL_NO_CFG_FUZZING'] = '1' - env['AFL_BENCH_UNTIL_CRASH'] = '1' - # persistent mode carries state across iterations, so calibration reports - # most of the corpus as unstable; the warning is pure noise here - env['AFL_NO_WARN_INSTABILITY'] = '1' - # drops one "attempting dry run" line per corpus entry; seeds are calibrated - # lazily instead, so broken ones surface when they are first fuzzed - env['AFL_NO_STARTUP_CALIBRATION'] = '1' - return env @@ -235,11 +227,14 @@ def _artifact_dir(target_dir: Path) -> Path: def _load_cargo_config( - rust_root_dir: Path, flags_key: str = 'cargo_test_flags' -) -> tuple[dict, dict[str, str], list[str]]: - """Load .ya-test-config.json and return (config, env, extra_flags).""" + ctx: genvm_tool.tests.stage.collection.Context, + rust_root_dir: Path, + flags_key: str = 'cargo_test_flags', +) -> tuple[dict, dict[str, str], list[str], dict[str, list[str]]]: + """Load .ya-test-config.json and return config, env, flags and case tags.""" test_env = _get_default_env() extra_flags: list[str] = [] + case_tags: dict[str, list[str]] = {} extra_config = rust_root_dir.joinpath('.ya-test-config.json') extra_conf: dict = {} @@ -249,8 +244,21 @@ def _load_cargo_config( for name in extra_conf.get('keep_env', []): if name in os.environ: test_env[name] = os.environ[name] - - return extra_conf, test_env, extra_flags + raw_case_tags = extra_conf.get('tags', {}) + if not isinstance(raw_case_tags, dict): + raise ValueError(f'{extra_config}: tags must be an object') + for case_name, raw_tags in raw_case_tags.items(): + if case_name != 'lib' and not case_name.startswith('bin/'): + raise ValueError(f'{extra_config}: invalid tag target {case_name!r}') + if not isinstance(raw_tags, list) or not all( + isinstance(tag, str) for tag in raw_tags + ): + raise ValueError( + f'{extra_config}: tags for {case_name!r} must be a list of strings' + ) + case_tags[case_name] = _validate_declared_tags(ctx, extra_config, raw_tags) + + return extra_conf, test_env, extra_flags, case_tags def _add_cargo_case( @@ -279,8 +287,53 @@ def _add_cargo_case( ctx.add_case(case) +_TAGS_MARKER = 'genvm-tool-test-tags:' +_TAGS_LINE = re.compile(r'^\s*//[/!]?\s*genvm-tool-test-tags:\s*(?P<tags>.*?)\s*$') + + +def _tag_source_name(ctx: genvm_tool.tests.stage.collection.Context, path: Path) -> str: + try: + return str(path.relative_to(ctx.shared.root_dir)) + except ValueError: + return str(path) + + +def _validate_declared_tags( + ctx: genvm_tool.tests.stage.collection.Context, path: Path, tags: list[str] +) -> list[str]: + declared = frozenset(tags) + registry = ctx.shared.tags_registry + if registry is None: + return sorted(declared) + invalid = test_tags.unknown(registry, declared) + if invalid: + ctx.add_tag_offender(_tag_source_name(ctx, path), invalid) + return sorted(declared - invalid) + + +def _tags_from_source( + ctx: genvm_tool.tests.stage.collection.Context, path: Path +) -> list[str]: + """Extra tags declared by a `// genvm-tool-test-tags: a,b,c` comment.""" + tags: list[str] = [] + for line_number, line in enumerate(path.read_text().splitlines(), 1): + if not line.lstrip().startswith('//') or _TAGS_MARKER not in line: + continue + match = _TAGS_LINE.fullmatch(line) + if match is None: + ctx.add_tag_declaration_error( + f'{_tag_source_name(ctx, path)}:{line_number}', + f'malformed {_TAGS_MARKER} marker', + ) + continue + line_tags = [tag.strip() for tag in match.group('tags').split(',') if tag.strip()] + tags.extend(line_tags) + return _validate_declared_tags(ctx, path, tags) + + def _is_skipped(skip_conf, key: str, name: str | None = None) -> bool: - """Check if a test category or specific name is skipped. + """ + Check if a test category or specific name is skipped. skip_conf[key] can be: - true: skip all @@ -299,12 +352,27 @@ def cargo_test( *, rust_root_dir: Path, ): - extra_conf, test_env, extra_flags = _load_cargo_config(rust_root_dir) + extra_conf, test_env, extra_flags, case_tags = _load_cargo_config(ctx, rust_root_dir) skip_conf = extra_conf.get('skip', {}) rel_dir = rust_root_dir.relative_to(local_ctx.shared.root_dir) cargo_toml = tomllib.loads(rust_root_dir.joinpath('Cargo.toml').read_text()) + has_lib = 'lib' in cargo_toml or rust_root_dir.joinpath('src', 'lib.rs').exists() + bins: list[dict] = list(cargo_toml.get('bin', [])) + if rust_root_dir.joinpath('src', 'main.rs').exists(): + pkg_name = cargo_toml.get('package', {}).get('name', rust_root_dir.name) + if not any(binary.get('name') == pkg_name for binary in bins): + bins.append({'name': pkg_name, 'path': 'src/main.rs'}) + known_tag_targets = {'lib'} if has_lib else set() + known_tag_targets.update(f'bin/{binary["name"]}' for binary in bins) + unknown_tag_targets = set(case_tags) - known_tag_targets + if unknown_tag_targets: + extra_config = rust_root_dir / '.ya-test-config.json' + raise ValueError( + f'{extra_config}: tags declared for unknown Cargo cases: ' + f'{", ".join(sorted(unknown_tag_targets))}' + ) target_dir = _target_dir(rust_root_dir) @@ -351,7 +419,7 @@ def cargo_test( + extra_flags, rust_root_dir=rust_root_dir, test_env=test_env, - tags=['rust', 'example'], + tags=['rust', 'example'] + _tags_from_source(ctx, rust_root_dir / ex_path), ) # 2. Test files in tests/ @@ -367,11 +435,10 @@ def cargo_test( command=base_cmd + ['--test', test_name], rust_root_dir=rust_root_dir, test_env=test_env, - tags=['rust', 'unit'], + tags=['rust', 'unit'] + _tags_from_source(ctx, test_file), ) # 3. --lib test - has_lib = 'lib' in cargo_toml or rust_root_dir.joinpath('src', 'lib.rs').exists() if has_lib and not _is_skipped(skip_conf, 'lib'): _add_cargo_case( ctx, @@ -379,19 +446,11 @@ def cargo_test( command=base_cmd + ['--lib'], rust_root_dir=rust_root_dir, test_env=test_env, - tags=['rust', 'unit'], + tags=['rust', 'unit'] + case_tags.get('lib', []), ) # 4. --bin tests for every binary if not _is_skipped(skip_conf, 'bins'): - bins: list[dict] = list(cargo_toml.get('bin', [])) - - # Implicit binary from src/main.rs - if rust_root_dir.joinpath('src', 'main.rs').exists(): - pkg_name = cargo_toml.get('package', {}).get('name', rust_root_dir.name) - if not any(b.get('name') == pkg_name for b in bins): - bins.append({'name': pkg_name, 'path': 'src/main.rs'}) - for bin_entry in bins: bin_name = bin_entry['name'] if _is_skipped(skip_conf, 'bins', bin_name): @@ -402,7 +461,7 @@ def cargo_test( command=base_cmd + ['--bin', bin_name], rust_root_dir=rust_root_dir, test_env=test_env, - tags=['rust', 'unit'], + tags=['rust', 'unit'] + case_tags.get(f'bin/{bin_name}', []), ) @@ -413,138 +472,177 @@ def cargo_fuzz( rust_root_dir: Path, name: str, ): - desc = desc.with_tags(['rust', 'fuzz'])._replace( - console_pool=True, + desc = desc.with_tags( + ['rust', 'fuzz', 'needs-fuzz'] + + _tags_from_source(ctx, rust_root_dir / 'fuzz' / f'{name}.rs') + )._replace( + **afl.pool_usage(ctx), ) - _extra_conf, test_env, extra_flags = _load_cargo_config( - rust_root_dir, 'cargo_afl_build_flags' + _extra_conf, test_env, extra_flags, _case_tags = _load_cargo_config( + ctx, rust_root_dir, 'cargo_afl_build_flags' ) target_dir = _target_dir(rust_root_dir) - # Keyed by crate, not by target name: the lines share fuzz target stems, and - # one AFL output dir per stem would mix their corpora β€” which `cmin` then - # writes back into `fuzz/inputs-<name>` under `--fuzz-update-corpus`. - out_dir = ( - local_ctx.shared.artifacts_dir - / 'fuzz' - / rust_root_dir.relative_to(local_ctx.shared.root_dir) - / name + # Fake entropy, so that a saved crash replays: `std`'s randomness seeds + # tokio's scheduler and every `HashMap`. `test_env` is a fresh copy, keeping + # this out of `cargo test`; the shim reaches the target through AFL, which + # turns `AFL_PRELOAD` into an `LD_PRELOAD` for the child only. + preload_manifest = ( + local_ctx.shared.root_dir / 'crates' / 'fuzzing' / 'preload' / 'Cargo.toml' ) + test_env['AFL_PRELOAD'] = str(_artifact_dir(target_dir) / 'libgenvm_fuzz_preload.so') + # A target with a mutator crate next to it takes its input as a serialized + # value, and the crate mutates that value structurally instead of editing + # its bytes. + # + # AFL's own mutations are deliberately left on. `AFL_CUSTOM_MUTATOR_ONLY` + # looks right β€” a havoc'd buffer usually fails to decode, and the target + # rejects it β€” but it measures worse: on `genvm-common-encode` over 15s, + # 20.55% coverage with `_ONLY` against 27.48% without, and 24.07% for the + # `arbitrary` generator this replaced. A rejected input costs one cheap + # execution, while the ones that do decode reach values the structural + # mutator does not propose. Set `GENVM_FUZZ_MUTATOR_ONLY=1` to compare. + mutator_manifest = rust_root_dir / 'fuzz' / 'mutators' / name / 'Cargo.toml' + if mutator_manifest.is_file(): + library = f'libfuzz_mutator_{name.replace("-", "_")}.so' + test_env['AFL_CUSTOM_MUTATOR_LIBRARY'] = str(_artifact_dir(target_dir) / library) + if os.environ.get('GENVM_FUZZ_MUTATOR_ONLY') == '1': + test_env['AFL_CUSTOM_MUTATOR_ONLY'] = '1' + else: + mutator_manifest = None + + out_dir = afl.output_dir(local_ctx.shared, rust_root_dir, name) + + fuzz_binary = _artifact_dir(target_dir) / 'examples' / f'fuzz-{name}' # Track fuzz binary for coverage if _is_coverage_enabled(): - fuzz_binary = _artifact_dir(target_dir) / 'examples' / f'fuzz-{name}' if fuzz_binary not in _profile_objects: _profile_objects.append(fuzz_binary) + inputs_dir = rust_root_dir / 'fuzz' / f'inputs-{name}' + fuzz_env = afl.environment(test_env) + # Drops one "attempting dry run" line per corpus entry; seeds are calibrated + # lazily instead, so broken ones surface when they are first fuzzed. Rust + # only: python-afl's forkserver does not survive skipping the dry run + fuzz_env['AFL_NO_STARTUP_CALIBRATION'] = '1' + # `fuzz_env` is the whole environment of this case, secondaries included, so + # what the builds below need lives in it too + fuzz_env['CARGO'] = 'cargo' + steps: list[genvm_tool.tests.exec.step.Step] = [] steps.extend( [ genvm_tool.tests.exec.step.SetCwd(path=rust_root_dir), ] - + [genvm_tool.tests.exec.step.SetEnv(key=k, value=v) for k, v in test_env.items()] - + [genvm_tool.tests.exec.step.SetEnv(key='CARGO', value='cargo')] + # The builds below need the AFL env too: without `AFL_NO_CFG_FUZZING` the + # example is built with `--cfg fuzzing`, which pulls `libfuzzer-sys` in + # through a dependency and collides with AFL's own coverage runtime + + [genvm_tool.tests.exec.step.SetEnv(key=k, value=v) for k, v in fuzz_env.items()] + [ genvm_tool.tests.exec.step.Run( args=[ - 'cargo-afl', - 'afl', + 'cargo', 'build', + '--manifest-path', + str(preload_manifest), '--target', RUST_TARGET, '--target-dir', target_dir, - '--example', - f'fuzz-{name}', '--color=always', - ] - + extra_flags, + ], mode=genvm_tool.tests.exec.command.RunMode.INTERACTIVE, ), genvm_tool.tests.test.CommandToResultStep(), genvm_tool.tests.test.ResultStopIfErrorStep(), - genvm_tool.tests.exec.step.MkDir(path=out_dir), + ] + + ( + [] + if mutator_manifest is None + else [ + genvm_tool.tests.exec.step.Run( + args=[ + 'cargo', + 'build', + '--manifest-path', + str(mutator_manifest), + '--target', + RUST_TARGET, + '--target-dir', + target_dir, + '--color=always', + ], + mode=genvm_tool.tests.exec.command.RunMode.INTERACTIVE, + ), + genvm_tool.tests.test.CommandToResultStep(), + genvm_tool.tests.test.ResultStopIfErrorStep(), + ] + ) + + [ genvm_tool.tests.exec.step.Run( args=[ 'cargo-afl', 'afl', - 'fuzz', - '-c', - '-', - '-M', - 'main', - '-i', - f'./fuzz/inputs-{name}', - '-o', - str(out_dir), - '-V', - str(getattr(ctx.configuration.args, 'fuzz_timeout', 30)), - '-t', - '5000', - str(_artifact_dir(target_dir) / f'examples/fuzz-{name}'), - ], - mode=genvm_tool.tests.exec.command.RunMode.INTERACTIVE_TTY, + 'build', + '--target', + RUST_TARGET, + '--target-dir', + target_dir, + '--example', + f'fuzz-{name}', + '--color=always', + ] + + extra_flags, + mode=genvm_tool.tests.exec.command.RunMode.INTERACTIVE, ), genvm_tool.tests.test.CommandToResultStep(), genvm_tool.tests.test.ResultStopIfErrorStep(), ] ) - - if getattr(ctx.configuration.args, 'fuzz_update_corpus', False): - opt_dir = out_dir.parent / f'{name}-opt' - - async def remove_opt_dir(_): - if opt_dir.exists(): - shutil.rmtree(opt_dir, ignore_errors=True) - opt_dir.mkdir(parents=True, exist_ok=True) - - inputs_dir = rust_root_dir.joinpath('fuzz', f'inputs-{name}') - - async def remove_inputs_dir(_): - if inputs_dir.exists(): - shutil.rmtree(inputs_dir, ignore_errors=True) - inputs_dir.mkdir(parents=True, exist_ok=True) - - steps.append(genvm_tool.tests.exec.step.PythonFunction(remove_opt_dir)) - steps.append( - genvm_tool.tests.exec.step.Run( - args=[ - 'cargo-afl', - 'afl', - 'cmin', - '-T', - 'all', - '-o', - str(opt_dir), - '-i', - str(out_dir), - '--', - str(_artifact_dir(target_dir) / f'examples/fuzz-{name}'), - ], - mode=genvm_tool.tests.exec.command.RunMode.INTERACTIVE, - ) - ) - steps.extend( - [ - genvm_tool.tests.test.CommandToResultStep(), - genvm_tool.tests.test.ResultStopIfErrorStep(), - genvm_tool.tests.exec.step.PythonFunction(remove_inputs_dir), - ] + steps.extend( + afl.fleet_steps( + ctx, + cwd=rust_root_dir, + env=fuzz_env, + inputs_dir=inputs_dir, + out_dir=out_dir, + launcher=['cargo-afl', 'afl', 'fuzz'], + target=[fuzz_binary], + extra_args=['-t', '5000'], + status_command=['cargo-afl', 'afl', 'whatsup', '-s', out_dir], + # A persistent-mode target expects a forkserver, so replaying an input + # means going through AFL rather than piping it into the binary. The + # preload has to come along, or the replay reseeds itself + replay=( + f'AFL_PRELOAD={fuzz_env["AFL_PRELOAD"]} ' + f'cargo-afl afl showmap -o /dev/null -- {fuzz_binary} <' + ), ) - steps.append( - genvm_tool.tests.exec.step.Run( - args=[ - sys.executable, - f'{local_ctx.shared.root_dir}/runners/genlayer-py-std/fuzz/resave.py', - opt_dir, - inputs_dir, - ], - mode=genvm_tool.tests.exec.command.RunMode.INTERACTIVE, - ) + ) + + steps.extend( + afl.corpus_update_steps( + ctx, + inputs_dir=inputs_dir, + out_dir=out_dir, + cmin_command=lambda queue_dir, opt_dir: [ + 'cargo-afl', + 'afl', + 'cmin', + '-T', + 'all', + '-o', + opt_dir, + '-i', + queue_dir, + '--', + fuzz_binary, + ], ) - steps.append(CONST_PASSED) + ) case = genvm_tool.tests.test.StepsCase( description=desc, diff --git a/tests/runner/genvm_tool_plugins/docker.py b/tests/runner/genvm_tool_plugins/docker.py index 38a9b77a..c48812f1 100644 --- a/tests/runner/genvm_tool_plugins/docker.py +++ b/tests/runner/genvm_tool_plugins/docker.py @@ -122,9 +122,18 @@ def _stop_cleanup(container_id: str) -> functools.partial: class ContainerHandle(genvm_tool.tests.exec.service.Handle): + # The webdriver image asks for a 10s start-period and probes every 30s, so a + # first verdict lands well after half a minute and each retry costs another + # interval. Room for several of them, not for all ten the image allows. + STARTUP_TIMEOUT = 180.0 + # Outlasts the teardown chain: the watchdog's ack budget, and behind it a + # `docker stop` capped at 30s. + INTERRUPT_TIMEOUT = 120.0 + def __init__(self, container_id: str, *, ci: bool): self._container_id = container_id self._ci = ci + self._last_inspect: 'genvm_tool.tests.exec.command.Result | None' = None self._watchdog_token = local_ctx.shared.watchdog.register( _stop_cleanup(container_id) ) @@ -187,22 +196,39 @@ async def run( container_id = res.stdout.strip() return ContainerHandle(container_id=container_id, ci=ci) + async def _inspect(self) -> 'genvm_tool.tests.exec.command.Result': + """ + State and health of the container in one call, cached for one probe. + + :meth:`healthy` runs immediately before :meth:`death_reason` on every + poll, so the second reads what the first fetched rather than forking a + docker of its own. + """ + cmd = genvm_tool.tests.exec.command.Command( + args=[ + 'docker', + 'inspect', + '--format', + '{{.State.Status}}|' + '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}', + self._container_id, + ], + cwd=local_ctx.shared.root_dir, + env=DEFAULT_ENV, + ) + self._last_inspect = await cmd.run( + ctx=local_ctx.shared, mode=genvm_tool.tests.exec.command.RunMode.SILENT + ) + return self._last_inspect + + @staticmethod + def _split_inspect(res: 'genvm_tool.tests.exec.command.Result') -> tuple[str, str]: + state, _, health = res.stdout.strip().partition('|') + return state, health + async def healthy(self) -> bool: try: - cmd = genvm_tool.tests.exec.command.Command( - args=[ - 'docker', - 'inspect', - '--format', - '{{.State.Health.Status}}', - self._container_id, - ], - cwd=local_ctx.shared.root_dir, - env=DEFAULT_ENV, - ) - res = await cmd.run( - ctx=local_ctx.shared, mode=genvm_tool.tests.exec.command.RunMode.SILENT - ) + res = await self._inspect() if res.exit_code != 0: if self._ci: logs = await self.get_logs() @@ -213,17 +239,17 @@ async def healthy(self) -> bool: logs=logs, ) return False - status = res.stdout.strip() - if status != 'healthy': + _, health = self._split_inspect(res) + if health != 'healthy': if self._ci: logs = await self.get_logs() local_ctx.shared.logger.warning( 'Container not healthy', container_id=self._container_id, - status=status, + status=health, logs=logs, ) - return status == 'healthy' + return health == 'healthy' except Exception as e: local_ctx.shared.logger.error( 'Failed to check container health', @@ -232,6 +258,28 @@ async def healthy(self) -> bool: ) return False + async def death_reason(self) -> str | None: + res, self._last_inspect = self._last_inspect, None + try: + if res is None: + res = await self._inspect() + # a probe of our own is not a probe the next call may reuse + self._last_inspect = None + except Exception: + return None # healthy() reports the probe itself failing + if res.exit_code != 0: + # `docker run --rm` drops the container the moment it exits; any + # other inspect failure is the daemon being flaky, so keep waiting + if 'no such' in (res.stdout + res.stderr).lower(): + return f'container {self._container_id} is gone' + return None + state, health = self._split_inspect(res) + if state in ('exited', 'dead'): + return f'container {state}, logs: {await self.get_logs()}' + if health == 'unhealthy': + return f'container unhealthy, logs: {await self.get_logs()}' + return None + async def interrupt(self) -> None: # Tear the container down through the watchdog so creation and teardown # always go through the same owner; blocks until the stop completes. diff --git a/tests/runner/genvm_tool_plugins/genvm.py b/tests/runner/genvm_tool_plugins/genvm.py index 5c72890d..c0513e2c 100644 --- a/tests/runner/genvm_tool_plugins/genvm.py +++ b/tests/runner/genvm_tool_plugins/genvm.py @@ -2,10 +2,13 @@ import json import os import typing +from collections.abc import Mapping from pathlib import Path import aiohttp +import genvm_tool.io as gvm_io import genvm_tool.tests +import genvm_tool.tests.exec.process import genvm_tool.tests.exec.service import genvm_tool_plugins.docker as docker @@ -86,52 +89,80 @@ async def _read_log_pipe( class ManagerHandle(genvm_tool.tests.exec.service.Handle): def __init__( self, - port: int, + port: int | None, process: asyncio.subprocess.Process | None, - log_file: typing.IO[str] | None, + log_file: gvm_io.AsyncFile[str] | None, log_tasks: list[asyncio.Task] | None, + socket_path: Path | None = None, + restart: typing.Callable[[], typing.Awaitable['ManagerHandle']] | None = None, ): self._port = port + self._socket_path = socket_path self._process = process self._log_file = log_file self._log_tasks = log_tasks + self._restart = restart @property - def port(self) -> int: + def port(self) -> int | None: return self._port @property def uri(self) -> str: + if self._socket_path is not None: + return f'unix://{self._socket_path}' return f'http://localhost:{self._port}' + @property + def socket_path(self) -> Path | None: + return self._socket_path + + @property + def process(self) -> asyncio.subprocess.Process | None: + return self._process + async def healthy(self) -> bool: try: - async with aiohttp.ClientSession() as session: + connector = ( + aiohttp.UnixConnector(path=str(self._socket_path)) + if self._socket_path is not None + else None + ) + async with aiohttp.ClientSession(connector=connector) as session: async with session.get( - f'{self.uri}/status', + 'http://localhost/status' + if self._socket_path is not None + else f'{self.uri}/status', timeout=aiohttp.ClientTimeout(total=5), ) as response: return response.status == 200 except Exception: return False + async def death_reason(self) -> str | None: + if self._process is None or self._process.returncode is None: + return None + return f'manager exited with code {self._process.returncode}' + async def interrupt(self) -> None: if self._process is not None: - try: - self._process.terminate() - except Exception: - pass - - try: - await asyncio.wait_for(self._process.wait(), timeout=5) - except asyncio.TimeoutError: - self._process.kill() - await self._process.wait() + await genvm_tool.tests.exec.process.stop(self._process) if self._log_tasks is not None: - for task in self._log_tasks: - await task + await genvm_tool.tests.exec.process.drain(self._log_tasks) if self._log_file is not None: - self._log_file.close() + await self._log_file.close() + + async def restart(self) -> None: + if self._restart is None: + raise RuntimeError('manager service is not restartable') + await self.interrupt() + replacement = await self._restart() + self._port = replacement._port + self._socket_path = replacement._socket_path + self._process = replacement._process + self._log_file = replacement._log_file + self._log_tasks = replacement._log_tasks + await self.await_startup() class ManagerService(genvm_tool.tests.exec.service.Service): @@ -142,46 +173,66 @@ def __init__( bin_path: Path, log_path: Path | None = None, ci: bool = False, + port: int | None = None, + config: Mapping[str, typing.Any] | None = None, + socket_path: Path | None = None, ): self._bin_path = bin_path self._log_path = log_path self._env = env self._ci = ci + self._port = port + self._config = dict(config or {}) + self._socket_path = socket_path + self._has_started = False async def start(self) -> ManagerHandle: + handle = await self._start_once() + handle._restart = self._start_once + return handle + + async def _start_once(self) -> ManagerHandle: log_file = None log_tasks = None use_pipe = self._log_path is not None if use_pipe and self._log_path: - log_file = open(self._log_path, 'w') + self._log_path.parent.mkdir(parents=True, exist_ok=True) + log_file = await gvm_io.open_file( + self._log_path, + 'a' if self._has_started else 'w', + ) + self._has_started = True - port = get_manager_port(self._env) + port = self._port if self._port is not None else get_manager_port(self._env) config_dir = self._log_path.parent if self._log_path is not None else Path('/tmp') config_dir.mkdir(parents=True, exist_ok=True) config_path = config_dir / 'genvm-manager.yaml' manifest_path = self._bin_path.parent.parent / 'data' / 'manifest.yaml' + config = { + 'threads': 4, + 'blocking_threads': 48, + 'log_disable': 'tracing*,polling*,tungstenite*,tokio_tungstenite*', + 'log_level': 'info', + 'manifest_path': str(manifest_path), + 'permits': None, + 'execution_retention': '5m', + 'max_message_bytes': 67108864, + } + config.update(self._config) config_path.write_text( - '\n'.join( - [ - 'threads: 4', - 'blocking_threads: 48', - 'log_disable: tracing*,polling*,tungstenite*,tokio_tungstenite*', - 'log_level: info', - f'manifest_path: {json.dumps(str(manifest_path))}', - 'permits: null', - 'execution_retention: 5m', - 'max_message_bytes: 67108864', - '', - ] - ) + '\n'.join(f'{key}: {json.dumps(value)}' for key, value in config.items()) + '\n' ) + listener_args = ( + ['--socket', str(self._socket_path)] + if self._socket_path is not None + else ['--port', str(port)] + ) process = await asyncio.subprocess.create_subprocess_exec( str(self._bin_path), 'manager', - '--port', - str(port), + *listener_args, '--die-with-parent', '--config', str(config_path), @@ -195,14 +246,20 @@ async def start(self) -> ManagerHandle: logger = local_ctx.shared.logger log_tasks = [ asyncio.create_task( - _read_log_pipe(process.stdout, log_file, logger, ci=self._ci) + _read_log_pipe(process.stdout, log_file.raw, logger, ci=self._ci) ), asyncio.create_task( - _read_log_pipe(process.stderr, log_file, logger, ci=self._ci) + _read_log_pipe(process.stderr, log_file.raw, logger, ci=self._ci) ), ] - handle = ManagerHandle(port, process, log_file, log_tasks) + handle = ManagerHandle( + None if self._socket_path is not None else port, + process, + log_file, + log_tasks, + socket_path=self._socket_path, + ) return handle diff --git a/tests/runner/genvm_tool_plugins/integration.py b/tests/runner/genvm_tool_plugins/integration.py index de2db945..20335dab 100644 --- a/tests/runner/genvm_tool_plugins/integration.py +++ b/tests/runner/genvm_tool_plugins/integration.py @@ -25,10 +25,14 @@ from pathlib import Path import genvm_tool.gvm32 +import genvm_tool.io as gvm_io import genvm_tool.tests +import genvm_tool_plugins.runners as runners +import gvm_extra.case_inputs as case_inputs import origin.base_host as base_host import origin.calldata as gvm_calldata import origin.fees as fees +import origin.host_fns as host_fns import origin.log_asserts as log_asserts import origin.logger as origin_logger import origin.public_abi as public_abi @@ -210,7 +214,7 @@ def log(self, level: str, msg: str, **kwargs) -> None: def _unfold_conf(x: typing.Any, vars: dict[str, str]) -> typing.Any: """Recursively substitute variables in configuration.""" if isinstance(x, str): - return re.sub(r'\$\{[a-zA-Z\-_]+\}', lambda m: vars[m.group()[2:-1]], x) + return re.sub(r'\$\{[a-zA-Z0-9\-_]+\}', lambda m: vars[m.group()[2:-1]], x) if isinstance(x, list): return [_unfold_conf(item, vars) for item in x] if isinstance(x, dict): @@ -219,7 +223,8 @@ def _unfold_conf(x: typing.Any, vars: dict[str, str]) -> typing.Any: def _routing_payload(route: int | str) -> bytes: - """Where a host places a callee: a major, or a version naming a line. + """ + Where a host places a callee: a major, or a version naming a line. A major is resolved by the manifest's rules, which cannot separate the live lines while every one of them is semver major 0 β€” a version string is what @@ -258,6 +263,11 @@ def _flatten_tree( # as `py-genlayer:test`, and that alias only resolves from `unsafe` up. _DEBUG_MODES = ('disabled', 'safe', 'safe-unbounded', 'unsafe', 'unsafe-tracing') _DEBUG_MODE_DEFAULT = 'unsafe' +_PHASE_TAG_BY_MODE = { + 'l': 'phase-leader', + 'v': 'phase-validator', + 's': 'phase-sync', +} @dataclass @@ -416,7 +426,7 @@ async def run( # Set up base storage base_mock_storage = MockStorage() if storage_json := top_level_conf.get('storage_json'): - storage_b64 = json.loads(Path(storage_json).read_text()) + storage_b64 = json.loads(await gvm_io.read_file_text(Path(storage_json))) base_mock_storage._storages = { Address(a): { base64.b64decode(k): bytearray(base64.b64decode(v)) for k, v in kv.items() @@ -425,8 +435,7 @@ async def run( } empty_storage = tmp_dir.joinpath('empty-storage.pickle') - with open(empty_storage, 'wb') as f: - pickle.dump(base_mock_storage, f) + await gvm_io.write_file_bytes(empty_storage, pickle.dumps(base_mock_storage)) return genvm_tool.tests.test.Result( passed=True, @@ -575,18 +584,7 @@ async def _run_single_step( my_tmp_dir.joinpath('config.json').write_text(config_str) # Prepare calldata - calldata_eval_vars = {} - calldata_eval_vars['Address'] = gvm_calldata.Address - calldata_eval_vars['true'] = True - calldata_eval_vars['false'] = False - - calldata_bytes = gvm_calldata.encode( - eval( - single_conf['calldata'], - calldata_eval_vars, - single_conf.get('vars', {}).copy(), - ) - ) + calldata_bytes = case_inputs.encode_calldata(single_conf) # Process code file code_path = single_conf.get('code') @@ -619,7 +617,9 @@ async def _run_single_step( }, } code_path = str(out_path) - code = Path(code_path).read_bytes() + code = runners.substitute( + await gvm_io.read_file_bytes(Path(code_path)), local_ctx.shared.root_dir + ) # Process message addresses single_conf['message']['contract_address'] = Address( @@ -652,8 +652,9 @@ async def _run_single_step( else: leader_nondet = single_conf.get('leader_nondet', None) if leader_nondet is None: - with open(path_to_which_leader_puts_result, 'rb') as f: - leader_nondet = pickle.load(f) + leader_nondet = pickle.loads( + await gvm_io.read_file_bytes(path_to_which_leader_puts_result) + ) if leader_nondet is not None: encoded_nondet = [] for res in leader_nondet: @@ -700,6 +701,18 @@ async def _run_single_step( # manifest keys rather than name a directory). routes = {Address(k): v for k, v in single_conf.get('executor_routes', {}).items()} hook_calls = routes != {} or single_conf.get('hook_cross_contract_calls', False) + route_requests: list[dict[str, typing.Any]] = [] + + def resolve_executor(address, state, advisory_major): + route_requests.append( + { + 'contract_address': address.as_hex, + 'state_mode': int(state), + 'advisory_major': advisory_major, + } + ) + return _routing_payload(routes[address]) if address in routes else None + host = MockHost( path=str(mock_sock_path), storage_path_post=post_storage, @@ -707,11 +720,7 @@ async def _run_single_step( balances={Address(k): v for k, v in single_conf.get('balances', {}).items()}, running_address=running_address, ctx=ctx, - resolve_callcontract_executor_hook=( - lambda address, _state, _major: ( - _routing_payload(routes[address]) if address in routes else None - ) - ), + resolve_callcontract_executor_hook=resolve_executor, ) host.balances.setdefault(running_address, 0) @@ -778,7 +787,9 @@ async def _run_single_step( gas_data=single_conf.get('gas_data'), leader_nondet_results=leader_nondet, message_fee_allocation=message_fee_allocation, - unsafe_overrides=base_host.UnsafeOverrides(reroute_to=reroute_to), + unsafe_overrides=base_host.UnsafeOverrides( + reroute_to=single_conf.get('reroute_to', reroute_to) + ), request_extra=request_extra, # A case may state the major the host claims, for the # cases whose subject is what the executor does with a @@ -788,14 +799,21 @@ async def _run_single_step( # Nested executors dial the same listener; wind their loops down # so a failure on that side surfaces here. await mock_host.stop_connections() + expected_route_requests = single_conf.get('expected_executor_route_requests') + assert ( + expected_route_requests is None or route_requests == expected_route_requests + ), ( + route_requests, + expected_route_requests, + ) return_part = '' - if res.result_kind == public_abi.ResultCode.RETURN: + if res.result_kind == host_fns.ResultCode.RETURN: return_part += ( f'executed with `Return({gvm_calldata.to_str(res.result_data)})`\n' ) - elif res.result_kind == public_abi.ResultCode.VM_ERROR: + elif res.result_kind == host_fns.ResultCode.VM_ERROR: return_part += f'executed with `VMError("{res.result_data}")`\n' - elif res.result_kind == public_abi.ResultCode.USER_ERROR: + elif res.result_kind == host_fns.ResultCode.USER_ERROR: if isinstance(res.result_data, str): return_part += f'executed with `UserError("{res.result_data}")`\n' else: @@ -836,8 +854,9 @@ async def _run_single_step( if is_leader: nondet_list = res.result_nondet_results - with open(path_to_which_leader_puts_result, 'wb') as f: - pickle.dump(nondet_list, f) + await gvm_io.write_file_bytes( + path_to_which_leader_puts_result, pickle.dumps(nondet_list) + ) # Save outputs my_tmp_dir.joinpath('stdout.txt').write_text(res.stdout) @@ -851,11 +870,10 @@ async def _run_single_step( # Save RunHostAndProgramRes pickle result_path = Path(single_conf['result_path']) - result_path.parent.mkdir(exist_ok=True, parents=True) - with open(result_path, 'wb') as f: - pickle.dump(res, f) + await gvm_io.make_dir(result_path.parent) + await gvm_io.write_file_bytes(result_path, pickle.dumps(res)) - if res.result_kind == public_abi.ResultCode.INTERNAL_ERROR: + if res.result_kind == host_fns.ResultCode.INTERNAL_ERROR: return { 'passed': False, 'context': { @@ -894,8 +912,8 @@ async def _run_single_step( semantics_path.write_text(semantics) exp_semantics_path = Path(single_conf['expected_semantics_path']) - if exp_semantics_path.exists(): - exp_text = exp_semantics_path.read_text() + if await gvm_io.path_exists(exp_semantics_path): + exp_text = await gvm_io.read_file_text(exp_semantics_path) diff = _get_diffs(exp_text, semantics, lambda x: x) if diff is not None: return { @@ -912,7 +930,7 @@ async def _run_single_step( } else: # Create expected output file - exp_semantics_path.write_text(semantics) + await gvm_io.write_file_text(exp_semantics_path, semantics) # ADR-012 load-action log-grep assertions. A case may declare # `runner_load_asserts` on a step to assert on the executor's stable @@ -957,9 +975,7 @@ async def _run_single_step( compares_golden = False check_expected_hash = expected_hash_path is not None - if ( - res.result_kind == public_abi.ResultCode.VM_ERROR and res.result_data == 'timeout' - ): + if res.result_kind == host_fns.ResultCode.VM_ERROR and res.result_data == 'timeout': check_expected_hash = False # Don't check hash for timeouts, as they can be flaky if check_expected_hash: @@ -1096,7 +1112,8 @@ def _digest_file(path: Path) -> str: def _resolve_import(base: str, rel: str, jpathdir: list[str]) -> Path: - """Where jsonnet would find ``rel``, imported from a file in ``base``. + """ + Where jsonnet would find ``rel``, imported from a file in ``base``. Unresolved: a nested import resolves against this path. """ @@ -1169,7 +1186,8 @@ def _connection(self) -> sqlite3.Connection: @staticmethod def _deps_hold(deps: list[list[str]], jpathdir: list[str]) -> bool: - """Whether every import still resolves where it did, to the same bytes. + """ + Whether every import still resolves where it did, to the same bytes. Re-resolved, not just re-digested: a new file of that name beside the case steals an import that fell through to the search path. @@ -1289,10 +1307,37 @@ def integration_test_single_executor( save_hashes = executor_integration_conf.get( 'save-hashes', not executor_integration_conf.get('ignore-hash', False) ) + integration_test_directory( + ctx, + cases_dir=CASES_DIR, + reroute_to=reroute_to, + save_hashes=save_hashes, + manager_service=manager_service, + modules_service=modules_service, + webdriver_service=webdriver_service, + ci=ci, + ) + + +def integration_test_directory( + ctx: genvm_tool.tests.stage.collection.Context, + *, + cases_dir: Path, + reroute_to: str, + save_hashes: bool, + manager_service: genvm_tool.tests.stage.collection.Service, + modules_service: genvm_tool.tests.stage.collection.Service, + webdriver_service: genvm_tool.tests.stage.collection.Service, + ci: bool, +) -> None: + """Collect one Jsonnet suite with the standard integration machinery.""" + import _jsonnet + + globals()['_jsonnet'] = _jsonnet # type: ignore glob_start = time.monotonic() jsonnet_files = list( - x for x in CASES_DIR.glob('**/*.jsonnet') if not x.name.startswith('_') + x for x in cases_dir.glob('**/*.jsonnet') if not x.name.startswith('_') ) glob_end = time.monotonic() # Bumped, not assigned: this runs once per executor line. @@ -1310,7 +1355,7 @@ def integration_test_single_executor( ctx, jsonnet_file, jsonnet_cache, - cases_dir=CASES_DIR, + cases_dir=cases_dir, reroute_to=reroute_to, save_hashes=save_hashes, manager_service=manager_service, @@ -1425,6 +1470,10 @@ def _single_integration_test( 'jsonnetDir': str(jsonnet_file.parent), 'fileBaseName': jsonnet_file.stem, 'tmpDir': str(tmp_dir), + **{ + 'executor' + key.replace('.', '').upper(): value + for key, value in build_info['executor_versions'].items() + }, }, ) @@ -1445,7 +1494,7 @@ def _single_integration_test( prepare_desc = genvm_tool.tests.test.Description( name=prepare_name, needed_services=frozen_services, - tags=frozen_tags, + tags=frozen_tags | {'phase-prepare'}, ) ctx.add_case( IntegrationPrepareCase( @@ -1467,6 +1516,13 @@ def _single_integration_test( for tree_path, depends_on_tree_path, single_conf in flat_steps: step_name = step_names[tree_path] is_benchmark = single_conf.get('benchmark', False) + mode = single_conf.get('mode') + try: + phase_tag = _PHASE_TAG_BY_MODE[mode] + except (KeyError, TypeError): + raise ValueError( + f'{jsonnet_file}: step {tree_path}: unexpected mode {mode!r}' + ) from None if depends_on_tree_path is None: deps = frozenset([prepare_name]) @@ -1476,7 +1532,7 @@ def _single_integration_test( step_desc = genvm_tool.tests.test.Description( name=step_name, needed_services=frozen_services, - tags=frozen_tags, + tags=frozen_tags | {phase_tag}, depends_on=deps, ) diff --git a/tests/runner/genvm_tool_plugins/ninja.py b/tests/runner/genvm_tool_plugins/ninja.py index 4d34c31d..6eefdbef 100644 --- a/tests/runner/genvm_tool_plugins/ninja.py +++ b/tests/runner/genvm_tool_plugins/ninja.py @@ -1,4 +1,5 @@ -"""Ninja build-graph DSL + per-line `configure` helpers (importable plugin). +""" +Ninja build-graph DSL + per-line `configure` helpers (importable plugin). Extracted from ``genvm-tool configure`` so each executor line can own its own build configuration. The shared machinery lives here β€” the ninja DSL @@ -36,7 +37,8 @@ def target_dir_for_line(rust_target_dir: Path, key: str) -> Path: - """Cargo target dir for one executor line. + """ + Cargo target dir for one executor line. Lines ship crates with identical names and versions; cargo's artifact hash does not cover where a package came from, so in a shared target dir two lines @@ -83,7 +85,8 @@ def _escape(s: str) -> str: def glob(base: Path, pattern: str) -> list[Path]: - """Mirror Ruby's `Pathname#glob`: a depth-first, per-component sorted walk + """ + Mirror Ruby's `Pathname#glob`: a depth-first, per-component sorted walk that skips dotfile entries (no FNM_DOTMATCH). Sorting by path *components* (not the joined string) is what makes @@ -110,7 +113,8 @@ def is_subpath(path: Path, base: Path) -> bool: # Nix expressions that evaluate a checkout's runner derivations to their # {id: hash} (latest.json) and {id: [hash]} (all.json) maps. def runner_manifest_expr(version: str, kind: str) -> str: - """Nix expr for one executor's `latest`/`all` runner manifest. + """ + Nix expr for one executor's `latest`/`all` runner manifest. The umbrella's ./runners owns this: `all` is every runner compatible up to `version`, `latest` is that executor's own current runners. Evaluated with @@ -276,7 +280,8 @@ def build(self, rule: str, *outputs) -> Build: def codegen( self, out: Path, lang: str, data: Path, extra_flags: list[str] | None = None ) -> None: - """Emit one `codegen` edge (`genvm-tool codegen --lang <lang>`). + """ + Emit one `codegen` edge (`genvm-tool codegen --lang <lang>`). Requires the `codegen` rule. The generated file depends on its data JSON and on the codegen backend sources (``codegen_deps``, set by the command) so an @@ -331,8 +336,10 @@ def register_cargo( clippy_lints = ['--', '-A', 'clippy::upper_case_acronyms', '-Dwarnings'] + # Lint/format edges declare no inputs and produce no output file: cargo does + # its own staleness tracking, and a stale ninja stamp hid real diagnostics. + clippy = self.build('cargo', 'target/' + rel_path + '/clippy') - clippy.add_dependency(files_trg) clippy.var('subcommand', 'clippy') clippy.var('wd', crate_dir) clippy.var('extra_args', extra_args + clippy_lints) @@ -381,7 +388,8 @@ def register_cargo( @dataclass class LineContext: - """Everything one executor line's ``configure(line)`` hook needs. + """ + Everything one executor line's ``configure(line)`` hook needs. Built by the configure command per active line. The hook composes the ``register_standard_*`` / ``install_tree`` steps and chooses a runner-registry @@ -435,16 +443,24 @@ def register_standard_codegen(self) -> None: 'python', data / 'public-abi.json', ) - pending_abi = self.exec_root / 'executor/codegen/data/public-abi-pending.json' + pending_abi = data / 'public-abi-pending.json' if pending_abi.exists(): self.codegen( self.exec_root / 'executor/crates/common/src/public_abi_pending.rs', 'rust', - self.exec_root / 'executor/codegen/data/public-abi-pending.json', + pending_abi, + ) + internal_constants = data / 'internal-constants.json' + if internal_constants.exists(): + self.codegen( + self.exec_root / 'executor/crates/common/src/internal_constants.rs', + 'rust', + internal_constants, ) def register_standard_crates(self) -> None: - """Build this line's `genvm` binary plus its `common`/`sdk-rs` crates. + """ + Build this line's `genvm` binary plus its `common`/`sdk-rs` crates. `calldata` and `calldata-derive` live in this line's tree too but get no edge of their own β€” they are path deps of the crates listed here. @@ -459,7 +475,8 @@ def register_standard_crates(self) -> None: self.n.register_cargo(f'{self.exec_rel}/executor/crates/sdk-rs', target_dir=td) def frozen_registry(self) -> None: - """Copy this line's committed ``executor/registry`` runner manifests verbatim. + """ + Copy this line's committed ``executor/registry`` runner manifests verbatim. Frozen legacy lines ship the exact ``latest.json``/``all.json`` from their release, which the nix build copies verbatim (see that line's @@ -474,7 +491,8 @@ def frozen_registry(self) -> None: self.data_phony.add_dependency(out) def nix_manifests(self) -> None: - """Derive this line's `latest`/`all` runner manifests via the nix machinery. + """ + Derive this line's `latest`/`all` runner manifests via the nix machinery. Manifests are derived by the umbrella machinery (which imports every active line's runners), so both the line's and the umbrella's nix inputs matter. @@ -499,7 +517,8 @@ def install_tree(self) -> None: def configure_line_default(line: LineContext) -> None: - """Fallback per-line configuration for a line whose `.genvm-tool.py` has no + """ + Fallback per-line configuration for a line whose `.genvm-tool.py` has no `configure` hook: the standard codegen + crates + install, resolving the runner registry from a committed ``executor/registry`` when present, else nix. """ diff --git a/tests/runner/genvm_tool_plugins/pytest.py b/tests/runner/genvm_tool_plugins/pytest.py index 5cb172f0..3d0398ab 100644 --- a/tests/runner/genvm_tool_plugins/pytest.py +++ b/tests/runner/genvm_tool_plugins/pytest.py @@ -4,6 +4,7 @@ from pathlib import Path import genvm_tool.tests +import genvm_tool_plugins.afl as afl default_env = { k: v @@ -11,9 +12,6 @@ if genvm_tool.tests.util.environ.DEFAULT_FILTER(k, v) } -default_env['AFL_FUZZER_LOOPCOUNT'] = '20' # without it no coverage will be written! -default_env['AFL_NO_CFG_FUZZING'] = '1' -default_env['AFL_BENCH_UNTIL_CRASH'] = '1' default_env.pop('VIRTUAL_ENV', None) local_ctx = genvm_tool.tests.stage.configuration.current_context() @@ -21,7 +19,7 @@ # The Python unit-test interpreter is provided by a standalone, pinned nix flake # (executors/v0.3.x/support/nix/py-test) that replaced genlayer-py-std's poetry # env. It is realised lazily at run time (not collection time, so `test show` -# stays cheap); pytest / py-afl-fuzz are exec'd from its bin/. +# stays cheap); tools are exec'd from its bin/. py_test_flake_dir = local_ctx.shared.root_dir.joinpath( 'executors', 'v0.3.x', 'support', 'nix', 'py-test' ) @@ -30,34 +28,62 @@ ) -def nix_env_command(argv: collections.abc.Sequence[str | Path]) -> list[str]: - """Wrap ``argv`` so it runs against the py-test flake's Python env. +def env_flake_dir(project_root_dir: Path) -> Path: + """ + The flake whose env a project's tests run under. + + A project carrying its own `flake.nix` gets that one: its tests need its own + dependencies, and the shared env holds only what genlayer-py-std's need. + """ + if project_root_dir.joinpath('flake.nix').is_file(): + return project_root_dir + return py_test_flake_dir + + +def nix_env_command( + argv: collections.abc.Sequence[str | Path], + flake_dir: Path | None = None, +) -> list[str]: + """ + Wrap ``argv`` so it runs against a py-test flake's Python env. Builds (or reuses, via the gc-root out-link) the flake's env under ``build/ya-build/py-test-env``, then execs ``<env>/bin/<argv[0]>`` with the - remaining args. ``argv[0]`` is a bin name (``pytest``, ``py-afl-fuzz``); the + remaining args. ``argv[0]`` is a bin name (for example ``pytest``); the rest pass through verbatim. """ + flake_dir = flake_dir or py_test_flake_dir + # One out-link per flake, or the envs would evict each other's gc-root and + # the next run would rebuild + out_link = py_test_env_link.with_name( + f'{py_test_env_link.name}-{flake_dir.name}' + if flake_dir != py_test_flake_dir + else py_test_env_link.name + ) script = ( 'set -eu\n' - f'mkdir -p {shlex.quote(str(py_test_env_link.parent))}\n' + f'mkdir -p {shlex.quote(str(out_link.parent))}\n' 'env_dir="$(nix build --print-out-paths' - f' --out-link {shlex.quote(str(py_test_env_link))}' - f' {shlex.quote("path:" + str(py_test_flake_dir))})"\n' + f' --out-link {shlex.quote(str(out_link))}' + f' {shlex.quote("path:" + str(flake_dir))})"\n' + 'export PATH="$env_dir/bin:$PATH"\n' 'exec "$env_dir/bin/$0" "$@"' ) return ['bash', '-c', script, *map(str, argv)] def pytest_env(project_root_dir: Path) -> dict: - """``default_env`` plus PYTHONPATH for the in-tree packages under test. + """ + ``default_env`` plus PYTHONPATH for the in-tree packages under test. - The nix env ships only the test deps; ``genlayer`` (src/) and - ``genlayer_embeddings`` / ``onnx`` (src-emb/) are imported via PYTHONPATH, - mirroring the editable install poetry used to provide. + The nix env ships only the dependencies; the package under test is imported + from the working tree, mirroring the editable install poetry used to + provide. That is ``src/`` + ``src-emb/`` for genlayer-py-std, and the + project root itself for a flat layout such as `genvm_tool`. """ env = dict(default_env) pythonpath = [ + str(project_root_dir), str(project_root_dir.joinpath('src')), str(project_root_dir.joinpath('src-emb')), ] @@ -77,7 +103,7 @@ def pytest( desc = desc.with_tags(['python', 'unit'])._replace(console_pool=True) case = genvm_tool.tests.test.SimpleCommandCase( description=desc, - command=nix_env_command(['pytest', '--color=yes']), + command=nix_env_command(['pytest', '--color=yes'], env_flake_dir(project_root_dir)), cwd=project_root_dir, env=pytest_env(project_root_dir), mode=genvm_tool.tests.exec.command.RunMode.INTERACTIVE, @@ -93,34 +119,82 @@ def py_fuzz( project_root_dir: Path, name: str, ): - desc = desc.with_tags(['python', 'fuzz'])._replace( - console_pool=True, + desc = desc.with_tags(['python', 'fuzz', 'needs-fuzz'])._replace( + **afl.pool_usage(ctx), ) inputs_dir = project_root_dir.joinpath('fuzz', 'inputs', name) - outputs_dir = project_root_dir.joinpath('fuzz', 'outputs', name) + out_dir = afl.output_dir(local_ctx.shared, project_root_dir, name) src_file = project_root_dir.joinpath('fuzz', 'src', f'{name}.py') - # NB: the py-test flake omits python-afl (not in nixpkgs), so `py-afl-fuzz` - # is absent from its bin/. The fuzz collector is disabled; re-add python-afl - # to the flake before enabling this. - case = genvm_tool.tests.test.SimpleCommandCase( - description=desc, - command=nix_env_command( - [ - 'py-afl-fuzz', - '-i', - inputs_dir, - '-o', - outputs_dir, - '-V', - str(getattr(ctx.configuration.args, 'fuzz_timeout', 30)), - '--', - src_file, - ] - ), - cwd=project_root_dir, - env=pytest_env(project_root_dir), - mode=genvm_tool.tests.exec.command.RunMode.INTERACTIVE, + # A target imports its project's own packages, so it runs under the same env + # as that project's unit tests; python-afl and AFL++ are in both + flake_dir = env_flake_dir(project_root_dir) + + def env_command(argv: collections.abc.Sequence[str | Path]) -> list[str]: + return nix_env_command(argv, flake_dir) + + env = afl.environment(pytest_env(project_root_dir)) + # What `py-afl-fuzz` exports before handing over to `afl-fuzz`. `AFL_SKIP_CHECKS` + # is left out: it is the pre-1.20b spelling, which the pinned AFL++ only warns + # about. The rest of the wrapper is the PATH lookup `nix_env_command` does + env.update( + { + 'AFL_DUMB_FORKSRV': '1', + 'AFL_SKIP_BIN_CHECK': '1', + 'PYTHON_AFL_PERSISTENT': '1', + 'PYTHON_AFL_SIGNAL': 'SIGUSR1', + } ) + steps: list[genvm_tool.tests.exec.step.Step] = list( + afl.fleet_steps( + ctx, + cwd=project_root_dir, + env=env, + inputs_dir=inputs_dir, + out_dir=out_dir, + launcher=['afl-fuzz'], + target=['--', src_file], + wrap=env_command, + # Realising the env once: every fleet member would otherwise pay a + # `nix build` before `exec`, staggering the instances' `-V` windows + prepare_command=env_command(['python3', '-c', '']), + status_command=env_command(['afl-whatsup', '-s', out_dir]), + # The target is a forkserver, so a saved input replays through AFL + # rather than by piping it into the script + replay=shlex.join( + str(arg) + for arg in env_command(['afl-showmap', '-o', '/dev/null', '--', src_file]) + ) + + ' <', + ) + ) + + steps.extend( + afl.corpus_update_steps( + ctx, + inputs_dir=inputs_dir, + out_dir=out_dir, + cmin_command=lambda queue_dir, opt_dir: env_command( + [ + 'afl-cmin', + '-T', + 'all', + '-o', + opt_dir, + '-i', + queue_dir, + '--', + src_file, + ] + ), + # `afl-cmin` measures one input per run through `afl-showmap`, so it + # gets the plain forkserver `py-afl-cmin` gives it; kept, the + # persistent loop shifts which inputs survive minimization + unset_env=('AFL_DUMB_FORKSRV', 'PYTHON_AFL_PERSISTENT'), + ) + ) + + case = genvm_tool.tests.test.StepsCase(description=desc, steps=steps) + ctx.add_case(case) diff --git a/tests/runner/genvm_tool_plugins/runners.py b/tests/runner/genvm_tool_plugins/runners.py new file mode 100644 index 00000000..e558febd --- /dev/null +++ b/tests/runner/genvm_tool_plugins/runners.py @@ -0,0 +1,55 @@ +import functools +import json +import re +from pathlib import Path + +# `@RUNNER_LATEST_<line>_<runner>@`, where `<line>` is `v0.3` or `v0.3.x` and +# `<runner>` is a key of that line's `latest.json`, e.g. `py-genlayer`. +TOKEN = re.compile(rb'@RUNNER_LATEST_(v\d+\.\d+(?:\.x)?)_([A-Za-z0-9_.-]+)@') + + +def _line(line: str) -> str: + return line[:-2] if line.endswith('.x') else line + + +@functools.cache +def latest(root_dir: Path, line: str) -> dict[str, str]: + """Runner id -> uid for the newest runners `line`'s executor was built with.""" + info_path = root_dir / 'build' / 'info.json' + versions = json.loads(info_path.read_text())['executor_versions'] + version = versions.get(_line(line)) + if version is None: + known = ', '.join(sorted(versions)) or '<none>' + raise KeyError(f'unknown executor line {line!r}, built lines are {known}') + path = root_dir / 'build' / 'out' / 'executor' / version / 'data' / 'latest.json' + if not path.exists(): + raise FileNotFoundError( + f'{path} is absent: build the runners before running this test' + ) + return json.loads(path.read_text()) + + +def uid(root_dir: Path, line: str, runner: str) -> str: + """Uid of one runner, as `latest.json` has it after the last build.""" + known = latest(root_dir, line) + if runner not in known: + raise KeyError( + f'{line} has no runner {runner!r}, it has {", ".join(sorted(known))}' + ) + return known[runner] + + +def substitute(data: bytes, root_dir: Path) -> bytes: + """ + Replaces every `@RUNNER_LATEST_<line>_<runner>@` with the built uid. + + A runner's uid changes with its contents, so a test that pins one by hand + goes stale on every runner edit. The token spares the test that pin. + """ + + def replace(match: re.Match[bytes]) -> bytes: + line = match.group(1).decode('utf-8') + runner = match.group(2).decode('utf-8') + return uid(root_dir, line, runner).encode('utf-8') + + return TOKEN.sub(replace, data) diff --git a/tests/runner/gvm_extra/case_inputs.py b/tests/runner/gvm_extra/case_inputs.py new file mode 100644 index 00000000..e3318677 --- /dev/null +++ b/tests/runner/gvm_extra/case_inputs.py @@ -0,0 +1,17 @@ +import origin.calldata as gvm_calldata + + +def encode_calldata(conf: dict) -> bytes: + """ + Calldata bytes an integration case config asks for. + + ``calldata`` is a python expression, evaluated against the case's ``vars``: + the jsonnet side keeps it as source so a case can build an + :class:`~origin.calldata.Address` or reuse a var. + """ + names = { + 'Address': gvm_calldata.Address, + 'true': True, + 'false': False, + } + return gvm_calldata.encode(eval(conf['calldata'], names, dict(conf.get('vars', {})))) diff --git a/tests/runner/origin/base_host.py b/tests/runner/origin/base_host.py index 2b2cd4bb..f35cf414 100644 --- a/tests/runner/origin/base_host.py +++ b/tests/runner/origin/base_host.py @@ -102,7 +102,8 @@ def __init__(self, error_code: host_fns.Errors, message: str = ''): @dataclass(frozen=True) class UnsafeOverrides: - """Request overrides that reach boundaries production traffic cannot. + """ + Request overrides that reach boundaries production traffic cannot. Each member states the `debug_mode` the manager requires before it applies: `reroute_to` from `safe`, `initial_recursion` from `unsafe`. With debugging @@ -477,7 +478,8 @@ def emit_host_loop_stats(): class ConsumedResultDecodeError(Exception): - """`consumed_result` bytes did not parse into a `ConsumedResult` at all. + """ + `consumed_result` bytes did not parse into a `ConsumedResult` at all. Distinct from `ConsumedResult.internal_error(...)`, which is itself a valid (if unhappy) result value produced from bytes that *did* parse: @@ -493,7 +495,7 @@ class ConsumedResult: """The decoded `consumed_result` blob: a `ResultCode` byte plus calldata.""" execution_hash: bytes - result_kind: public_abi.ResultCode + result_kind: host_fns.ResultCode result_data: gvm_calldata.Decoded result_fingerprint: ResultFingerprint | None = None result_storage_changes: list[tuple[bytes, bytes]] = field(default_factory=list) @@ -505,7 +507,7 @@ class ConsumedResult: def internal_error(cls, message: str) -> 'ConsumedResult': return cls( execution_hash=b'', - result_kind=public_abi.ResultCode.INTERNAL_ERROR, + result_kind=host_fns.ResultCode.INTERNAL_ERROR, result_data=message, ) @@ -522,7 +524,7 @@ def decode(cls, raw: typing.Any) -> 'ConsumedResult': ) empty = not as_bytes if not empty: - result_kind = public_abi.ResultCode(as_bytes[0]) + result_kind = host_fns.ResultCode(as_bytes[0]) decoded = gvm_calldata.decode(as_bytes[1:]) except Exception as exc: # Unreadable bytes are a protocol violation rather than a result, so @@ -538,6 +540,10 @@ def decode(cls, raw: typing.Any) -> 'ConsumedResult': return cls.internal_error('empty_result') if not isinstance(decoded, dict): return cls.internal_error('result is not a mapping') + # The executor reports fatality; degrading it to an ordinary VM error + # is the host's job, so that a caller in another major can still see it + if result_kind == host_fns.ResultCode.FATAL_VM_ERROR: + result_kind = host_fns.ResultCode.VM_ERROR return cls( execution_hash=decoded.get('execution_hash', b''), result_kind=result_kind, @@ -560,7 +566,7 @@ class RunHostAndProgramRes: execution_hash: bytes - result_kind: public_abi.ResultCode + result_kind: host_fns.ResultCode result_data: gvm_calldata.Decoded result_fingerprint: ResultFingerprint | None result_storage_changes: list[tuple[bytes, bytes]] @@ -615,7 +621,8 @@ class ManagerRunLost(Exception): class ManagerRunNotStarted(Exception): - """A run never reached execution, so retrying it re-executes nothing. + """ + A run never reached execution, so retrying it re-executes nothing. Raised for the two ways the manager can refuse before the genvm runs -- a rejected RUN request and a `failed_to_start` terminal. Retry policy belongs @@ -633,7 +640,8 @@ def __init__(self, message: str, *, retryable: bool, reason: str): class TerminalResultUnavailable(Exception): - """A run reached a terminal `finished` state, but its result could not be + """ + A run reached a terminal `finished` state, but its result could not be retrieved or decoded -- artifact transfer failed, or `consumed_result` was empty/malformed. @@ -763,7 +771,8 @@ async def attach(self, boot_id: int, genvm_id: int) -> RunState: def _require_current_generation( self, boot_id: int, genvm_id: int, *, op: str ) -> None: - """Refuse to act on a run from a manager generation we have since left. + """ + Refuse to act on a run from a manager generation we have since left. `CANCEL`/`ACK`/`GET_ARTIFACT` identify a run to the manager by `genvm_id` alone -- the wire protocol has no `boot_id` field for them @@ -1124,7 +1133,8 @@ def _duration_string(seconds: float | None) -> str | None: def _decode_genvm_log(data: bytes) -> list[dict[str, typing.Any]]: - """Decodes the genvm log artifact, which is json lines. + """ + Decodes the genvm log artifact, which is json lines. Split on newline bytes only. `str.splitlines` also breaks on U+2028, U+2029 and U+0085, none of which json escapes and all of which therefore appear raw @@ -1324,9 +1334,9 @@ async def cancel_on_shutdown(): ) from exc if ( status.get('cause') == 'deadline' - and consumed.result_kind != public_abi.ResultCode.RETURN + and consumed.result_kind != host_fns.ResultCode.RETURN ): - consumed.result_kind = public_abi.ResultCode.VM_ERROR + consumed.result_kind = host_fns.ResultCode.VM_ERROR consumed.result_data = str(public_abi.VmError.timeout()) else: # A failed_to_start terminal means the genvm was accepted but never @@ -1346,7 +1356,7 @@ async def cancel_on_shutdown(): raise ManagerRunNotStarted(error, retryable=retryable, reason=reason) vm_error_description: str | None = None - if consumed.result_kind == public_abi.ResultCode.VM_ERROR and isinstance( + if consumed.result_kind == host_fns.ResultCode.VM_ERROR and isinstance( consumed.result_data, str ): try: diff --git a/tests/runner/origin/fees.py b/tests/runner/origin/fees.py index ed3b3069..47ba4019 100644 --- a/tests/runner/origin/fees.py +++ b/tests/runner/origin/fees.py @@ -1,4 +1,5 @@ -"""Message-fee allocation tree types. +""" +Message-fee allocation tree types. Mirrors the executor's `genvm_common::domain::fees` module: the fee parameters and the nested `MessageAllocationNode` tree that is passed alongside an diff --git a/tests/runner/origin/host_fns.py b/tests/runner/origin/host_fns.py index 316d11e8..9715e1be 100644 --- a/tests/runner/origin/host_fns.py +++ b/tests/runner/origin/host_fns.py @@ -19,25 +19,19 @@ class Methods(IntEnum): RUN_NESTED = 8 +class ResultCode(IntEnum): + RETURN = 0 + USER_ERROR = 1 + VM_ERROR = 2 + INTERNAL_ERROR = 3 + FATAL_VM_ERROR = 4 + + class Errors(IntEnum): OK = 0 EVM_REVERTED = 1 FORBIDDEN = 2 -class VmErrorDetail: - __slots__ = ('value',) - def __init__(self, value: str): - self.value = value - def __str__(self) -> str: - return self.value - @staticmethod - def internal() -> 'VmErrorDetail': - return VmErrorDetail('internal') - @staticmethod - def external() -> 'VmErrorDetail': - return VmErrorDetail('external') - - CURRENT_MAJOR: typing.Final[int] = 0 diff --git a/tests/runner/origin/log_asserts.py b/tests/runner/origin/log_asserts.py index a46507a6..21c0618a 100644 --- a/tests/runner/origin/log_asserts.py +++ b/tests/runner/origin/log_asserts.py @@ -1,4 +1,5 @@ -"""Assertions over the executor's structured log records. +""" +Assertions over the executor's structured log records. The manager captures every executor-emitted log line and surfaces them as ``RunHostAndProgramRes.genvm_log`` β€” a list of JSON objects shaped like @@ -6,12 +7,10 @@ under ``debug_mode >= safe-unbounded`` (integration tests run ``unsafe``), so no record is evicted regardless of how chatty the run is. -The ADR-012 load action emits one stable ``"runner load"`` record per load, -carrying: +The load action emits one stable ``"runner load"`` record per load, carrying: - ``runner`` β€” the canonical runner id (``chain:``/``custom:``/``name:hash``); -- ``runner_load_cost`` β€” the flat per-load constant - (``public_abi::memory_limiter_consts::RUNNER_LOAD_COST``); +- ``runner_load_cost`` β€” the flat per-load memory limiter constant; - ``size`` β€” the charged content size (archive ``total_size``); - ``status`` β€” ``"charged"`` (first load in this VM) or ``"cached"`` (already in the VM's loaded set β€” free). @@ -62,7 +61,8 @@ def extract( match: dict | None = None, runner_prefix: str | None = None, ) -> list[dict]: - """Return the ``genvm_log`` records matching ``match`` (defaulting to the + """ + Return the ``genvm_log`` records matching ``match`` (defaulting to the ``runner load`` message) and an optional ``runner`` prefix.""" return _select(genvm_log, match or {}, runner_prefix) diff --git a/tests/runner/origin/public_abi.py b/tests/runner/origin/public_abi.py index cc3445bb..abb7a496 100644 --- a/tests/runner/origin/public_abi.py +++ b/tests/runner/origin/public_abi.py @@ -11,7 +11,6 @@ class ResultCode(IntEnum): RETURN = 0 USER_ERROR = 1 VM_ERROR = 2 - INTERNAL_ERROR = 3 class StorageType(IntEnum): @@ -30,16 +29,6 @@ class Permissions(IntEnum): CAN_USE_BALANCE_FOR_MESSAGE_FEES = 1 -class _MemoryLimiterConsts(typing.NamedTuple): - TABLE_ENTRY: int = 64 - FILE_MAPPING: int = 256 - FD_ALLOCATION: int = 96 - RUNNER_LOAD_COST: int = 4096 - VM_SPAWN_COST: int = 134217728 - -memory_limiter_consts: typing.Final = _MemoryLimiterConsts() - - class _RootOffsets(typing.NamedTuple): MAJOR: int = 0 CONTRACT: int = 1 @@ -52,24 +41,39 @@ class _RootOffsets(typing.NamedTuple): root_offsets: typing.Final = _RootOffsets() -class _TopLimits(typing.NamedTuple): - NONDET_BLOCKS: int = 4096 - LOCKED_SLOTS: int = 256 - UPGRADERS: int = 32 - VM_RECURSION: int = 512 - WEB_REQUEST_MIN_SPACE: int = 65536 - WEB_RENDER_MIN_SPACE: int = 134217728 - MAX_FDS: int = 1024 - WASM_CALL_DEPTH: int = 1024 - WASM_STACK_VALUE_SLOTS: int = 65535 - -top_limits: typing.Final = _TopLimits() - - class SpecialMethod(StrEnum): GET_SCHEMA = '#get-schema' ERRORED_MESSAGE = '#error' +class _VmErrorLeaderFaultNondetOutput: + @staticmethod + def absent() -> 'VmError': + return VmError('leader_fault nondet_output absent') + @staticmethod + def malformed() -> 'VmError': + return VmError('leader_fault nondet_output malformed') + @staticmethod + def uses_this_error() -> '_VmErrorLeaderFaultNondetOutputUsesThisError': + return _VmErrorLeaderFaultNondetOutputUsesThisError() + @staticmethod + def extra() -> '_VmErrorLeaderFaultNondetOutputExtra': + return _VmErrorLeaderFaultNondetOutputExtra() + +class _VmErrorLeaderFaultNondetOutputUsesThisError: + @staticmethod + def val_str(v: str) -> 'VmError': + return VmError(f'leader_fault nondet_output uses_this_error {v}') + +class _VmErrorLeaderFaultNondetOutputExtra: + @staticmethod + def val_str(v: str) -> 'VmError': + return VmError(f'leader_fault nondet_output extra {v}') + +class _VmErrorLeaderFault: + @staticmethod + def nondet_output() -> '_VmErrorLeaderFaultNondetOutput': + return _VmErrorLeaderFaultNondetOutput() + class _VmErrorWasmTrap: @staticmethod def val() -> 'VmError': @@ -195,6 +199,14 @@ class _VmErrorEvm: def reverted() -> 'VmError': return VmError('evm reverted') +class _VmErrorInvalidContractRunner: + @staticmethod + def absent() -> 'VmError': + return VmError('invalid_contract runner absent') + @staticmethod + def malformed() -> 'VmError': + return VmError('invalid_contract runner malformed') + class _VmErrorInvalidContractWasm: @staticmethod def validating() -> 'VmError': @@ -211,18 +223,15 @@ class _VmErrorInvalidContract: def val() -> 'VmError': return VmError('invalid_contract') @staticmethod - def absent_runner_comment() -> 'VmError': - return VmError('invalid_contract absent_runner_comment') - @staticmethod def not_utf8_text() -> 'VmError': return VmError('invalid_contract not_utf8_text') @staticmethod - def malformed_runner() -> 'VmError': - return VmError('invalid_contract malformed_runner') - @staticmethod def major_mismatch() -> 'VmError': return VmError('invalid_contract major_mismatch') @staticmethod + def runner() -> '_VmErrorInvalidContractRunner': + return _VmErrorInvalidContractRunner() + @staticmethod def wasm() -> '_VmErrorInvalidContractWasm': return _VmErrorInvalidContractWasm() @@ -241,11 +250,14 @@ def __str__(self) -> str: def timeout() -> 'VmError': return VmError('timeout') @staticmethod - def absent_leader_nondet_output() -> 'VmError': - return VmError('absent_leader_nondet_output') + def malformed_entry() -> 'VmError': + return VmError('malformed_entry') + @staticmethod + def forbidden() -> 'VmError': + return VmError('forbidden') @staticmethod - def host_forbidden() -> 'VmError': - return VmError('host_forbidden') + def leader_fault() -> '_VmErrorLeaderFault': + return _VmErrorLeaderFault() @staticmethod def exit_code() -> '_VmErrorExitCode': return _VmErrorExitCode() @@ -264,6 +276,12 @@ def evm() -> '_VmErrorEvm': @staticmethod def invalid_contract() -> '_VmErrorInvalidContract': return _VmErrorInvalidContract() + def internal(self) -> 'VmError': + assert ' # ' not in self.value + return VmError(f'{self.value} # internal') + def external(self) -> 'VmError': + assert ' # ' not in self.value + return VmError(f'{self.value} # external') diff --git a/tests/system/cross-major-observability/assets/v0.2/caller.py b/tests/system/cross-major-observability/assets/v0.2/caller.py new file mode 100644 index 00000000..784eb1eb --- /dev/null +++ b/tests/system/cross-major-observability/assets/v0.2/caller.py @@ -0,0 +1,27 @@ +# { "Depends": "py-genlayer:@RUNNER_LATEST_v0.2_py-genlayer@" } +from genlayer import * + + +class Contract(gl.Contract): + def __init__(self): + pass + + @gl.public.view + def context(self, target: Address): + return gl.get_contract_at(target).view().context() + + @gl.public.view + def permissions(self, target: Address, helper: Address): + return gl.get_contract_at(target).view().permissions(helper) + + @gl.public.view + def debug_alias(self, target: Address): + return gl.get_contract_at(target).view().debug_alias() + + @gl.public.view + def read(self, target: Address) -> int: + return gl.get_contract_at(target).view().read() + + @gl.public.view + def answer(self, target: Address) -> int: + return gl.get_contract_at(target).view().answer() diff --git a/tests/system/cross-major-observability/assets/v0.2/chain.py b/tests/system/cross-major-observability/assets/v0.2/chain.py new file mode 100644 index 00000000..a3ab6c98 --- /dev/null +++ b/tests/system/cross-major-observability/assets/v0.2/chain.py @@ -0,0 +1,15 @@ +# { "Depends": "py-genlayer:@RUNNER_LATEST_v0.2_py-genlayer@" } +from genlayer import * + + +class Contract(gl.Contract): + def __init__(self): + pass + + @gl.public.view + def hop(self, depth: int, other: Address) -> int: + if depth <= 0: + return 2 + return 2 + gl.get_contract_at(other).view().hop( + depth - 1, gl.message.contract_address + ) diff --git a/tests/system/cross-major-observability/assets/v0.3/chain.py b/tests/system/cross-major-observability/assets/v0.3/chain.py new file mode 100644 index 00000000..44b83fc4 --- /dev/null +++ b/tests/system/cross-major-observability/assets/v0.3/chain.py @@ -0,0 +1,16 @@ +# { "Depends": "py-genlayer:@RUNNER_LATEST_v0.3_py-genlayer@" } +import genlayer as gl +from genlayer.types import Address + + +class Contract(gl.contract.Contract): + def __init__(self): + pass + + @gl.public.view + def hop(self, depth: int, other: Address) -> int: + if depth <= 0: + return 3 + return 3 + gl.contract.get_at(other).view().hop( + depth - 1, gl.message.contract_address + ) diff --git a/tests/system/cross-major-observability/assets/v0.3/helper.py b/tests/system/cross-major-observability/assets/v0.3/helper.py new file mode 100644 index 00000000..ee36e49c --- /dev/null +++ b/tests/system/cross-major-observability/assets/v0.3/helper.py @@ -0,0 +1,11 @@ +# { "Depends": "py-genlayer:@RUNNER_LATEST_v0.3_py-genlayer@" } +import genlayer as gl + + +class Contract(gl.contract.Contract): + def __init__(self): + pass + + @gl.public.view + def answer(self) -> int: + return 11 diff --git a/tests/system/cross-major-observability/assets/v0.3/observer.py b/tests/system/cross-major-observability/assets/v0.3/observer.py new file mode 100644 index 00000000..eb4fd657 --- /dev/null +++ b/tests/system/cross-major-observability/assets/v0.3/observer.py @@ -0,0 +1,60 @@ +# { "Depends": "py-genlayer:@RUNNER_LATEST_v0.3_py-genlayer@" } +import genlayer as gl +import _genlayer_wasi as wasi +from genlayer.types import Address, u32 + + +class Contract(gl.contract.Contract): + value: u32 + + def __init__(self): + self.value = 777 + + @gl.public.view + def context(self): + return { + 'contract': gl.message.contract_address.as_hex, + 'sender': gl.message.sender_address.as_hex, + 'origin': gl.message.origin_address.as_hex, + 'signer': gl.message.signer_address.as_hex, + 'stack': [address.as_hex for address in gl.message.stack], + 'value': int(gl.message.value), + 'is_init': gl.message.is_init, + } + + @gl.public.view + def read(self) -> int: + return self.value + + @gl.public.view + def permissions(self, helper: Address): + def attempt(action): + try: + value = action() + return {'kind': 'allowed', 'value': str(value)} + except OSError as exc: + return {'kind': 'oserror', 'errno': exc.errno} + except Exception as exc: + return {'kind': type(exc).__name__, 'value': str(exc)} + + write = attempt(lambda: wasi.storage_write(b'\x77' * 32, 0, b'forbidden')) + send = attempt(lambda: gl.contract.get_at(Address(b'\x66' * 20)).emit().ping()) + nondet = attempt(lambda: gl.vm.run_nondet(lambda: 1, lambda _result: True)) + + registered = gl.vm.register_runner( + b'# { "Depends": "py-genlayer:@RUNNER_LATEST_v0.3_py-genlayer@" }\n' + b'print("registered")\n' + ) + called = gl.contract.get_at(helper).view().answer() + return { + 'write': write, + 'send': send, + 'nondet': nondet, + 'registered': registered, + 'called': called, + } + + @gl.public.view + def debug_alias(self) -> int: + result = gl.vm.spawn_sandbox(lambda: 1, runner='py-genlayer:test') + return gl.vm.unpack_result(result) diff --git a/tests/system/cross-major-observability/assets/v0.3/self_recursion.py b/tests/system/cross-major-observability/assets/v0.3/self_recursion.py new file mode 100644 index 00000000..8e1e2370 --- /dev/null +++ b/tests/system/cross-major-observability/assets/v0.3/self_recursion.py @@ -0,0 +1,20 @@ +# { "Depends": "py-genlayer:@RUNNER_LATEST_v0.3_py-genlayer@" } +import genlayer as gl + + +class Contract(gl.contract.Contract): + def __init__(self): + pass + + @gl.public.view + def recurse(self, depth: int) -> int: + if depth == 0: + return 1 + return 1 + gl.contract.get_at(self.address).view().recurse(depth - 1) + + @gl.public.view + def observe_recursion(self, depth: int) -> str: + try: + return 'ok:' + str(self.recurse(depth)) + except gl.vm.UserError as exc: + return 'caught:' + str(exc.data) diff --git a/tests/system/cross-major-observability/assets/v0.3/trap.py b/tests/system/cross-major-observability/assets/v0.3/trap.py new file mode 100644 index 00000000..f143796d --- /dev/null +++ b/tests/system/cross-major-observability/assets/v0.3/trap.py @@ -0,0 +1,11 @@ +# { "Depends": "py-genlayer:@RUNNER_LATEST_v0.3_py-genlayer@" } +import genlayer as gl + + +class Contract(gl.contract.Contract): + def __init__(self): + pass + + @gl.public.view + def answer(self) -> int: + raise RuntimeError('callee trap') diff --git a/tests/system/cross-major-observability/assets/v0.3/user_error.py b/tests/system/cross-major-observability/assets/v0.3/user_error.py new file mode 100644 index 00000000..e50f8d5a --- /dev/null +++ b/tests/system/cross-major-observability/assets/v0.3/user_error.py @@ -0,0 +1,11 @@ +# { "Depends": "py-genlayer:@RUNNER_LATEST_v0.3_py-genlayer@" } +import genlayer as gl + + +class Contract(gl.contract.Contract): + def __init__(self): + pass + + @gl.public.view + def answer(self) -> int: + gl.vm.UserError.immediate('nested user error') diff --git a/tests/system/cross-major-observability/test.py b/tests/system/cross-major-observability/test.py index 99df3864..0055fa12 100644 --- a/tests/system/cross-major-observability/test.py +++ b/tests/system/cross-major-observability/test.py @@ -1,9 +1,11 @@ -"""Cross-major CallContract observability and determinism integration tests. +""" +Cross-major CallContract observability and determinism integration tests. This case reuses the real-manager harness from ``tests/system/cross-major`` without adding more concerns to that already-large case. """ +import functools import importlib.util import json import pickle @@ -12,10 +14,11 @@ from dataclasses import dataclass from pathlib import Path +import genvm_tool.io as gvm_io import genvm_tool.tests import genvm_tool.tests.stage.collection from gvm_extra.mock_host import MockStorage -from origin import public_abi +from origin import host_fns, public_abi from origin.calldata import Address @@ -41,229 +44,45 @@ def _load_cross_major_harness(): ADDR_CHAIN_V02 = Address('0x' + '48' * 20) ABSENT = Address('0x' + '99' * 20) +# Mirrors `CROSS_MAJOR_RECURSION` in implementation/src/manager/run.rs: how many +# delegated hops one chain of contract calls may make. +CROSS_MAJOR_RECURSION = 6 -CALLER_V02 = base._contract_source( - 2, - """ - from genlayer import * - - - class Contract(gl.Contract): - def __init__(self): - pass - - @gl.public.view - def context(self, target: Address): - return gl.get_contract_at(target).view().context() - - @gl.public.view - def permissions(self, target: Address, helper: Address): - return gl.get_contract_at(target).view().permissions(helper) - - @gl.public.view - def debug_alias(self, target: Address): - return gl.get_contract_at(target).view().debug_alias() - - @gl.public.view - def read(self, target: Address) -> int: - return gl.get_contract_at(target).view().read() - - @gl.public.view - def answer(self, target: Address) -> int: - return gl.get_contract_at(target).view().answer() - """, -) - -OBSERVER_V03 = base._contract_source( - 3, - f""" - import genlayer as gl - import _genlayer_wasi as wasi - from genlayer.types import Address, u32 - - - class Contract(gl.contract.Contract): - value: u32 - - def __init__(self): - self.value = 777 - - @gl.public.view - def context(self): - return {{ - 'contract': gl.message.contract_address.as_hex, - 'sender': gl.message.sender_address.as_hex, - 'origin': gl.message.origin_address.as_hex, - 'signer': gl.message.signer_address.as_hex, - 'stack': [address.as_hex for address in gl.message.stack], - 'value': int(gl.message.value), - 'is_init': gl.message.is_init, - }} - - @gl.public.view - def read(self) -> int: - return self.value - - @gl.public.view - def permissions(self, helper: Address): - def attempt(action): - try: - value = action() - return {{'kind': 'allowed', 'value': str(value)}} - except OSError as exc: - return {{'kind': 'oserror', 'errno': exc.errno}} - except Exception as exc: - return {{'kind': type(exc).__name__, 'value': str(exc)}} - - write = attempt( - lambda: wasi.storage_write(b'\\x77' * 32, 0, b'forbidden') - ) - send = attempt( - lambda: gl.contract.get_at(Address(b'\\x66' * 20)).emit().ping() - ) - nondet = attempt( - lambda: gl.vm.run_nondet(lambda: 1, lambda _result: True) - ) - - registered = gl.vm.register_runner( - b'# {{ "Depends": "py-genlayer:{base.PY_GENLAYER_V03}" }}\\n' - b'print("registered")\\n' - ) - called = gl.contract.get_at(helper).view().answer() - return {{ - 'write': write, - 'send': send, - 'nondet': nondet, - 'registered': registered, - 'called': called, - }} - - @gl.public.view - def debug_alias(self) -> int: - result = gl.vm.spawn_sandbox(lambda: 1, runner='py-genlayer:test') - return gl.vm.unpack_result(result) - """, -) - -HELPER_V03 = base._contract_source( - 3, - """ - import genlayer as gl - - - class Contract(gl.contract.Contract): - def __init__(self): - pass - - @gl.public.view - def answer(self) -> int: - return 11 - """, -) - -SELF_V03 = base._contract_source( - 3, - """ - import genlayer as gl - - - class Contract(gl.contract.Contract): - def __init__(self): - pass - - @gl.public.view - def recurse(self, depth: int) -> int: - if depth == 0: - return 1 - return 1 + gl.contract.get_at(self.address).view().recurse(depth - 1) - - @gl.public.view - def observe_recursion(self, depth: int) -> str: - try: - return 'ok:' + str(self.recurse(depth)) - except gl.vm.UserError as exc: - return 'caught:' + str(exc.data) - """, -) - -TRAP_V03 = base._contract_source( - 3, - """ - import genlayer as gl - - - class Contract(gl.contract.Contract): - def __init__(self): - pass - @gl.public.view - def answer(self) -> int: - raise RuntimeError('callee trap') - """, +_contract_source = functools.partial( + base._contract_source, + assets_dir=Path(__file__).resolve().parent / 'assets', ) -USER_ERROR_V03 = base._contract_source( - 3, - """ - import genlayer as gl - - - class Contract(gl.contract.Contract): - def __init__(self): - pass - - @gl.public.view - def answer(self) -> int: - gl.vm.UserError.immediate('nested user error') - """, -) - -CHAIN_V03 = base._contract_source( - 3, - """ - import genlayer as gl - from genlayer.types import Address - - - class Contract(gl.contract.Contract): - def __init__(self): - pass - - @gl.public.view - def hop(self, depth: int, other: Address) -> int: - if depth <= 0: - return 3 - return 3 + gl.contract.get_at(other).view().hop( - depth - 1, gl.message.contract_address - ) - """, -) - -CHAIN_V02 = base._contract_source( - 2, - """ - from genlayer import * - - - class Contract(gl.Contract): - def __init__(self): - pass - - @gl.public.view - def hop(self, depth: int, other: Address) -> int: - if depth <= 0: - return 2 - return 2 + gl.get_contract_at(other).view().hop( - depth - 1, gl.message.contract_address - ) - """, -) +CALLER_V02 = _contract_source(2, 'caller') +OBSERVER_V03 = _contract_source(3, 'observer') +HELPER_V03 = _contract_source(3, 'helper') +SELF_V03 = _contract_source(3, 'self_recursion') +TRAP_V03 = _contract_source(3, 'trap') +USER_ERROR_V03 = _contract_source(3, 'user_error') +CHAIN_V03 = _contract_source(3, 'chain') +CHAIN_V02 = _contract_source(2, 'chain') + +FIXTURES = { + 'caller-v02': (2, ADDR_CALLER_V02, CALLER_V02), + 'observer-v03': (3, ADDR_OBSERVER_V03, OBSERVER_V03), + 'helper-v03': (3, ADDR_HELPER_V03, HELPER_V03), + 'self-v03': (3, ADDR_SELF_V03, SELF_V03), + 'trap-v03': (3, ADDR_TRAP_V03, TRAP_V03), + 'user-error-v03': (3, ADDR_USER_ERROR_V03, USER_ERROR_V03), + 'chain-v03': (3, ADDR_CHAIN_V03, CHAIN_V03), + 'chain-v02': (2, ADDR_CHAIN_V02, CHAIN_V02), +} @dataclass class ObservabilityCase(genvm_tool.tests.test.Case): description: genvm_tool.tests.test.Description shared: genvm_tool.tests.SharedContext + manager_service: genvm_tool.tests.stage.collection.Service + method: str + fixtures: tuple[str, ...] + arguments: tuple[typing.Any, ...] = () async def into_steps(self) -> list[genvm_tool.tests.exec.step.Step]: return [ObservabilityStep(self)] @@ -278,54 +97,21 @@ async def _run_all(self): 2: build_info['executor_versions']['v0.2'], 3: build_info['executor_versions']['v0.3'], } - work_dir = self.case.shared.artifacts_dir / 'cross-major-observability' + work_dir = self.case.shared.case_dir_for(self.case.description.name) self.storage_path = work_dir / 'storage.pickle' work_dir.mkdir(parents=True, exist_ok=True) - with open(self.storage_path, 'wb') as storage_file: - pickle.dump(MockStorage(), storage_file) + await gvm_io.write_file_bytes(self.storage_path, pickle.dumps(MockStorage())) - async with base.ManagerProc( - build_dir=self.build_dir, - work_dir=work_dir / 'manager', - ) as manager: - self.manager = manager + manager = self.case.manager_service.handle + assert manager is not None + self.manager = manager - self.phase = 'deploy observability fixtures' - for line, address, code in ( - (2, ADDR_CALLER_V02, CALLER_V02), - (3, ADDR_OBSERVER_V03, OBSERVER_V03), - (3, ADDR_HELPER_V03, HELPER_V03), - (3, ADDR_SELF_V03, SELF_V03), - (3, ADDR_TRAP_V03, TRAP_V03), - (3, ADDR_USER_ERROR_V03, USER_ERROR_V03), - (3, ADDR_CHAIN_V03, CHAIN_V03), - (2, ADDR_CHAIN_V02, CHAIN_V02), - ): - await self._deploy(line, address, code) + self.phase = 'deploy observability fixtures' + for line, address, code in (FIXTURES[name] for name in self.case.fixtures): + await self._deploy(line, address, code) - self.phase = 'leader and validator modes are genuine' - await self._assert_validator_mode() - - self.phase = 'callee observes the preserved nested message' - await self._assert_context() - - self.phase = 'nested permissions and disabled debug are observable' - await self._assert_permissions_and_debug() - - self.phase = 'read-only roots and nested storage reads agree' - await self._assert_read_only_storage() - - self.phase = 'self-call recursion budgets agree in lvs' - await self._assert_self_recursion() - - self.phase = 'deep alternating nesting agrees in lvs' - await self._assert_deep_nesting() - - self.phase = 'contract-controlled nested errors are not internal' - await self._assert_error_propagation() - - self.phase = 'imported deterministic fuel agrees in lvs' - await self._assert_fuel() + self.phase = self.case.method + await getattr(self, self.case.method)(*self.case.arguments) async def _execute( self, @@ -361,7 +147,10 @@ async def _execute( ), host_data='{"node_address":"test","tx_id":"cross-major-observe"}', host='unix://' + mock_host.path, - code=kwargs.get('code'), + code=base.resolve_runners( + typing.cast('bytes | None', kwargs.get('code')), + self.case.shared.root_dir, + ), calldata=typing.cast(bytes, kwargs['calldata']), timeout=kwargs.get('timeout', 30), debug_mode='unsafe', @@ -380,7 +169,7 @@ async def _execute( ) if ( kwargs.get('apply_changes', True) - and result.result_kind == public_abi.ResultCode.RETURN + and result.result_kind == host_fns.ResultCode.RETURN ): assert mock_host.storage is not None base._apply_storage_changes( @@ -425,7 +214,7 @@ async def one(suffix: str, **mode): ('validator', validator), ('sync', sync), ): - assert result.result_kind != public_abi.ResultCode.INTERNAL_ERROR, ( + assert result.result_kind != host_fns.ResultCode.INTERNAL_ERROR, ( name, label, result, @@ -465,7 +254,7 @@ async def _assert_validator_mode(self): is_sync=False, leader_nondet_results=[base.gvm_calldata.encode({})], ) - assert bogus.result_kind == public_abi.ResultCode.VM_ERROR, bogus + assert bogus.result_kind == host_fns.ResultCode.VM_ERROR, bogus async def _assert_context(self): routes: list[Address] = [] @@ -491,12 +280,12 @@ def resolve(address, state, major): 'is_init': False, } for result in (leader, validator, sync): - assert result.result_kind == public_abi.ResultCode.RETURN, result + assert result.result_kind == host_fns.ResultCode.RETURN, result assert result.result_data == expected, (result.result_data, expected) assert routes == [ADDR_OBSERVER_V03] * 3, routes self.notes.append(('nested context', expected)) - async def _assert_permissions_and_debug(self): + async def _assert_permissions(self): leader, validator, sync = await self._lvs_extended( name='permissions', line=2, @@ -506,7 +295,7 @@ async def _assert_permissions_and_debug(self): permissions='wscnu', ) for result in (leader, validator, sync): - assert result.result_kind == public_abi.ResultCode.RETURN, result + assert result.result_kind == host_fns.ResultCode.RETURN, result observed = result.result_data for denied in ('write', 'send', 'nondet'): assert observed[denied]['kind'] == 'SystemError', observed @@ -515,6 +304,7 @@ async def _assert_permissions_and_debug(self): assert observed['called'] == 11, observed self.notes.append(('nested permission observations', leader.result_data)) + async def _assert_debug_alias(self): top_level = await self._execute( name='debug-alias-top-level', line=3, @@ -525,12 +315,15 @@ async def _assert_permissions_and_debug(self): resolve_hook=self._route_v03, apply_changes=False, ) - # Unsafe mode resolves the alias and gets as far as running it. The - # deliberately mismatched sandbox payload is then an ordinary user-visible - # invalid-contract result, rather than the absent/malformed runner emitted - # before startup under disabled debug. - assert top_level.result_kind == public_abi.ResultCode.USER_ERROR, top_level - assert top_level.result_data == 'vm error: invalid_contract', top_level + # Unsafe mode resolves the alias and gets as far as running it, so the + # deliberately mismatched sandbox payload surfaces as a user error the + # contract can see. Under disabled debug the same runner fails before + # startup and is a vm error instead; the kind, not the code, is what + # separates the two. + assert top_level.result_kind == host_fns.ResultCode.USER_ERROR, top_level + assert top_level.result_data == 'vm error: ' + str( + public_abi.VmError.invalid_contract().runner().malformed() + ), top_level leader, validator, sync = await self._lvs_extended( name='debug-alias-nested', @@ -540,9 +333,9 @@ async def _assert_permissions_and_debug(self): resolve_hook=self._route_v03, ) for result in (leader, validator, sync): - assert result.result_kind == public_abi.ResultCode.VM_ERROR, result + assert result.result_kind == host_fns.ResultCode.VM_ERROR, result assert result.result_data == str( - public_abi.VmError.invalid_contract().malformed_runner() + public_abi.VmError.invalid_contract().runner().malformed() ), result self.notes.append( ( @@ -551,124 +344,213 @@ async def _assert_permissions_and_debug(self): ) ) - async def _assert_read_only_storage(self): - for permissions in ('c', 'wscn'): - reads: list[tuple[Address, public_abi.StorageType]] = [] - leader, validator, sync = await self._lvs_extended( - name=f'storage-{permissions}', - line=2, - address=ADDR_CALLER_V02, - calldata=base._calldata('read', ADDR_OBSERVER_V03), - resolve_hook=self._route_v03, - permissions=permissions, - read_log=reads, - ) - for result in (leader, validator, sync): - assert result.result_kind == public_abi.ResultCode.RETURN, result - assert result.result_data == 777, result - assert any(account == ADDR_OBSERVER_V03 for account, _mode in reads), reads - self.notes.append(('read-only nested storage', 'c and wscn both returned 777')) + async def _assert_read_only_storage(self, permissions: str): + reads: list[tuple[Address, public_abi.StorageType]] = [] + leader, validator, sync = await self._lvs_extended( + name=f'storage-{permissions}', + line=2, + address=ADDR_CALLER_V02, + calldata=base._calldata('read', ADDR_OBSERVER_V03), + resolve_hook=self._route_v03, + permissions=permissions, + read_log=reads, + ) + for result in (leader, validator, sync): + assert result.result_kind == host_fns.ResultCode.RETURN, result + assert result.result_data == 777, result + assert any(account == ADDR_OBSERVER_V03 for account, _mode in reads), reads + self.notes.append(('read-only nested storage', permissions)) - async def _assert_self_recursion(self): + async def _assert_self_recursion(self, budget: int): expected_error = str(public_abi.VmError.out_of().vm_recursion()) - outcomes = [] - for budget in (0, 1, 2, 4, 5, 6): - leader, validator, sync = await self._lvs_extended( - name=f'self-recursion-{budget}', - line=3, - address=ADDR_SELF_V03, - calldata=base._calldata('observe_recursion', 4), - resolve_hook=self._route_v03, - debug_initial_recursion=budget, - ) - for result in (leader, validator, sync): - if budget < 5: - assert result.result_kind == public_abi.ResultCode.VM_ERROR, result - assert result.result_data == expected_error, ( - budget, - result.result_data, - expected_error, - ) - else: - assert result.result_kind == public_abi.ResultCode.RETURN, result - assert result.result_data == 'ok:5', (budget, result) - outcomes.append( + leader, validator, sync = await self._lvs_extended( + name=f'self-recursion-{budget}', + line=3, + address=ADDR_SELF_V03, + calldata=base._calldata('observe_recursion', 4), + resolve_hook=self._route_v03, + debug_initial_recursion=budget, + ) + for result in (leader, validator, sync): + if budget < 5: + assert result.result_kind == host_fns.ResultCode.VM_ERROR, result + assert result.result_data == expected_error, ( + budget, + result.result_data, + expected_error, + ) + else: + assert result.result_kind == host_fns.ResultCode.RETURN, result + assert result.result_data == 'ok:5', (budget, result) + self.notes.append( + ( + 'self recursion outcome', ( budget, str(leader.result_kind), str(leader.result_data), leader.execution_hash.hex(), - ) - ) - self.notes.append(('self recursion outcomes', outcomes)) - - async def _assert_deep_nesting(self): - outcomes = [] - for depth in (1, 4, 8): - leader, validator, sync = await self._lvs_extended( - name=f'deep-{depth}', - line=3, - address=ADDR_CHAIN_V03, - calldata=base._calldata('hop', depth, ADDR_CHAIN_V02), - resolve_hook=self._route_chain, + ), ) + ) + + async def _assert_deep_nesting(self, depth: int): + # A chain deeper than the manager's cross-major bound is refused rather + # than served, and every node refuses it at the same place. + leader, validator, sync = await self._lvs_extended( + name=f'deep-{depth}', + line=3, + address=ADDR_CHAIN_V03, + calldata=base._calldata('hop', depth, ADDR_CHAIN_V02), + resolve_hook=self._route_chain, + ) + if depth > CROSS_MAJOR_RECURSION: + for result in (leader, validator, sync): + assert result.result_kind == host_fns.ResultCode.VM_ERROR, result + assert result.result_data == 'out_of vm_recursion', (depth, result) + outcome: int | str = 'out_of vm_recursion' + else: expected = sum(3 if index % 2 == 0 else 2 for index in range(depth + 1)) for result in (leader, validator, sync): - assert result.result_kind == public_abi.ResultCode.RETURN, result + assert result.result_kind == host_fns.ResultCode.RETURN, result assert result.result_data == expected, (depth, result, expected) - outcomes.append((depth, expected, leader.execution_hash.hex())) - self.notes.append(('deep alternating hashes', outcomes)) - - async def _assert_error_propagation(self): - cases = ( - ('undeployed', ABSENT), - ('non-contract', base.SENDER), - ('trap', ADDR_TRAP_V03), - ('user-error', ADDR_USER_ERROR_V03), + outcome = expected + self.notes.append( + ('deep alternating hash', (depth, outcome, leader.execution_hash.hex())) ) - observed = [] - for label, target in cases: - leader, validator, sync = await self._lvs_extended( - name=f'error-{label}', - line=2, - address=ADDR_CALLER_V02, - calldata=base._calldata('answer', target), - resolve_hook=lambda address, state, major, target=target: ( - self._route(3) - if address == target - else self._route_v03(address, state, major) - ), + + async def _assert_error(self, label: str, target: Address): + leader, validator, sync = await self._lvs_extended( + name=f'error-{label}', + line=2, + address=ADDR_CALLER_V02, + calldata=base._calldata('answer', target), + resolve_hook=lambda address, state, major: ( + self._route(3) if address == target else self._route_v03(address, state, major) + ), + ) + for result in (leader, validator, sync): + assert result.result_kind in { + host_fns.ResultCode.VM_ERROR, + host_fns.ResultCode.USER_ERROR, + }, (label, result) + self.notes.append( + ( + 'nested error outcome', + (label, str(leader.result_kind), str(leader.result_data)[:120]), ) - for result in (leader, validator, sync): - assert result.result_kind in { - public_abi.ResultCode.VM_ERROR, - public_abi.ResultCode.USER_ERROR, - }, (label, result) - observed.append((label, str(leader.result_kind), str(leader.result_data)[:120])) - self.notes.append(('nested error outcomes', observed)) - - async def _assert_fuel(self): - outcomes = [] - for fuel in (0, 1, 1000, 2**32): - leader, validator, sync = await self._lvs_extended( - name=f'fuel-{fuel}', - line=2, - address=ADDR_CALLER_V02, - calldata=base._calldata('read', ADDR_OBSERVER_V03), - resolve_hook=self._route_v03, - host_fuel=fuel, + ) + + async def _assert_undeployed_error(self): + await self._assert_error('undeployed', ABSENT) + + async def _assert_non_contract_error(self): + await self._assert_error('non-contract', base.SENDER) + + async def _assert_trap_error(self): + await self._assert_error('trap', ADDR_TRAP_V03) + + async def _assert_user_error(self): + await self._assert_error('user-error', ADDR_USER_ERROR_V03) + + async def _assert_fuel(self, fuel: int): + leader, validator, sync = await self._lvs_extended( + name=f'fuel-{fuel}', + line=2, + address=ADDR_CALLER_V02, + calldata=base._calldata('read', ADDR_OBSERVER_V03), + resolve_hook=self._route_v03, + host_fuel=fuel, + ) + for result in (leader, validator, sync): + assert result.result_kind == host_fns.ResultCode.RETURN, result + assert result.result_data == 777, result + self.notes.append( + ('nested deterministic fuel hash', (fuel, leader.execution_hash.hex())) + ) + + +def collect( + ctx: genvm_tool.tests.stage.collection.Context, + *, + manager_service: genvm_tool.tests.stage.collection.Service, +) -> None: + cases = [ + ('validator-mode', '_assert_validator_mode', ('chain-v03', 'chain-v02'), ()), + ('message-context', '_assert_context', ('caller-v02', 'observer-v03'), ()), + ( + 'permissions', + '_assert_permissions', + ('caller-v02', 'observer-v03', 'helper-v03'), + (), + ), + ('debug-alias', '_assert_debug_alias', ('caller-v02', 'observer-v03'), ()), + *[ + ( + f'read-only-storage-{permissions}', + '_assert_read_only_storage', + ('caller-v02', 'observer-v03'), + (permissions,), ) - for result in (leader, validator, sync): - assert result.result_kind == public_abi.ResultCode.RETURN, result - assert result.result_data == 777, result - outcomes.append((fuel, leader.execution_hash.hex())) - self.notes.append(('nested deterministic fuel hashes', outcomes)) - - -def collect(ctx: genvm_tool.tests.stage.collection.Context) -> None: - desc = genvm_tool.tests.test.Description( - name='tests/system/cross-major-observability', - tags=frozenset({'integration', 'stable', 'cross-major'}), - console_pool=True, - ) - ctx.add_case(ObservabilityCase(description=desc, shared=ctx.shared)) + for permissions in ('c', 'wscn') + ], + *[ + ( + f'self-recursion-{budget}', + '_assert_self_recursion', + ('self-v03',), + (budget,), + ) + for budget in (0, 1, 2, 4, 5, 6) + ], + *[ + ( + f'deep-nesting-{depth}', + '_assert_deep_nesting', + ('chain-v03', 'chain-v02'), + (depth,), + ) + for depth in (1, 4, 8) + ], + ('error-undeployed', '_assert_undeployed_error', ('caller-v02',), ()), + ('error-non-contract', '_assert_non_contract_error', ('caller-v02',), ()), + ('error-trap', '_assert_trap_error', ('caller-v02', 'trap-v03'), ()), + ( + 'error-user', + '_assert_user_error', + ('caller-v02', 'user-error-v03'), + (), + ), + *[ + ( + f'fuel-{fuel}', + '_assert_fuel', + ('caller-v02', 'observer-v03'), + (fuel,), + ) + for fuel in (0, 1, 1000, 2**32) + ], + ] + for slug, method, fixtures, arguments in cases: + desc = genvm_tool.tests.test.Description( + name=f'tests/system/cross-major-observability/{slug}', + needed_services=frozenset({manager_service}), + tags=frozenset( + { + 'integration', + 'stable', + 'feature-version-routing-cross-major', + 'feature-observability', + } + ), + ) + ctx.add_case( + ObservabilityCase( + description=desc, + shared=ctx.shared, + manager_service=manager_service, + method=method, + fixtures=fixtures, + arguments=arguments, + ) + ) diff --git a/tests/system/cross-major/assets/v0.2/blob_caller.py b/tests/system/cross-major/assets/v0.2/blob_caller.py new file mode 100644 index 00000000..7039b4a8 --- /dev/null +++ b/tests/system/cross-major/assets/v0.2/blob_caller.py @@ -0,0 +1,11 @@ +# { "Depends": "py-genlayer:@RUNNER_LATEST_v0.2_py-genlayer@" } +from genlayer import * + + +class Contract(gl.Contract): + def __init__(self): + pass + + @gl.public.view + def send(self, target: Address, size: int) -> int: + return gl.get_contract_at(target).view().sink('x' * size) diff --git a/tests/system/cross-major/assets/v0.2/busy.py b/tests/system/cross-major/assets/v0.2/busy.py new file mode 100644 index 00000000..7f866277 --- /dev/null +++ b/tests/system/cross-major/assets/v0.2/busy.py @@ -0,0 +1,12 @@ +# { "Depends": "py-genlayer:@RUNNER_LATEST_v0.2_py-genlayer@" } +from genlayer import * + + +class Contract(gl.Contract): + def __init__(self): + pass + + @gl.public.view + def answer(self) -> int: + while True: + pass diff --git a/tests/system/cross-major/assets/v0.2/callee.py b/tests/system/cross-major/assets/v0.2/callee.py new file mode 100644 index 00000000..0180656a --- /dev/null +++ b/tests/system/cross-major/assets/v0.2/callee.py @@ -0,0 +1,11 @@ +# { "Depends": "py-genlayer:@RUNNER_LATEST_v0.2_py-genlayer@" } +from genlayer import * + + +class Contract(gl.Contract): + def __init__(self): + pass + + @gl.public.view + def answer(self) -> int: + return 202 diff --git a/tests/system/cross-major/assets/v0.2/caller.py b/tests/system/cross-major/assets/v0.2/caller.py new file mode 100644 index 00000000..ec212a69 --- /dev/null +++ b/tests/system/cross-major/assets/v0.2/caller.py @@ -0,0 +1,11 @@ +# { "Depends": "py-genlayer:@RUNNER_LATEST_v0.2_py-genlayer@" } +from genlayer import * + + +class Contract(gl.Contract): + def __init__(self): + pass + + @gl.public.view + def call(self, target: Address) -> int: + return gl.get_contract_at(target).view().answer() diff --git a/tests/system/cross-major/assets/v0.2/chain.py b/tests/system/cross-major/assets/v0.2/chain.py new file mode 100644 index 00000000..7f0e6fd4 --- /dev/null +++ b/tests/system/cross-major/assets/v0.2/chain.py @@ -0,0 +1,14 @@ +# { "Depends": "py-genlayer:@RUNNER_LATEST_v0.2_py-genlayer@" } +from genlayer import * + + +class Contract(gl.Contract): + def __init__(self): + pass + + @gl.public.view + def hop(self, depth: int, other: Address) -> int: + if depth <= 0: + return 2 + me = gl.message.contract_address + return 2 + gl.get_contract_at(other).view().hop(depth - 1, me) diff --git a/tests/system/cross-major/assets/v0.2/loop.py b/tests/system/cross-major/assets/v0.2/loop.py new file mode 100644 index 00000000..e8e4fe59 --- /dev/null +++ b/tests/system/cross-major/assets/v0.2/loop.py @@ -0,0 +1,11 @@ +# { "Depends": "py-genlayer:@RUNNER_LATEST_v0.2_py-genlayer@" } +from genlayer import * + + +class Contract(gl.Contract): + def __init__(self): + pass + + @gl.public.view + def loop(self, first: Address, second: Address) -> int: + return gl.get_contract_at(first).view().loop(0, first, second) diff --git a/tests/system/cross-major/assets/v0.2/signer_caller.py b/tests/system/cross-major/assets/v0.2/signer_caller.py new file mode 100644 index 00000000..4d593747 --- /dev/null +++ b/tests/system/cross-major/assets/v0.2/signer_caller.py @@ -0,0 +1,11 @@ +# { "Depends": "py-genlayer:@RUNNER_LATEST_v0.2_py-genlayer@" } +from genlayer import * + + +class Contract(gl.Contract): + def __init__(self): + pass + + @gl.public.view + def signer(self, target: Address) -> str: + return gl.get_contract_at(target).view().signer() diff --git a/tests/system/cross-major/assets/v0.2/user_error.py b/tests/system/cross-major/assets/v0.2/user_error.py new file mode 100644 index 00000000..bac20cba --- /dev/null +++ b/tests/system/cross-major/assets/v0.2/user_error.py @@ -0,0 +1,11 @@ +# { "Depends": "py-genlayer:@RUNNER_LATEST_v0.2_py-genlayer@" } +from genlayer import * + + +class Contract(gl.Contract): + def __init__(self): + pass + + @gl.public.view + def answer(self) -> int: + raise gl.vm.UserError('boom') diff --git a/tests/system/cross-major/assets/v0.3/blob.py b/tests/system/cross-major/assets/v0.3/blob.py new file mode 100644 index 00000000..f06a1b0a --- /dev/null +++ b/tests/system/cross-major/assets/v0.3/blob.py @@ -0,0 +1,11 @@ +# { "Depends": "py-genlayer:@RUNNER_LATEST_v0.3_py-genlayer@" } +import genlayer as gl + + +class Contract(gl.contract.Contract): + def __init__(self): + pass + + @gl.public.view + def sink(self, blob: str) -> int: + return len(blob) diff --git a/tests/system/cross-major/assets/v0.3/callee.py b/tests/system/cross-major/assets/v0.3/callee.py new file mode 100644 index 00000000..1c2b2c1b --- /dev/null +++ b/tests/system/cross-major/assets/v0.3/callee.py @@ -0,0 +1,11 @@ +# { "Depends": "py-genlayer:@RUNNER_LATEST_v0.3_py-genlayer@" } +import genlayer as gl + + +class Contract(gl.contract.Contract): + def __init__(self): + pass + + @gl.public.view + def answer(self) -> int: + return 303 diff --git a/tests/system/cross-major/assets/v0.3/caller.py b/tests/system/cross-major/assets/v0.3/caller.py new file mode 100644 index 00000000..3e9e1f26 --- /dev/null +++ b/tests/system/cross-major/assets/v0.3/caller.py @@ -0,0 +1,12 @@ +# { "Depends": "py-genlayer:@RUNNER_LATEST_v0.3_py-genlayer@" } +import genlayer as gl +from genlayer.types import Address + + +class Contract(gl.contract.Contract): + def __init__(self): + pass + + @gl.public.view + def call(self, target: Address) -> int: + return gl.contract.get_at(target).view().answer() diff --git a/tests/system/cross-major/assets/v0.3/chain.py b/tests/system/cross-major/assets/v0.3/chain.py new file mode 100644 index 00000000..e166801f --- /dev/null +++ b/tests/system/cross-major/assets/v0.3/chain.py @@ -0,0 +1,15 @@ +# { "Depends": "py-genlayer:@RUNNER_LATEST_v0.3_py-genlayer@" } +import genlayer as gl +from genlayer.types import Address + + +class Contract(gl.contract.Contract): + def __init__(self): + pass + + @gl.public.view + def hop(self, depth: int, other: Address) -> int: + if depth <= 0: + return 3 + me = gl.message.contract_address + return 3 + gl.contract.get_at(other).view().hop(depth - 1, me) diff --git a/tests/system/cross-major/assets/v0.3/loop.py b/tests/system/cross-major/assets/v0.3/loop.py new file mode 100644 index 00000000..0c4514db --- /dev/null +++ b/tests/system/cross-major/assets/v0.3/loop.py @@ -0,0 +1,14 @@ +# { "Depends": "py-genlayer:@RUNNER_LATEST_v0.3_py-genlayer@" } +import genlayer as gl +from genlayer.types import Address + + +class Contract(gl.contract.Contract): + def __init__(self): + pass + + @gl.public.view + def loop(self, remaining: int, first: Address, second: Address) -> int: + if remaining > 0: + return gl.contract.get_at(first).view().loop(remaining - 1, first, second) + return gl.contract.get_at(second).view().loop(first, second) diff --git a/tests/system/cross-major/assets/v0.3/signer.py b/tests/system/cross-major/assets/v0.3/signer.py new file mode 100644 index 00000000..d8792643 --- /dev/null +++ b/tests/system/cross-major/assets/v0.3/signer.py @@ -0,0 +1,11 @@ +# { "Depends": "py-genlayer:@RUNNER_LATEST_v0.3_py-genlayer@" } +import genlayer as gl + + +class Contract(gl.contract.Contract): + def __init__(self): + pass + + @gl.public.view + def signer(self) -> str: + return gl.message.signer_address.as_hex diff --git a/tests/system/cross-major/assets/v0.3/state_caller.py b/tests/system/cross-major/assets/v0.3/state_caller.py new file mode 100644 index 00000000..c612dce2 --- /dev/null +++ b/tests/system/cross-major/assets/v0.3/state_caller.py @@ -0,0 +1,17 @@ +# { "Depends": "py-genlayer:@RUNNER_LATEST_v0.3_py-genlayer@" } +import genlayer as gl +from genlayer.types import Address + + +class Contract(gl.contract.Contract): + def __init__(self): + pass + + @gl.public.view + def call(self, target: Address, final: bool) -> int: + mode = ( + gl.vm.public_abi.StorageType.LATEST_FINAL + if final + else gl.vm.public_abi.StorageType.LATEST_NON_FINAL + ) + return gl.contract.get_at(target).view(state=mode).answer() diff --git a/tests/system/cross-major/assets/v0.3/structured_error.py b/tests/system/cross-major/assets/v0.3/structured_error.py new file mode 100644 index 00000000..4e2c26fa --- /dev/null +++ b/tests/system/cross-major/assets/v0.3/structured_error.py @@ -0,0 +1,11 @@ +# { "Depends": "py-genlayer:@RUNNER_LATEST_v0.3_py-genlayer@" } +import genlayer as gl + + +class Contract(gl.contract.Contract): + def __init__(self): + pass + + @gl.public.view + def answer(self) -> int: + gl.vm.UserError.immediate({'code': 7}) diff --git a/tests/system/cross-major/assets/v0.3/user_error.py b/tests/system/cross-major/assets/v0.3/user_error.py new file mode 100644 index 00000000..5cf9cdcb --- /dev/null +++ b/tests/system/cross-major/assets/v0.3/user_error.py @@ -0,0 +1,11 @@ +# { "Depends": "py-genlayer:@RUNNER_LATEST_v0.3_py-genlayer@" } +import genlayer as gl + + +class Contract(gl.contract.Contract): + def __init__(self): + pass + + @gl.public.view + def answer(self) -> int: + gl.vm.UserError.immediate('boom') diff --git a/tests/system/cross-major/cases/routing.jsonnet b/tests/system/cross-major/cases/routing.jsonnet new file mode 100644 index 00000000..b0f0c485 --- /dev/null +++ b/tests/system/cross-major/cases/routing.jsonnet @@ -0,0 +1,95 @@ +local util = import 'templates/util.jsonnet'; +local message = import 'templates/message.json'; + +local route02 = '${executorV02}'; +local route03 = '${executorV03}'; + +local deploy(slug, address, code) = { + slug: slug, + code: '${jsonnetDir}/../assets/' + code, + message: message + { + contract_address: address, + is_init: true, + }, + calldata: '{}', + modes: 'l', + stable_hash: false, + expected_semantics_components: [], + reroute_to: if std.startsWith(code, 'v0.2/') then route02 else route03, +}; + +local call(slug, caller, target, method, route, extra={}) = { + slug: slug, + code: null, + vars: { target: target }, + message: message + { contract_address: caller }, + calldata: '{"": "' + method + '", "args": [Address(target)]}', + executor_routes: if route == null then {} else { [target]: route }, + hook_cross_contract_calls: true, + stable_hash: false, + expected_semantics_components: ['return', 'kind'], +} + extra; + +local caller03 = '0x1111111111111111111111111111111111111111'; +local caller02 = '0x1212121212121212121212121212121212121212'; +local callee03 = '0x1313131313131313131313131313131313131313'; +local callee02 = '0x1414141414141414141414141414141414141414'; +local signer03 = '0x1717171717171717171717171717171717171717'; +local signerCaller02 = '0x1818181818181818181818181818181818181818'; +local structured03 = '0x1919191919191919191919191919191919191919'; +local userError03 = '0x2020202020202020202020202020202020202020'; +local userError02 = '0x2121212121212121212121212121212121212121'; + +local scenarios = [ + util.chain([ + deploy('v03-to-v02-caller', caller03, 'v0.3/caller.py'), + deploy('v03-to-v02-callee', callee02, 'v0.2/callee.py'), + call('v03-to-v02', caller03, callee02, 'call', route02), + ]), + util.chain([ + deploy('v02-to-v03-caller', caller02, 'v0.2/caller.py'), + deploy('v02-to-v03-callee', callee03, 'v0.3/callee.py'), + call('v02-to-v03', caller02, callee03, 'call', route03, { reroute_to: route02 }), + ]), + util.chain([ + deploy('signer-caller', signerCaller02, 'v0.2/signer_caller.py'), + deploy('signer-callee', signer03, 'v0.3/signer.py'), + call('signer', signerCaller02, signer03, 'signer', route03, { reroute_to: route02 }), + ]), + util.chain([ + deploy('structured-error-caller', caller02, 'v0.2/caller.py'), + deploy('structured-error-callee', structured03, 'v0.3/structured_error.py'), + call('structured-error', caller02, structured03, 'call', route03, { reroute_to: route02 }), + ]), + util.chain([ + deploy('plain-error-caller', caller02, 'v0.2/caller.py'), + deploy('plain-error-callee', userError03, 'v0.3/user_error.py'), + call('plain-error', caller02, userError03, 'call', route03, { reroute_to: route02 }), + ]), + util.chain([ + deploy('plain-error-reverse-caller', caller03, 'v0.3/caller.py'), + deploy('plain-error-reverse-callee', userError02, 'v0.2/user_error.py'), + call('plain-error-reverse', caller03, userError02, 'call', route02), + ]), + util.chain([ + deploy('null-same-caller', caller03, 'v0.3/caller.py'), + deploy('null-same-callee', callee03, 'v0.3/callee.py'), + call('null-same-major', caller03, callee03, 'call', null, { + expected_executor_route_requests: [{ + contract_address: callee03, + state_mode: 2, + advisory_major: 0, + }], + }), + ]), + util.chain([ + deploy('null-mismatch-caller', caller03, 'v0.3/caller.py'), + deploy('null-mismatch-callee', callee02, 'v0.2/callee.py'), + call('null-mismatch', caller03, callee02, 'call', null), + ]), +]; + +{ + tags: util.features([['version-routing', 'cross-major']], 'stable') + ['python'], + entry: util.addPaths(scenarios), +} diff --git a/tests/system/cross-major/cases/routing.null-mismatch-caller_null-mismatch-callee_null-mismatch.stdout b/tests/system/cross-major/cases/routing.null-mismatch-caller_null-mismatch-callee_null-mismatch.stdout new file mode 100644 index 00000000..07c2a520 --- /dev/null +++ b/tests/system/cross-major/cases/routing.null-mismatch-caller_null-mismatch-callee_null-mismatch.stdout @@ -0,0 +1,2 @@ +executed with `VMError("invalid_contract runner absent")` +VM_ERROR diff --git a/tests/system/cross-major/cases/routing.null-same-caller_null-same-callee_null-same-major.stdout b/tests/system/cross-major/cases/routing.null-same-caller_null-same-callee_null-same-major.stdout new file mode 100644 index 00000000..d95482b9 --- /dev/null +++ b/tests/system/cross-major/cases/routing.null-same-caller_null-same-callee_null-same-major.stdout @@ -0,0 +1,2 @@ +executed with `Return(303)` +RETURN diff --git a/tests/system/cross-major/cases/routing.plain-error-caller_plain-error-callee_plain-error.stdout b/tests/system/cross-major/cases/routing.plain-error-caller_plain-error-callee_plain-error.stdout new file mode 100644 index 00000000..90bfa685 --- /dev/null +++ b/tests/system/cross-major/cases/routing.plain-error-caller_plain-error-callee_plain-error.stdout @@ -0,0 +1,2 @@ +executed with `UserError("boom")` +USER_ERROR diff --git a/tests/system/cross-major/cases/routing.plain-error-reverse-caller_plain-error-reverse-callee_plain-error-reverse.stdout b/tests/system/cross-major/cases/routing.plain-error-reverse-caller_plain-error-reverse-callee_plain-error-reverse.stdout new file mode 100644 index 00000000..90bfa685 --- /dev/null +++ b/tests/system/cross-major/cases/routing.plain-error-reverse-caller_plain-error-reverse-callee_plain-error-reverse.stdout @@ -0,0 +1,2 @@ +executed with `UserError("boom")` +USER_ERROR diff --git a/tests/system/cross-major/cases/routing.signer-caller_signer-callee_signer.stdout b/tests/system/cross-major/cases/routing.signer-caller_signer-callee_signer.stdout new file mode 100644 index 00000000..4a9bc9ff --- /dev/null +++ b/tests/system/cross-major/cases/routing.signer-caller_signer-callee_signer.stdout @@ -0,0 +1,2 @@ +executed with `Return("0x0200000000000000000000000000000000000000")` +RETURN diff --git a/tests/system/cross-major/cases/routing.structured-error-caller_structured-error-callee_structured-error.stdout b/tests/system/cross-major/cases/routing.structured-error-caller_structured-error-callee_structured-error.stdout new file mode 100644 index 00000000..47776f49 --- /dev/null +++ b/tests/system/cross-major/cases/routing.structured-error-caller_structured-error-callee_structured-error.stdout @@ -0,0 +1,2 @@ +executed with `VMError("invalid_contract major_mismatch")` +VM_ERROR diff --git a/tests/system/cross-major/cases/routing.v02-to-v03-caller_v02-to-v03-callee_v02-to-v03.stdout b/tests/system/cross-major/cases/routing.v02-to-v03-caller_v02-to-v03-callee_v02-to-v03.stdout new file mode 100644 index 00000000..d95482b9 --- /dev/null +++ b/tests/system/cross-major/cases/routing.v02-to-v03-caller_v02-to-v03-callee_v02-to-v03.stdout @@ -0,0 +1,2 @@ +executed with `Return(303)` +RETURN diff --git a/tests/system/cross-major/cases/routing.v03-to-v02-caller_v03-to-v02-callee_v03-to-v02.stdout b/tests/system/cross-major/cases/routing.v03-to-v02-caller_v03-to-v02-callee_v03-to-v02.stdout new file mode 100644 index 00000000..ef72678b --- /dev/null +++ b/tests/system/cross-major/cases/routing.v03-to-v02-caller_v03-to-v02-callee_v03-to-v02.stdout @@ -0,0 +1,2 @@ +executed with `Return(202)` +RETURN diff --git a/tests/system/cross-major/test.py b/tests/system/cross-major/test.py index 5636ecbb..5f57b641 100644 --- a/tests/system/cross-major/test.py +++ b/tests/system/cross-major/test.py @@ -1,31 +1,28 @@ -"""Cross-major CallContract system test. - -The case owns its manager because it exercises manager cancellation and -deadlines while executor processes are nested beneath it. -""" +"""Cross-major CallContract assertions that need Python host inspection.""" import asyncio import contextlib import json import os import pickle -import socket -import textwrap import typing from dataclasses import dataclass from pathlib import Path -import aiohttp +import genvm_tool.io as gvm_io import genvm_tool.tests import genvm_tool.tests.stage.collection +import genvm_tool_plugins.runners as runners import origin.base_host as base_host import origin.calldata as gvm_calldata from gvm_extra.mock_host import MockHost, MockStorage -from origin import public_abi +from origin import host_fns, public_abi from origin.calldata import Address -PY_GENLAYER_V03 = '9b8kjyda2ycxyq4ea6g4yfpnydxhd52gqba5rb8dw7krkh5mn9p0' -PY_GENLAYER_V02 = '1jb45aa8ynh2a9c9xn3b7qqh8sm5q93hwfp7jqmwsfhh8jpz09h6' +# One directory per line, because a contract is written against its own sdk. +# Their `@RUNNER_LATEST_...@` tokens stand for the uids of the last build; +# `_execute` resolves them before any code reaches an executor. +ASSETS_DIR = Path(__file__).resolve().parent / 'assets' ADDR_CALLER_V03 = Address('0x' + '31' * 20) ADDR_CALLER_V02 = Address('0x' + '21' * 20) @@ -34,324 +31,65 @@ ADDR_BUSY_V02 = Address('0x' + '23' * 20) ADDR_LOOP_V03 = Address('0x' + '33' * 20) ADDR_LOOP_V02 = Address('0x' + '24' * 20) -ADDR_SIGNER_V03 = Address('0x' + '34' * 20) -ADDR_SIGNER_CALLER_V02 = Address('0x' + '25' * 20) -ADDR_STRUCTURED_V03 = Address('0x' + '35' * 20) ADDR_BLOB_V03 = Address('0x' + '36' * 20) ADDR_BLOB_CALLER_V02 = Address('0x' + '26' * 20) -ADDR_UERR_V03 = Address('0x' + '37' * 20) -ADDR_UERR_V02 = Address('0x' + '27' * 20) ADDR_STATE_CALLER_V03 = Address('0x' + '38' * 20) ADDR_CHAIN_V03 = Address('0x' + '39' * 20) ADDR_CHAIN_V02 = Address('0x' + '28' * 20) +ABSENT_CALLEE = Address('0x' + '99' * 20) SENDER = Address('0x' + '01' * 20) # Larger than the manager cap the test configures below, so the nested envelope # cannot fit through the socketpair. -"""Client frame cap for `/ws`; kept low so a nested envelope can exceed it.""" MANAGER_MAX_MESSAGE_BYTES = 1 * 1024 * 1024 BLOB_BYTES = 2 * 1024 * 1024 TIMESTAMP = '2024-11-26T06:42:42.424242Z' -def _contract_source(line: int, body: str) -> bytes: - runner = PY_GENLAYER_V03 if line == 3 else PY_GENLAYER_V02 - return ( - f'# {{ "Depends": "py-genlayer:{runner}" }}\n' + textwrap.dedent(body).lstrip() - ).encode() - - -CALLER_V03 = _contract_source( - 3, - """ - import genlayer as gl - from genlayer.types import Address - - - class Contract(gl.contract.Contract): - def __init__(self): - pass - - @gl.public.view - def call(self, target: Address) -> int: - return gl.contract.get_at(target).view().answer() - """, -) - -CALLER_V02 = _contract_source( - 2, - """ - from genlayer import * - - - class Contract(gl.Contract): - def __init__(self): - pass - - @gl.public.view - def call(self, target: Address) -> int: - return gl.get_contract_at(target).view().answer() - """, -) - -STATE_CALLER_V03 = _contract_source( - 3, - """ - import genlayer as gl - from genlayer.types import Address - - - class Contract(gl.contract.Contract): - def __init__(self): - pass - - @gl.public.view - def call(self, target: Address, final: bool) -> int: - mode = ( - gl.vm.public_abi.StorageType.LATEST_FINAL - if final - else gl.vm.public_abi.StorageType.LATEST_NON_FINAL - ) - return gl.contract.get_at(target).view(state=mode).answer() - """, -) - -CALLEE_V03 = _contract_source( - 3, - """ - import genlayer as gl - - - class Contract(gl.contract.Contract): - def __init__(self): - pass - - @gl.public.view - def answer(self) -> int: - return 303 - """, -) - -CALLEE_V02 = _contract_source( - 2, - """ - from genlayer import * - - - class Contract(gl.Contract): - def __init__(self): - pass - - @gl.public.view - def answer(self) -> int: - return 202 - """, -) - -SIGNER_V03 = _contract_source( - 3, - """ - import genlayer as gl - - - class Contract(gl.contract.Contract): - def __init__(self): - pass - - @gl.public.view - def signer(self) -> str: - return gl.message.signer_address.as_hex - """, -) - -SIGNER_CALLER_V02 = _contract_source( - 2, - """ - from genlayer import * - - - class Contract(gl.Contract): - def __init__(self): - pass - - @gl.public.view - def signer(self, target: Address) -> str: - return gl.get_contract_at(target).view().signer() - """, -) - -STRUCTURED_ERROR_V03 = _contract_source( - 3, - """ - import genlayer as gl - - - class Contract(gl.contract.Contract): - def __init__(self): - pass - - @gl.public.view - def answer(self) -> int: - gl.vm.UserError.immediate({'code': 7}) - """, -) - -BLOB_V03 = _contract_source( - 3, - """ - import genlayer as gl - - - class Contract(gl.contract.Contract): - def __init__(self): - pass - - @gl.public.view - def sink(self, blob: str) -> int: - return len(blob) - """, -) - -BLOB_CALLER_V02 = _contract_source( - 2, - """ - from genlayer import * - - - class Contract(gl.Contract): - def __init__(self): - pass - - @gl.public.view - def send(self, target: Address, size: int) -> int: - return gl.get_contract_at(target).view().sink('x' * size) - """, -) - -USER_ERROR_V03 = _contract_source( - 3, - """ - import genlayer as gl - - - class Contract(gl.contract.Contract): - def __init__(self): - pass - - @gl.public.view - def answer(self) -> int: - gl.vm.UserError.immediate('boom') - """, -) - -USER_ERROR_V02 = _contract_source( - 2, +def resolve_runners(code: bytes | None, root_dir: Path) -> bytes | None: """ - from genlayer import * - - - class Contract(gl.Contract): - def __init__(self): - pass - - @gl.public.view - def answer(self) -> int: - raise gl.vm.UserError('boom') - """, -) + Turns the `@RUNNER_LATEST_...@` tokens of a contract source into uids. -CHAIN_V03 = _contract_source( - 3, + Every comparison against a deployed contract's bytes has to go through this + too, because storage holds what the executor was given. """ - import genlayer as gl - from genlayer.types import Address + return None if code is None else runners.substitute(code, root_dir) - class Contract(gl.contract.Contract): - def __init__(self): - pass - - @gl.public.view - def hop(self, depth: int, other: Address) -> int: - if depth <= 0: - return 3 - me = gl.message.contract_address - return 3 + gl.contract.get_at(other).view().hop(depth - 1, me) - """, -) - -CHAIN_V02 = _contract_source( - 2, +def _contract_source(line: int, name: str, assets_dir: Path = ASSETS_DIR) -> bytes: """ - from genlayer import * - - - class Contract(gl.Contract): - def __init__(self): - pass - - @gl.public.view - def hop(self, depth: int, other: Address) -> int: - if depth <= 0: - return 2 - me = gl.message.contract_address - return 2 + gl.get_contract_at(other).view().hop(depth - 1, me) - """, -) - -BUSY_V02 = _contract_source( - 2, + Reads a contract asset written against `line`'s sdk. """ - from genlayer import * - - - class Contract(gl.Contract): - def __init__(self): - pass - - @gl.public.view - def answer(self) -> int: - while True: - pass - """, -) - -LOOP_V03 = _contract_source( - 3, - """ - import genlayer as gl - from genlayer.types import Address - - - class Contract(gl.contract.Contract): - def __init__(self): - pass - - @gl.public.view - def loop(self, remaining: int, first: Address, second: Address) -> int: - if remaining > 0: - return gl.contract.get_at(first).view().loop( - remaining - 1, first, second - ) - return gl.contract.get_at(second).view().loop(first, second) - """, -) - -LOOP_V02 = _contract_source( - 2, - """ - from genlayer import * - - - class Contract(gl.Contract): - def __init__(self): - pass - - @gl.public.view - def loop(self, first: Address, second: Address) -> int: - return gl.get_contract_at(first).view().loop(0, first, second) - """, -) + return (assets_dir / f'v0.{line}' / f'{name}.py').read_bytes() + + +CALLER_V03 = _contract_source(3, 'caller') +CALLER_V02 = _contract_source(2, 'caller') +STATE_CALLER_V03 = _contract_source(3, 'state_caller') +CALLEE_V03 = _contract_source(3, 'callee') +CALLEE_V02 = _contract_source(2, 'callee') +BLOB_V03 = _contract_source(3, 'blob') +BLOB_CALLER_V02 = _contract_source(2, 'blob_caller') +CHAIN_V03 = _contract_source(3, 'chain') +CHAIN_V02 = _contract_source(2, 'chain') +BUSY_V02 = _contract_source(2, 'busy') +LOOP_V03 = _contract_source(3, 'loop') +LOOP_V02 = _contract_source(2, 'loop') + +FIXTURES = { + 'caller-v03': (3, ADDR_CALLER_V03, CALLER_V03), + 'caller-v02': (2, ADDR_CALLER_V02, CALLER_V02), + 'callee-v03': (3, ADDR_CALLEE_V03, CALLEE_V03), + 'callee-v02': (2, ADDR_CALLEE_V02, CALLEE_V02), + 'busy-v02': (2, ADDR_BUSY_V02, BUSY_V02), + 'loop-v03': (3, ADDR_LOOP_V03, LOOP_V03), + 'loop-v02': (2, ADDR_LOOP_V02, LOOP_V02), + 'blob-v03': (3, ADDR_BLOB_V03, BLOB_V03), + 'blob-caller-v02': (2, ADDR_BLOB_CALLER_V02, BLOB_CALLER_V02), + 'state-caller-v03': (3, ADDR_STATE_CALLER_V03, STATE_CALLER_V03), + 'chain-v03': (3, ADDR_CHAIN_V03, CHAIN_V03), + 'chain-v02': (2, ADDR_CHAIN_V02, CHAIN_V02), +} def _wat_data(data: bytes) -> str: @@ -432,89 +170,6 @@ def add_stat(self, key: str, value: typing.Any, /): self.stats[key] = value -class ManagerProc: - def __init__(self, *, build_dir: Path, work_dir: Path): - self.build_dir = build_dir - self.work_dir = work_dir - self.process: asyncio.subprocess.Process | None = None - self.log_file = None - self.port = self._unused_port() - - @staticmethod - def _unused_port() -> int: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: - sock.bind(('127.0.0.1', 0)) - return sock.getsockname()[1] - - @property - def uri(self) -> str: - return f'http://127.0.0.1:{self.port}' - - async def __aenter__(self): - self.work_dir.mkdir(parents=True, exist_ok=True) - config_path = self.work_dir / 'genvm-manager.yaml' - config_path.write_text( - '\n'.join( - [ - 'threads: 4', - 'blocking_threads: 48', - 'log_disable: tracing*,polling*,tungstenite*,tokio_tungstenite*', - 'log_level: info', - f'manifest_path: {self.build_dir / "out" / "data" / "manifest.yaml"}', - 'permits: 2', - 'execution_retention: 30s', - f'max_message_bytes: {MANAGER_MAX_MESSAGE_BYTES}', - '', - ] - ) - ) - self.log_file = open(self.work_dir / 'manager.log', 'wb') - self.process = await asyncio.subprocess.create_subprocess_exec( - str(self.build_dir / 'out' / 'bin' / 'genvm-modules'), - 'manager', - '--port', - str(self.port), - '--die-with-parent', - '--config', - str(config_path), - stdin=asyncio.subprocess.DEVNULL, - stdout=self.log_file, - stderr=self.log_file, - start_new_session=True, - ) - await self._wait_healthy() - return self - - async def __aexit__(self, *_args): - if self.process is not None: - try: - self.process.terminate() - await asyncio.wait_for(self.process.wait(), timeout=5) - except asyncio.TimeoutError: - self.process.kill() - await self.process.wait() - if self.log_file is not None: - self.log_file.close() - - async def _wait_healthy(self): - last_error: BaseException | None = None - for _ in range(80): - if self.process is not None and self.process.returncode is not None: - raise RuntimeError(f'manager exited early with {self.process.returncode}') - try: - async with aiohttp.request( - 'GET', - f'{self.uri}/status', - timeout=aiohttp.ClientTimeout(total=1), - ) as response: - if response.status == 200: - return - except BaseException as exc: - last_error = exc - await asyncio.sleep(0.1) - raise TimeoutError(f'manager did not become healthy: {last_error}') - - def _message(address: Address, *, is_init: bool) -> base_host.Message: return { 'contract_address': address, @@ -532,6 +187,17 @@ def _calldata(method: str, *args: typing.Any) -> bytes: return gvm_calldata.encode({'': method, 'args': list(args)}) +def _assert_hashes_agree(label: str, runs: list[tuple[str, typing.Any]]) -> None: + """ + A disagreement is only actionable once the failure says who disagreed. + """ + hashes = [(name, run.execution_hash.hex()) for name, run in runs] + assert len({digest for _name, digest in hashes}) == 1, '\n'.join( + [f'{label}: execution hashes disagree'] + + [f' {name}: {digest}' for name, digest in hashes] + ) + + def _apply_storage_changes( storage: MockStorage, address: Address, @@ -581,7 +247,7 @@ async def _wait_for_descendants( async def _wait_for_no_descendants(pid: int, timeout: float = 10) -> None: async with asyncio.timeout(timeout): - while _descendants(pid): + while _descendants(pid): # noqa: ASYNC110 β€” polls the OS process table, no event to await await asyncio.sleep(0.05) @@ -589,6 +255,10 @@ async def _wait_for_no_descendants(pid: int, timeout: float = 10) -> None: class CrossMajorCase(genvm_tool.tests.test.Case): description: genvm_tool.tests.test.Description shared: genvm_tool.tests.SharedContext + manager_service: genvm_tool.tests.stage.collection.Service + method: str + fixtures: tuple[str, ...] + arguments: tuple[typing.Any, ...] = () async def into_steps(self) -> list[genvm_tool.tests.exec.step.Step]: return [CrossMajorStep(self)] @@ -614,7 +284,7 @@ async def run( except BaseException as exc: return genvm_tool.tests.test.Result( passed=False, - context={'phase': self.phase, 'error': repr(exc)}, + context={'phase': self.phase, 'error': exc, 'notes': self.notes}, elapsed_seconds=0, ) @@ -626,36 +296,19 @@ async def _run_all(self): 2: build_info['executor_versions']['v0.2'], 3: build_info['executor_versions']['v0.3'], } - work_dir = self.case.shared.artifacts_dir / 'cross-major' + work_dir = self.case.shared.case_dir_for(self.case.description.name) self.storage_path = work_dir / 'storage.pickle' work_dir.mkdir(parents=True, exist_ok=True) - with open(self.storage_path, 'wb') as storage_file: - pickle.dump(MockStorage(), storage_file) - - async with ManagerProc( - build_dir=self.build_dir, - work_dir=work_dir / 'manager', - ) as manager: - self.manager = manager - - self.phase = 'deploy fixtures' - await self._deploy(3, ADDR_CALLER_V03, CALLER_V03) - await self._deploy(2, ADDR_CALLER_V02, CALLER_V02) - await self._deploy(3, ADDR_CALLEE_V03, CALLEE_V03) - await self._deploy(2, ADDR_CALLEE_V02, CALLEE_V02) - await self._deploy(2, ADDR_BUSY_V02, BUSY_V02) - await self._deploy(3, ADDR_LOOP_V03, LOOP_V03) - await self._deploy(2, ADDR_LOOP_V02, LOOP_V02) - await self._deploy(3, ADDR_SIGNER_V03, SIGNER_V03) - await self._deploy(2, ADDR_SIGNER_CALLER_V02, SIGNER_CALLER_V02) - await self._deploy(3, ADDR_STRUCTURED_V03, STRUCTURED_ERROR_V03) - await self._deploy(3, ADDR_BLOB_V03, BLOB_V03) - await self._deploy(2, ADDR_BLOB_CALLER_V02, BLOB_CALLER_V02) - await self._deploy(3, ADDR_UERR_V03, USER_ERROR_V03) - await self._deploy(2, ADDR_UERR_V02, USER_ERROR_V02) - await self._deploy(3, ADDR_STATE_CALLER_V03, STATE_CALLER_V03) - await self._deploy(3, ADDR_CHAIN_V03, CHAIN_V03) - await self._deploy(2, ADDR_CHAIN_V02, CHAIN_V02) + await gvm_io.write_file_bytes(self.storage_path, pickle.dumps(MockStorage())) + + manager = self.case.manager_service.handle + assert manager is not None + self.manager = manager + + self.phase = 'deploy fixtures' + for line, address, code in (FIXTURES[name] for name in self.case.fixtures): + await self._deploy(line, address, code) + if 'loop-v03' in self.case.fixtures: loop_v03 = await self._compile_wat( _loop_wat(ADDR_LOOP_V02), work_dir / 'loop-v03' ) @@ -665,111 +318,14 @@ async def _run_all(self): self._replace_code(ADDR_LOOP_V03, LOOP_V03, loop_v03) self._replace_code(ADDR_LOOP_V02, LOOP_V02, loop_v02) - self.phase = 'v0.3 to v0.2' - forward = await self._call( - 3, - ADDR_CALLER_V03, - ADDR_CALLEE_V02, - lambda address, _state, _major: ( - self._route(2) if address == ADDR_CALLEE_V02 else None - ), - ) - assert forward.result_kind == public_abi.ResultCode.RETURN, forward - assert forward.result_data == 202, forward - assert len(forward.execution_hash) == 32, forward.execution_hash - assert forward.execution_hash.hex() == ( - 'c0e5932da9b2e9bf79579079f1fec46f74b83f02114bcd2f711a6798ee20bdd6' - ), forward.execution_hash.hex() - - self.phase = 'v0.2 to v0.3' - backward = await self._call( - 2, - ADDR_CALLER_V02, - ADDR_CALLEE_V03, - lambda address, _state, _major: ( - self._route(3) if address == ADDR_CALLEE_V03 else None - ), - ) - assert backward.result_kind == public_abi.ResultCode.RETURN, backward - assert backward.result_data == 303, backward - assert len(backward.execution_hash) == 32, backward.execution_hash - assert backward.execution_hash.hex() == ( - '6aad898ed66c94e7bc9fc1f6bb7d615b9b4cf01d1e1205ca0c640712a8c6792d' - ), backward.execution_hash.hex() - - self.phase = 'v0.2 forwards the signer it never exposes' - await self._assert_signer_preserved() - - self.phase = 'a nested envelope is not bounded by the websocket cap' - await self._assert_envelope_ignores_websocket_cap() - - self.phase = 'a structured user error survives the boundary' - await self._assert_structured_user_error() - - self.phase = 'null preserves same-major behavior' - resolved: list[tuple[Address, int]] = [] - - def resolve_null(address, _state, advisory_major): - resolved.append((address, advisory_major)) - return None - - same_major = await self._call( - 3, - ADDR_CALLER_V03, - ADDR_CALLEE_V03, - resolve_null, - ) - assert same_major.result_kind == public_abi.ResultCode.RETURN, same_major - assert same_major.result_data == 303, same_major - assert resolved == [(ADDR_CALLEE_V03, 0)], resolved - - self.phase = 'null preserves mismatch behavior' - mismatch = await self._call( - 3, - ADDR_CALLER_V03, - ADDR_CALLEE_V02, - lambda _address, _state, _major: None, - ) - assert mismatch.result_kind == public_abi.ResultCode.VM_ERROR, mismatch - assert str(mismatch.result_data).startswith( - str(public_abi.VmError.invalid_contract().absent_runner_comment()) - ), mismatch.result_data - - self.phase = 'a host that does not hook is never asked' - await self._assert_unhooked_stays_in_process() - - self.phase = 'leader, validator and sync agree on the nested hash' - await self._assert_lvs_agrees() - - self.phase = 'the route a host picks does not change the hash' - await self._assert_route_is_hash_invariant() - - self.phase = 'a plain user error survives the boundary' - await self._assert_plain_user_error() - - self.phase = 'the state mode a caller picks crosses the boundary' - await self._assert_state_mode_crosses() + self.phase = self.case.method + await getattr(self, self.case.method)(*self.case.arguments) - self.phase = 'a deep alternating chain stays deterministic' - await self._assert_deep_chain() + async def _assert_cancel_stops_chain(self): + await self._assert_chain_stopped('cancel') - self.phase = 'a starved fuel budget never becomes an internal error' - await self._assert_starved_fuel() - - self.phase = 'a contract cannot manufacture an internal error' - await self._assert_child_failures_are_canonical() - - self.phase = 'cancel kills process chain' - await self._assert_chain_stopped('cancel') - - self.phase = 'deadline kills process chain' - await self._assert_chain_stopped('deadline') - - self.phase = 'a nested chain that runs out of time reports timeout' - await self._assert_nested_deadline_is_timeout() - - self.phase = 'resolve loop bottoms out on the recursion budget' - await self._assert_resolve_loop_recursion() + async def _assert_deadline_stops_chain(self): + await self._assert_chain_stopped('deadline') async def _new_host( self, @@ -847,7 +403,7 @@ async def _execute( message=_message(address, is_init=is_init), host_data='{"node_address":"test","tx_id":"cross-major"}', host='unix://' + mock_host.path, - code=code, + code=resolve_runners(code, self.case.shared.root_dir), calldata=calldata, timeout=timeout, debug_mode='unsafe', @@ -861,7 +417,7 @@ async def _execute( }, bucket_totals=[2**200] * 20, ) - if apply_changes and result.result_kind == public_abi.ResultCode.RETURN: + if apply_changes and result.result_kind == host_fns.ResultCode.RETURN: assert mock_host.storage is not None _apply_storage_changes( mock_host.storage, @@ -881,14 +437,14 @@ async def _deploy(self, line: int, address: Address, code: bytes): code=code, is_init=True, ) - assert result.result_kind == public_abi.ResultCode.RETURN, result + assert result.result_kind == host_fns.ResultCode.RETURN, result assert len(result.execution_hash) == 32, result.execution_hash async def _compile_wat(self, wat: str, path: Path) -> bytes: - path.mkdir(parents=True, exist_ok=True) + await gvm_io.make_dir(path) wat_path = path / 'contract.wat' wasm_path = path / 'contract.wasm' - wat_path.write_text(wat) + await gvm_io.write_file_text(wat_path, wat) process = await asyncio.create_subprocess_exec( 'wat2wasm', '--enable-annotations', @@ -900,9 +456,12 @@ async def _compile_wat(self, wat: str, path: Path) -> bytes: ) stdout, stderr = await process.communicate() assert process.returncode == 0, (stdout, stderr) - return wasm_path.read_bytes() + return await gvm_io.read_file_bytes(wasm_path) def _replace_code(self, address: Address, old_code: bytes, new_code: bytes) -> None: + root_dir = self.case.shared.root_dir + old_code = typing.cast(bytes, resolve_runners(old_code, root_dir)) + new_code = typing.cast(bytes, resolve_runners(new_code, root_dir)) with open(self.storage_path, 'rb') as storage_file: storage: MockStorage = pickle.load(storage_file) slots = storage._storages[address] @@ -923,7 +482,8 @@ def _replace_code(self, address: Address, old_code: bytes, new_code: bytes) -> N pickle.dump(storage, storage_file) def _route(self, line: int) -> bytes: - """Routing payload placing a callee on `line`'s executor. + """ + Routing payload placing a callee on `line`'s executor. It names the directory rather than the major, because a major cannot pick between the live lines: every manifest key is semver major 0, so @@ -931,17 +491,6 @@ def _route(self, line: int) -> bytes: """ return gvm_calldata.encode({'kind': 'version', 'version': self.versions[line]}) - async def _call(self, line: int, caller: Address, target: Address, resolve_hook): - return await self._execute( - name=f'call-{line}-{caller.as_bytes.hex()}-{target.as_bytes.hex()}', - line=line, - address=caller, - calldata=_calldata('call', target), - code=None, - is_init=False, - resolve_hook=resolve_hook, - ) - def _raw_request( self, *, @@ -1000,7 +549,7 @@ def resolve(address, _state, _major): line=3, address=ADDR_CALLER_V03, host_path=mock_host.path, - deadline='30s' if mode == 'cancel' else '3s', + deadline='30s' if mode == 'cancel' else '10s', ) ) await asyncio.wait_for(nested_resolved.wait(), timeout=10) @@ -1019,45 +568,9 @@ def resolve(address, _state, _major): await host_task await host.stop_connections() - async def _assert_signer_preserved(self): - """This line's contract-facing message has no signer, but the executor - still receives one and must hand it to a callee that does expose it.""" - result = await self._execute( - name='signer-v02-to-v03', - line=2, - address=ADDR_SIGNER_CALLER_V02, - calldata=_calldata('signer', ADDR_SIGNER_V03), - code=None, - is_init=False, - resolve_hook=lambda address, _state, _major: ( - self._route(3) if address == ADDR_SIGNER_V03 else None - ), - ) - assert result.result_kind == public_abi.ResultCode.RETURN, result - assert result.result_data == SENDER.as_hex, (result.result_data, SENDER.as_hex) - - async def _assert_structured_user_error(self): - """A callee's user error is contract data, so it must stay contract-visible - on the caller. v0.2 user errors are strings, so a structured payload cannot - be carried across as one and degrades to the vm error naming the boundary - that could not represent it β€” never an internal error.""" - result = await self._execute( - name='structured-error-v02-to-v03', - line=2, - address=ADDR_CALLER_V02, - calldata=_calldata('call', ADDR_STRUCTURED_V03), - code=None, - is_init=False, - resolve_hook=lambda address, _state, _major: ( - self._route(3) if address == ADDR_STRUCTURED_V03 else None - ), - ) - assert result.result_kind == public_abi.ResultCode.VM_ERROR, result - expected = str(public_abi.VmError.invalid_contract().major_mismatch()) - assert result.result_data == expected, (result.result_data, expected) - async def _assert_envelope_ignores_websocket_cap(self): - """A contract picks the calldata it forwards, so nothing an operator + """ + A contract picks the calldata it forwards, so nothing an operator configures may decide whether the call goes through. This manager runs with `max_message_bytes` deliberately below the envelope this builds: the cap bounds client frames on `/ws`, never the executor socketpair.""" @@ -1074,11 +587,12 @@ async def _assert_envelope_ignores_websocket_cap(self): ), timeout=60, ) - assert result.result_kind == public_abi.ResultCode.RETURN, result + assert result.result_kind == host_fns.ResultCode.RETURN, result assert result.result_data == BLOB_BYTES, result async def _assert_nested_deadline_is_timeout(self): - """The chain dies on a signal, so no executor reports anything. What the + """ + The chain dies on a signal, so no executor reports anything. What the caller must still observe is a timeout, not the internal error the dead boundary would otherwise produce.""" result = await self._execute( @@ -1093,13 +607,14 @@ async def _assert_nested_deadline_is_timeout(self): ), timeout=10, ) - assert result.result_kind == public_abi.ResultCode.VM_ERROR, result + assert result.result_kind == host_fns.ResultCode.VM_ERROR, result assert result.result_data == str(public_abi.VmError.timeout()), result async def _lvs( self, *, name: str, line: int, address: Address, calldata: bytes, resolve_hook ): - """Run the same step as leader, validator and sync run. + """ + Run the same step as leader, validator and sync run. The three must agree on the execution hash; anything else is a determinism violation an honest validator would see as a disagreement. @@ -1125,7 +640,7 @@ async def one(suffix: str, **kwargs): sync = await one('sync', is_sync=True) return leader, validator, sync - async def _assert_lvs_agrees(self): + async def _assert_lvs_validator_mode(self): # A leader/validator run that silently degraded to a sync one would make # the agreement below vacuous, so first pin that the validator mode is # real: handing it a leader result nothing consumes must be rejected. @@ -1143,38 +658,34 @@ async def _assert_lvs_agrees(self): is_sync=False, leader_nondet_results=[gvm_calldata.encode({})], ) - assert bogus.result_kind == public_abi.ResultCode.VM_ERROR, bogus - - for suffix, line, caller, callee, major in ( - ('fwd', 3, ADDR_CALLER_V03, ADDR_CALLEE_V02, 2), - ('bwd', 2, ADDR_CALLER_V02, ADDR_CALLEE_V03, 3), - ): - target = callee - leader, validator, sync = await self._lvs( - name=f'lvs-{suffix}', - line=line, - address=caller, - calldata=_calldata('call', callee), - resolve_hook=lambda address, _state, _major, target=target, major=major: ( - self._route(major) if address == target else None - ), - ) - for label, run in (('leader', leader), ('validator', validator), ('sync', sync)): - assert run.result_kind == public_abi.ResultCode.RETURN, (suffix, label, run) - assert len(run.execution_hash) == 32, (suffix, label, run.execution_hash) - assert leader.execution_hash == validator.execution_hash, ( - suffix, - leader.execution_hash.hex(), - validator.execution_hash.hex(), - ) - assert leader.execution_hash == sync.execution_hash, ( - suffix, - leader.execution_hash.hex(), - sync.execution_hash.hex(), - ) + assert bogus.result_kind == host_fns.ResultCode.VM_ERROR, bogus + + async def _assert_lvs_direction( + self, + suffix: str, + line: int, + caller: Address, + callee: Address, + major: int, + ): + leader, validator, sync = await self._lvs( + name=f'lvs-{suffix}', + line=line, + address=caller, + calldata=_calldata('call', callee), + resolve_hook=lambda address, _state, _major: ( + self._route(major) if address == callee else None + ), + ) + runs = [('leader', leader), ('validator', validator), ('sync', sync)] + for label, run in runs: + assert run.result_kind == host_fns.ResultCode.RETURN, (suffix, label, run) + assert len(run.execution_hash) == 32, (suffix, label, run.execution_hash) + _assert_hashes_agree(f'nested call, {suffix}', runs) async def _assert_unhooked_stays_in_process(self): - """Without `hook_cross_contract_calls` the manager answers the resolve + """ + Without `hook_cross_contract_calls` the manager answers the resolve method itself, so a host is never consulted even when the callee really does live on another major, and the call stays in-process.""" asked: list[Address] = [] @@ -1196,13 +707,14 @@ def resolve(address, _state, _major): ) assert asked == [], asked # The in-process outcome for a v0.2 callee, identical to answering null. - assert result.result_kind == public_abi.ResultCode.VM_ERROR, result + assert result.result_kind == host_fns.ResultCode.VM_ERROR, result assert str(result.result_data).startswith( - str(public_abi.VmError.invalid_contract().absent_runner_comment()) + str(public_abi.VmError.invalid_contract().runner().absent()) ), result.result_data async def _assert_route_is_hash_invariant(self): - """A host picks the route, a contract does not. A same-major call that a + """ + A host picks the route, a contract does not. A same-major call that a host chooses to send through the nested boundary must hash exactly as the in-process one, otherwise the route is consensus-visible.""" in_process = await self._execute( @@ -1227,81 +739,57 @@ async def _assert_route_is_hash_invariant(self): ), apply_changes=False, ) - assert in_process.result_kind == public_abi.ResultCode.RETURN, in_process - assert nested.result_kind == public_abi.ResultCode.RETURN, nested + assert in_process.result_kind == host_fns.ResultCode.RETURN, in_process + assert nested.result_kind == host_fns.ResultCode.RETURN, nested assert in_process.result_data == nested.result_data == 303, (in_process, nested) # The parent's own execution hash folds the callee's small hash, which # commits to the outcome and to nothing about where the callee ran. - assert in_process.execution_hash == nested.execution_hash, ( + _assert_hashes_agree( 'the route is consensus-visible', - in_process.execution_hash.hex(), - nested.execution_hash.hex(), + [('in process', in_process), ('nested', nested)], ) - async def _assert_plain_user_error(self): - """A string user error is representable on both lines, so it must reach - the caller as a user error in either direction.""" - for suffix, line, caller, callee, major in ( - ('v02-to-v03', 2, ADDR_CALLER_V02, ADDR_UERR_V03, 3), - ('v03-to-v02', 3, ADDR_CALLER_V03, ADDR_UERR_V02, 2), - ): - target = callee + async def _assert_state_mode_crosses( + self, + final: bool, + expected: public_abi.StorageType, + ): + """ + The caller's state mode selects which chain view the callee reads, so + the boundary must hand the callee exactly the mode the in-process route + would have used.""" + modes: dict[bool, set[public_abi.StorageType]] = {} + for nested in (False, True): + resolved: list[public_abi.StorageType] = [] + read_log: list[tuple[Address, public_abi.StorageType]] = [] + + def resolve(address, state, _major, nested=nested, resolved=resolved): + if address != ADDR_CALLEE_V03: + return None + resolved.append(state) + return self._route(3) if nested else None + result = await self._execute( - name=f'plain-error-{suffix}', - line=line, - address=caller, - calldata=_calldata('call', callee), + name=f'state-mode-{"final" if final else "nonfinal"}-{nested}', + line=3, + address=ADDR_STATE_CALLER_V03, + calldata=_calldata('call', ADDR_CALLEE_V03, final), code=None, is_init=False, - resolve_hook=lambda address, _state, _major, target=target, major=major: ( - self._route(major) if address == target else None - ), + resolve_hook=resolve, apply_changes=False, + read_log=read_log, ) - assert result.result_kind == public_abi.ResultCode.USER_ERROR, (suffix, result) - - async def _assert_state_mode_crosses(self): - """The caller's state mode selects which chain view the callee reads, so - the boundary must hand the callee exactly the mode the in-process route - would have used.""" - for final, expected in ( - (False, public_abi.StorageType.LATEST_NON_FINAL), - (True, public_abi.StorageType.LATEST_FINAL), - ): - modes: dict[bool, set[public_abi.StorageType]] = {} - for nested in (False, True): - resolved: list[public_abi.StorageType] = [] - read_log: list[tuple[Address, public_abi.StorageType]] = [] - - def resolve(address, state, _major, nested=nested, resolved=resolved): - if address != ADDR_CALLEE_V03: - return None - resolved.append(state) - return self._route(3) if nested else None - - result = await self._execute( - name=f'state-mode-{"final" if final else "nonfinal"}-{nested}', - line=3, - address=ADDR_STATE_CALLER_V03, - calldata=_calldata('call', ADDR_CALLEE_V03, final), - code=None, - is_init=False, - resolve_hook=resolve, - apply_changes=False, - read_log=read_log, - ) - assert result.result_kind == public_abi.ResultCode.RETURN, ( - final, - nested, - result, - ) - assert result.result_data == 303, (final, nested, result) - assert resolved == [expected], (final, nested, resolved) - modes[nested] = { - mode for account, mode in read_log if account == ADDR_CALLEE_V03 - } - assert modes[False] == modes[True], (final, modes) - assert modes[True] == {expected}, (final, modes) + assert result.result_kind == host_fns.ResultCode.RETURN, ( + final, + nested, + result, + ) + assert result.result_data == 303, (final, nested, result) + assert resolved == [expected], (final, nested, resolved) + modes[nested] = {mode for account, mode in read_log if account == ADDR_CALLEE_V03} + assert modes[False] == modes[True], (final, modes) + assert modes[True] == {expected}, (final, modes) def _chain_resolve(self, address, _state, _major): if address == ADDR_CHAIN_V03: @@ -1311,7 +799,8 @@ def _chain_resolve(self, address, _state, _major): return None async def _assert_deep_chain(self): - """Every hop of an alternating chain crosses the boundary, so the whole + """ + Every hop of an alternating chain crosses the boundary, so the whole chain is a stack of nested processes. Its answer and its hash must not depend on which of the three runs produced them.""" depth = 6 @@ -1325,7 +814,7 @@ async def _assert_deep_chain(self): # The chain alternates v0.3, v0.2, ..., so it sums 3 and 2 in turn. expected = sum(3 if index % 2 == 0 else 2 for index in range(depth + 1)) for label, run in (('leader', leader), ('validator', validator), ('sync', sync)): - assert run.result_kind == public_abi.ResultCode.RETURN, (label, run) + assert run.result_kind == host_fns.ResultCode.RETURN, (label, run) assert run.result_data == expected, (label, run.result_data, expected) assert leader.execution_hash == validator.execution_hash, ( leader.execution_hash.hex(), @@ -1336,79 +825,77 @@ async def _assert_deep_chain(self): sync.execution_hash.hex(), ) - async def _assert_starved_fuel(self): - """The caller hands the callee whatever fuel the host says is left. A + async def _assert_starved_fuel(self, fuel: int): + """ + The caller hands the callee whatever fuel the host says is left. A callee that cannot finish on it must report a canonical vm error, and two hosts reporting the same figure must agree on the outcome.""" - outcomes = [] - for fuel in (0, 1, 1000, 10**6): - result = await self._execute( - name=f'starved-fuel-{fuel}', - line=3, - address=ADDR_CALLER_V03, - calldata=_calldata('call', ADDR_CALLEE_V02), - code=None, - is_init=False, - resolve_hook=lambda address, _state, _major: ( - self._route(2) if address == ADDR_CALLEE_V02 else None - ), - apply_changes=False, - host_fuel=fuel, - ) - assert result.result_kind != public_abi.ResultCode.INTERNAL_ERROR, (fuel, result) - outcomes.append((fuel, result.result_kind, result.result_data)) - repeat = await self._execute( - name=f'starved-fuel-{fuel}-again', - line=3, - address=ADDR_CALLER_V03, - calldata=_calldata('call', ADDR_CALLEE_V02), - code=None, - is_init=False, - resolve_hook=lambda address, _state, _major: ( - self._route(2) if address == ADDR_CALLEE_V02 else None - ), - apply_changes=False, - host_fuel=fuel, - ) - assert repeat.execution_hash == result.execution_hash, ( - fuel, - result.execution_hash.hex(), - repeat.execution_hash.hex(), - ) - self.notes.append(('starved fuel outcomes', [str(entry) for entry in outcomes])) - - async def _assert_child_failures_are_canonical(self): - """A contract must not be able to manufacture an - internal error. Every failure a contract can pick the input for β€” a - method the callee lacks, an address nothing was ever deployed to, a - callee routed to a line its stored major disagrees with β€” must land on a - user or vm error instead.""" - absent = Address('0x' + '99' * 20) - cases = ( - # The callee has `sink`, not `answer`. - ('missing-method', 2, ADDR_CALLER_V02, ADDR_BLOB_V03, 3), - ('absent-callee', 3, ADDR_CALLER_V03, absent, 2), - # The callee's stored major is 2; the host routes it to v0.3 anyway. - ('routed-to-wrong-line', 3, ADDR_CALLER_V03, ADDR_CALLEE_V02, 3), + result = await self._execute( + name=f'starved-fuel-{fuel}', + line=3, + address=ADDR_CALLER_V03, + calldata=_calldata('call', ADDR_CALLEE_V02), + code=None, + is_init=False, + resolve_hook=lambda address, _state, _major: ( + self._route(2) if address == ADDR_CALLEE_V02 else None + ), + apply_changes=False, + host_fuel=fuel, ) - observed = [] - for label, line, caller, callee, major in cases: - target = callee - result = await self._execute( - name=f'canonical-{label}', - line=line, - address=caller, - calldata=_calldata('call', callee), - code=None, - is_init=False, - resolve_hook=lambda address, _state, _major, target=target, major=major: ( - self._route(major) if address == target else None - ), - apply_changes=False, + assert result.result_kind != host_fns.ResultCode.INTERNAL_ERROR, (fuel, result) + repeat = await self._execute( + name=f'starved-fuel-{fuel}-again', + line=3, + address=ADDR_CALLER_V03, + calldata=_calldata('call', ADDR_CALLEE_V02), + code=None, + is_init=False, + resolve_hook=lambda address, _state, _major: ( + self._route(2) if address == ADDR_CALLEE_V02 else None + ), + apply_changes=False, + host_fuel=fuel, + ) + assert repeat.execution_hash == result.execution_hash, ( + fuel, + result.execution_hash.hex(), + repeat.execution_hash.hex(), + ) + self.notes.append( + ('starved fuel outcome', (fuel, str(result.result_kind), str(result.result_data))) + ) + + async def _assert_child_failure_is_canonical( + self, + label: str, + line: int, + caller: Address, + callee: Address, + major: int, + ): + """ + A contract-controlled child failure must land on a user or VM error, + never an internal error.""" + result = await self._execute( + name=f'canonical-{label}', + line=line, + address=caller, + calldata=_calldata('call', callee), + code=None, + is_init=False, + resolve_hook=lambda address, _state, _major: ( + self._route(major) if address == callee else None + ), + apply_changes=False, + ) + assert result.result_kind != host_fns.ResultCode.INTERNAL_ERROR, (label, result) + self.notes.append( + ( + 'child failure outcome', + (label, str(result.result_kind), str(result.result_data)[:80]), ) - assert result.result_kind != public_abi.ResultCode.INTERNAL_ERROR, (label, result) - observed.append((label, str(result.result_kind), str(result.result_data)[:80])) - self.notes.append(('child failure outcomes', [str(entry) for entry in observed])) + ) async def _assert_resolve_loop_recursion(self): resolved: list[tuple[Address, int]] = [] @@ -1434,7 +921,7 @@ def resolve(address, _state, advisory_major): # cost 511 spawns to reach; this hits the same code path in two. debug_initial_recursion=2, ) - assert result.result_kind == public_abi.ResultCode.VM_ERROR, ( + assert result.result_kind == host_fns.ResultCode.VM_ERROR, ( result, len(resolved), resolved[-3:], @@ -1450,10 +937,140 @@ def resolve(address, _state, advisory_major): ], resolved -def collect(ctx: genvm_tool.tests.stage.collection.Context) -> None: - desc = genvm_tool.tests.test.Description( - name='tests/system/cross-major', - tags=frozenset({'integration', 'stable', 'cross-major'}), - console_pool=True, - ) - ctx.add_case(CrossMajorCase(description=desc, shared=ctx.shared)) +def collect( + ctx: genvm_tool.tests.stage.collection.Context, + *, + manager_service: genvm_tool.tests.stage.collection.Service, + limited_manager_service: genvm_tool.tests.stage.collection.Service | None, + managed_manager: bool, +) -> None: + cases = [ + ( + 'large-envelope', + '_assert_envelope_ignores_websocket_cap', + ('blob-caller-v02', 'blob-v03'), + (), + ), + ( + 'unhooked', + '_assert_unhooked_stays_in_process', + ('caller-v03', 'callee-v02'), + (), + ), + ( + 'lvs-validator-mode', + '_assert_lvs_validator_mode', + ('caller-v03', 'callee-v02'), + (), + ), + ( + 'lvs-v03-to-v02', + '_assert_lvs_direction', + ('caller-v03', 'callee-v02'), + ('v03-to-v02', 3, ADDR_CALLER_V03, ADDR_CALLEE_V02, 2), + ), + ( + 'lvs-v02-to-v03', + '_assert_lvs_direction', + ('caller-v02', 'callee-v03'), + ('v02-to-v03', 2, ADDR_CALLER_V02, ADDR_CALLEE_V03, 3), + ), + ( + 'route-hash', + '_assert_route_is_hash_invariant', + ('caller-v03', 'callee-v03'), + (), + ), + ( + 'state-mode-nonfinal', + '_assert_state_mode_crosses', + ('state-caller-v03', 'callee-v03'), + (False, public_abi.StorageType.LATEST_NON_FINAL), + ), + ( + 'state-mode-final', + '_assert_state_mode_crosses', + ('state-caller-v03', 'callee-v03'), + (True, public_abi.StorageType.LATEST_FINAL), + ), + ('deep-chain', '_assert_deep_chain', ('chain-v03', 'chain-v02'), ()), + *[ + ( + f'starved-fuel-{fuel}', + '_assert_starved_fuel', + ('caller-v03', 'callee-v02'), + (fuel,), + ) + for fuel in (0, 1, 1000, 10**6) + ], + ( + 'child-failure-missing-method', + '_assert_child_failure_is_canonical', + ('caller-v02', 'blob-v03'), + ('missing-method', 2, ADDR_CALLER_V02, ADDR_BLOB_V03, 3), + ), + ( + 'child-failure-absent-callee', + '_assert_child_failure_is_canonical', + ('caller-v03',), + ('absent-callee', 3, ADDR_CALLER_V03, ABSENT_CALLEE, 2), + ), + ( + 'child-failure-wrong-line', + '_assert_child_failure_is_canonical', + ('caller-v03', 'callee-v02'), + ('routed-to-wrong-line', 3, ADDR_CALLER_V03, ADDR_CALLEE_V02, 3), + ), + ( + 'cancel-process-tree', + '_assert_cancel_stops_chain', + ('caller-v03', 'busy-v02'), + (), + ), + ( + 'deadline-process-tree', + '_assert_deadline_stops_chain', + ('caller-v03', 'busy-v02'), + (), + ), + ( + 'nested-timeout', + '_assert_nested_deadline_is_timeout', + ('caller-v03', 'busy-v02'), + (), + ), + ( + 'resolve-recursion', + '_assert_resolve_loop_recursion', + ('loop-v03', 'loop-v02'), + (), + ), + ] + for slug, method, fixtures, arguments in cases: + if not managed_manager and slug in { + 'cancel-process-tree', + 'deadline-process-tree', + }: + continue + if slug == 'large-envelope': + if limited_manager_service is None: + continue + case_manager = limited_manager_service + else: + case_manager = manager_service + desc = genvm_tool.tests.test.Description( + name=f'tests/system/cross-major/{slug}', + needed_services=frozenset({case_manager}), + tags=frozenset({'integration', 'stable', 'feature-version-routing-cross-major'}), + console_pool=slug in {'cancel-process-tree', 'deadline-process-tree'}, + ) + ctx.add_case( + CrossMajorCase( + description=desc, + shared=ctx.shared, + manager_service=case_manager, + method=method, + fixtures=fixtures, + arguments=arguments, + ) + ) diff --git a/tests/system/manager-socket/test.py b/tests/system/manager-socket/test.py index a489cc41..f375043c 100644 --- a/tests/system/manager-socket/test.py +++ b/tests/system/manager-socket/test.py @@ -1,17 +1,17 @@ import asyncio import contextlib -import json import os import pickle -import socket import stat import typing from dataclasses import dataclass from pathlib import Path import aiohttp +import genvm_tool.io as gvm_io import genvm_tool.tests import genvm_tool.tests.stage.collection +import genvm_tool_plugins.genvm as genvm import origin.base_host as base_host import origin.calldata as gvm_calldata from gvm_extra.mock_host import MockHost, MockStorage @@ -126,120 +126,36 @@ async def close(self) -> None: self.session = None -class ManagerProc: - def __init__( - self, - *, - root: Path, - build_dir: Path, - artifacts_dir: Path, - name: str, - socket_path: Path, - max_message_bytes: int = 67108864, - execution_retention: str = '30s', - use_unix_listener: bool = False, - ): - self.root = root - self.build_dir = build_dir - self.work_dir = artifacts_dir / name - self.socket_path = socket_path - self.max_message_bytes = max_message_bytes - self.execution_retention = execution_retention - self.use_unix_listener = use_unix_listener - self.process: asyncio.subprocess.Process | None = None - self.port = self._unused_port() - self.log_file = None - - @staticmethod - def _unused_port() -> int: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: - sock.bind(('127.0.0.1', 0)) - return sock.getsockname()[1] +class _ManagerFixture: + def __init__(self, handle: genvm.ManagerHandle, work_dir: Path): + self.handle = handle + self.work_dir = work_dir + self.work_dir.mkdir(parents=True, exist_ok=True) @property def uri(self) -> str: - if self.use_unix_listener: - return f'unix://{self.socket_path}' - return f'http://127.0.0.1:{self.port}' + return self.handle.uri + + @property + def socket_path(self) -> Path | None: + return self.handle.socket_path + + async def restart(self) -> None: + await self.handle.restart() async def __aenter__(self): - self.work_dir.mkdir(parents=True, exist_ok=True) - config_path = self.work_dir / 'genvm-manager.yaml' - config_path.write_text( - '\n'.join( - [ - 'threads: 4', - 'blocking_threads: 48', - 'log_disable: tracing*,polling*,tungstenite*,tokio_tungstenite*', - 'log_level: info', - f'manifest_path: {self.build_dir / "out" / "data" / "manifest.yaml"}', - 'permits: 2', - f'execution_retention: {self.execution_retention}', - f'max_message_bytes: {self.max_message_bytes}', - '', - ] - ) - ) - listener_args = ( - ['--socket', str(self.socket_path)] - if self.use_unix_listener - else ['--port', str(self.port)] - ) - self.log_file = open(self.work_dir / 'manager.log', 'wb') - self.process = await asyncio.subprocess.create_subprocess_exec( - str(self.build_dir / 'out' / 'bin' / 'genvm-modules'), - 'manager', - *listener_args, - '--die-with-parent', - '--config', - str(config_path), - stdin=asyncio.subprocess.DEVNULL, - stdout=self.log_file, - stderr=self.log_file, - start_new_session=True, - ) - await self._wait_healthy() return self async def __aexit__(self, *_args): - if self.process is not None: - try: - self.process.terminate() - await asyncio.wait_for(self.process.wait(), timeout=5) - except asyncio.TimeoutError: - self.process.kill() - await self.process.wait() - if self.log_file is not None: - self.log_file.close() - - async def _wait_healthy(self): - last_error: BaseException | None = None - for _ in range(80): - if self.process is not None and self.process.returncode is not None: - raise RuntimeError(f'manager exited early with {self.process.returncode}') - try: - if self.use_unix_listener: - async with aiohttp.ClientSession( - connector=aiohttp.UnixConnector(path=str(self.socket_path)) - ) as session: - async with session.get( - 'http://localhost/status', - timeout=aiohttp.ClientTimeout(total=1), - ) as resp: - if resp.status == 200: - return - else: - async with aiohttp.request( - 'GET', - f'{self.uri}/status', - timeout=aiohttp.ClientTimeout(total=1), - ) as resp: - if resp.status == 200: - return - except BaseException as exc: - last_error = exc - await asyncio.sleep(0.1) - raise TimeoutError(f'manager did not become healthy: {last_error}') + pass + + +class _StaleSocketManagerService(genvm.ManagerService): + async def _start_once(self) -> genvm.ManagerHandle: + assert self._socket_path is not None + self._socket_path.parent.mkdir(parents=True, exist_ok=True) + self._socket_path.write_text('stale') + return await super()._start_once() def _run_request( @@ -302,7 +218,8 @@ async def _wait_event(client: ManagerWsClient, variant: str, timeout: float = 10 async def _wait_terminal(client: ManagerWsClient, timeout: float = 10.0): - """Wait for whichever terminal event the run produces. + """ + Wait for whichever terminal event the run produces. Which one it is depends on how far the run got, and the point of the test is that *some* terminal event arrives without the host having to time out. @@ -371,7 +288,7 @@ async def _get_artifact(client: ManagerWsClient, genvm_id: int, field: str) -> b def _make_mock_host( - manager: ManagerProc, + manager: _ManagerFixture, name: str, ctx: _TestContext, running_address: Address | None = None, @@ -400,6 +317,8 @@ def _make_mock_host( class ManagerSocketCase(genvm_tool.tests.test.Case): description: genvm_tool.tests.test.Description shared: genvm_tool.tests.SharedContext + manager_service: genvm_tool.tests.stage.collection.Service + method: str async def into_steps(self) -> list[genvm_tool.tests.exec.step.Step]: return [ManagerSocketStep(self)] @@ -408,7 +327,7 @@ async def into_steps(self) -> list[genvm_tool.tests.exec.step.Step]: class ManagerSocketStep(genvm_tool.tests.exec.step.Python): def __init__(self, case: ManagerSocketCase): self.case = case - self.phase = 'start' + self.phase = case.method def to_str(self) -> str: return '<manager socket protocol test>' @@ -417,7 +336,7 @@ async def run( self, previous_results: list[typing.Any] ) -> genvm_tool.tests.test.Result: try: - await self._run_all() + await getattr(self, self.case.method)() return genvm_tool.tests.test.Result(passed=True, context={}, elapsed_seconds=0) except BaseException as exc: return genvm_tool.tests.test.Result( @@ -433,69 +352,17 @@ async def _manager( max_message_bytes: int = 67108864, use_unix_listener: bool = False, ): - artifacts = self.case.shared.artifacts_dir / 'manager_socket' - socket_path = Path('/tmp') / f'gvm-ms-{os.getpid()}' / f'{name}.sock' - return ManagerProc( - root=self.case.shared.root_dir, - build_dir=Path( - json.loads((self.case.shared.root_dir / 'build' / 'info.json').read_text())[ - 'build_dir' - ] - ), - artifacts_dir=artifacts, - name=name, - socket_path=socket_path, - max_message_bytes=max_message_bytes, - use_unix_listener=use_unix_listener, + del max_message_bytes, use_unix_listener + handle = self.case.manager_service.handle + assert isinstance(handle, genvm.ManagerHandle) + return _ManagerFixture( + handle, + self.case.shared.case_dir_for(self.case.description.name) / name, ) async def _manager_with_retention(self, name: str, retention: str): - artifacts = self.case.shared.artifacts_dir / 'manager_socket' - socket_path = Path('/tmp') / f'gvm-ms-{os.getpid()}' / f'{name}.sock' - return ManagerProc( - root=self.case.shared.root_dir, - build_dir=Path( - json.loads((self.case.shared.root_dir / 'build' / 'info.json').read_text())[ - 'build_dir' - ] - ), - artifacts_dir=artifacts, - name=name, - socket_path=socket_path, - execution_retention=retention, - ) - - async def _run_all(self): - self.phase = 'protocol errors' - await self._malformed_unknown_and_survival() - self.phase = 'oversized close' - await self._oversized_closes_connection() - self.phase = 'startup failures' - await self._startup_failure_events_and_permits() - self.phase = 'happy path' - await self._happy_path_artifact_ack_attach() - self.phase = 'reconnect mid-run' - await self._disconnect_mid_run_reconnect_attach() - self.phase = 'finish before ack reconnect' - await self._disconnect_after_finish_before_ack() - self.phase = 'multi attach' - await self._two_clients_receive_terminal() - self.phase = 'ack and ttl' - await self._ack_and_ttl_drop_results() - self.phase = 'boot mismatch' - await self._manager_restart_boot_mismatch() - self.phase = 'restart loses run' - await self._manager_restart_fails_waiter() - self.phase = 'cancel after start' - await self._cancel_after_start_single_terminal() - self.phase = 'deadline event' - await self._deadline_pushes_terminal() - self.phase = 'ttl drain' - await self._client_never_returns_ttl_drains() - self.phase = 'http socket parity' - await self._http_and_socket_same_fixture() - self.phase = 'unix stale' - await self._unix_parent_and_stale_socket() + del retention + return await self._manager(name) async def _malformed_unknown_and_survival(self): async with await self._manager('protocol-errors') as manager: @@ -577,9 +444,9 @@ async def _happy_path_artifact_ack_attach(self): tmp_dir = manager.work_dir / 'host' tmp_dir.mkdir(parents=True, exist_ok=True) storage_path = tmp_dir / 'storage.pickle' - with open(storage_path, 'wb') as f: - pickle.dump(MockStorage(), f) - host_path = tmp_dir / 'host.sock' + await gvm_io.write_file_bytes(storage_path, pickle.dumps(MockStorage())) + host_path = Path('/tmp') / f'gvm-ms-{os.getpid()}' / 'happy-host.sock' + host_path.parent.mkdir(parents=True, exist_ok=True) ctx = _TestContext(self.case.shared.logger) host = MockHost( path=str(host_path), @@ -797,6 +664,7 @@ async def _manager_restart_boot_mismatch(self): async with ManagerWsClient(manager.uri) as client: boot_id = await _read_hello(client) genvm_id = 1 + await manager.restart() async with manager: async with ManagerWsClient(manager.uri) as client: await _read_hello(client) @@ -828,8 +696,7 @@ async def _manager_restart_fails_waiter(self): # The manager that owns the run dies and a fresh one takes its # place, handing out the same ids again -- the run cannot be # re-attached, so the waiter has to fail rather than reconnect. - await manager.__aexit__() - await manager.__aenter__() + await manager.restart() try: terminal = await asyncio.wait_for(client.wait_terminal(state), timeout=15) except base_host.ManagerRunLost: @@ -948,7 +815,7 @@ async def _http_and_socket_same_fixture(self): assert http_res['result_kind'] == socket_res.result_kind assert http_res['result_data'] == socket_res.result_data - async def _run_fixture_over_http(self, manager: ManagerProc, name: str): + async def _run_fixture_over_http(self, manager: _ManagerFixture, name: str): ctx = _TestContext(self.case.shared.logger) host, host_path = _make_mock_host(manager, name, ctx) cancel_host = asyncio.Event() @@ -985,7 +852,7 @@ async def _run_fixture_over_http(self, manager: ManagerProc, name: str): 'result_data': consumed.result_data, } - async def _run_fixture_over_socket(self, manager: ManagerProc, name: str): + async def _run_fixture_over_socket(self, manager: _ManagerFixture, name: str): ctx = _TestContext(self.case.shared.logger) host, host_path = _make_mock_host(manager, name, ctx) with host as mock_host: @@ -1011,24 +878,141 @@ async def _run_fixture_over_socket(self, manager: ManagerProc, name: str): ) async def _unix_parent_and_stale_socket(self): - artifacts = Path('/tmp') / f'gvm-ms-{os.getpid()}' - socket_path = artifacts / 'unix-stale.sock' - socket_path.parent.mkdir(parents=True, exist_ok=True) - socket_path.write_text('stale') async with await self._manager('unix-stale', use_unix_listener=True) as manager: - assert manager.socket_path == socket_path + assert manager.socket_path is not None + socket_path = manager.socket_path mode = os.stat(socket_path).st_mode assert stat.S_ISSOCK(mode) -def collect(ctx: genvm_tool.tests.stage.collection.Context) -> None: - # console_pool: this case starts, restarts and kills managers of its own, and - # waits out retention sweeps. Sharing the runner with other cases makes their - # manager unreachable partway through, which they report as a connection - # refused far away from the cause. - desc = genvm_tool.tests.test.Description( - name='tests/system/manager-socket', - tags=frozenset({'integration', 'stable', 'manager-socket'}), - console_pool=True, - ) - ctx.add_case(ManagerSocketCase(description=desc, shared=ctx.shared)) +def collect( + ctx: genvm_tool.tests.stage.collection.Context, + *, + manager_semaphore: genvm_tool.tests.stage.collection.Semaphore, + build_dir: Path, + ci: bool, +) -> None: + service_root = ctx.shared.artifacts_dir / 'manager_socket' / 'services' + base_config = { + 'permits': {'total': 2}, + 'execution_retention': '30s', + } + + def service( + name: str, + *, + config: dict[str, typing.Any] | None = None, + socket_path: Path | None = None, + implementation: type[genvm.ManagerService] = genvm.ManagerService, + ): + merged_config = {**base_config, **(config or {})} + return ctx.new_service( + name=f'manager-socket-{name}', + manager=implementation( + bin_path=build_dir / 'out' / 'bin' / 'genvm-modules', + log_path=service_root / name / 'manager.log', + env=ctx.configuration, + ci=ci, + config=merged_config, + socket_path=socket_path, + ), + semaphores=[manager_semaphore], + ) + + cases = [ + ( + 'protocol-errors', + '_malformed_unknown_and_survival', + {}, + None, + genvm.ManagerService, + ), + ( + 'oversized-close', + '_oversized_closes_connection', + {'max_message_bytes': 32}, + None, + genvm.ManagerService, + ), + ( + 'startup-failures', + '_startup_failure_events_and_permits', + {}, + None, + genvm.ManagerService, + ), + ('happy-path', '_happy_path_artifact_ack_attach', {}, None, genvm.ManagerService), + ( + 'reconnect-mid-run', + '_disconnect_mid_run_reconnect_attach', + {}, + None, + genvm.ManagerService, + ), + ( + 'finish-before-ack', + '_disconnect_after_finish_before_ack', + {}, + None, + genvm.ManagerService, + ), + ( + 'multiple-attachments', + '_two_clients_receive_terminal', + {}, + None, + genvm.ManagerService, + ), + ( + 'ack-drops-result', + '_ack_and_ttl_drop_results', + {'execution_retention': '500ms'}, + None, + genvm.ManagerService, + ), + ('boot-mismatch', '_manager_restart_boot_mismatch', {}, None, genvm.ManagerService), + ( + 'restart-loses-run', + '_manager_restart_fails_waiter', + {}, + None, + genvm.ManagerService, + ), + ('cancel', '_cancel_after_start_single_terminal', {}, None, genvm.ManagerService), + ('deadline', '_deadline_pushes_terminal', {}, None, genvm.ManagerService), + ( + 'retention-drain', + '_client_never_returns_ttl_drains', + {'execution_retention': '500ms'}, + None, + genvm.ManagerService, + ), + ('http-parity', '_http_and_socket_same_fixture', {}, None, genvm.ManagerService), + ( + 'unix-stale-socket', + '_unix_parent_and_stale_socket', + {}, + Path('/tmp') / f'gvm-ms-{os.getpid()}' / 'unix-stale.sock', + _StaleSocketManagerService, + ), + ] + for slug, method, config, socket_path, implementation in cases: + manager_service = service( + slug, + config=config, + socket_path=socket_path, + implementation=implementation, + ) + desc = genvm_tool.tests.test.Description( + name=f'tests/system/manager-socket/{slug}', + needed_services=frozenset({manager_service}), + tags=frozenset({'integration', 'stable', 'feature-manager-socket'}), + ) + ctx.add_case( + ManagerSocketCase( + description=desc, + shared=ctx.shared, + manager_service=manager_service, + method=method, + ) + ) diff --git a/tests/system/parse_version/test.py b/tests/system/parse_version/test.py index 9cdbd943..cd887492 100644 --- a/tests/system/parse_version/test.py +++ b/tests/system/parse_version/test.py @@ -1,4 +1,5 @@ -"""parse-version-pattern system tests. +""" +parse-version-pattern system tests. Each ``*.wat`` here is compiled to wasm and piped into ``genvm parse-version-pattern``; its ``*.expected`` sibling holds the expected @@ -34,7 +35,7 @@ def _add_case( ) -> None: name = f'tests/system/parse_version/{wat_file.stem}' desc = genvm_tool.tests.test.Description(name).with_tags( - ['integration', 'parse_version'] + frozenset({'integration', 'feature-version-parse'}) ) test_env = _default_env() diff --git a/tests/system/permits/test.py b/tests/system/permits/test.py index b5ab128a..50885c0a 100644 --- a/tests/system/permits/test.py +++ b/tests/system/permits/test.py @@ -1,4 +1,5 @@ -"""Permits semaphore system test. +""" +Permits semaphore system test. Tests that the manager's permit semaphore correctly blocks new genvm executions when all permits are exhausted. Evaluated by ``collection.Context.collect_dir``; @@ -12,6 +13,7 @@ from pathlib import Path import aiohttp +import genvm_tool.io as gvm_io import genvm_tool.tests import genvm_tool.tests.stage.collection import origin.base_host as base_host @@ -49,6 +51,7 @@ class PermitsTestCase(genvm_tool.tests.test.Case): description: genvm_tool.tests.test.Description manager_service: genvm_tool.tests.stage.collection.Service shared: genvm_tool.tests.SharedContext + method: str async def into_steps(self) -> list[genvm_tool.tests.exec.step.Step]: return [PermitsTestStep(self)] @@ -70,8 +73,13 @@ async def run( shared = self._case.shared logger = shared.logger - N = 1 - num_permits = N * 2 # minimum enforced by manager is 2 + # the manager refuses to go below the most expensive run, so take that + # many permits and fill them with sync runs exactly + costs = (await self._get_status(manager_uri))['permits'] + sync_cost = costs['per_sync_run'] + min_permits = max(sync_cost, costs['per_nondet_run']) + N = -(-min_permits // sync_cost) + num_permits = N * sync_cost original_permits = await self._get_permits(manager_uri) genvm_ids: list[str] = [] @@ -80,23 +88,21 @@ async def run( shut_downer = asyncio.Event() async with base_host.ManagerClient(manager_uri) as manager_client: try: - # Set permits to N * 2 await self._set_permits(manager_uri, num_permits) logger.debug('permits set', permits=num_permits) - # Start N*2 background genvms (each sync, 1 permit) + # Start N background genvms, each sync tmp_dir = shared.artifacts_dir / 'permits_test' tmp_dir.mkdir(parents=True, exist_ok=True) code = BUSY_CONTRACT.read_bytes() - for i in range(num_permits): + for i in range(N): sock_path = f'/tmp/genvm-test-permits-{i}.sock' storage_path = tmp_dir / f'storage_{i}.pickle' # Create empty storage - with open(storage_path, 'wb') as f: - pickle.dump(MockStorage(), f) + await gvm_io.write_file_bytes(storage_path, pickle.dumps(MockStorage())) ctx = _TestContext(logger) host = MockHost( @@ -140,23 +146,22 @@ async def run( logger.debug('all permits consumed, testing blocking behavior') # Try to start another genvm; it should block on semaphore. - blocked = await self._assert_blocks(manager_uri, code) - if not blocked: - return genvm_tool.tests.test.Result( - passed=False, - context={'reason': 'POST /genvm/run did not block when permits exhausted'}, - elapsed_seconds=0, + if self._case.method == 'http-blocks': + passed = await self._assert_blocks(manager_uri, code) + reason = 'POST /genvm/run did not block when permits exhausted' + elif self._case.method == 'socket-cancel': + passed = await self._assert_socket_cancel_queued( + manager_uri, manager_client, code ) - - socket_cancelled = await self._assert_socket_cancel_queued( - manager_uri, manager_client, code - ) - if not socket_cancelled: + reason = ( + 'socket cancel while queued consumed a permit or missed terminal event' + ) + else: + raise ValueError(f'unknown permits test method: {self._case.method}') + if not passed: return genvm_tool.tests.test.Result( passed=False, - context={ - 'reason': 'socket cancel while queued consumed a permit or missed terminal event' - }, + context={'reason': reason}, elapsed_seconds=0, ) @@ -340,16 +345,18 @@ def collect( *, manager_service: genvm_tool.tests.stage.collection.Service, ) -> None: - desc = genvm_tool.tests.test.Description( - name='tests/system/permits', - needed_services=frozenset({manager_service}), - tags=frozenset({'integration', 'stable', 'permits'}), - console_pool=True, - ) - ctx.add_case( - PermitsTestCase( - description=desc, - manager_service=manager_service, - shared=ctx.shared, + for slug in ('http-blocks', 'socket-cancel'): + desc = genvm_tool.tests.test.Description( + name=f'tests/system/permits/{slug}', + needed_services=frozenset({manager_service}), + tags=frozenset({'integration', 'stable', 'feature-permit'}), + console_pool=True, + ) + ctx.add_case( + PermitsTestCase( + description=desc, + manager_service=manager_service, + shared=ctx.shared, + method=slug, + ) ) - ) diff --git a/tests/tags.json b/tests/tags.json new file mode 100644 index 00000000..525f7c29 --- /dev/null +++ b/tests/tags.json @@ -0,0 +1,148 @@ +{ + "general": { + "unit": "In-crate or single-file test of one component", + "integration": "Manager-driven case: a jsonnet case or a tests/system suite", + "example": "Compilation-only check of an example", + "fuzz": "Randomized or property test", + "bench": "Benchmark, measured rather than asserted", + "rust": "Rust code under test", + "python": "Python code under test", + "bash": "Shell code under test", + "wasm": "WebAssembly artifact under test", + "smoke": "Manually selected fast representative of a feature subtree", + "needs-llm": "Requires LLM provider keys in the environment", + "needs-web": "Requires outbound web access", + "needs-time": "Correctness relies on real clock progression, sleeps or timeouts", + "needs-fuzz": "Requires a fuzz engine, corpus or fuzz toolchain", + "slow": "Expensive to run, independently of real time", + "stable": "Deterministic, no retries", + "semi-stable": "Occasionally flaky", + "unstable": "Flaky, retried", + "octane": "Selected by the external Octane benchmark suite", + "phase-prepare": "Generated setup phase of a case", + "phase-leader": "Generated leader phase of a case", + "phase-validator": "Generated validator phase of a case", + "phase-sync": "Generated sync phase of a case" + }, + "features": { + "nondet": "Nondeterministic execution", + "nondet-consensus": "Consensus over nondeterministic execution", + "nondet-consensus-leader": "Leader behavior", + "nondet-consensus-leader-error": "Leader error handling", + "nondet-consensus-leader-malicious": "Malicious leader handling", + "nondet-consensus-validator": "Validator behavior", + "nondet-consensus-validator-rollback": "Validator rollback", + "nondet-consensus-validator-sync": "Validator synchronization", + + "nasty-determinism": "Deterministic within the tested runner and version, but not a cross-version compatibility promise", + "nasty-determinism-float": "Floating-point behavior, same constraint", + "nasty-determinism-hash": "Hash-derived behavior, same constraint", + "nasty-determinism-id": "Runtime identity values such as Python id, same constraint", + "nasty-determinism-order": "Collection or iteration order, same constraint", + "nasty-determinism-random": "Random output, same constraint", + + "runner": "Runner loading and composition", + "runner-custom": "Custom runner sections", + "runner-dependency": "Runner dependencies", + "runner-load": "Runner loading", + "runner-lock": "Runner locking", + "runner-malformed": "Malformed runner rejection", + "runner-map-vm": "Runner VM mapping", + "runner-multi-file": "Multi-file runners", + "runner-permission": "Runner permissions", + "runner-register": "Runner registration", + "runner-slot": "Runner slots", + "runner-zip": "Zipped runners", + + "message": "Message handling", + "message-deploy": "Contract deployment", + "message-deploy-salt": "Deployment salt", + "message-external": "External calls", + "message-external-view": "External view calls", + "message-send": "Value-carrying sends", + "message-send-eth": "Eth sends", + "message-view": "View calls", + "message-payable": "Payable methods", + "message-eth": "Eth messages", + + "storage": "Contract storage", + "storage-allocation": "Storage allocation", + "storage-bootstrap": "Storage bootstrap", + "storage-dynamic": "Dynamically sized storage", + "storage-dynamic-array": "Dynamic arrays", + "storage-float": "Floats in storage", + "storage-lock": "Storage locking", + "storage-nested": "Nested storage types", + "storage-persistence": "Persistence across runs", + "storage-root": "Storage root", + "storage-tree-map": "TreeMap", + "storage-vector-db": "Vector DB", + + "prompt": "Prompting, including tokenizers and mocks", + "prompt-text": "Text prompts", + "prompt-json": "JSON prompts", + "prompt-vision": "Vision prompts", + "prompt-embedding": "Embeddings", + "prompt-tokenizer": "Tokenizers", + "prompt-comparative": "Comparative prompts", + "prompt-non-comparative": "Non-comparative prompts", + + "web": "Web module", + "web-render": "Page rendering", + "web-render-screenshot": "Screenshots", + "web-render-wait-js": "Waiting on JS", + "web-request": "HTTP requests", + "web-request-body": "Request and response bodies", + "web-request-signed": "Signed requests", + "web-request-status": "Status handling", + + "sandbox": "Sandboxing", + "sandbox-det": "Deterministic sandbox", + "sandbox-non-det": "Nondeterministic sandbox", + + "schema": "Calldata and method schemas", + "schema-primitive": "Primitive types", + "schema-complex": "Complex types", + "schema-float": "Floats", + "schema-tuple": "Tuples", + + "wasi": "WASI surface", + "wasi-environment": "Arguments and environment", + "wasi-path-normalization": "Path normalization", + "wasi-fd-seek": "File descriptor seeking", + "wasi-fd-pread": "Positional file descriptor reads", + "wasi-fd-write": "Writes to standard streams", + "wasi-map-file": "Mapping runner files into the guest filesystem", + "wasi-clock": "Clocks", + "wasi-random": "Randomness", + + "balance": "Balances", + "fees": "Fees", + "fees-balance": "Fee effects on balances", + + "user-error": "Errors attributable to contract code", + "exploit": "Hostile contract behavior", + "exploit-close-stdio": "Closing stdio", + "exploit-deep-calldata": "Deeply nested calldata", + "exploit-disagreement": "Forced disagreement", + "exploit-fork-bomb": "Fork bombs", + "exploit-leader-fault": "Faulty leader", + "exploit-oom": "Memory exhaustion", + "exploit-recursion": "Unbounded recursion", + "exploit-storage-limit": "Storage limits", + "exploit-wasi-extra": "Undeclared WASI functions", + + "permission": "Permission checks", + "permission-method": "Method permissions", + "permission-module": "Module permissions", + "permission-runner": "Runner permissions", + "permit": "Permits", + + "manager-socket": "Manager socket protocol", + "version-parse": "Version parsing", + "version-routing": "Version routing", + "version-routing-cross-major": "Cross-major routing", + "observability": "Logs, metrics and traces", + "event": "Events" + } +} diff --git a/tests/templates/simple_deploy_then_write.jsonnet b/tests/templates/simple_deploy_then_write.jsonnet index ae6da9e6..ffe87ea6 100644 --- a/tests/templates/simple_deploy_then_write.jsonnet +++ b/tests/templates/simple_deploy_then_write.jsonnet @@ -1,13 +1,17 @@ local msg = import './message.json'; { - run(scriptfile, method, args=[]):: + // a deploy that errors persists nothing, so a contract whose constructor + // takes arguments must get them here or the `next` step has no contract + run(scriptfile, method, args=[], ctor_args=[]):: { "vars": {}, "code": scriptfile, "message": msg + { "is_init": true, }, - "calldata": "{}", + "calldata": if std.length(ctor_args) == 0 then "{}" else std.manifestJsonEx({ + "args": ctor_args, + }, " "), next: [ { "vars": {}, diff --git a/webdriver/src/prj/package.json b/webdriver/src/prj/package.json index 52706881..5bf4ff8e 100644 --- a/webdriver/src/prj/package.json +++ b/webdriver/src/prj/package.json @@ -7,7 +7,9 @@ "scripts": { "start": "node --loader ts-node/esm src/index.ts", "dev": "ts-node src/index.ts", - "build": "tsc" + "build": "tsc", + "test": "node --import ./ts-node-register.mjs --test src/**/*.test.ts", + "typecheck": "tsc --noEmit" }, "dependencies": { "puppeteer-core": "24.16.0", diff --git a/webdriver/src/prj/src/duration.test.ts b/webdriver/src/prj/src/duration.test.ts new file mode 100644 index 00000000..02f9ce03 --- /dev/null +++ b/webdriver/src/prj/src/duration.test.ts @@ -0,0 +1,40 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; + +import { parseSize } from './duration.js'; + +describe('parseSize', () => { + it('treats a bare number as octets', () => { + assert.equal(parseSize('512', undefined), 512); + }); + + it('reads decimal suffixes as powers of 1000', () => { + assert.equal(parseSize('1B', undefined), 1); + assert.equal(parseSize('2KB', undefined), 2000); + assert.equal(parseSize('3MB', undefined), 3_000_000); + assert.equal(parseSize('4GB', undefined), 4_000_000_000); + }); + + it('reads binary suffixes as powers of 1024', () => { + assert.equal(parseSize('2KiB', undefined), 2048); + assert.equal(parseSize('5MiB', undefined), 5 * 1024 * 1024); + assert.equal(parseSize('1GiB', undefined), 1024 ** 3); + }); + + it('accepts fractions, spacing and any casing', () => { + assert.equal(parseSize('1.5 mib', undefined), 1024 * 1024 * 1.5); + assert.equal(parseSize(' 2gIb ', undefined), 2 * 1024 ** 3); + }); + + it('falls back to the default only when nothing was given', () => { + assert.equal(parseSize(undefined, 7), 7); + assert.equal(parseSize('', 7), 7); + assert.equal(parseSize('0', 7), 0); + }); + + it('rejects garbage rather than guessing', () => { + for (const bad of ['MiB', '1 potato', '1KB2', '-1MiB', '1e3']) { + assert.throws(() => parseSize(bad, undefined), /invalid size/, bad); + } + }); +}); diff --git a/webdriver/src/prj/src/duration.ts b/webdriver/src/prj/src/duration.ts index ec7ddaad..7fc241c8 100644 --- a/webdriver/src/prj/src/duration.ts +++ b/webdriver/src/prj/src/duration.ts @@ -54,6 +54,56 @@ export function formatDurationMs(ms: number): string { return `${(ms / 3600000).toFixed(1)}h`; } +const SIZE_UNITS: Record<string, number> = { + b: 1, + kb: 1000, + mb: 1000 ** 2, + gb: 1000 ** 3, + kib: 1024, + mib: 1024 ** 2, + gib: 1024 ** 3, +}; + +/** + * Parse a size string with optional suffix: "512", "32KB", "5MiB", "1GB". + * Decimal suffixes are powers of 1000, `i` ones powers of 1024, matching what + * the units mean everywhere else. Plain numbers are treated as octets. + * Returns the value in octets, or `dflt` if the input is empty/undefined. + */ +export function parseSize<D = number>( + input: string | undefined, + dflt: D, +): number | D { + if (input === undefined || input === '') { + return dflt; + } + const match = input.trim().match(/^(\d+(?:\.\d+)?)\s*([a-zA-Z]+)?$/); + if (!match || !match[1]) { + throw new Error(`invalid size: "${input}"`); + } + const multiplier = SIZE_UNITS[(match[2] ?? 'b').toLowerCase()]; + if (multiplier === undefined) { + throw new Error(`invalid size suffix: "${match[2]}"`); + } + return Math.floor(parseFloat(match[1]) * multiplier); +} + +/** + * A size limit below one octet rejects everything it is supposed to bound, so + * it fails at startup rather than silently disabling what it guards. + */ +export function envSize(envName: string, dflt: number | string): number { + const raw = process.env[envName]; + const parsed = parseSize(raw, dflt); + const value = + typeof parsed === 'string' ? parseSize(parsed, undefined) : parsed; + if (value === undefined || value < 1) { + throw new Error(`env ${envName} must be a positive size, got "${raw}"`); + } + logger.log('info', 'env', { name: envName, raw: raw ?? null, value, dflt }); + return value; +} + export function envInt(envName: string, defaultValue: number): number { const raw = process.env[envName]; const value = raw !== undefined && raw !== '' ? parseInt(raw, 10) : NaN; @@ -67,6 +117,21 @@ export function envInt(envName: string, defaultValue: number): number { return result; } +/** + * Like `envInt`, but for limits where a non-positive value is nonsense rather + * than merely unusual β€” a zero response cap, for instance, would reject every + * page. Fails at startup instead of silently disabling the feature. + */ +export function envPositiveInt(envName: string, defaultValue: number): number { + const value = envInt(envName, defaultValue); + if (!Number.isInteger(value) || value < 1) { + throw new Error( + `env ${envName} must be a positive integer, got "${process.env[envName]}"`, + ); + } + return value; +} + export function envStr(envName: string, defaultValue: string): string { const raw = process.env[envName]; const value = raw !== undefined && raw !== '' ? raw : defaultValue; diff --git a/webdriver/src/prj/src/index.ts b/webdriver/src/prj/src/index.ts index d948c6ac..540faaba 100644 --- a/webdriver/src/prj/src/index.ts +++ b/webdriver/src/prj/src/index.ts @@ -5,7 +5,19 @@ import { Command } from 'commander'; import * as logger from './logging.js'; import * as chromeBrowser from './browser/chrome.js'; import * as ssrf from './ssrf.js'; -import { envDurationMs, envInt, formatDurationMs } from './duration.js'; +import { + envDurationMs, + envInt, + envPositiveInt, + envSize, + formatDurationMs, +} from './duration.js'; +import { + Semaphore, + SemaphoreAborted, + SemaphoreQueueFull, + SemaphoreTimeout, +} from './semaphore.js'; interface NavigationOptions { waitUntil?: pup.PuppeteerLifeCycleEvent; @@ -17,6 +29,7 @@ interface RenderOptions { waitAfterLoaded?: number; waitUntil?: pup.PuppeteerLifeCycleEvent; maxPageHeapMB?: number; + maxResponseChars?: number; } const program = new Command(); @@ -30,9 +43,43 @@ program const options = program.opts(); const STATUS_I_AM_A_TEAPOT = 418; +const STATUS_INSUFFICIENT_STORAGE = 507; +const STATUS_SERVICE_UNAVAILABLE = 503; const DEFAULT_MAX_PAGE_HEAP_MB = envInt('GVM_WEBDRIVER_MAX_PAGE_HEAP_MB', 1024); +// peak_mem ~= MAX_CONCURRENT_RENDERS * (page heap + extracted response) + browser baseline +// +// we deem that by default we are limited by manager permits, +// so this knob is for extra tuning and default 128 is essentially "unlimited" +const MAX_CONCURRENT_RENDERS = envPositiveInt( + 'GVM_WEBDRIVER_MAX_CONCURRENT_RENDERS', + 128, +); + +// Counted in characters, which is what bounds the string materialized in this +// process. One character encodes to at most 4 UTF-8 bytes on the wire, and +// occupies 2 bytes in V8 for the common (BMP) case. +const MAX_RESPONSE_CHARS = envSize('GVM_WEBDRIVER_MAX_RESPONSE', '5MiB'); + +// Waiting callers are cheap but not free: each holds a live connection. Past +// this depth the service is not going to work through the backlog before +// callers give up anyway, so refusing immediately beats refusing slowly. +const MAX_RENDER_QUEUE = envInt('GVM_WEBDRIVER_MAX_RENDER_QUEUE', 64); + +// Kept well below the caller's own transport timeout: a queue wait that +// outlives it converts a condition we can report cleanly into a transport +// failure, which callers treat far more harshly. +const RENDER_QUEUE_TIMEOUT_MS = envDurationMs( + 'GVM_WEBDRIVER_RENDER_QUEUE_TIMEOUT', + '120s', +); + +const renderSlots = new Semaphore({ + permits: MAX_CONCURRENT_RENDERS, + maxQueued: MAX_RENDER_QUEUE, +}); + const MAX_WAIT_AFTER_LOADED_MS = envDurationMs( 'GVM_WEBDRIVER_MAX_WAIT_AFTER_LOADED', '60s', @@ -119,22 +166,53 @@ async function navigateToPage( } } -async function asText(page: pup.Page) { - const bodyText = await page.evaluate(() => { - return document.body.innerText; - }); +/** + * Read `innerText` or `innerHTML`, rejecting oversized documents *in the page*. + * + * The size test has to happen on the browser side. Pulling the string across + * first and measuring it here would already have paid the memory cost we are + * trying to avoid β€” this process would hold the whole document no matter what + * any downstream limit says. + */ +async function extractBounded( + page: pup.Page, + kind: 'innerText' | 'innerHTML', + maxChars: number, +): Promise<string> { + const extracted = await page.evaluate( + (k, limit) => { + const raw = + k === 'innerText' ? document.body.innerText : document.body.innerHTML; + return raw.length > limit + ? { tooLarge: true as const, length: raw.length } + : { tooLarge: false as const, content: raw }; + }, + kind, + maxChars, + ); + + if (extracted.tooLarge) { + throw new ResponseLimitExceeded(extracted.length, maxChars); + } + return extracted.content; +} - return normalizeWhitespace(bodyText); +async function asText(page: pup.Page, maxChars: number) { + return normalizeWhitespace(await extractBounded(page, 'innerText', maxChars)); } -async function asHTML(page: pup.Page) { - return await page.evaluate(() => { - return document.body.innerHTML; - }); +async function asHTML(page: pup.Page, maxChars: number) { + return await extractBounded(page, 'innerHTML', maxChars); } -async function asScreenshot(page: pup.Page) { - return await page.screenshot(); +async function asScreenshot(page: pup.Page, maxChars: number) { + // Screenshots are viewport-sized rather than full-page, so this is a + // backstop rather than a limit anyone should reach. + const image = await page.screenshot(); + if (image.length > maxChars) { + throw new ResponseLimitExceeded(image.length, maxChars); + } + return image; } class HeapLimitExceeded extends Error { @@ -143,6 +221,12 @@ class HeapLimitExceeded extends Error { } } +class ResponseLimitExceeded extends Error { + constructor(size: number, maxSize: number) { + super(`Rendered response ${size} exceeds limit ${maxSize}`); + } +} + const HEAP_CHECK_INTERVAL_MS = envInt( 'GVM_WEBDRIVER_HEAP_CHECK_INTERVAL_MS', 200, @@ -212,18 +296,49 @@ async function renderPage( targetUrl: string, mode: 'text' | 'html' | 'screenshot', options: RenderOptions = {}, + signal?: AbortSignal, ): Promise<{ status: number; body: any }> { - const browserManager = await chromeBrowser.INSTANCE; - const browserInstance = browserManager.getBrowser(); + // The slot is taken before the browser is touched, so a queued request costs + // nothing but the open connection. try { - return renderPageWithBrowser( - browserInstance.get(), - targetUrl, - mode, - options, - ); + await renderSlots.acquire(RENDER_QUEUE_TIMEOUT_MS, signal); + } catch (e) { + // Saturation escapes as a transport failure rather than a render result, + // and the distinction is deliberate. How busy this host happens to be is + // not a property of the page, so it must not reach the contract: two + // hosts under different load would otherwise return different answers + // for the same request and each would look legitimate. Failing at the + // transport level keeps the outcome an honest "this host could not run + // it" instead of a fabricated observation about the web. + if (e instanceof SemaphoreTimeout || e instanceof SemaphoreQueueFull) { + logger.log('warn', 'render rejected: saturated', { + url: targetUrl, + reason: e.name, + inFlight: renderSlots.inFlight, + queued: renderSlots.queued, + timeout: formatDurationMs(RENDER_QUEUE_TIMEOUT_MS), + }); + } else if (e instanceof SemaphoreAborted) { + logger.log('debug', 'render abandoned while queued', { url: targetUrl }); + } + throw e; + } + + try { + const browserManager = await chromeBrowser.INSTANCE; + const browserInstance = browserManager.getBrowser(); + try { + return await renderPageWithBrowser( + browserInstance.get(), + targetUrl, + mode, + options, + ); + } finally { + browserInstance.close(); + } } finally { - browserInstance.close(); + renderSlots.release(); } } @@ -238,13 +353,25 @@ async function renderPageWithBrowser( waitAfterLoaded = 0, waitUntil = 'domcontentloaded', maxPageHeapMB = DEFAULT_MAX_PAGE_HEAP_MB, + maxResponseChars = MAX_RESPONSE_CHARS, } = options; // Each render runs in its own browser context so cookies, localStorage, // IndexedDB, service workers and HSTS state never leak between tenants // sharing this long-lived browser. const context = await browserInstance.createBrowserContext(); - const page = await context.newPage(); + + // `newPage` fails when the shared browser is under pressure, and the cleanup + // below only runs once execution is inside the try. Without this the context + // survives for the browser's whole lifetime β€” and since pressure is exactly + // what makes `newPage` fail, the response to it would be to leak more. + let page: pup.Page; + try { + page = await context.newPage(); + } catch (e) { + await context.close().catch(() => {}); + throw e; + } try { await ssrf.installSsrfGuard(page); @@ -287,13 +414,13 @@ async function renderPageWithBrowser( let data; switch (mode) { case 'text': - data = await asText(page); + data = await asText(page, maxResponseChars); break; case 'html': - data = await asHTML(page); + data = await asHTML(page, maxResponseChars); break; case 'screenshot': - data = await asScreenshot(page); + data = await asScreenshot(page, maxResponseChars); break; default: data = 'Invalid mode'; @@ -302,14 +429,26 @@ async function renderPageWithBrowser( return { status: statusCode, body: data }; }); } catch (e) { + // Both limits are reported as render results, not transport failures: a + // transport failure aborts the caller's whole execution, and a page too + // big for this host is a thing the contract can reasonably be told about + // and handle, like any other page it could not load. if (e instanceof HeapLimitExceeded) { logger.log('warn', 'page heap limit exceeded', { url: targetUrl, error: e.message, }); - return { status: 507, body: e.message }; + } else if (e instanceof ResponseLimitExceeded) { + logger.log('warn', 'rendered response limit exceeded', { + url: targetUrl, + mode, + error: e.message, + }); + } else { + throw e; } - throw e; + + return { status: STATUS_INSUFFICIENT_STORAGE, body: e.message }; } finally { // Close the page and its context together; the context teardown is what // actually discards the per-request browsing state. @@ -317,6 +456,23 @@ async function renderPageWithBrowser( } } +/** + * Fires when the caller goes away before it has been answered, so a request + * nobody is waiting for stops occupying a queue slot. + */ +function disconnectSignal( + req: http.IncomingMessage, + res: http.ServerResponse, +): AbortSignal { + const controller = new AbortController(); + req.on('close', () => { + if (!res.writableEnded) { + controller.abort(); + } + }); + return controller.signal; +} + async function handleRenderRequest( parsedUrl: URL, req: http.IncomingMessage, @@ -355,7 +511,12 @@ async function handleRenderRequest( : {}), }; - const result = await renderPage(targetUrl, mode, options); + const result = await renderPage( + targetUrl, + mode, + options, + disconnectSignal(req, res), + ); if (statusIsGood(result.status)) { updateLastSuccessfulRenderTime(); @@ -370,6 +531,29 @@ async function handleRenderRequest( } res.end(result.body); } catch (error) { + if (error instanceof SemaphoreAborted) { + // The caller hung up while queued; there is nobody left to answer. + return; + } + // How loaded this host happens to be is not a property of the page, so it + // must not reach the contract: two hosts under different load would + // answer the same request differently, both answers looking legitimate. + // A transport failure says plainly that this host could not run it. + if ( + error instanceof SemaphoreTimeout || + error instanceof SemaphoreQueueFull + ) { + res.writeHead(STATUS_SERVICE_UNAVAILABLE, { + 'Content-Type': 'application/json', + }); + res.end( + JSON.stringify({ + error: 'Service unavailable', + message: error.message, + }), + ); + return; + } res.writeHead(500, { 'Content-Type': 'application/json' }); res.end( JSON.stringify({ @@ -380,7 +564,11 @@ async function handleRenderRequest( } } -async function handleHealthcheck(parsedUrl: URL, res: http.ServerResponse) { +async function handleHealthcheck( + parsedUrl: URL, + req: http.IncomingMessage, + res: http.ServerResponse, +) { const now = Date.now(); const sinceLastSuccessMs = now - lastSuccessfulRenderTime; if (sinceLastSuccessMs < HEALTHCHECK_CACHE_DURATION_MS) { @@ -415,7 +603,12 @@ async function handleHealthcheck(parsedUrl: URL, res: http.ServerResponse) { cacheDuration: formatDurationMs(HEALTHCHECK_CACHE_DURATION_MS), }); try { - const result = await renderPage(targetUrl, mode, {}); + const result = await renderPage( + targetUrl, + mode, + {}, + disconnectSignal(req, res), + ); if (statusIsGood(result.status)) { updateLastSuccessfulRenderTime(); logger.log('info', 'healthcheck ok', { status: result.status }); @@ -427,6 +620,13 @@ async function handleHealthcheck(parsedUrl: URL, res: http.ServerResponse) { res.end('unhealthy'); } } catch (error) { + if (error instanceof SemaphoreAborted) { + return; + } + // Saturation lands here too, and reporting unhealthy is right: the probe + // only renders when nothing has succeeded for the cache duration, so a + // host that is both saturated and has completed nothing in that window + // genuinely is not serving. logger.log('error', 'healthcheck error', { error: (error as Error).message, }); @@ -442,7 +642,7 @@ const server = http.createServer(async (req, res) => { if (pathname === '/render') { await handleRenderRequest(parsedUrl, req, res); } else if (pathname === '/healthcheck') { - await handleHealthcheck(parsedUrl, res); + await handleHealthcheck(parsedUrl, req, res); } else if (pathname === '/log-level') { if (req.method === 'GET') { res.writeHead(200, { 'Content-Type': 'text/plain' }); diff --git a/webdriver/src/prj/src/semaphore.test.ts b/webdriver/src/prj/src/semaphore.test.ts new file mode 100644 index 00000000..afc6d3fa --- /dev/null +++ b/webdriver/src/prj/src/semaphore.test.ts @@ -0,0 +1,270 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; + +import { + Semaphore, + SemaphoreAborted, + SemaphoreQueueFull, + SemaphoreTimeout, +} from './semaphore.js'; + +const NEVER = 60_000; + +/** Queue depth is irrelevant to most cases, so default it out of the way. */ +function sem(permits: number, maxQueued = 1024) { + return new Semaphore({ permits, maxQueued }); +} + +function deferred<T = void>() { + let resolve!: (v: T) => void; + const promise = new Promise<T>((r) => { + resolve = r; + }); + return { promise, resolve }; +} + +describe('Semaphore', () => { + it('rejects invalid construction', () => { + assert.throws(() => sem(0)); + assert.throws(() => sem(-1)); + assert.throws(() => sem(1.5)); + assert.throws(() => new Semaphore({ permits: 1, maxQueued: -1 })); + assert.throws(() => new Semaphore({ permits: 1, maxQueued: 1.5 })); + }); + + it('grants up to the permit count without waiting', async () => { + const s = sem(3); + await s.acquire(NEVER); + await s.acquire(NEVER); + await s.acquire(NEVER); + assert.equal(s.inFlight, 3); + assert.equal(s.queued, 0); + }); + + it('queues the caller past the limit until a permit is released', async () => { + const s = sem(1); + await s.acquire(NEVER); + + let granted = false; + const waiting = s.acquire(NEVER).then(() => { + granted = true; + }); + + await new Promise((r) => setTimeout(r, 10)); + assert.equal(granted, false, 'should still be queued'); + assert.equal(s.queued, 1); + + s.release(); + await waiting; + assert.equal(granted, true); + assert.equal(s.inFlight, 1, 'permit passed to the waiter, not returned'); + }); + + it('hands permits to waiters in FIFO order', async () => { + const s = sem(1); + await s.acquire(NEVER); + + const order: number[] = []; + const waiters = [1, 2, 3].map((n) => + s.acquire(NEVER).then(() => { + order.push(n); + }), + ); + + await new Promise((r) => setTimeout(r, 10)); + s.release(); + s.release(); + s.release(); + await Promise.all(waiters); + + assert.deepEqual(order, [1, 2, 3]); + }); + + it('times out a caller that waits too long, and stops queueing it', async () => { + const s = sem(1); + await s.acquire(NEVER); + + await assert.rejects(() => s.acquire(20), SemaphoreTimeout); + assert.equal(s.queued, 0, 'timed-out waiter must leave the queue'); + + s.release(); + await s.acquire(NEVER); + assert.equal(s.inFlight, 1); + }); + + it('does not grant a permit to a waiter that already timed out', async () => { + const s = sem(1); + await s.acquire(NEVER); + + const timedOut = assert.rejects(() => s.acquire(20), SemaphoreTimeout); + await new Promise((r) => setTimeout(r, 40)); + await timedOut; + + s.release(); + assert.equal(s.inFlight, 0, 'released permit must not be held by a ghost'); + }); + + it('never exceeds the permit count under concurrent load', async () => { + const permits = 4; + const s = sem(permits); + let active = 0; + let peak = 0; + + await Promise.all( + Array.from({ length: 50 }, () => + s.withPermit(NEVER, async () => { + active += 1; + peak = Math.max(peak, active); + await new Promise((r) => setTimeout(r, 1)); + active -= 1; + }), + ), + ); + + assert.equal(peak, permits, 'concurrency must saturate but not exceed'); + assert.equal(s.inFlight, 0); + assert.equal(s.queued, 0); + }); + + it('releases the permit when the guarded task throws', async () => { + const s = sem(1); + await assert.rejects( + () => + s.withPermit(NEVER, async () => { + throw new Error('boom'); + }), + /boom/, + ); + assert.equal(s.inFlight, 0, 'a failed task must not leak its permit'); + }); + + it('ignores a release with nothing outstanding', async () => { + const s = sem(2); + s.release(); + s.release(); + s.release(); + + // A leaked permit would let a third caller through here. + await s.acquire(NEVER); + await s.acquire(NEVER); + await assert.rejects(() => s.acquire(20), SemaphoreTimeout); + }); + + it('keeps a slow task from starving a queued one indefinitely', async () => { + const s = sem(1); + const slow = deferred(); + + const running = s.withPermit(NEVER, () => slow.promise); + const queued = s.acquire(NEVER); + + slow.resolve(); + await running; + await queued; + assert.equal(s.inFlight, 1); + }); + + describe('queue depth', () => { + it('refuses callers once the queue is full', async () => { + const s = new Semaphore({ permits: 1, maxQueued: 2 }); + await s.acquire(NEVER); + + const queued = [s.acquire(NEVER), s.acquire(NEVER)]; + assert.equal(s.queued, 2); + + await assert.rejects(() => s.acquire(NEVER), SemaphoreQueueFull); + assert.equal(s.queued, 2, 'a refused caller must not be queued'); + + s.release(); + s.release(); + await Promise.all(queued); + }); + + it('accepts again once the queue drains', async () => { + const s = new Semaphore({ permits: 1, maxQueued: 1 }); + await s.acquire(NEVER); + const first = s.acquire(NEVER); + await assert.rejects(() => s.acquire(NEVER), SemaphoreQueueFull); + + s.release(); + await first; + + // Slot freed: a new caller queues rather than being refused. + const second = s.acquire(NEVER); + assert.equal(s.queued, 1); + s.release(); + await second; + }); + + it('refuses immediately rather than waiting out the timeout', async () => { + const s = new Semaphore({ permits: 1, maxQueued: 0 }); + await s.acquire(NEVER); + + const started = Date.now(); + await assert.rejects(() => s.acquire(NEVER), SemaphoreQueueFull); + assert.ok( + Date.now() - started < 50, + 'a full queue must fail fast, not after the wait', + ); + }); + }); + + describe('abort', () => { + it('drops a waiter whose caller went away', async () => { + const s = sem(1); + await s.acquire(NEVER); + + const controller = new AbortController(); + const waiting = assert.rejects( + () => s.acquire(NEVER, controller.signal), + SemaphoreAborted, + ); + assert.equal(s.queued, 1); + + controller.abort(); + await waiting; + assert.equal(s.queued, 0, 'aborted waiter must leave the queue'); + + // The permit must go to a live caller, not the abandoned one. + s.release(); + assert.equal(s.inFlight, 0); + }); + + it('rejects at once when the signal is already aborted', async () => { + const s = sem(1); + await s.acquire(NEVER); + await assert.rejects( + () => s.acquire(NEVER, AbortSignal.abort()), + SemaphoreAborted, + ); + assert.equal(s.queued, 0); + }); + + it('does not reject a caller that already holds a permit', async () => { + const s = sem(1); + const controller = new AbortController(); + await s.acquire(NEVER, controller.signal); + controller.abort(); + // Aborting after the permit was granted is a no-op; release still works. + s.release(); + assert.equal(s.inFlight, 0); + }); + + it('frees the queue slot so a later caller is not refused', async () => { + const s = new Semaphore({ permits: 1, maxQueued: 1 }); + await s.acquire(NEVER); + + const controller = new AbortController(); + const abandoned = assert.rejects( + () => s.acquire(NEVER, controller.signal), + SemaphoreAborted, + ); + controller.abort(); + await abandoned; + + const queued = s.acquire(NEVER); + assert.equal(s.queued, 1, 'slot must have been reclaimed'); + s.release(); + await queued; + }); + }); +}); diff --git a/webdriver/src/prj/src/semaphore.ts b/webdriver/src/prj/src/semaphore.ts new file mode 100644 index 00000000..84197977 --- /dev/null +++ b/webdriver/src/prj/src/semaphore.ts @@ -0,0 +1,156 @@ +export interface SemaphoreOptions { + /** Concurrent holders. */ + permits: number; + /** Callers allowed to wait for a permit before further ones are refused. */ + maxQueued: number; +} + +/** + * Counting semaphore with FIFO waiters, a bounded queue and a bounded wait. + * + * Callers past the permit count queue rather than being refused outright: + * refusing on any contention would make the outcome depend on whatever else the + * host happened to be running at the time. + * + * The queue is still finite, and the wait is still bounded. Both exist because + * an unbounded queue only defers the problem β€” a caller parked indefinitely + * eventually trips its own transport timeout, which fails far more harshly than + * a refusal we issue ourselves, and until then it costs a live connection. + */ +export class Semaphore { + private readonly permits: number; + private readonly maxQueued: number; + private available: number; + private readonly waiters: Array<{ + resolve: () => void; + reject: (e: Error) => void; + timer: NodeJS.Timeout; + cleanup: () => void; + }> = []; + + constructor(options: SemaphoreOptions) { + const { permits, maxQueued } = options; + if (!Number.isInteger(permits) || permits < 1) { + throw new Error( + `semaphore permits must be a positive integer, got ${permits}`, + ); + } + if (!Number.isInteger(maxQueued) || maxQueued < 0) { + throw new Error( + `semaphore maxQueued must be a non-negative integer, got ${maxQueued}`, + ); + } + this.permits = permits; + this.maxQueued = maxQueued; + this.available = permits; + } + + get inFlight(): number { + return this.permits - this.available; + } + + get queued(): number { + return this.waiters.length; + } + + /** + * Acquire a permit. + * + * @param timeoutMs how long to wait before giving up. + * @param signal aborts the wait β€” pass the caller's disconnect signal so a + * vanished caller stops occupying a queue slot it will never use. + * @throws {SemaphoreQueueFull} if the queue is already at capacity. + * @throws {SemaphoreTimeout} if no permit became available in time. + * @throws {SemaphoreAborted} if `signal` fired while waiting. + */ + acquire(timeoutMs: number, signal?: AbortSignal): Promise<void> { + if (this.available > 0) { + this.available -= 1; + return Promise.resolve(); + } + if (this.waiters.length >= this.maxQueued) { + return Promise.reject(new SemaphoreQueueFull(this.maxQueued)); + } + if (signal?.aborted) { + return Promise.reject(new SemaphoreAborted()); + } + + return new Promise<void>((resolve, reject) => { + const remove = () => { + const idx = this.waiters.indexOf(entry); + if (idx !== -1) { + this.waiters.splice(idx, 1); + } + }; + const onAbort = () => { + remove(); + clearTimeout(entry.timer); + reject(new SemaphoreAborted()); + }; + const entry = { + resolve, + reject, + timer: setTimeout(() => { + remove(); + entry.cleanup(); + reject(new SemaphoreTimeout(timeoutMs)); + }, timeoutMs), + cleanup: () => signal?.removeEventListener('abort', onAbort), + }; + signal?.addEventListener('abort', onAbort, { once: true }); + this.waiters.push(entry); + }); + } + + release(): void { + const next = this.waiters.shift(); + if (next === undefined) { + // Guard against a double release leaking permits, which would silently + // raise the concurrency ceiling above the configured one. + if (this.available < this.permits) { + this.available += 1; + } + return; + } + clearTimeout(next.timer); + next.cleanup(); + // The permit passes straight to the waiter; `available` deliberately stays + // where it is. + next.resolve(); + } + + /** Run `fn` holding a permit, releasing it however `fn` settles. */ + async withPermit<T>( + timeoutMs: number, + fn: () => Promise<T>, + signal?: AbortSignal, + ): Promise<T> { + await this.acquire(timeoutMs, signal); + try { + return await fn(); + } finally { + this.release(); + } + } +} + +export class SemaphoreTimeout extends Error { + constructor(timeoutMs: number) { + super(`timed out after ${timeoutMs}ms waiting for a render slot`); + this.name = 'SemaphoreTimeout'; + } +} + +export class SemaphoreQueueFull extends Error { + constructor(maxQueued: number) { + super(`render queue is full (${maxQueued} waiting)`); + this.name = 'SemaphoreQueueFull'; + } +} + +export class SemaphoreAborted extends Error { + constructor() { + super('caller went away while waiting for a render slot'); + this.name = 'SemaphoreAborted'; + } +} diff --git a/webdriver/src/prj/ts-node-register.mjs b/webdriver/src/prj/ts-node-register.mjs new file mode 100644 index 00000000..ef847107 --- /dev/null +++ b/webdriver/src/prj/ts-node-register.mjs @@ -0,0 +1,6 @@ +// `--loader ts-node/esm` is gone in current Node; registering the hook from +// inside the process is the supported spelling. +import { register } from 'node:module'; +import { pathToFileURL } from 'node:url'; + +register('ts-node/esm', pathToFileURL('./')); From 3b6aea1726b20252e272f4d249ae7fc4a1b34bbf Mon Sep 17 00:00:00 2001 From: kp2pml30 <kp2pml30@gmail.com> Date: Thu, 20 Aug 2026 17:00:04 +0900 Subject: [PATCH 11/45] =?UTF-8?q?fix(manager):=20guard=20fatal=20VM=20resu?= =?UTF-8?q?lt=20publication=20=F0=9F=94=92=EF=B8=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crates/modules-interfaces/src/domain.rs | 2 + ...pre-validating untrusted decoded inputs.md | 11 ++- docs/adr/014. manager host socket protocol.md | 10 +- .../src/impl-spec/02-vm/03-consensus.rst | 5 +- .../src/impl-spec/appendix/manager-socket.rst | 18 ++++ docs/website/src/spec/03-vm/05-result.rst | 14 ++- executors/v0.3.x | 2 +- implementation/src/manager/run.rs | 99 ++++++++++++++++--- implementation/src/manager/run_test.rs | 68 +++++++++++++ tests/runner/origin/base_host.py | 6 +- tests/system/manager-socket/test.py | 17 ++++ 11 files changed, 221 insertions(+), 31 deletions(-) diff --git a/crates/modules-interfaces/src/domain.rs b/crates/modules-interfaces/src/domain.rs index a083b58c..ca16e8fd 100644 --- a/crates/modules-interfaces/src/domain.rs +++ b/crates/modules-interfaces/src/domain.rs @@ -397,6 +397,8 @@ pub struct ReportedResult { /// what a caller in another executor folds. pub small_hash: bytes::Bytes, + /// `FatalVmError` is legal only while transporting a nested result; a + /// top-level report must publish the same payload as `VmError` pub kind: ResultCode, pub data: genlayer_calldata::unparsed::Maybe<genlayer_calldata::Value>, pub backtrace: Option<Backtrace>, diff --git a/docs/adr/013. pre-validating untrusted decoded inputs.md b/docs/adr/013. pre-validating untrusted decoded inputs.md index 1317889b..af14d049 100644 --- a/docs/adr/013. pre-validating untrusted decoded inputs.md +++ b/docs/adr/013. pre-validating untrusted decoded inputs.md @@ -99,7 +99,7 @@ hiding a leader whose execution diverged into extra blocks. Now: if the leader supplied more results than blocks run, the run's result is replaced by `leader_fault nondet_output extra <h>`, `<h>` = first 6 chars of `gvm32(sha3_256(b))` where `b` is the **full `[result_code][data]` buffer** the -run would otherwise have returned (`RunOk::as_bytes()`) β€” fingerprinting the +run would otherwise have returned β€” fingerprinting the discarded result rather than losing it. This is **not** a non-deterministic disagreement (it's a property of the counts, not a re-executed block). No fix-point escape is needed: a `leader_fault nondet_output` code is never @@ -114,13 +114,16 @@ Before the computed result is pushed to the host (and hashed): - A `VMError`'s fused ` # <detail>` suffix is **stripped** β€” the nondet channel is detail-free (diagnostic, no compat promise, and a free-form byte channel into the hash otherwise). The local `cause` is kept for logs. -- **Nothing else.** The leader must *not* run Rule 1 on its own output: +- A fatal VM error is not encoded into `nondet_results`; it propagates through + the caller and is downgraded to `VMError` only at the topmost publication + boundary +- **Nothing else is rewritten.** The leader must *not* run Rule 1 on its own output: self-filtering can only rewrite an honest result into a derived-namespace code that validators replace again β€” a guaranteed honest-vs-honest hash split. Rule 1 is for hostile input, Rule 1b for trusted output. -Stripping suffices because the rest is valid by construction: codes come from -the generated constructors and canonical `exit_code <i32>` (a generated test +The remaining encodable results are valid by construction: codes come from the +generated constructors and canonical `exit_code <i32>` (a generated test asserts every constructor passes the validity check). The one arbitrary-string site, the `vmError "…"` fee builtin, is unreachable from a nondet child (every bucket that raises it needs a permission the child lacks); the child also can't diff --git a/docs/adr/014. manager host socket protocol.md b/docs/adr/014. manager host socket protocol.md index c7036c2c..804b61af 100644 --- a/docs/adr/014. manager host socket protocol.md +++ b/docs/adr/014. manager host socket protocol.md @@ -174,9 +174,13 @@ flushes its buffered host writers before exiting, which the ACK round-trip used to force implicitly). The method is deleted from the shared `host-fns` enum and from both executor lines. -`consume_result` is untouched: it routes to host 1 and is how the manager -obtains a trusted copy of the result without round-tripping it through the -node. Orthogonal to this change. +`consume_result` still routes to host 1 and is how the manager obtains the +result without round-tripping it through the node. For a top-level run, the +manager validates the outer framing and decoded `ReportedResult` before +retaining it. Malformed reports are refused; a fatal result triggers a debug +assertion and is logged and downgraded to `VMError` in release builds. Embedded +`nondet_results` remain opaque because their encoding belongs to the executor +line ### HTTP surface diff --git a/docs/website/src/impl-spec/02-vm/03-consensus.rst b/docs/website/src/impl-spec/02-vm/03-consensus.rst index b6cf04b6..a07226ce 100644 --- a/docs/website/src/impl-spec/02-vm/03-consensus.rst +++ b/docs/website/src/impl-spec/02-vm/03-consensus.rst @@ -45,8 +45,9 @@ followed by an encoded payload: - ``VmError`` β€” UTF-8 error code from :doc:`/spec/appendix/constants` ``vm_error``. -The validator reconstructs an ``rt::vm::RunOk`` from these bytes -(``genlayer_sdk.rs:1502``) and the contract observes the same value as the leader did. +The validator reconstructs a catchable contract outcome from these bytes and +the contract observes the same value as the leader did. Any other result code, +including ``FatalVmError``, is treated as malformed leader input Validator Comparison -------------------- diff --git a/docs/website/src/impl-spec/appendix/manager-socket.rst b/docs/website/src/impl-spec/appendix/manager-socket.rst index 12d49409..5f22661a 100644 --- a/docs/website/src/impl-spec/appendix/manager-socket.rst +++ b/docs/website/src/impl-spec/appendix/manager-socket.rst @@ -255,6 +255,24 @@ started with one, ``host_genvm_id``. Variants: "artifact_sizes": { "stdout": u64, "stderr": u64, "genvm_log": u64 } } } +For a top-level run, ``consumed_result`` is one outer ``ResultCode`` byte +followed by a calldata-encoded ``ReportedResult`` map. Before retaining it, the +manager checks that: + +#. The outer byte is a known result code and agrees with the map's ``kind`` +#. The map decodes completely +#. ``execution_hash`` and ``small_hash`` are each 32 bytes unless the result is + ``InternalError`` + +An invalid report is refused without an acknowledgement and is not published +as ``consumed_result``. ``FatalVmError`` is also illegal at this boundary: a +debug manager asserts, while a release manager logs the executor violation and +rewrites both result-code locations to ``VmError`` before publication. Clients +therefore never receive a top-level ``FatalVmError`` + +The manager does not decode entries of the reported ``nondet_results`` vector. +Those bytes remain opaque, executor-line-specific consensus proposals + Lifecycle guarantees: - Exactly one terminal event (``failed_to_start`` or ``finished``) per run, diff --git a/docs/website/src/spec/03-vm/05-result.rst b/docs/website/src/spec/03-vm/05-result.rst index 572d30ac..50cda1d1 100644 --- a/docs/website/src/spec/03-vm/05-result.rst +++ b/docs/website/src/spec/03-vm/05-result.rst @@ -31,6 +31,13 @@ set. A non-fatal error of a :term:`sub-VM` is returned to its caller as terminates with the same VM error, and propagation continues until the topmost VM boundary +Nested transport encodes a fatal VM error with result code ``4``. At the +topmost publication boundary, the executor MUST downgrade it to an ordinary +:ref:`gvm-def-vm-error` with the same payload before producing the reported +result. Result code ``4`` is therefore forbidden in a top-level reported +result. Both its :ref:`gvm-def-execution-hash` and its +:ref:`gvm-def-subvm-hash` use ``VMError`` as the result kind + .. _gvm-def-vm-error-code: VM Error Code Format @@ -105,7 +112,9 @@ Non-Deterministic Block Result Encoding - :ref:`gvm-def-vm-error`\: utf-8 string These three are the only codes a leader-proposed non-deterministic block result -may carry; validators treat every other byte as a malformed leader result +may carry; validators treat every other byte as a malformed leader result. A +fatal VM error computed by a leader's non-deterministic child propagates to its +caller and MUST NOT be encoded into ``nondet_results`` Contract Result Encoding ------------------------ @@ -176,6 +185,9 @@ with the following keys (in this order): Two runs that agree on the deterministic result produce the same execution hash, so consensus can compare a single 32-byte value instead of the full result. +A fatal VM error is committed with ``VMError`` as ``kind``. Fatality controls +propagation and is not a distinct consensus-visible outcome + ``emissions`` covers the whole content of every emitted message and event, not just its metered cost: two emissions can carry different calldata or different event topics for the same fee, so a fee-only commitment would let nodes agree on diff --git a/executors/v0.3.x b/executors/v0.3.x index a54b15b7..55b13368 160000 --- a/executors/v0.3.x +++ b/executors/v0.3.x @@ -1 +1 @@ -Subproject commit a54b15b7f3d4d81e362b80e6d2a678f89ae52b37 +Subproject commit 55b133688edd8950b8873aad9ce5c2f1040a8c98 diff --git a/implementation/src/manager/run.rs b/implementation/src/manager/run.rs index beea74b0..90cd7399 100644 --- a/implementation/src/manager/run.rs +++ b/implementation/src/manager/run.rs @@ -1387,29 +1387,84 @@ fn nested_internal_error() -> genvm_modules_interfaces::NestedRunReply { } } -fn nested_reply_from_consumed_result( +fn result_code_from_byte( + value: u8, + source: &str, +) -> anyhow::Result<genvm_modules_interfaces::ResultCode> { + match value { + 0 => Ok(genvm_modules_interfaces::ResultCode::Return), + 1 => Ok(genvm_modules_interfaces::ResultCode::UserError), + 2 => Ok(genvm_modules_interfaces::ResultCode::VmError), + 3 => Ok(genvm_modules_interfaces::ResultCode::InternalError), + 4 => Ok(genvm_modules_interfaces::ResultCode::FatalVmError), + value => anyhow::bail!("{source} returned unknown result code {value}"), + } +} + +fn decode_reported_result( data: &[u8], -) -> anyhow::Result<genvm_modules_interfaces::NestedRunReply> { + source: &str, +) -> anyhow::Result<( + genvm_modules_interfaces::ResultCode, + genvm_modules_interfaces::ReportedResult, +)> { let (&kind, encoded) = data .split_first() - .ok_or_else(|| anyhow::anyhow!("nested executor returned an empty result"))?; - let kind = match kind { - 0 => genvm_modules_interfaces::ResultCode::Return, - 1 => genvm_modules_interfaces::ResultCode::UserError, - 2 => genvm_modules_interfaces::ResultCode::VmError, - 3 => genvm_modules_interfaces::ResultCode::InternalError, - 4 => genvm_modules_interfaces::ResultCode::FatalVmError, - value => anyhow::bail!("nested executor returned unknown result code {value}"), - }; + .ok_or_else(|| anyhow::anyhow!("{source} returned an empty result"))?; + let kind = result_code_from_byte(kind, source)?; let reported: genvm_modules_interfaces::ReportedResult = calldata::decode_obj(encoded)?; - // The code is stated twice: once as the framing byte, once inside the - // reported map that the execution hash commits to. A disagreement means the - // callee is not the implementation we think it is. + // The framing byte and reported map must name the same committed result anyhow::ensure!( kind == reported.kind, - "nested executor result code {kind:?} disagrees with the reported {:?}", + "{source} result code {kind:?} disagrees with the reported {:?}", reported.kind ); + + Ok((kind, reported)) +} + +fn downgrade_fatal_reported_result( + mut reported: genvm_modules_interfaces::ReportedResult, +) -> Vec<u8> { + debug_assert_eq!( + reported.kind, + genvm_modules_interfaces::ResultCode::FatalVmError + ); + reported.kind = genvm_modules_interfaces::ResultCode::VmError; + let mut normalized = vec![genvm_modules_interfaces::ResultCode::VmError as u8]; + normalized.extend(calldata::encode_obj(&reported)); + normalized +} + +fn guard_top_level_consumed_result(data: Vec<u8>, genvm_id: GenVMId) -> anyhow::Result<Vec<u8>> { + let (kind, reported) = decode_reported_result(&data, "top-level executor")?; + if kind != genvm_modules_interfaces::ResultCode::InternalError { + anyhow::ensure!( + reported.execution_hash.len() == 32, + "top-level executor returned an invalid execution hash length" + ); + anyhow::ensure!( + reported.small_hash.len() == 32, + "top-level executor returned an invalid small hash length" + ); + } + if kind != genvm_modules_interfaces::ResultCode::FatalVmError { + return Ok(data); + } + + debug_assert_ne!( + kind, + genvm_modules_interfaces::ResultCode::FatalVmError, + "top-level executor returned a fatal VM error after its publication boundary" + ); + log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0; "top-level executor returned a fatal VM error; downgrading it to vm_error"); + Ok(downgrade_fatal_reported_result(reported)) +} + +fn nested_reply_from_consumed_result( + data: &[u8], +) -> anyhow::Result<genvm_modules_interfaces::NestedRunReply> { + let (kind, reported) = decode_reported_result(data, "nested executor")?; if kind != genvm_modules_interfaces::ResultCode::InternalError { anyhow::ensure!( reported.small_hash.len() == 32, @@ -1507,6 +1562,7 @@ fn read_manager_host_stream( parent_req: Arc<Request>, consumed_result: sync::DArc<tokio::sync::OnceCell<Vec<u8>>>, genvm_id: GenVMId, + is_top_level: bool, stream_state: Arc<ManagerHostStreamState>, ) -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send>> { Box::pin(async move { @@ -1548,6 +1604,17 @@ fn read_manager_host_stream( return; } }; + let data = if is_top_level { + match guard_top_level_consumed_result(data, genvm_id) { + Ok(data) => data, + Err(e) => { + log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, error:ah = &e; "refusing invalid top-level consume_result"); + return; + } + } + } else { + data + }; log_debug_into!(&LoggerWithId, genvm_id:id = genvm_id.0, len = data.len(); "manager received consume_result"); let _ = consumed_result.set(data); @@ -2330,7 +2397,6 @@ async fn run_genvm_process( _ => req.selector.clone(), }; let version = resolve_selector(&full_ctx.ver_ctx, &selector, req.timestamp, genvm_id).await?; - let ctx = full_ctx.clone().into_gep(|x| &x.run_ctx); // Capture controls how logs and stdout/stderr are kept: disabled (forwarded @@ -2443,6 +2509,7 @@ async fn run_genvm_process( Arc::new(req.clone()), consumed_result, genvm_id, + is_top_level, manager_stream_state.clone(), )); diff --git a/implementation/src/manager/run_test.rs b/implementation/src/manager/run_test.rs index 1c8a2be8..098d36b9 100644 --- a/implementation/src/manager/run_test.rs +++ b/implementation/src/manager/run_test.rs @@ -567,6 +567,74 @@ fn encoded_nested_result(reported: &genvm_modules_interfaces::ReportedResult) -> encoded } +#[test] +fn top_level_guard_accepts_a_valid_report() { + let encoded = encoded_nested_result(&clean_reported()); + + assert_eq!( + guard_top_level_consumed_result(encoded.clone(), GenVMId(1)).unwrap(), + encoded + ); +} + +#[test] +fn top_level_guard_rejects_disagreeing_result_codes() { + let mut encoded = encoded_nested_result(&clean_reported()); + encoded[0] = genvm_modules_interfaces::ResultCode::VmError as u8; + + assert!(guard_top_level_consumed_result(encoded, GenVMId(1)).is_err()); +} + +#[test] +fn top_level_guard_rejects_invalid_framing() { + for encoded in [Vec::new(), vec![5], vec![0]] { + assert!(guard_top_level_consumed_result(encoded, GenVMId(1)).is_err()); + } +} + +#[test] +fn top_level_guard_rejects_invalid_hash_lengths() { + let mut reported = clean_reported(); + reported.execution_hash = bytes::Bytes::new(); + + assert!(guard_top_level_consumed_result(encoded_nested_result(&reported), GenVMId(1)).is_err()); +} + +#[cfg(debug_assertions)] +#[test] +#[should_panic(expected = "fatal VM error after its publication boundary")] +fn top_level_guard_asserts_on_fatal_in_debug_builds() { + let mut reported = clean_reported(); + reported.kind = genvm_modules_interfaces::ResultCode::FatalVmError; + + let _ = guard_top_level_consumed_result(encoded_nested_result(&reported), GenVMId(1)); +} + +#[cfg(not(debug_assertions))] +#[test] +fn top_level_guard_downgrades_fatal_in_release_builds() { + let mut reported = clean_reported(); + reported.kind = genvm_modules_interfaces::ResultCode::FatalVmError; + + let encoded = + guard_top_level_consumed_result(encoded_nested_result(&reported), GenVMId(1)).unwrap(); + let (kind, reported) = decode_reported_result(&encoded, "test").unwrap(); + + assert_eq!(kind, genvm_modules_interfaces::ResultCode::VmError); + assert_eq!(reported.kind, genvm_modules_interfaces::ResultCode::VmError); +} + +#[test] +fn fatal_downgrade_updates_both_result_codes() { + let mut reported = clean_reported(); + reported.kind = genvm_modules_interfaces::ResultCode::FatalVmError; + let encoded = downgrade_fatal_reported_result(reported); + let (kind, reported) = decode_reported_result(&encoded, "test").unwrap(); + + assert_eq!(kind, genvm_modules_interfaces::ResultCode::VmError); + assert_eq!(reported.kind, genvm_modules_interfaces::ResultCode::VmError); +} + fn some_storage_delta() -> genvm_modules_interfaces::StorageDelta { genvm_modules_interfaces::StorageDelta::new([0; 36], vec![1]) } diff --git a/tests/runner/origin/base_host.py b/tests/runner/origin/base_host.py index f35cf414..23debcd5 100644 --- a/tests/runner/origin/base_host.py +++ b/tests/runner/origin/base_host.py @@ -525,6 +525,8 @@ def decode(cls, raw: typing.Any) -> 'ConsumedResult': empty = not as_bytes if not empty: result_kind = host_fns.ResultCode(as_bytes[0]) + if result_kind == host_fns.ResultCode.FATAL_VM_ERROR: + raise ValueError('fatal_vm_error crossed the top-level result boundary') decoded = gvm_calldata.decode(as_bytes[1:]) except Exception as exc: # Unreadable bytes are a protocol violation rather than a result, so @@ -540,10 +542,6 @@ def decode(cls, raw: typing.Any) -> 'ConsumedResult': return cls.internal_error('empty_result') if not isinstance(decoded, dict): return cls.internal_error('result is not a mapping') - # The executor reports fatality; degrading it to an ordinary VM error - # is the host's job, so that a caller in another major can still see it - if result_kind == host_fns.ResultCode.FATAL_VM_ERROR: - result_kind = host_fns.ResultCode.VM_ERROR return cls( execution_hash=decoded.get('execution_hash', b''), result_kind=result_kind, diff --git a/tests/system/manager-socket/test.py b/tests/system/manager-socket/test.py index f375043c..96300778 100644 --- a/tests/system/manager-socket/test.py +++ b/tests/system/manager-socket/test.py @@ -14,6 +14,7 @@ import genvm_tool_plugins.genvm as genvm import origin.base_host as base_host import origin.calldata as gvm_calldata +import origin.host_fns as host_fns from gvm_extra.mock_host import MockHost, MockStorage from origin.calldata import Address from origin.manager_api import CURRENT_MAJOR, Errors, Methods @@ -376,6 +377,15 @@ async def _malformed_unknown_and_survival(self): await client.send(Methods.CANCEL, 3, {'cancel': {'genvm_id': 999}}) await _read_error(client, 3, Errors.UNKNOWN_ID) + async def _fatal_consumed_result_is_rejected(self): + raw = bytes([host_fns.ResultCode.FATAL_VM_ERROR]) + gvm_calldata.encode({}) + try: + base_host.ConsumedResult.decode(raw) + except base_host.ConsumedResultDecodeError as exc: + assert 'fatal_vm_error crossed the top-level result boundary' in str(exc) + else: + raise AssertionError('fatal consumed_result was accepted') + async def _oversized_closes_connection(self): async with await self._manager('oversized', max_message_bytes=32) as manager: async with ManagerWsClient(manager.uri) as client: @@ -920,6 +930,13 @@ def service( ) cases = [ + ( + 'fatal-result-rejected', + '_fatal_consumed_result_is_rejected', + {}, + None, + genvm.ManagerService, + ), ( 'protocol-errors', '_malformed_unknown_and_survival', From d07f21dab9c4061d5422b73e2535101e06fd5962 Mon Sep 17 00:00:00 2001 From: Kira <kp2pml30@gmail.com> Date: Tue, 25 Aug 2026 04:15:30 +0900 Subject: [PATCH 12/45] fix: vm fatal errors (#24) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(runners): update SDK APIs and runner hashes βœ¨β¬†οΈ * chore(abi): rename the pre-finalization state to decided in v0.3 🚚πŸ’₯ * fix(spec): align v0.3 WASI behavior πŸ› * refactor(codegen): scope trie details to their paths ♻️ `latest_non_final` and `accepted` named an implementation queue rather than the state-view contract: the view is the latest state-changing decided transaction, with finalized state as the fallback. Wire values change with no back-compat alias, so hosts and SDKs must move in lockstep. * feat(abi): align executor v1 interfaces * fix(runners): harden v0.3 packaging workflow * fix(tests): encode cross-major recursion per ABI * test(integration): skip leader_fault_uncatchable ⏭️ Bump the v0.3.x gitlink: the case's `A: compared ...` prints race between overlapping nondet queue consumers and a shared stdout, and they are the only observable that a comparison stage ran. --- .claude/settings.local.json | 3 - .genvm-tool.py | 5 + .gitignore | 1 + .../codegen/data/host-fns.json | 10 +- crates/modules-interfaces/src/abi_stub.rs | 24 +- crates/modules-interfaces/src/domain.rs | 14 +- .../modules-interfaces/src/domain/fees/abi.rs | 2 +- crates/modules-interfaces/src/host_fns.rs | 40 ++-- crates/modules-interfaces/src/lib.rs | 4 +- crates/modules-interfaces/src/nested.rs | 4 +- ...11. runner ids and runtime registration.md | 6 + ...pre-validating untrusted decoded inputs.md | 19 +- docs/adr/015. cross-major contract calls.md | 8 +- docs/contributing/howto/committing/runners.md | 18 +- .../howto/committing/submodules.md | 6 +- .../howto/extending/modify-runner.md | 16 +- docs/contributing/howto/testing/fuzzing.md | 6 + .../contributing/howto/testing/integration.md | 2 +- docs/schemas/default-config.json | 2 +- docs/schemas/runners.json | 2 +- .../01-core-architecture/04-executor.rst | 2 +- .../impl-spec/02-vm/02-version-management.rst | 5 +- docs/website/src/impl-spec/04-fees.rst | 10 +- .../src/impl-spec/appendix/host-loop.rst | 36 +-- .../src/impl-spec/appendix/manager-api.yaml | 4 +- .../src/impl-spec/appendix/manager-socket.rst | 2 +- .../02-execution-environment/02-wasip1.rst | 85 ++++--- .../03-wasi_genlayer_sdk/01-functions.rst | 2 +- .../03-wasi_genlayer_sdk/02-gl_call.rst | 95 ++++---- .../03-wasi_genlayer_sdk/03-schemas.rst | 6 +- .../03-wasi_genlayer_sdk/index.rst | 2 +- .../02-execution-environment/04-runners.rst | 19 +- docs/website/src/spec/03-vm/01-startup.rst | 9 +- .../src/spec/03-vm/02-meta-properties.rst | 20 +- .../03-vm/04-determinism-mode-switching.rst | 18 +- docs/website/src/spec/03-vm/05-result.rst | 4 +- .../src/spec/04-contract-interface/02-abi.rst | 2 +- .../spec/04-contract-interface/03-storage.rst | 2 +- .../spec/04-contract-interface/06-gvm32.rst | 2 +- docs/website/src/spec/appendix/constants.rst | 73 ++++-- .../src/spec/appendix/internal-constants.rst | 7 + docs/website/src/spec/changelog.rst | 16 ++ executors/v0.2.x | 2 +- executors/v0.3.x | 2 +- flake.nix | 2 +- implementation/src/llm/handler.rs | 20 +- implementation/src/manager/run.rs | 28 +-- implementation/src/manager/run_test.rs | 18 +- runners/views/universal.nix | 6 +- .../tools/genvm-tool/genvm_tool/codegen/go.py | 18 +- .../genvm-tool/genvm_tool/codegen/model.py | 63 ++++-- .../genvm-tool/genvm_tool/codegen/python.py | 10 +- .../genvm-tool/genvm_tool/codegen/rst.py | 17 +- .../genvm-tool/genvm_tool/codegen/rust.py | 21 +- .../unit_tests/test_codegen_detail_suffix.py | 85 ++++--- tests/runner/genvm_tool_plugins/afl.py | 41 +++- tests/runner/genvm_tool_plugins/cargo.py | 1 - .../runner/genvm_tool_plugins/integration.py | 6 +- tests/runner/gvm_extra/mock_host.py | 35 +-- tests/runner/origin/base_host.py | 100 ++++---- tests/runner/origin/fees.py | 6 +- tests/runner/origin/host_fns.py | 10 +- tests/runner/origin/public_abi.py | 77 +++++-- .../assets/v0.3/observer.py | 4 +- .../system/cross-major-observability/test.py | 12 +- .../cross-major/assets/v0.3/state_caller.py | 4 +- tests/system/cross-major/test.py | 53 +++-- tests/system/make_zip/test.py | 214 ++++++++++++++++++ 68 files changed, 953 insertions(+), 515 deletions(-) delete mode 100644 .claude/settings.local.json create mode 100644 tests/system/make_zip/test.py diff --git a/.claude/settings.local.json b/.claude/settings.local.json deleted file mode 100644 index d61c7860..00000000 --- a/.claude/settings.local.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "outputStyle": "Brief Style" -} diff --git a/.genvm-tool.py b/.genvm-tool.py index 7b4ca8ed..2c9a6057 100644 --- a/.genvm-tool.py +++ b/.genvm-tool.py @@ -285,3 +285,8 @@ def collect_parse_version(ctx: genvm_tool.tests.stage.collection.Context): ctx.collect_dir('tests/system/parse_version') ctx.add_collector(collect_parse_version) + + def collect_make_zip(ctx: genvm_tool.tests.stage.collection.Context): + ctx.collect_dir('tests/system/make_zip') + + ctx.add_collector(collect_make_zip) diff --git a/.gitignore b/.gitignore index eab50f70..771520e7 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ /.pre-commit-config.yaml .claude/hooks +.claude/settings.local.json .idea .envrc *.log diff --git a/crates/modules-interfaces/codegen/data/host-fns.json b/crates/modules-interfaces/codegen/data/host-fns.json index e9d9e0e5..028e1682 100644 --- a/crates/modules-interfaces/codegen/data/host-fns.json +++ b/crates/modules-interfaces/codegen/data/host-fns.json @@ -5,13 +5,13 @@ "name": "methods", "values": { "storage_read": 0, - "consume_fuel": 1, - "eth_call": 2, - "get_balance": 3, - "remaining_fuel_as_gen": 4, + "consume_time_fee_gen_wei": 1, + "external_call": 2, + "get_balance_gen_wei": 3, + "get_remaining_time_fee_gen_wei": 4, "notify_nondet_disagreement": 5, "consume_result": 6, - "resolve_callcontract_executor": 7, + "resolve_call_contract_executor": 7, "run_nested": 8 } }, diff --git a/crates/modules-interfaces/src/abi_stub.rs b/crates/modules-interfaces/src/abi_stub.rs index c0dd0892..6f5010af 100644 --- a/crates/modules-interfaces/src/abi_stub.rs +++ b/crates/modules-interfaces/src/abi_stub.rs @@ -15,9 +15,9 @@ pub enum On { #[serde(rename = "finalized")] #[calldata(rename = "finalized")] Finalized, - #[serde(rename = "accepted")] - #[calldata(rename = "accepted")] - Accepted, + #[serde(rename = "decided")] + #[calldata(rename = "decided")] + Decided, } #[derive( @@ -57,21 +57,21 @@ pub struct MessageData { pub value: num_bigint::BigInt, pub is_init: bool, /// Transaction timestamp - #[serde(default = "default_datetime")] + #[serde(default = "default_transaction_timestamp")] #[calldata( - serialize_with = encode_datetime_rfc3339, - deserialize_with = decode_datetime_rfc3339 + serialize_with = encode_transaction_timestamp_rfc3339, + deserialize_with = decode_transaction_timestamp_rfc3339 )] - #[calldata(default = default_datetime)] - pub datetime: chrono::DateTime<chrono::Utc>, + #[calldata(default = default_transaction_timestamp)] + pub transaction_timestamp: chrono::DateTime<chrono::Utc>, } -fn decode_datetime_rfc3339( +fn decode_transaction_timestamp_rfc3339( val: genlayer_calldata::Value, ) -> Result<chrono::DateTime<chrono::Utc>, genlayer_calldata::codec::DecodeError> { let genlayer_calldata::Value::Str(s) = val else { return Err(genlayer_calldata::codec::DecodeError::Unexpected( - "expected string for datetime", + "expected string for transaction_timestamp", )); }; chrono::DateTime::parse_from_rfc3339(&s) @@ -79,7 +79,7 @@ fn decode_datetime_rfc3339( .map_err(|e| genlayer_calldata::codec::DecodeError::UserError(Box::new(e))) } -fn encode_datetime_rfc3339<W: genlayer_calldata::Writer>( +fn encode_transaction_timestamp_rfc3339<W: genlayer_calldata::Writer>( dt: &chrono::DateTime<chrono::Utc>, enc: &mut genlayer_calldata::Encoder<W>, ) -> Result<(), W::Error> { @@ -88,7 +88,7 @@ fn encode_datetime_rfc3339<W: genlayer_calldata::Writer>( enc.push_str(&s) } -fn default_datetime() -> chrono::DateTime<chrono::Utc> { +fn default_transaction_timestamp() -> chrono::DateTime<chrono::Utc> { chrono::DateTime::parse_from_rfc3339("2024-11-26T06:42:42.424242Z") .unwrap() .to_utc() diff --git a/crates/modules-interfaces/src/domain.rs b/crates/modules-interfaces/src/domain.rs index ca16e8fd..94b47423 100644 --- a/crates/modules-interfaces/src/domain.rs +++ b/crates/modules-interfaces/src/domain.rs @@ -91,7 +91,7 @@ pub struct NestedExecutionData { #[derive(Debug, Clone, PartialEq, Eq, genlayer_calldata::Encode, genlayer_calldata::Decode)] #[calldata(tag = "type")] pub enum ExecutionEmission { - EthSend { + ExternalMessage { address: genlayer_calldata::Address, calldata: Bytes, value: U256, @@ -100,7 +100,7 @@ pub enum ExecutionEmission { fee_params: fees::ExternalMessageParams, }, - PostMessage { + InternalMessage { call_key: crate::CallKey, address: genlayer_calldata::Address, calldata: genlayer_calldata::codec::Maybe<genlayer_calldata::Value>, @@ -115,7 +115,7 @@ pub enum ExecutionEmission { /// balance rather than the sender's prefunded message-fee pool. use_balance: bool, }, - DeployContract { + InternalDeployMessage { calldata: genlayer_calldata::codec::Maybe<genlayer_calldata::Value>, code: Bytes, value: U256, @@ -126,10 +126,10 @@ pub enum ExecutionEmission { fee_params: fees::InternalMessageParams, subtree: bytes::Bytes, - /// Chain `useBalance`; see `ExecutionEmission::PostMessage::use_balance`. + /// Chain `useBalance`; see `ExecutionEmission::InternalMessage::use_balance`. use_balance: bool, }, - EmitEvent { + Event { topics: Vec<Bytes>, blob: genlayer_calldata::codec::Maybe<genlayer_calldata::Map<genlayer_calldata::Value>>, storage_fee: U256, @@ -403,7 +403,7 @@ pub struct ReportedResult { pub data: genlayer_calldata::unparsed::Maybe<genlayer_calldata::Value>, pub backtrace: Option<Backtrace>, pub wasm_store_hashes: WasmStoreHashes, - pub storage_changes: Vec<StorageDelta>, + pub storage_deltas: Vec<StorageDelta>, pub emissions: Vec<ExecutionEmission>, @@ -413,5 +413,5 @@ pub struct ReportedResult { pub data_fees_remaining: Vec<primitive_types::U256>, pub data_fees_consumed: BucketsConsumed, - pub llm_consumption: primitive_types::U256, + pub llm_consumed_gen_wei: primitive_types::U256, } diff --git a/crates/modules-interfaces/src/domain/fees/abi.rs b/crates/modules-interfaces/src/domain/fees/abi.rs index 93f293e7..89d938c8 100644 --- a/crates/modules-interfaces/src/domain/fees/abi.rs +++ b/crates/modules-interfaces/src/domain/fees/abi.rs @@ -109,7 +109,7 @@ fn encode_node(node: &MessageAllocationNode, parent_index: U256) -> Vec<u8> { let (message_type, on_acceptance, fee_params) = match &node.fee_params { MessageAllocationNodeParams::Internal(params) => ( MESSAGE_TYPE_INTERNAL, - matches!(node.on, On::Accepted), + matches!(node.on, On::Decided), encode_internal_params(params), ), // External messages have no acceptance/finalize lifecycle. diff --git a/crates/modules-interfaces/src/host_fns.rs b/crates/modules-interfaces/src/host_fns.rs index ae6be202..91121027 100644 --- a/crates/modules-interfaces/src/host_fns.rs +++ b/crates/modules-interfaces/src/host_fns.rs @@ -17,13 +17,13 @@ use serde::{Deserialize, Serialize}; #[repr(u8)] pub enum Methods { StorageRead = 0, - ConsumeFuel = 1, - EthCall = 2, - GetBalance = 3, - RemainingFuelAsGen = 4, + ConsumeTimeFeeGenWei = 1, + ExternalCall = 2, + GetBalanceGenWei = 3, + GetRemainingTimeFeeGenWei = 4, NotifyNondetDisagreement = 5, ConsumeResult = 6, - ResolveCallcontractExecutor = 7, + ResolveCallContractExecutor = 7, RunNested = 8, } @@ -32,26 +32,26 @@ impl Methods { pub fn value(self) -> u8 { match self { Methods::StorageRead => 0, - Methods::ConsumeFuel => 1, - Methods::EthCall => 2, - Methods::GetBalance => 3, - Methods::RemainingFuelAsGen => 4, + Methods::ConsumeTimeFeeGenWei => 1, + Methods::ExternalCall => 2, + Methods::GetBalanceGenWei => 3, + Methods::GetRemainingTimeFeeGenWei => 4, Methods::NotifyNondetDisagreement => 5, Methods::ConsumeResult => 6, - Methods::ResolveCallcontractExecutor => 7, + Methods::ResolveCallContractExecutor => 7, Methods::RunNested => 8, } } pub fn str_snake_case(self) -> &'static str { match self { Methods::StorageRead => "storage_read", - Methods::ConsumeFuel => "consume_fuel", - Methods::EthCall => "eth_call", - Methods::GetBalance => "get_balance", - Methods::RemainingFuelAsGen => "remaining_fuel_as_gen", + Methods::ConsumeTimeFeeGenWei => "consume_time_fee_gen_wei", + Methods::ExternalCall => "external_call", + Methods::GetBalanceGenWei => "get_balance_gen_wei", + Methods::GetRemainingTimeFeeGenWei => "get_remaining_time_fee_gen_wei", Methods::NotifyNondetDisagreement => "notify_nondet_disagreement", Methods::ConsumeResult => "consume_result", - Methods::ResolveCallcontractExecutor => "resolve_callcontract_executor", + Methods::ResolveCallContractExecutor => "resolve_call_contract_executor", Methods::RunNested => "run_nested", } } @@ -63,13 +63,13 @@ impl TryFrom<u8> for Methods { fn try_from(value: u8) -> Result<Self, ()> { match value { 0 => Ok(Methods::StorageRead), - 1 => Ok(Methods::ConsumeFuel), - 2 => Ok(Methods::EthCall), - 3 => Ok(Methods::GetBalance), - 4 => Ok(Methods::RemainingFuelAsGen), + 1 => Ok(Methods::ConsumeTimeFeeGenWei), + 2 => Ok(Methods::ExternalCall), + 3 => Ok(Methods::GetBalanceGenWei), + 4 => Ok(Methods::GetRemainingTimeFeeGenWei), 5 => Ok(Methods::NotifyNondetDisagreement), 6 => Ok(Methods::ConsumeResult), - 7 => Ok(Methods::ResolveCallcontractExecutor), + 7 => Ok(Methods::ResolveCallContractExecutor), 8 => Ok(Methods::RunNested), _ => Err(()), } diff --git a/crates/modules-interfaces/src/lib.rs b/crates/modules-interfaces/src/lib.rs index 9217ad97..d4b364c0 100644 --- a/crates/modules-interfaces/src/lib.rs +++ b/crates/modules-interfaces/src/lib.rs @@ -537,11 +537,11 @@ pub mod llm { pub enum Message { Prompt { payload: PromptPayload, - remaining_fuel_as_gen: primitive_types::U256, + remaining_time_fee_gen_wei: primitive_types::U256, }, PromptTemplate { payload: PromptTemplatePayload, - remaining_fuel_as_gen: primitive_types::U256, + remaining_time_fee_gen_wei: primitive_types::U256, }, } diff --git a/crates/modules-interfaces/src/nested.rs b/crates/modules-interfaces/src/nested.rs index 94577089..0c518ff2 100644 --- a/crates/modules-interfaces/src/nested.rs +++ b/crates/modules-interfaces/src/nested.rs @@ -72,8 +72,8 @@ pub enum ExecutorSelector { #[serde(rename_all = "snake_case")] pub enum NestedStorageType { Default, - LatestFinal, - LatestNonFinal, + LatestFinalized, + LatestDecided, } /// Permission bits carried across an executor boundary. diff --git a/docs/adr/011. runner ids and runtime registration.md b/docs/adr/011. runner ids and runtime registration.md index 1d7bb3b2..2c7bf4ae 100644 --- a/docs/adr/011. runner ids and runtime registration.md +++ b/docs/adr/011. runner ids and runtime registration.md @@ -55,6 +55,12 @@ returns the resulting runner id. It is allowed only in deterministic mode and on holds the new `register_runners` permission (permission char `u`). A friendly `genlayer.vm.register_runner(code) -> str` wrapper is exposed in the Python SDK. +> **Superseded.** The `register_runners` permission was removed from v0.3: no +> `u` char was ever parsed, the root VM always held the bit, and containment +> (a `custom:` id resolves only from the registering VM's loaded set, which dies +> with it) already carries what the permission was meant to buy. +> `RegisterRunner` now requires deterministic mode alone. + Add a companion `gl_call` `MapFile { runner, path_in_runner, path_in_vfs }` that maps a file (or, when `path_in_runner` ends with `/`, a directory subtree) from any runner into the VM filesystem at runtime, sharing the exact logic of the `MapFile` diff --git a/docs/adr/013. pre-validating untrusted decoded inputs.md b/docs/adr/013. pre-validating untrusted decoded inputs.md index af14d049..532f0d21 100644 --- a/docs/adr/013. pre-validating untrusted decoded inputs.md +++ b/docs/adr/013. pre-validating untrusted decoded inputs.md @@ -130,10 +130,10 @@ bucket that raises it needs a permission the child lacks); the child also can't reach the derived namespace (it has `spawn_nondet: false`). Both invariants must be re-checked if the nondet child's permissions change β€” a non-trie code trips the `debug_assert!` in debug and, in release, is an honest-node hash split. -Consequence: `leaders_result` is always detail-free while a validator's own +Consequence: `leader_result` is always detail-free while a validator's own error keeps its detail; the default `compare_vm_errors` compares `public_code` only, so it is unaffected β€” custom comparators must not read a detail from -`leaders_result`. +`leader_result`. ### Rule 2 β€” message payload shape @@ -150,12 +150,13 @@ untrusted entry calldata is decoded: - **Top-level host entry** β€” decoded in `run_with_impl` before permissions are read or any runner loads; failure (or a `""` present on `is_init`) is a - `malformed_entry` VM error **result**. `PostMessage`/`DeployContract` emit - messages that later execute as top-level entries, so they pass here too. -- **Every `gl_call` message** β€” `CallContract`, `PostMessage` and - `DeployContract` carry the calldata as a typed field, so a malformed payload - fails the `gl_call::Message` decode and answers **`Errno::Inval`**, like the - other argument checks; the caller can recover. + `malformed_entry` VM error **result**. + `EmitInternalMessage`/`EmitInternalDeployMessage` emit messages that later + execute as top-level entries, so they pass here too. +- **Every `gl_call` message** β€” `CallContract`, `EmitInternalMessage` and + `EmitInternalDeployMessage` carry the calldata as a typed field, so a + malformed payload fails the `gl_call::Message` decode and answers + **`Errno::Inval`**, like the other argument checks; the caller can recover. ### Run modes @@ -218,7 +219,7 @@ calldata, extra keys or observable nondet details. - **Re-encode the `Return` payload instead of rejecting** β€” silently normalizes malformed consensus input; the validator's bytes would differ from the proposal while claiming agreement. -- **Validate lazily when the contract reads `leaders_result`** β€” too late, the +- **Validate lazily when the contract reads `leader_result`** β€” too late, the payload is already in the hash. - **Enforce the message shape in each SDK** β€” once per language, differently, and a runner can't reject a call before it starts. diff --git a/docs/adr/015. cross-major contract calls.md b/docs/adr/015. cross-major contract calls.md index 08c75fc1..d1ed78ca 100644 --- a/docs/adr/015. cross-major contract calls.md +++ b/docs/adr/015. cross-major contract calls.md @@ -63,7 +63,7 @@ observed mismatch, asks the host which executor should run the callee. The resolution and the execution are separate calls, to different peers, because they answer to different authorities. -1. **`resolve_callcontract_executor`**, on host connection 0 β€” the host that +1. **`resolve_call_contract_executor`**, on host connection 0 β€” the host that initiated the transaction β€” when the run request sets `hook_cross_contract_calls`. Otherwise it goes to the manager on host 1, which answers null; see the implementation notes. @@ -135,7 +135,7 @@ computed by the caller and carried on the wire. The table is complete against | `ExecutionData` field | Nested value | | --- | --- | | `calldata` | Derived `CallContract` calldata. The child is always `Main`; v0.2 rewrites its legacy `method` key to the current empty key at the boundary. | -| `message` | Derived child message: target contract, inherited sender/origin/signer, chain and datetime, `value = 0`, `is_init = false`. v0.2's contract-facing message has no signer field, so its executor carries the signer beside the message and forwards it unchanged; contracts on that line still cannot observe it. | +| `message` | Derived child message: target contract, inherited sender/origin/signer, chain and `transaction_timestamp`, `value = 0`, `is_init = false`. Each executor maps `transaction_timestamp` back to the contract-facing `datetime`; v0.2's contract-facing message has no signer field, so its executor carries the signer beside the message and forwards it unchanged; contracts on that line still cannot observe it. | | `code` | `None`; the child reads the target code through host 0 storage. | | `host_data`, `gas_data` | Copied by the manager from the outer request, not carried in the envelope. | | `method_hosts` | Constructed by the manager: host 0 by default, `consume_result` and `run_nested` on host 1. | @@ -286,7 +286,7 @@ Infrastructure faults are internal errors; contract-produced outcomes stay contract-visible. The dividing line is whether a contract can reach the condition. -Internal: a host that cannot answer `resolve_callcontract_executor` (the host is +Internal: a host that cannot answer `resolve_call_contract_executor` (the host is the authority on which line owns an address); a nested callee reporting an internal error, which propagates unchanged; a callee reporting any effect, which the manager refuses outright since a `CallContract` child holds none of the @@ -355,7 +355,7 @@ determinism hazard named above. updates Rust, Python and RST β€” see [genvm-tool.md](../contributing/howto/genvm-tool.md). - **Resolution is opt-in.** A run request carries `hook_cross_contract_calls`, - default false. False routes `resolve_callcontract_executor` to host 1, where + default false. False routes `resolve_call_contract_executor` to host 1, where the manager answers null; true routes it to host 0 so the node decides. The default is "manager answers null" because a node that does not route across majors should pay neither the round-trip nor the cost of implementing a host diff --git a/docs/contributing/howto/committing/runners.md b/docs/contributing/howto/committing/runners.md index 55a298ab..927cece0 100644 --- a/docs/contributing/howto/committing/runners.md +++ b/docs/contributing/howto/committing/runners.md @@ -12,14 +12,16 @@ Before committing: 1. Set `dev-mode.nix` back to `false` 2. Set every `"test"` hash in `current.nix` to `null` -3. Run `runners/support/versions/hash-updater.py`, from anywhere inside the - repo. It builds the umbrella's `#runners-all` with `--keep-going`, writes - every reported `got:` value back into `current.nix`, and repeats until a - fixed point. Hashes are content-addressed and depend on resolved dependency - uids, so they must be discovered bottom-up, which the script handles. It - needs the executor nested under the manager umbrella; a standalone checkout - cannot build `runners-all` -4. Commit once every hash is a real `sha256-…` value +3. Stage the runner changes and run pre-commit; fix and restage them before + hash discovery so the hooks and Nix inspect the same source tree +4. Run `runners/support/versions/hash-updater.py`, from anywhere inside the + repo. It builds the umbrella's `#runners-all` with `--keep-going`, writes + every reported `got:` value back into `current.nix`, and repeats until a + fixed point. Hashes are content-addressed and depend on resolved dependency + uids, so they must be discovered bottom-up, which the script handles. It + needs the executor nested under the manager umbrella; a standalone checkout + cannot build `runners-all` +5. Stage `current.nix` and commit once every hash is a real `sha256-…` value **Never restore an old hash by hand.** It becomes wrong the moment any source or dependency changes, and only `hash-updater.py` produces a correct one diff --git a/docs/contributing/howto/committing/submodules.md b/docs/contributing/howto/committing/submodules.md index fb364d77..8a89f141 100644 --- a/docs/contributing/howto/committing/submodules.md +++ b/docs/contributing/howto/committing/submodules.md @@ -79,7 +79,7 @@ are normally ahead-only. Runner hash hygiene first: [runners.md](runners.md) ## Why There Are Two Runner Trees -Forward-rolling lines share the top-level `runners/<id>/<aa>/<rest>.tar` tree, -named by Crockford base32 of `sha256(tar)`; frozen v0.2.x keeps -`executor/<version>/legacy-runners/` with Nix base32 hashes. `flake.nix` splits +Forward-rolling lines share the top-level `runners/<id>/<aa>/<rest>.zip` tree, +named by Crockford base32 of `sha256(zip)`; frozen v0.2.x keeps +`executor/<version>/legacy-runners/` as ustar tars with Nix base32 hashes. `flake.nix` splits the runner list per line, and `genvm check` verifies each with its own scheme diff --git a/docs/contributing/howto/extending/modify-runner.md b/docs/contributing/howto/extending/modify-runner.md index da7f39f5..25ee5ba5 100644 --- a/docs/contributing/howto/extending/modify-runner.md +++ b/docs/contributing/howto/extending/modify-runner.md @@ -3,13 +3,15 @@ Paths are relative to `executors/v0.3.x/`, the line that owns runner sources 1. Set `runners/support/versions/dev-mode.nix` to `true` and the runner's hash - in `runners/support/versions/current.nix` to `"test"`, then develop and test. - Hashes move, so integration tests need `--ignore-hash` - ([integration.md](../testing/integration.md)) + in `runners/support/versions/current.nix` to `"test"`, then develop and test + Hashes move, so integration tests need `--ignore-hash` + ([integration.md](../testing/integration.md)) 2. Set dev-mode back to `false`, and the changed runner's hash, plus its - dependents', to `null` -3. Refresh the hashes with `runners/support/versions/hash-updater.py` -4. Rebuild to confirm no mismatch remains + dependents', to `null` +3. Stage the runner changes and run pre-commit; fix and restage them before + hash discovery so the hooks and Nix inspect the same source tree +4. Refresh the hashes with `runners/support/versions/hash-updater.py` +5. Stage `current.nix` and rebuild to confirm no mismatch remains -Step 3 in detail, and why a hash is never restored by hand: +Step 4 in detail, and why a hash is never restored by hand: [committing/runners.md](../committing/runners.md) diff --git a/docs/contributing/howto/testing/fuzzing.md b/docs/contributing/howto/testing/fuzzing.md index 621697e6..2ef8e999 100644 --- a/docs/contributing/howto/testing/fuzzing.md +++ b/docs/contributing/howto/testing/fuzzing.md @@ -53,6 +53,12 @@ named as a corpus entry) as soon as it is reported, so every later run replays it β€” the next run wipes the crash dir it was saved in. Commit it along with the fix; put it in `<corpus>-curated` instead if a corpus update must never drop it +Startup replays every committed entry, and AFL skips the ones it cannot use +rather than aborting the run: a `-t` ceiling of 5 s (it auto-scales below that) +keeps a seed that grew slower than AFL's 1 s default from taking the target down. +A seed that *crashes* is skipped by the same switch, so the case reads it back out +of the instance logs and fails with the entry's name under `crashing_seeds` + Findings of both languages live under `build/test-artifacts/fuzz/<project>/`. Reruns resume from that dir (`AFL_AUTORESUME`) and therefore ignore `-i`: remove it to pick up a corpus that changed. Corpus updates `cmin` the fleet's queues diff --git a/docs/contributing/howto/testing/integration.md b/docs/contributing/howto/testing/integration.md index 4c8c3b65..cdff315f 100644 --- a/docs/contributing/howto/testing/integration.md +++ b/docs/contributing/howto/testing/integration.md @@ -11,7 +11,7 @@ runs its own cases against its own built executor, via `build/info.json` A step may declare `executor_routes: {'<address>': <route>}` to send a `CallContract` on that address to another executor line instead of running it -in-process β€” the mock host answers `resolve_callcontract_executor` with that +in-process β€” the mock host answers `resolve_call_contract_executor` with that route, and the manager spawns the nested run. A route is a major (an integer, resolved by the manifest's rules) or a version string naming the line outright, `re:`-prefixed to match manifest keys rather than name a directory. Prefer a diff --git a/docs/schemas/default-config.json b/docs/schemas/default-config.json index 692e39fe..ccce9e0c 100644 --- a/docs/schemas/default-config.json +++ b/docs/schemas/default-config.json @@ -244,7 +244,7 @@ }, "max_wait_after_loaded": { "$ref": "#/definitions/timeout", - "description": "upper bound for `wait_after_loaded` requested by contracts; larger values are clamped (with a warning). Defaults to 60s" + "description": "upper bound for `post_load_wait` requested by contracts; larger values are clamped (with a warning). Defaults to 60s" }, "meta": { "type": ["object", "null"], diff --git a/docs/schemas/runners.json b/docs/schemas/runners.json index 22a543f0..dcb452da 100644 --- a/docs/schemas/runners.json +++ b/docs/schemas/runners.json @@ -9,7 +9,7 @@ "definitions": { "WasmMode": { - "enum": ["det", "nondet"] + "enum": ["det", "!det"] }, "runner-id": { "type": "string", diff --git a/docs/website/src/impl-spec/01-core-architecture/04-executor.rst b/docs/website/src/impl-spec/01-core-architecture/04-executor.rst index a5fb2519..0fa4cf6a 100644 --- a/docs/website/src/impl-spec/01-core-architecture/04-executor.rst +++ b/docs/website/src/impl-spec/01-core-architecture/04-executor.rst @@ -73,7 +73,7 @@ The levels are ordered; each *adds* to the previous one: - The ``:latest`` / ``:test`` runner ids may be resolved. **Unsafe across machines**: different nodes may resolve different code and diverge consensus, though a single node stays deterministic. * - ``unsafe-tracing`` - ``unbounded`` - - Real wall-clock time is exposed to the contract in deterministic mode (``RuntimeMicroSec`` returns actual elapsed time instead of ``0``; non-deterministic mode already returns real time regardless of debug level). **Can break determinism on a single machine.** Local debugging only. + - Real wall-clock time is exposed to the contract in deterministic mode (``RuntimeMicroseconds`` returns actual elapsed time instead of ``0``; non-deterministic mode already returns real time regardless of debug level). **Can break determinism on a single machine.** Local debugging only. Only ``unsafe`` and ``unsafe-tracing`` can affect determinism (across machines and on a single machine respectively); ``safe`` and ``safe-unbounded`` are fully diff --git a/docs/website/src/impl-spec/02-vm/02-version-management.rst b/docs/website/src/impl-spec/02-vm/02-version-management.rst index 06ce487d..b426907e 100644 --- a/docs/website/src/impl-spec/02-vm/02-version-management.rst +++ b/docs/website/src/impl-spec/02-vm/02-version-management.rst @@ -18,8 +18,9 @@ There are two independent notions of "version" in play: Detection from contract bytes ----------------------------- -``executor/src/runners/parse.rs::detect_version_from_wasm`` walks the WASM custom sections -and returns the contents of the ``genvm.version`` section as the runner version string. +``executor/src/runners/parse.rs::describe_wasm`` walks the WASM custom sections in one +pass and returns the contents of the ``genvm.version`` section as the runner version +string, alongside the optional ``genvm.runner.json`` section. When the contract is not a raw WASM but a zip-packaged archive the version is read from the ``version`` file in the archive. For single-file text contracts (Python source) the first comment line is inspected: if it starts with ``v`` it is taken as the version diff --git a/docs/website/src/impl-spec/04-fees.rst b/docs/website/src/impl-spec/04-fees.rst index 6b07f959..0aa0d5a8 100644 --- a/docs/website/src/impl-spec/04-fees.rst +++ b/docs/website/src/impl-spec/04-fees.rst @@ -259,10 +259,11 @@ Message-Fee Allocation Matching The ``a.matchedFeeParams`` above is selected per outbound message from the call's allocation list (``accumulator.message_fee_allocation``). A node (``domain/fees.rs``, ``matches_internal`` / ``matches_external``) matches on kind -(``External`` for ``EthSend``, else ``Internal``), ``on`` (internal only), and +(``External`` for ``EmitExternalMessage``, else ``Internal``), ``on`` (internal +only), and ``recipient`` / ``call_key`` (each a wildcard ``None`` or an exact match). Selection is **first-match-wins** in list order (``find_map``); no match yields the -``fee no_matching_node`` VM error. +``fee no_matching_allocation`` VM error. Wildcards are not reordered, so a wildcard node shadows every more-specific node after it: producers are advised to sort more-specific nodes ahead of wildcard ones. @@ -275,13 +276,14 @@ An outgoing internal message is funded one of two ways: - **Allocation-matched (default).** The fee is matched against the allocation tree as above, capped by the matched node's ``budget``, and consumes both the ``message_fee`` and ``message_receipt`` buckets. -- **Balance-funded (``use_balance``).** When a ``PostMessage`` / ``DeployContract`` +- **Balance-funded (``use_balance``).** When an ``EmitInternalMessage`` / + ``EmitInternalDeployMessage`` sets ``use_balance`` (the chain's ``useBalance``, gated on :ref:`gvm-perm-use-balance-for-message-fees`), allocation matching is skipped entirely. The fee is metered from the guest-supplied ``fee_params`` and that metered amount is the child's ``declaredBudget``, reserved from the emitting contract's balance (jointly with ``value``; insufficient balance yields - ``Inbalance``). The ``message_fee`` bucket is **not** consumed (the message is + ``InsufficientBalance``). The ``message_fee`` bucket is **not** consumed (the message is excluded from the sender pool on-chain); only ``message_receipt`` is. The emitted allocation subtree is empty, so nested child messages must each fund themselves. diff --git a/docs/website/src/impl-spec/appendix/host-loop.rst b/docs/website/src/impl-spec/appendix/host-loop.rst index d79258aa..54c6ef2a 100644 --- a/docs/website/src/impl-spec/appendix/host-loop.rst +++ b/docs/website/src/impl-spec/appendix/host-loop.rst @@ -35,9 +35,9 @@ independently runs the protocol described below. In the manager deployment, host 0 is the node's host loop and host 1 is a socketpair to the manager. The manager serves ``consume_result``, ``run_nested`` -and -- by default -- ``resolve_callcontract_executor`` on host 1. +and -- by default -- ``resolve_call_contract_executor`` on host 1. -``resolve_callcontract_executor`` moves to host 0 only when the run request sets +``resolve_call_contract_executor`` moves to host 0 only when the run request sets ``hook_cross_contract_calls`` (see :doc:`manager-socket`). Otherwise the manager answers it with a null reply, which keeps every ``CallContract`` in-process. A host that does not route calls across major boundaries therefore need not @@ -84,12 +84,12 @@ result, clean-finish vs crash) is reported by the manager's terminal event method_id := read_byte match method_id json/methods/storage_read: - read_type := read_byte as json/storage_type + read_type := read_byte as json/storage_view address := read_bytes(ACCOUNT_ADDR_SIZE) slot := read_bytes(SLOT_ID_SIZE) - index := read_u32_le + offset := read_u32_le len := read_u32_le - data, err := host_storage_read(read_type, address, slot, index, len) + data, err := host_storage_read(read_type, address, slot, offset, len) if err != json/errors/ok: write_byte err else: @@ -101,35 +101,35 @@ result, clean-finish vs crash) is reported by the manager's terminal event # this is needed to ensure that genvm doesn't close socket before all data is read write_byte 0x00 - json/methods/consume_fuel: - gas := read_u64_le - host_consume_fuel(gas) + json/methods/consume_time_fee_gen_wei: + time_fee_gen_wei := read_u256_le + host_consume_time_fee_gen_wei(time_fee_gen_wei) # note: this method doesn't send any response - json/methods/eth_call: + json/methods/external_call: address := read_bytes(ACCOUNT_ADDR_SIZE) calldata := read_slice() - result, err := host_eth_call(address, calldata) + result, err := host_external_call(address, calldata) if err != json/errors/ok: write_byte err else: write_byte json/errors/ok write_byte_slice result - json/methods/get_balance: + json/methods/get_balance_gen_wei: address := read_bytes(ACCOUNT_ADDR_SIZE) - balance, err := host_get_balance(address) + balance, err := host_get_balance_gen_wei(address) if err != json/errors/ok: write_byte err else: write_byte json/errors/ok write_bytes balance.to_le_bytes(32) # 256-bit integer - json/methods/resolve_callcontract_executor: + json/methods/resolve_call_contract_executor: address := read_bytes(ACCOUNT_ADDR_SIZE) - state := read_byte as json/storage_type + state := read_byte as json/storage_view advisory_major := read_byte - payload, err := host_resolve_callcontract_executor( + payload, err := host_resolve_call_contract_executor( address, state, advisory_major) if err != json/errors/ok: write_byte err @@ -158,13 +158,13 @@ result, clean-finish vs crash) is reported by the manager's terminal event } write_byte_slice calldata_encode(reply) - json/methods/remaining_fuel_as_gen: - fuel, err := host_remaining_fuel_as_gen() + json/methods/get_remaining_time_fee_gen_wei: + time_fee_gen_wei, err := host_get_remaining_time_fee_gen_wei() if err != json/errors/ok: write_byte err else: write_byte json/errors/ok - write_bytes fuel.to_le_bytes(32) # 256-bit unsigned, little-endian, always 32 bytes + write_bytes time_fee_gen_wei.to_le_bytes(32) # 256-bit unsigned, little-endian, always 32 bytes json/methods/notify_nondet_disagreement: call_no := read_u32_le diff --git a/docs/website/src/impl-spec/appendix/manager-api.yaml b/docs/website/src/impl-spec/appendix/manager-api.yaml index e2a25f08..aa0ee8c9 100644 --- a/docs/website/src/impl-spec/appendix/manager-api.yaml +++ b/docs/website/src/impl-spec/appendix/manager-api.yaml @@ -760,7 +760,7 @@ components: is_init: type: boolean description: Whether this is a contract initialization call - datetime: + transaction_timestamp: type: string format: date-time description: Transaction timestamp @@ -920,7 +920,7 @@ components: type: boolean default: false description: | - Whether the host wants to answer `resolve_callcontract_executor`. When + Whether the host wants to answer `resolve_call_contract_executor`. When false the manager answers it with a null reply and every `CallContract` stays in-process. unsafe_overrides: diff --git a/docs/website/src/impl-spec/appendix/manager-socket.rst b/docs/website/src/impl-spec/appendix/manager-socket.rst index 5f22661a..60a759c1 100644 --- a/docs/website/src/impl-spec/appendix/manager-socket.rst +++ b/docs/website/src/impl-spec/appendix/manager-socket.rst @@ -103,7 +103,7 @@ Request payload: the same logical structure as the deprecated (currently index 1, the ``consume_result`` socketpair). - ``hook_cross_contract_calls`` (bool, optional, default ``false``) -- whether this host wants to be asked where a ``CallContract`` runs. When false the - manager answers ``resolve_callcontract_executor`` itself with a null reply, + manager answers ``resolve_call_contract_executor`` itself with a null reply, so every call stays in-process and the host need not implement that method. When true the question is routed to host 0 and the host may send the caller across a major boundary (see :doc:`host-loop`). A nested run inherits the diff --git a/docs/website/src/spec/02-execution-environment/02-wasip1.rst b/docs/website/src/spec/02-execution-environment/02-wasip1.rst index 51e2bdf4..8ef9f26a 100644 --- a/docs/website/src/spec/02-execution-environment/02-wasip1.rst +++ b/docs/website/src/spec/02-execution-environment/02-wasip1.rst @@ -181,36 +181,47 @@ File Metadata The :ref:`gvm-def-vfs` stores no metadata, so every ``Filestat`` reports ``dev``, ``ino``, ``atim``, ``mtim`` and ``ctim`` as ``0``; ``size`` is the -file's length in octets and ``0`` for a directory. Timestamps are **not** -derived from the transaction timestamp β€” a file has no modification time at -all. +file's length in octets and ``0`` for a directory, and ``nlink`` is ``1``. +Timestamps are **not** derived from the transaction timestamp β€” a file has no +modification time at all. + +Operations represented in a target type's supported base mask require the +corresponding right on a descriptor returned by ``path_open`` and fail with +``Notcapable`` when it is absent. Supported masks are defined by +:ref:`gvm-def-wasi-descriptor-rights`. ``path_open`` Function ~~~~~~~~~~~~~~~~~~~~~~ #. The path is resolved as in `Path Resolution`_. -#. ``oflags``, ``fs_rights_base``, ``fs_rights_inheriting`` and ``fdflags`` are - ignored. The :ref:`gvm-def-vfs` is read-only, so ``creat``, ``excl`` and +#. The directory descriptor requires the ``path_open`` base right, otherwise + the call fails with ``Notcapable``. +#. If ``oflags::directory`` targets a regular file, the call fails with + ``Notdir``. The :ref:`gvm-def-vfs` is read-only, so ``creat``, ``excl`` and ``trunc`` never take effect β€” opening a non-existent path fails with - ``Noent`` rather than creating it β€” and ``directory`` does not restrict the - result. + ``Noent`` rather than creating it. ``fdflags`` are ignored. +#. The new descriptor's base and inheriting rights are limited by the + corresponding requested rights and the parent descriptor's inheriting + rights. Unsupported target-type rights are then removed as defined in + :ref:`gvm-def-wasi-descriptor-rights`. #. On success a fresh descriptor is allocated (see :ref:`gvm-def-fd-allocation`) that refers to the resolved file or directory. A file descriptor starts at offset ``0``. -#. The descriptor is released again if the call fails, so a failed open leaks - neither a descriptor nor its :ref:`gvm-def-ram-consumption`. +#. A failed call allocates no descriptor and consumes no descriptor + :ref:`gvm-def-ram-consumption`. ``path_filestat_get`` Function ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Resolves the path as in `Path Resolution`_ and returns the ``Filestat`` described in `File Metadata`_ with ``filetype`` ``regular_file`` or -``directory`` and ``nlink`` ``0``. +``directory``. ``fd_readdir`` Function ~~~~~~~~~~~~~~~~~~~~~~~ -A descriptor that is not a directory fails with ``Badf``. +After the rights check, a descriptor that is not a directory fails with +``Badf``. The entry sequence of a directory is fixed and does not depend on how the :ref:`gvm-def-vfs` was populated: @@ -241,12 +252,12 @@ Returns the descriptor's current offset. ``stdout``/``stderr`` fail with ``fd_datasync`` Function ~~~~~~~~~~~~~~~~~~~~~~~~ -Does nothing and always returns success. +Does nothing and returns success after the rights check. ``fd_sync`` Function ~~~~~~~~~~~~~~~~~~~~ -Does nothing and always returns success. +Does nothing and returns success after the rights check. ``fd_seek`` Function ~~~~~~~~~~~~~~~~~~~~ @@ -262,27 +273,29 @@ Moves a file descriptor's offset and returns the new value. ``fd_prestat_dir_name`` Function ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Writes the preopened directory's name β€” the last component of its path, or ``/`` -for the root. ``fd_prestat_get`` returns the ``Prestat`` whose ``pr_name_len`` -determines the name's length. A buffer shorter than that name fails with -``Overflow``; any non-directory descriptor fails with ``Badf``. +The only preopened directory is the VFS root. This function writes its +guest-visible name, ``/``. ``fd_prestat_get`` returns the ``Prestat`` whose +``pr_name_len`` is therefore ``1``. A buffer shorter than that fails with +``Overflow``; every descriptor other than the root preopen, including a +directory returned by ``path_open``, fails with ``Badf``. ``fd_prestat_get`` Function ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -#. If the descriptor refers to a preopened directory, returns its ``Prestat`` - (a ``Dir`` whose ``pr_name_len`` is the length of the directory's last path - component). This is required for libc preopen discovery. +#. For the root preopen, returns a ``Dir`` whose ``pr_name_len`` is ``1``, the + length of its guest-visible name ``/``. This is required for libc preopen + discovery. #. For any other descriptor (``stdin``/``stdout``/``stderr``, a regular file, or a - non-existent descriptor) returns ``Badf``. + directory returned by ``path_open``, or a non-existent descriptor) returns + ``Badf``. ``fd_write`` Function ~~~~~~~~~~~~~~~~~~~~~ -Only ``stdout`` and ``stderr`` are writable; a file or directory descriptor -fails with ``Rofs``. The written octets are diagnostic output: they are not part -of the :ref:`gvm-def-vm-result` and are not covered by the -:ref:`gvm-def-execution-hash`. +After the rights check, only ``stdout`` and ``stderr`` are writable; a file or +directory descriptor fails with ``Rofs``. The written octets are diagnostic +output: they are not part of the :ref:`gvm-def-vm-result` and are not covered +by the :ref:`gvm-def-execution-hash`. The call reports every supplied octet as written and never fails on the underlying stream, so a contract cannot observe whether the :term:`Host` @@ -312,18 +325,24 @@ Returns the ``Filestat`` described in `File Metadata`_ with ``nlink`` ``1`` and - ``directory`` for a directory descriptor - ``character_device`` for ``stdout``/``stderr`` +.. _gvm-def-wasi-descriptor-rights: + ``fd_fdstat_get`` Function ~~~~~~~~~~~~~~~~~~~~~~~~~~ ``fs_flags`` is always empty β€” no descriptor is appending, non-blocking or -synchronous β€” and ``fs_rights_inheriting`` always equals ``fs_rights_base``. -The rights are fixed per descriptor kind and never include a mutating right: +synchronous. The maximum read-only rights are: + +- A regular-file descriptor has the read, seek, tell, advise, sync, datasync + and fd-filestat-get base rights, and no inheriting rights +- A directory descriptor has the ``path_open``, readdir, path-filestat-get and + fd-filestat-get base rights; its supported inheriting rights are the union of + the regular-file and directory base rights +- ``stdout``/``stderr`` have ``unknown`` filetype and the write right in both + masks -- a file descriptor: ``regular_file`` with the read, seek, tell, advise, sync, - datasync, readdir, readlink, ``path_open`` and filestat-get rights -- a directory descriptor: ``directory`` with the read, ``path_open``, readdir, - readlink and filestat-get rights -- ``stdout``/``stderr``: ``unknown`` filetype with the write right only +The root preopen receives the full directory masks. For a descriptor returned +by ``path_open``, both masks are further limited as described in that function. ``fd_close`` Function ~~~~~~~~~~~~~~~~~~~~~ @@ -335,7 +354,7 @@ descriptor fails with ``Badf``. ``fd_advise`` Function ~~~~~~~~~~~~~~~~~~~~~~ -Does nothing and always returns success. +Does nothing and returns success after the rights check. ``clock_time_get`` Function ~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/01-functions.rst b/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/01-functions.rst index 8b3077fa..d8cf25f3 100644 --- a/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/01-functions.rst +++ b/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/01-functions.rst @@ -59,6 +59,6 @@ Returns - ``error_success`` on success - ``error_inval`` for invalid requests - ``error_forbidden`` for permission violations -- ``error_inbalance`` for insufficient balance +- ``error_insufficient_balance`` for insufficient balance See :doc:`02-gl_call` for the list of available messages. diff --git a/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/02-gl_call.rst b/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/02-gl_call.rst index 026faa5d..2710685e 100644 --- a/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/02-gl_call.rst +++ b/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/02-gl_call.rst @@ -1,8 +1,8 @@ ``gl_call`` Messages ==================== -``EthSend`` Message -------------------- +``EmitExternalMessage`` Message +------------------------------- Sends transaction to Ethereum address with optional value transfer. @@ -12,7 +12,7 @@ Payload .. code-block:: { - "EthSend": { + "EmitExternalMessage": { "address": Address, // 20-byte target address "calldata": Bytes, // EVM calldata "value": U256 // Wei to transfer @@ -26,8 +26,8 @@ Requirements #. :ref:`gvm-perm-send-messages` #. Sufficient contract balance for value transfer -``EthCall`` Message -------------------- +``ExternalCall`` Message +------------------------ Calls Ethereum contract method (read-only operation). @@ -37,7 +37,7 @@ Payload .. code-block:: { - "EthCall": { + "ExternalCall": { "address": Address, // 20-byte target contract address "calldata": Bytes // EVM calldata } @@ -65,7 +65,8 @@ Payload "CallContract": { "address": Address, // 20-byte target contract address "calldata": Calldata, // Method call in calldata format - "state": Number // Storage type: 0=default, 1=latest_final, 2=latest_non_final + "storage_view": Number, // Storage view: 0=default, 1=latest_finalized, 2=latest_decided + "catch_vm_error": Bool // optional (default false): take a VM error as the result } } @@ -78,6 +79,11 @@ Requirements Creates a :term:`sub-VM`. See :ref:`gvm-meta-property-derivation`. +A :term:`sub-VM` that ends in a :ref:`gvm-def-vm-error` normally ends its caller +too. With *param* ``catch_vm_error`` set, the caller reads that error as the +call's result instead. A fatal :ref:`gvm-def-vm-error` is never caught: the flag +does not apply to it, and the caller ends regardless. + Before every call, :term:`GenVM` asks the :term:`host` whether to delegate the callee to another executor. A null answer preserves the local path, including :ref:`gvm-def-str-trie-value-vm-error-invalid-contract-major-mismatch`. A @@ -92,10 +98,10 @@ answered with ``Errno::Inval`` like the other argument checks and the caller can recover β€” unlike a top-level entry, where the same violation is the execution's result (:ref:`gvm-vm-startup-entry-validation`). -.. _gvm-def-post-message: +.. _gvm-def-emit-internal-message: -``PostMessage`` Message ------------------------ +``EmitInternalMessage`` Message +------------------------------- Posts message to GenLayer contract for later execution. @@ -110,11 +116,11 @@ Payload .. code-block:: { - "PostMessage": { + "EmitInternalMessage": { "address": Address, // 20-byte target contract address "calldata": Calldata, // Method call in calldata format "value": U256, // Wei to transfer - "on": String, // "finalized" or "accepted" + "on": String, // "finalized" or "decided" "use_balance": Bool, // optional (default false), see below "fee_params": FeeParams // optional (default absent), required iff use_balance } @@ -144,8 +150,8 @@ transaction's fee configuration and mirrors the chain's .. code-block:: FeeParams { - "leader_timeunits_allocation": U256, // per-round leader time units - "validator_timeunits_allocation": U256, // per-round validator time units + "leader_time_units_allocation": U256, // per-round leader time units + "validator_time_units_allocation": U256, // per-round validator time units "execution_budget_per_round": U256, // unified budget per leader round "rotations": [U256], // per-round rotations; non-empty. // rotations[0] is the initial round, @@ -166,7 +172,7 @@ Semantics: - The message is excluded from allocation matching, so no matching node is required (and none is consulted). - The contract must be able to cover ``value + metered_fee`` from its balance; - otherwise the call fails with ``Inbalance``. + otherwise the call fails with ``InsufficientBalance``. - The emitted allocation subtree is **empty**: nesting is fail-closed, so a child message must itself set ``use_balance`` or it fails to fund. @@ -181,8 +187,8 @@ Semantics: - Out-of-bounds magnitudes: prices and budgets (``max_price_gen_per_time_unit``, ``storage_fee_max_gas_price``, ``receipt_fee_max_gas_price``, ``execution_budget_per_round``) must be below - 2\ :sup:`96`; counts (``leader_timeunits_allocation``, - ``validator_timeunits_allocation``, each ``rotations`` entry) below + 2\ :sup:`96`; counts (``leader_time_units_allocation``, + ``validator_time_units_allocation``, each ``rotations`` entry) below 2\ :sup:`32`. These bounds keep the metered floor within ``U256``. Metering additionally enforces node-configured floors, surfaced as ``VMError``\ s: @@ -193,8 +199,8 @@ Metering additionally enforces node-configured floors, surfaced as ``VMError``\ - ``fee too_many_rounds`` β€” ``rotations`` implies more consensus rounds than the node's validator table supports (on-chain ``MAX_ROUNDS``). -``DeployContract`` Message --------------------------- +``EmitInternalDeployMessage`` Message +------------------------------------ Deploys new intelligent contract to blockchain. @@ -204,11 +210,11 @@ Payload .. code-block:: { - "DeployContract": { + "EmitInternalDeployMessage": { "calldata": Calldata, // Constructor arguments in calldata format "code": Bytes, // Contract bytecode "value": U256, // Wei to transfer - "on": String, // "finalized" or "accepted" + "on": String, // "finalized" or "decided" "salt_nonce": U256, // Salt for CREATE2-style deterministic addressing "use_balance": Bool, // optional (default false) "fee_params": FeeParams // optional (default absent), required iff use_balance @@ -246,7 +252,8 @@ Payload "data_leader": Bytes, // Code/data for leader execution "data_validator": Bytes, // Code/data for validator execution "runner": String, // optional (default "contract"): runner to execute - "custom_runners": [String] // optional (default absent): custom runners to grant + "custom_runners": [String], // optional (default absent): custom runners to grant + "catch_vm_error": Bool // optional (default false): take a VM error as the result } } @@ -262,6 +269,11 @@ Creates a non-deterministic :term:`sub-VM`. Derivation of its meta-properties, including the *param* ``runner`` and *param* ``custom_runners`` semantics, is specified in :ref:`gvm-meta-property-derivation`. +A :term:`sub-VM` that ends in a :ref:`gvm-def-vm-error` normally ends its caller +too. With *param* ``catch_vm_error`` set, the caller reads that error as the +call's result instead. A fatal :ref:`gvm-def-vm-error` is never caught: the flag +does not apply to it, and the caller ends regardless. + .. _gvm-def-gl-call-sandbox: ``Sandbox`` Message @@ -281,8 +293,8 @@ Payload "runner": String, // runner to execute; becomes the child's "contract" "allow_write_storage": Bool, // Whether to allow storage writes "allow_send_messages": Bool, // Whether to allow sending messages - "allow_register_runners": Bool, // Whether to allow registering runners - "custom_runners": [String] // optional (default absent): custom runners to grant + "custom_runners": [String], // optional (default absent): custom runners to grant + "changes_on_error": String // fate of the child's changes on a non-return } } @@ -295,10 +307,14 @@ meta-properties, including the *param* ``runner`` and *param* :ref:`gvm-meta-property-derivation`. The caller receives the sandbox result (:ref:`gvm-def-subvm-result-encoding`) -and may handle both :ref:`gvm-def-vm-error` and :ref:`gvm-def-user-error`; -storage writes performed by the sandbox are not reverted on error. If the -sandbox terminates with :ref:`gvm-def-fatal-vm-error`, the caller terminates -with the same fatal VM error instead. +and may handle both :ref:`gvm-def-vm-error` and :ref:`gvm-def-user-error`. +If the sandbox terminates with :ref:`gvm-def-fatal-vm-error`, the caller +terminates with the same fatal VM error instead. + +*param* ``changes_on_error`` says what becomes of the storage writes and +emissions of a sandbox that does not :ref:`gvm-def-return`. ``"inherit"`` is +its only accepted value: the caller keeps them, exactly as it keeps the ones it +made itself. Any other value is a malformed message .. _gvm-def-gl-call-register-runner: @@ -317,7 +333,7 @@ Payload { "RegisterRunner": { - "code": Bytes // runner archive (ustar/zip or commented text) + "code": Bytes // runner archive (zip, raw wasm or commented text) } } @@ -325,7 +341,6 @@ Requirements ~~~~~~~~~~~~ #. :ref:`gvm-perm-deterministic` -#. :ref:`gvm-perm-register-runners` Semantics ~~~~~~~~~ @@ -339,8 +354,8 @@ enters the caller's loaded set. The outcomes, in check order, are: -#. Missing :ref:`gvm-def-det-mode` or :ref:`gvm-perm-register-runners`: the call - fails with ``Forbidden``. Nothing is charged and no state changes. +#. Missing :ref:`gvm-def-det-mode`: the call fails with ``Forbidden``. Nothing + is charged and no state changes. #. Insufficient memory for the charge: the :term:`sub-VM` exits with an out-of-memory :ref:`gvm-def-vm-error`. Nothing is charged and the runner is not registered. @@ -393,7 +408,7 @@ Payload "WebRender": { "mode": String, // "text", "html", or "screenshot" "url": String, // URL to render - "wait_after_loaded": String // Wait duration, e.g. "5s" or "500ms" + "post_load_wait": String // Wait duration, e.g. "5s" or "500ms" } } @@ -623,11 +638,14 @@ Requirements .. _tracing-runtime-microsec: -``Trace.RuntimeMicroSec`` Sub-Message -------------------------------------- +``Trace.RuntimeMicroseconds`` Sub-Message +----------------------------------------- In :ref:`gvm-def-non-det-mode` returns the elapsed execution time in microseconds since VM start. -In :ref:`gvm-def-det-mode`, it returns ``0`` β€” exposing real elapsed time there would break determinism. The sole exception is the ``unsafe-tracing`` debug level (see the executor "Debug modes" section), which returns real elapsed time even in deterministic mode; that level is for local debugging only and must never be used on a consensus network. +In :ref:`gvm-def-det-mode`, it returns ``0`` β€” exposing real elapsed time there +would break determinism. An implementation MAY support a debug mode that +returns real elapsed time instead; such a mode is for local debugging only and +MUST NOT be used on a consensus network. Payload ~~~~~~~ @@ -635,13 +653,14 @@ Payload .. code-block:: { - "Trace": "RuntimeMicroSec" + "Trace": "RuntimeMicroseconds" } .. note:: The returned value is not at an implementation's discretion β€” in - :ref:`gvm-def-det-mode` it is exactly ``0``, and the call never fails (see + :ref:`gvm-def-det-mode` it is exactly ``0`` unless an implementation's debug + mode permits real elapsed time, and the call never fails (see :ref:`gvm-def-gl-call-observable-discretion`). Requirements diff --git a/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/03-schemas.rst b/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/03-schemas.rst index 2aeff786..0a5a0243 100644 --- a/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/03-schemas.rst +++ b/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/03-schemas.rst @@ -38,9 +38,9 @@ Validity Of A Proposed Result A result the executor did not compute itself β€” the leader-proposed non-deterministic result consumed in :ref:`gvm-def-sync-mode` and :ref:`gvm-def-validator-mode` β€” is accepted only if it satisfies all of the -following. A rejected proposal never traps and never bypasses the comparison -stage; it is replaced by a derived :ref:`gvm-def-vm-error` and handed to the -comparison as if it had been proposed. +following. A rejected proposal is replaced by a derived :ref:`gvm-def-vm-error`. +Sync mode returns that replacement without a vote; validator mode records a +disagreement without running the comparison stage #. The buffer is non-empty. An absent or empty proposal yields :ref:`gvm-def-str-trie-value-vm-error-leader-fault-nondet-output-absent`. diff --git a/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/index.rst b/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/index.rst index aa014ffe..c6b731d8 100644 --- a/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/index.rst +++ b/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/index.rst @@ -34,7 +34,7 @@ Interface Definition static const uint32_t error_io = 5 static const uint32_t error_forbidden = 6 - static const uint32_t error_inbalance = 7 + static const uint32_t error_insufficient_balance = 7 __attribute__((import_module("genlayer_sdk"))) uint32_t storage_read(char const* slot, uint32_t index, char* buf, uint32_t buf_len); diff --git a/docs/website/src/spec/02-execution-environment/04-runners.rst b/docs/website/src/spec/02-execution-environment/04-runners.rst index b45f1edf..725ea338 100644 --- a/docs/website/src/spec/02-execution-environment/04-runners.rst +++ b/docs/website/src/spec/02-execution-environment/04-runners.rst @@ -17,11 +17,11 @@ forms (see the ``runner-id`` definition in the runner.json schema): - ``<human-readable-id>:<hash>`` β€” a packaged runner. ``human-readable-id`` is provided for convenience; ``hash`` is a hash of its contents (see `Hash Format`_). - ``contract`` β€” the runner of the contract currently being executed. -- ``chain:<address>[:<a|f>[:<slot>]]`` β€” a runner code blob read from a storage - slot of an arbitrary contract (``a`` = accepted, ``f`` = finalized). ``<address>`` +- ``chain:<address>[:<d|f>[:<slot>]]`` β€” a runner code blob read from a storage + slot of an arbitrary contract (``d`` = decided, ``f`` = finalized). ``<address>`` is a ``0x`` 20 byte hex address and ``<slot>`` is a :term:`SlotID` encoded with - :doc:`../04-contract-interface/06-gvm32`. Both ``<a|f>`` and ``<slot>`` are - optional: ``<a|f>`` defaults to ``a`` and ``<slot>`` to the target contract's + :doc:`../04-contract-interface/06-gvm32`. Both ``<d|f>`` and ``<slot>`` are + optional: ``<d|f>`` defaults to ``d`` and ``<slot>`` to the target contract's root code slot. - ``custom:<hash>`` β€” a runner registered at runtime via the ``RegisterRunner`` ``gl_call``, looked up by its hash. @@ -41,8 +41,8 @@ A ``chain:`` id is code, but its resolution is an ordinary by consensus, not against whatever a node's chain tip happens to be. Every validator of the transaction therefore reads the same octets, whichever :term:`Host` serves them. -#. ``f`` (finalized) reads the state at the last finalized block; ``a`` - (accepted, the default) reads the accepted state β€” the same view a +#. ``f`` (finalized) reads the state at the last finalized block; ``d`` + (decided, the default) reads the decided state β€” the same view a read-only :ref:`gvm-def-gl-call-call-contract` child observes by default (see :ref:`contract-execution-flow`). Neither view includes the executing transaction's own uncommitted writes, so a contract cannot deploy code and @@ -111,6 +111,11 @@ Creates a minimal :term:`runner` configuration runner.json = { "StartWasm": "file" } file = # source bytes +Both defaults can be overridden by the module itself, through custom sections: +``genvm.version`` supplies the version string and ``genvm.runner.json`` the +whole ``runner.json``. This is how a single wasm file declares dependencies +without being repackaged as a ZIP + 3. Text-based ~~~~~~~~~~~~~ @@ -320,7 +325,7 @@ Conditionally executes an action based on WebAssembly execution mode. Properties ^^^^^^^^^^ -- ``cond``: WebAssembly mode, either ``det`` (deterministic) or ``nondet`` (non-deterministic) +- ``cond``: WebAssembly mode, either ``det`` (deterministic) or ``!det`` (non-deterministic) - ``action``: Action to execute when condition is met Example diff --git a/docs/website/src/spec/03-vm/01-startup.rst b/docs/website/src/spec/03-vm/01-startup.rst index b580d192..358bf604 100644 --- a/docs/website/src/spec/03-vm/01-startup.rst +++ b/docs/website/src/spec/03-vm/01-startup.rst @@ -43,7 +43,8 @@ runner loads. A payload that violates the convention produces the :ref:`gvm-def-str-trie-value-vm-error-malformed-entry` as the execution result, like any other failure at this point. -Messages emitted by :ref:`gvm-def-post-message` and ``DeployContract`` are +Messages emitted by :ref:`gvm-def-emit-internal-message` and +``EmitInternalDeployMessage`` are executed later as top-level entries, so they are validated here too. .. _gvm-vm-startup-message: @@ -139,7 +140,7 @@ Initial Entry the contract's root slot. - :ref:`gvm_vm_field_state_mode` is the default storage view. - :ref:`gvm_vm_field_topmost_runner_id` is the deployment runner or the - contract's accepted code runner. + contract's decided code runner. - :ref:`gvm_vm_field_det_subvm_hashes` and :ref:`gvm_vm_field_granted_custom` are empty. @@ -160,7 +161,7 @@ The child is read-only. keeps the parent's value (the plain copy rule), so by default the callee observes a view at least as recent as its caller's. Because the child cannot write, its :ref:`gvm-def-enum-value-storage-type-default` view is - the accepted state: it never includes the calling transaction's uncommitted + the decided state: it never includes the calling transaction's uncommitted writes (see :ref:`contract-execution-flow`). - :ref:`gvm_vm_field_topmost_runner_id` is the callee's contract runner. - :ref:`gvm_vm_field_granted_custom` is the caller's entire loaded @@ -195,8 +196,6 @@ inherited permission, never add one. - :ref:`gvm-perm-use-balance-for-message-fees` is cleared unless *param* ``allow_send_messages`` is set: balance-funded fees only affect message emission, so they are gated by the same flag - - :ref:`gvm-perm-register-runners` is cleared unless *param* - ``allow_register_runners`` is set - :ref:`gvm_vm_field_topmost_runner_id` is the *param* ``runner``, resolved in the caller's scope. diff --git a/docs/website/src/spec/03-vm/02-meta-properties.rst b/docs/website/src/spec/03-vm/02-meta-properties.rst index d66e3bf1..50d7a774 100644 --- a/docs/website/src/spec/03-vm/02-meta-properties.rst +++ b/docs/website/src/spec/03-vm/02-meta-properties.rst @@ -45,7 +45,7 @@ Internal field containing the permission meta-properties granted to the ~~~~~~~~~~~~~~ Internal field selecting the storage view used by reads -(a :ref:`gvm-def-enum-storage-type`). +(a :ref:`gvm-def-enum-storage-view`). .. _gvm_vm_field_topmost_runner_id: @@ -102,8 +102,8 @@ iff it can write storage. ``send_messages`` ~~~~~~~~~~~~~~~~~ -Allows sending messages to other addresses. Required by ``EthSend``, -``PostMessage``, and ``DeployContract``. Requires +Allows sending messages to other addresses. Required by ``EmitExternalMessage``, +``EmitInternalMessage``, and ``EmitInternalDeployMessage``. Requires :ref:`gvm-perm-deterministic`. .. _gvm-perm-call-others: @@ -111,7 +111,7 @@ Allows sending messages to other addresses. Required by ``EthSend``, ``call_others`` ~~~~~~~~~~~~~~~ -Allows calling other contracts. Required by ``EthCall`` and +Allows calling other contracts. Required by ``ExternalCall`` and :ref:`gvm-def-gl-call-call-contract`. Requires :ref:`gvm-perm-deterministic`. @@ -128,22 +128,14 @@ A :term:`sub-VM` that a host delegated to another executor (see caller holds. Such a :term:`sub-VM` has no connection to the non-deterministic modules, so the permission cannot be served there. -.. _gvm-perm-register-runners: - -``register_runners`` -~~~~~~~~~~~~~~~~~~~~ - -Allows registering runner archives at runtime via -:ref:`gvm-def-gl-call-register-runner`. Requires :ref:`gvm-perm-deterministic`. - .. _gvm-perm-use-balance-for-message-fees: ``can_use_balance_for_message_fees`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Allows the contract to draw on its own balance to pay fees for outgoing -internal messages (``PostMessage``, ``DeployContract``). The flag has no effect -on external ``EthSend`` messages. +internal messages (``EmitInternalMessage``, ``EmitInternalDeployMessage``). The +flag has no effect on ``EmitExternalMessage``. Unlike the meta-properties above, this one is stored as a bit in the root slot's inline ``permissions`` bitfield and read before execution begins. The bit offset diff --git a/docs/website/src/spec/03-vm/04-determinism-mode-switching.rst b/docs/website/src/spec/03-vm/04-determinism-mode-switching.rst index 2d295c77..5554a0ba 100644 --- a/docs/website/src/spec/03-vm/04-determinism-mode-switching.rst +++ b/docs/website/src/spec/03-vm/04-determinism-mode-switching.rst @@ -37,16 +37,14 @@ so the resulting :ref:`gvm-def-vm-result` is simply the call's result. Validator Mode -------------- -The proposed result is accepted or replaced per -:ref:`gvm-def-proposed-result-validity`, then handed to the :term:`sub-VM` for -comparison. That :term:`sub-VM` must :ref:`gvm-def-return` a ``bool`` value: -whether the validator accepts the leader's result. Any other result has the same -effect as producing ``bool(false)``. - -A replaced proposal takes this same path β€” the derived code is compared like any -other, so the vote stays contract-controlled and no branch bypasses the -comparison. The replacement is a pure function of the proposed bytes, so every -honest validator derives the same value and votes identically. +An accepted proposal is handed to the :term:`sub-VM` for comparison. That +:term:`sub-VM` must :ref:`gvm-def-return` a ``bool`` value: whether the validator +accepts the leader's result. Any other result has the same effect as producing +``bool(false)`` + +A rejected proposal records a disagreement without running the comparison +stage. The contract cannot vote away bytes that no honest leader could have +produced Returns the accepted or replaced result. diff --git a/docs/website/src/spec/03-vm/05-result.rst b/docs/website/src/spec/03-vm/05-result.rst index 50cda1d1..0e0c0659 100644 --- a/docs/website/src/spec/03-vm/05-result.rst +++ b/docs/website/src/spec/03-vm/05-result.rst @@ -89,7 +89,7 @@ Represents a user-produced error in utf-8 format. Only a :ref:`gvm-def-return` carries effects. A topmost run that ends in :ref:`gvm-def-user-error` or :ref:`gvm-def-vm-error` reports no -``storage_changes`` and no ``emissions``, whatever it wrote or emitted before +``storage_deltas`` and no ``emissions``, whatever it wrote or emitted before failing, and its :ref:`gvm-def-execution-hash` covers those empty fields. .. _gvm-def-internal-error: @@ -178,7 +178,7 @@ with the following keys (in this order): #. ``data_fees_remaining`` #. ``emissions`` β€” emitted messages and events, in emission order #. ``kind`` β€” the :ref:`gvm-def-vm-result` result code -#. ``storage_changes`` +#. ``storage_deltas`` #. ``subvm_hashes`` β€” see :ref:`gvm-def-subvm-hash` #. ``wasm_store_hashes`` diff --git a/docs/website/src/spec/04-contract-interface/02-abi.rst b/docs/website/src/spec/04-contract-interface/02-abi.rst index 83c06ef3..1b16a3f2 100644 --- a/docs/website/src/spec/04-contract-interface/02-abi.rst +++ b/docs/website/src/spec/04-contract-interface/02-abi.rst @@ -60,7 +60,7 @@ and :ref:`gvm-def-gl-call-call-contract`. Call Key -------- -Every emitted message (see :ref:`gvm-def-post-message`) carries a ``call_key``: +Every emitted message (see :ref:`gvm-def-emit-internal-message`) carries a ``call_key``: a 256-bit unsigned integer that identifies which method the message targets. It serves the same role as a function selector in EVM, but is derived differently and is not truncated. diff --git a/docs/website/src/spec/04-contract-interface/03-storage.rst b/docs/website/src/spec/04-contract-interface/03-storage.rst index 61a6b130..7685c50e 100644 --- a/docs/website/src/spec/04-contract-interface/03-storage.rst +++ b/docs/website/src/spec/04-contract-interface/03-storage.rst @@ -59,7 +59,7 @@ the following data: - ``code_slot``: (offset 5) A raw 32-byte :term:`SlotID`. If it is all-zero (the default), the contract code is read from the ``code`` slot (offset 2); otherwise the code is read from the slot it points to (same 4-byte-length-prefixed layout). This lets a contract serve its - code from an arbitrary slot, including one shared via a ``chain:<address>:<a|f>:<slot>`` runner id. + code from an arbitrary slot, including one shared via a ``chain:<address>:<d|f>:<slot>`` runner id. - ``permissions``: (offset 37) A 32-byte (``u256``) little-endian permission bitfield read by the executor at the start of every load. Bit ``n`` corresponds to the permission whose value is ``n`` (currently only bit ``0``, ``can_use_balance_for_message_fees``). It is not reserved: diff --git a/docs/website/src/spec/04-contract-interface/06-gvm32.rst b/docs/website/src/spec/04-contract-interface/06-gvm32.rst index 8b208434..cf063919 100644 --- a/docs/website/src/spec/04-contract-interface/06-gvm32.rst +++ b/docs/website/src/spec/04-contract-interface/06-gvm32.rst @@ -67,4 +67,4 @@ Usage - **Custom runner ids** β€” ``custom:<hash>`` where ``<hash>`` is the GVM32 encoding of the SHA3-256 of the registered code. - **Slot ids** β€” a :term:`SlotID` is rendered in GVM32, including the ``<slot>`` - component of a ``chain:<address>:<a|f>:<slot>`` runner id. + component of a ``chain:<address>:<d|f>:<slot>`` runner id. diff --git a/docs/website/src/spec/appendix/constants.rst b/docs/website/src/spec/appendix/constants.rst index d744f684..0e7ad72d 100644 --- a/docs/website/src/spec/appendix/constants.rst +++ b/docs/website/src/spec/appendix/constants.rst @@ -29,31 +29,31 @@ vm_error Value: ``2`` -.. _gvm-def-enum-storage-type: +.. _gvm-def-enum-storage-view: -storage_type +storage_view ------------ Type: u8 -.. _gvm-def-enum-value-storage-type-default: +.. _gvm-def-enum-value-storage-view-default: default ~~~~~~~ Value: ``0`` -.. _gvm-def-enum-value-storage-type-latest-final: +.. _gvm-def-enum-value-storage-view-latest-finalized: -latest_final -~~~~~~~~~~~~ +latest_finalized +~~~~~~~~~~~~~~~~ Value: ``1`` -.. _gvm-def-enum-value-storage-type-latest-non-final: +.. _gvm-def-enum-value-storage-view-latest-decided: -latest_non_final -~~~~~~~~~~~~~~~~ +latest_decided +~~~~~~~~~~~~~~ Value: ``2`` @@ -344,15 +344,15 @@ Param: i32 ``out_of message_fee total`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. _gvm-def-str-trie-value-vm-error-out-of-message-fee-node: +.. _gvm-def-str-trie-value-vm-error-out-of-message-fee-allocation-budget: -``out_of message_fee node`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``out_of message_fee allocation_budget`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. _gvm-def-str-trie-value-vm-error-out-of-vm-recursion: +.. _gvm-def-str-trie-value-vm-error-out-of-subvm-recursion: -``out_of vm_recursion`` -~~~~~~~~~~~~~~~~~~~~~~~ +``out_of subvm_recursion`` +~~~~~~~~~~~~~~~~~~~~~~~~~~ .. _gvm-def-str-trie-value-vm-error-out-of-nondet-blocks: @@ -374,10 +374,10 @@ Param: i32 ``out_of fds`` ~~~~~~~~~~~~~~ -.. _gvm-def-str-trie-value-vm-error-fee-no-matching-node: +.. _gvm-def-str-trie-value-vm-error-fee-no-matching-allocation: -``fee no_matching_node`` -~~~~~~~~~~~~~~~~~~~~~~~~ +``fee no_matching_allocation`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. _gvm-def-str-trie-value-vm-error-fee-below-minimum: @@ -441,15 +441,40 @@ This error remains terminal for top-level and runner loads. During :ref:`gvm-def ``invalid_contract wasm entrypoint`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. _gvm-def-str-trie-detail-vm-error-internal: +.. _gvm-def-str-trie-detail-vm-error-out-of-receipt-message-internal: -``# internal`` -~~~~~~~~~~~~~~ +``out_of receipt message # internal`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. _gvm-def-str-trie-detail-vm-error-external: +.. _gvm-def-str-trie-detail-vm-error-out-of-message-fee-total-internal: -``# external`` -~~~~~~~~~~~~~~ +``out_of message_fee total # internal`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. _gvm-def-str-trie-detail-vm-error-out-of-message-fee-total-external: + +``out_of message_fee total # external`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. _gvm-def-str-trie-detail-vm-error-out-of-message-fee-allocation-budget-internal: + +``out_of message_fee allocation_budget # internal`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. _gvm-def-str-trie-detail-vm-error-out-of-message-fee-allocation-budget-external: + +``out_of message_fee allocation_budget # external`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. _gvm-def-str-trie-detail-vm-error-fee-no-matching-allocation-internal: + +``fee no_matching_allocation # internal`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. _gvm-def-str-trie-detail-vm-error-fee-no-matching-allocation-external: + +``fee no_matching_allocation # external`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. _gvm-def-const-event-max-topics: diff --git a/docs/website/src/spec/appendix/internal-constants.rst b/docs/website/src/spec/appendix/internal-constants.rst index 44b72f37..2323d901 100644 --- a/docs/website/src/spec/appendix/internal-constants.rst +++ b/docs/website/src/spec/appendix/internal-constants.rst @@ -127,6 +127,13 @@ wasm_stack_value_slots Value: ``65535`` +.. _gvm-def-consts-value-top-limits-vfs-path-components: + +vfs_path_components +~~~~~~~~~~~~~~~~~~~ + +Value: ``128`` + .. _gvm-def-consts-runner-limits: runner_limits diff --git a/docs/website/src/spec/changelog.rst b/docs/website/src/spec/changelog.rst index 2751adcb..7af5d709 100644 --- a/docs/website/src/spec/changelog.rst +++ b/docs/website/src/spec/changelog.rst @@ -13,6 +13,22 @@ v0.3 Breaking ~~~~~~~~ +#. The pre-finalization state is spelled *decided* everywhere it is named. + :ref:`gvm-def-enum-storage-view` reads ``latest_finalized`` (1) and + ``latest_decided`` (2) instead of ``latest_final`` and ``latest_non_final``, + the ``on`` field of the ``EmitInternalMessage`` and + ``EmitInternalDeployMessage`` ``gl_call`` + payloads (:doc:`02-execution-environment/03-wasi_genlayer_sdk/02-gl_call`) + takes ``"decided"`` instead of ``"accepted"``, and a ``chain:`` runner id + selects it with ``d`` rather than ``a`` (see + :ref:`gvm-def-chain-runner-state`). Numeric enum values are unchanged; none + of the old spellings is accepted +#. A ``chain:`` runner id resolved while a contract is being deployed + canonicalizes to ``i``, which previously spelled the deploy state as ``d`` +#. The ``When`` action's ``cond`` field spells the non-deterministic mode + ``!det``; the previous ``nondet`` spelling is rejected, with no + back-compat alias. See the ``When`` action in + :doc:`02-execution-environment/04-runners` #. ``runner.json`` rejects unknown top-level and nested fields; a runner that relied on an extra field being silently ignored fails to load. The single top-level ``$schema`` string annotation is still accepted. See diff --git a/executors/v0.2.x b/executors/v0.2.x index 9da582c9..2a32a840 160000 --- a/executors/v0.2.x +++ b/executors/v0.2.x @@ -1 +1 @@ -Subproject commit 9da582c9eb7b8f801cc00a4fd8a20f5dedd33da3 +Subproject commit 2a32a84066012eaa887a1e62d345a92e8b37c562 diff --git a/executors/v0.3.x b/executors/v0.3.x index 55b13368..79e89ad5 160000 --- a/executors/v0.3.x +++ b/executors/v0.3.x @@ -1 +1 @@ -Subproject commit 55b133688edd8950b8873aad9ce5c2f1040a8c98 +Subproject commit 79e89ad57188fd0c6ae8d939fc0056461d5d0d21 diff --git a/flake.nix b/flake.nix index dee0485b..de34f94d 100644 --- a/flake.nix +++ b/flake.nix @@ -525,7 +525,7 @@ done ''; - # Merge a { uid -> `<id>/<aa>/<rest>.tar` tree } set into one tree. + # Merge a { uid -> `<id>/<aa>/<rest>.zip` tree } set into one tree. merge-runner-trees = name: uni: pkgs.runCommand name { srcs = builtins.attrValues uni; } '' diff --git a/implementation/src/llm/handler.rs b/implementation/src/llm/handler.rs index 30b01c4a..76b843ab 100644 --- a/implementation/src/llm/handler.rs +++ b/implementation/src/llm/handler.rs @@ -89,7 +89,7 @@ impl crate::common::MessageHandler<llm_iface::Message, llm_iface::PromptAnswer> match message { llm_iface::Message::Prompt { payload, - remaining_fuel_as_gen, + remaining_time_fee_gen_wei, } => { if payload.images.len() > 2 { return Err(ModuleError { @@ -121,15 +121,15 @@ impl crate::common::MessageHandler<llm_iface::Message, llm_iface::PromptAnswer> } } self.0 - .exec_prompt(self.0.clone(), payload, remaining_fuel_as_gen) + .exec_prompt(self.0.clone(), payload, remaining_time_fee_gen_wei) .await } llm_iface::Message::PromptTemplate { payload, - remaining_fuel_as_gen, + remaining_time_fee_gen_wei, } => { self.0 - .exec_prompt_template(self.0.clone(), payload, remaining_fuel_as_gen) + .exec_prompt_template(self.0.clone(), payload, remaining_time_fee_gen_wei) .await } } @@ -208,7 +208,7 @@ impl Inner { &self, _zelf: Arc<Inner>, payload: llm_iface::PromptPayload, - remaining_fuel_as_gen: primitive_types::U256, + remaining_time_fee_gen_wei: primitive_types::U256, ) -> ModuleResult<llm_iface::PromptAnswer> { log_debug_into!(&LoggerWithId, payload:serde = payload, genvm_id:id = self.genvm_id.0; "exec_prompt start"); @@ -216,13 +216,13 @@ impl Inner { .user_vm .vm .to_value_with(&payload, scripting::DEFAULT_LUA_SER_OPTIONS)?; - let fuel = self.u256_to_lua_rat(remaining_fuel_as_gen)?; + let time_fee_gen_wei = self.u256_to_lua_rat(remaining_time_fee_gen_wei)?; let res: Result<mlua::Value, _> = self .user_vm .call_fn( &self.user_vm.data.exec_prompt, - (self.ctx_val.clone(), payload, fuel), + (self.ctx_val.clone(), payload, time_fee_gen_wei), ) .await; @@ -240,7 +240,7 @@ impl Inner { &self, _zelf: Arc<Inner>, payload: llm_iface::PromptTemplatePayload, - remaining_fuel_as_gen: primitive_types::U256, + remaining_time_fee_gen_wei: primitive_types::U256, ) -> ModuleResult<llm_iface::PromptAnswer> { log_debug_into!(&LoggerWithId, payload:serde = payload, genvm_id:id = self.genvm_id.0; "exec_prompt_template start"); @@ -248,13 +248,13 @@ impl Inner { .user_vm .vm .to_value_with(&payload, scripting::DEFAULT_LUA_SER_OPTIONS)?; - let fuel = self.u256_to_lua_rat(remaining_fuel_as_gen)?; + let time_fee_gen_wei = self.u256_to_lua_rat(remaining_time_fee_gen_wei)?; let res: Result<mlua::Value, _> = self .user_vm .call_fn( &self.user_vm.data.exec_prompt_template, - (self.ctx_val.clone(), payload, fuel), + (self.ctx_val.clone(), payload, time_fee_gen_wei), ) .await; diff --git a/implementation/src/manager/run.rs b/implementation/src/manager/run.rs index 90cd7399..de98dd67 100644 --- a/implementation/src/manager/run.rs +++ b/implementation/src/manager/run.rs @@ -1028,7 +1028,7 @@ pub struct Request { #[calldata(default = default_host_hello_data)] pub host_hello_data: Vec<bytes::Bytes>, /// Whether the host wants to be asked where a `CallContract` should run. - /// When false the manager answers `resolve_callcontract_executor` itself + /// When false the manager answers `resolve_call_contract_executor` itself /// with a null reply, so every call stays in-process and the host never /// has to implement that method. #[serde(default)] @@ -1509,7 +1509,7 @@ fn nested_effect(reported: &genvm_modules_interfaces::ReportedResult) -> Option< wasm_store_hashes: _, // A remaining budget is a report, not a consumption. data_fees_remaining: _, - storage_changes, + storage_deltas, emissions, nondet_disagreement, nondet_results, @@ -1521,11 +1521,11 @@ fn nested_effect(reported: &genvm_modules_interfaces::ReportedResult) -> Option< message_fee, event, }, - llm_consumption, + llm_consumed_gen_wei, } = reported; - if !storage_changes.is_empty() { - return Some("storage_changes"); + if !storage_deltas.is_empty() { + return Some("storage_deltas"); } if !emissions.is_empty() { return Some("emissions"); @@ -1536,8 +1536,8 @@ fn nested_effect(reported: &genvm_modules_interfaces::ReportedResult) -> Option< if !nondet_results.is_empty() { return Some("nondet_results"); } - if !llm_consumption.is_zero() { - return Some("llm_consumption"); + if !llm_consumed_gen_wei.is_zero() { + return Some("llm_consumed_gen_wei"); } for (bucket, name) in [ @@ -1678,23 +1678,23 @@ fn read_manager_host_stream( log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, error:ah = &e; "failed to send run_nested reply"); } } - Ok(host_fns::Methods::ResolveCallcontractExecutor) => { + Ok(host_fns::Methods::ResolveCallContractExecutor) => { // The do-nothing route: a request that did not opt into // `hook_cross_contract_calls` gets this arm instead of the // node's host, and a null reply means "stay in-process". let mut request = [0u8; calldata::ADDRESS_SIZE + 2]; if let Err(e) = reader.read_exact(&mut request).await { - log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, error:err = e; "failed to read resolve_callcontract_executor request"); + log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, error:err = e; "failed to read resolve_call_contract_executor request"); return; } let encoded = calldata::encode_obj(&calldata::Value::Null); let mut writer = writer.lock().await; if let Err(e) = writer.write_all(&[host_fns::Errors::Ok as u8]).await { - log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, error:err = e; "failed to send resolve_callcontract_executor status"); + log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, error:err = e; "failed to send resolve_call_contract_executor status"); return; } if let Err(e) = write_length_prefixed(&mut *writer, &encoded).await { - log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, error:ah = &e; "failed to send resolve_callcontract_executor reply"); + log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, error:ah = &e; "failed to send resolve_call_contract_executor reply"); return; } } @@ -1936,7 +1936,7 @@ fn execution_data_from_request(req: &Request) -> genvm_modules_interfaces::Execu method_hosts[host_fns::Methods::ConsumeResult as usize] = 1; method_hosts[host_fns::Methods::RunNested as usize] = 1; if !req.hook_cross_contract_calls { - method_hosts[host_fns::Methods::ResolveCallcontractExecutor as usize] = 1; + method_hosts[host_fns::Methods::ResolveCallContractExecutor as usize] = 1; } genvm_modules_interfaces::ExecutionData { @@ -2157,7 +2157,7 @@ impl Ctx { max_execution_minutes: parent_req.max_execution_minutes, bucket_totals: Vec::new(), host_data: parent_req.host_data.clone(), - timestamp: envelope.message.datetime, + timestamp: envelope.message.transaction_timestamp, host: parent_req.host.clone(), extra_args: Vec::new(), calldata: envelope.calldata.clone(), @@ -2434,7 +2434,7 @@ async fn run_genvm_process( let mut proc = build_genvm_command(command_path, &req, genvm_id, &write_fd); // Setup manager host socketpair (host id=1 for consume_result, run_nested - // and -- unless the request hooks them -- resolve_callcontract_executor) + // and -- unless the request hooks them -- resolve_call_contract_executor) let (manager_parent, manager_child) = FdWrapper::socketpair()?; manager_parent.set_cloexec(true)?; manager_child.set_cloexec(false)?; diff --git a/implementation/src/manager/run_test.rs b/implementation/src/manager/run_test.rs index 098d36b9..7f8ffae1 100644 --- a/implementation/src/manager/run_test.rs +++ b/implementation/src/manager/run_test.rs @@ -550,13 +550,13 @@ fn clean_reported() -> genvm_modules_interfaces::ReportedResult { data: calldata::Value::Null.into(), backtrace: None, wasm_store_hashes: Default::default(), - storage_changes: Vec::new(), + storage_deltas: Vec::new(), emissions: Vec::new(), nondet_disagreement: None, nondet_results: Vec::new(), data_fees_remaining: Vec::new(), data_fees_consumed: Default::default(), - llm_consumption: primitive_types::U256::zero(), + llm_consumed_gen_wei: primitive_types::U256::zero(), } } @@ -640,7 +640,7 @@ fn some_storage_delta() -> genvm_modules_interfaces::StorageDelta { } fn some_emission() -> genvm_modules_interfaces::ExecutionEmission { - genvm_modules_interfaces::ExecutionEmission::EmitEvent { + genvm_modules_interfaces::ExecutionEmission::Event { topics: Vec::new(), blob: calldata::Map::new().into(), storage_fee: primitive_types::U256::zero(), @@ -660,16 +660,16 @@ fn nested_reply_refuses_every_reported_effect() { // end to end -- the boundary derivation clears the permission behind each of // them -- so this is the only place the refusal is exercised. let mutate: [(&str, fn(&mut genvm_modules_interfaces::ReportedResult)); 10] = [ - ("storage_changes", |r| { - r.storage_changes.push(some_storage_delta()) + ("storage_deltas", |r| { + r.storage_deltas.push(some_storage_delta()) }), ("emissions", |r| r.emissions.push(some_emission())), ("nondet_disagreement", |r| r.nondet_disagreement = Some(0)), ("nondet_results", |r| { r.nondet_results.push(bytes::Bytes::from_static(&[1])) }), - ("llm_consumption", |r| { - r.llm_consumption = primitive_types::U256::one() + ("llm_consumed_gen_wei", |r| { + r.llm_consumed_gen_wei = primitive_types::U256::one() }), ("storage", |r| { r.data_fees_consumed.storage = primitive_types::U256::one() @@ -892,7 +892,7 @@ fn message_schema_matches_the_rust_struct() { chain_id, value, is_init, - datetime, + transaction_timestamp, } = message; }; @@ -904,7 +904,7 @@ fn message_schema_matches_the_rust_struct() { "chain_id", "value", "is_init", - "datetime", + "transaction_timestamp", ] .map(str::to_owned) .to_vec(); diff --git a/runners/views/universal.nix b/runners/views/universal.nix index 22f09e29..edf49ae8 100644 --- a/runners/views/universal.nix +++ b/runners/views/universal.nix @@ -1,4 +1,4 @@ -# Builds a runner list into a `runners/<id>/<aa>/<rest>.tar` tree, one derivation +# Builds a runner list into a `runners/<id>/<aa>/<rest>.zip` tree, one derivation # per uid (so identical runners shared across executor lines de-duplicate). { pkgs, @@ -24,9 +24,9 @@ builtins.foldl' ( throw "invalid runner uid `${x.uid}`; expected `<prefix>:<hash32>`" else builtins.elemAt uidMatch 1; - # `<id>/<aa>/<rest>.tar` β€” consumers place this tree *under* their own + # `<id>/<aa>/<rest>.zip` β€” consumers place this tree *under* their own # `runners/` dir, so do NOT prefix another `runners/` here. - result-path = "${x.id}/${builtins.substring 0 2 hash32}/${builtins.substring 2 50 hash32}.tar"; + result-path = "${x.id}/${builtins.substring 0 2 hash32}/${builtins.substring 2 50 hash32}.zip"; in '' mkdir -p $out/$(dirname -- ${result-path}) diff --git a/support/tools/genvm-tool/genvm_tool/codegen/go.py b/support/tools/genvm-tool/genvm_tool/codegen/go.py index 2561fde7..5feab6f2 100644 --- a/support/tools/genvm-tool/genvm_tool/codegen/go.py +++ b/support/tools/genvm-tool/genvm_tool/codegen/go.py @@ -42,6 +42,11 @@ def _trie_inner(node: TrieNode, root_camel: str, buf: list[str]) -> None: # Walk children in source order: unlike the struct-based backends, go emits a # flat function per terminal/param exactly where it appears in the JSON. prefix = root_camel + node.suffix + for detail in node.details: + val = f'{" ".join(node.parts)} # {detail}' + buf.append( + f'func {prefix}{to_camel(detail)}() {root_camel} {{ return {_dump(val)} }}\n' + ) for kind, head, payload in node.order: if kind == 'leaf': val = ' '.join(payload) @@ -74,10 +79,7 @@ def render(defs: list[Definition], *, go_package: str = 'genvm', **_opts) -> str buf.append(f'package {go_package}\n') needs_fmt = any(isinstance(d, StrTrie) and _has_param(d.root) for d in defs) - needs_strings = any(isinstance(d, StrTrie) and d.suffix is not None for d in defs) - imports = [ - n for n, needed in (('fmt', needs_fmt), ('strings', needs_strings)) if needed - ] + imports = [n for n, needed in (('fmt', needs_fmt),) if needed] if len(imports) == 1: buf.append(f'\nimport "{imports[0]}"\n') elif imports: @@ -109,14 +111,6 @@ def render(defs: list[Definition], *, go_package: str = 'genvm', **_opts) -> str root_camel = to_camel(d.name) buf.append(f'\ntype {root_camel} string\n\n') _trie_inner(d.root, root_camel, buf) - for name, _parts in d.suffix.leaves if d.suffix else []: - # Mirrors the rust/python guard: a value carries at most one detail. - buf.append( - f'func (v {root_camel}) {to_camel(name)}() {root_camel} {{ ' - f'if strings.Contains(string(v), " # ") {{ ' - f'panic("a value carries at most one detail") }}; ' - f'return v + {_dump(f" # {name}")} }}\n' - ) return ''.join(buf) diff --git a/support/tools/genvm-tool/genvm_tool/codegen/model.py b/support/tools/genvm-tool/genvm_tool/codegen/model.py index e9f319c0..80f9ae63 100644 --- a/support/tools/genvm-tool/genvm_tool/codegen/model.py +++ b/support/tools/genvm-tool/genvm_tool/codegen/model.py @@ -56,7 +56,8 @@ class TrieNode: ``terminal`` is set does ``' '.join(parts)`` name a leaf. A node is either a *param* node (``param`` set: consumes one typed argument) or a regular node carrying ``leaves`` (terminal strings) and ``methods`` (child nodes), mirroring - the original ruby generators. + the original ruby generators. ``details`` contains the ``#`` child's terminal + values when that child exists. """ suffix: str @@ -65,6 +66,7 @@ class TrieNode: param: tuple[str, list[str]] | None leaves: list[tuple[str, list[str]]] methods: list[tuple[str, 'TrieNode']] + details: list[str] # Children in original JSON order, tagged ``('leaf', head, parts)`` or # ``('method', head, child)``. ``leaves``/``methods`` are the grouped views the # struct-based backends (rust, python) want; the go backend, which emits flat @@ -78,10 +80,6 @@ class StrTrie: root: TrieNode entries: list # unfolded entries, kept for flat path enumeration (rst) docs: dict[str, str] - # Details reachable from any built value, spelled ``<path> # <detail>``. A - # root-level ``#`` entry declares them; they are not paths, so they stay out - # of ``root``/``entries`` and out of the generated validity check. - suffix: TrieNode | None = None Definition = Enum | Const | Consts | StrTrie @@ -117,8 +115,14 @@ def _build(entries, prefix_parts: list[str], suffix: str, terminal: bool) -> Tri param=None, leaves=[], methods=[], + details=[], order=[], ) + entries, node.details = _split_details(entries) + if node.details and not terminal: + raise ValueError( + f'{" ".join(prefix_parts) or "root"}: details require a public path' + ) for entry in entries: if entry is None: continue @@ -135,15 +139,17 @@ def _build(entries, prefix_parts: list[str], suffix: str, terminal: bool) -> Tri param=(param_type, current), leaves=[], methods=[], + details=[], order=[], ) node.methods.append((head, child)) node.order.append(('method', head, child)) elif isinstance(tail, list): non_null = [e for e in tail if e is not None] + path_children = [e for e in non_null if e['head'] != DETAIL_HEAD] # A trie level is terminal if any sibling was a `null` placeholder (or it - # has no real children): the path itself is a valid leaf string. - is_terminal = (len(tail) != len(non_null)) or len(non_null) == 0 + # has no path children): the path itself is a valid leaf string. + is_terminal = (len(tail) != len(non_null)) or len(path_children) == 0 if len(non_null) == 0: node.leaves.append((head, current)) node.order.append(('leaf', head, current)) @@ -161,40 +167,58 @@ def enumerate_paths(entries, prefix: str = ''): if entry is None: continue head = entry['head'] + if head == DETAIL_HEAD: + continue tail = entry['tail'] current = head if prefix == '' else f'{prefix} {head}' if isinstance(tail, str) and tail.startswith('$'): result.append((current, tail[1:])) elif isinstance(tail, list): non_null = [e for e in tail if e is not None] - is_terminal = (len(tail) != len(non_null)) or len(non_null) == 0 + path_children = [e for e in non_null if e['head'] != DETAIL_HEAD] + is_terminal = (len(tail) != len(non_null)) or len(path_children) == 0 if is_terminal: result.append((current, None)) result.extend(enumerate_paths(non_null, current)) return result +def enumerate_details(entries, prefix: str = ''): + """Flatten detail paths to ``<public path> # <detail>`` strings.""" + result = [] + for entry in entries: + if entry is None or entry['head'] == DETAIL_HEAD: + continue + head = entry['head'] + tail = entry['tail'] + current = head if prefix == '' else f'{prefix} {head}' + if not isinstance(tail, list): + continue + paths, details = _split_details(tail) + result.extend(f'{current} # {detail}' for detail in details) + result.extend(enumerate_details(paths, current)) + return result + + DETAIL_HEAD = '#' -"""Head of the root entry declaring a trie's detail suffixes.""" +"""Head of an entry declaring details for its parent public path.""" -def _split_details(name: str, entries: list) -> tuple[list, TrieNode | None]: - """Carve the ``#`` entry out of a trie's root, returning ``(paths, details)``.""" +def _split_details(entries: list) -> tuple[list, list[str]]: + """Carve a ``#`` child out of one trie node.""" found = [e for e in entries if e is not None and e['head'] == DETAIL_HEAD] if not found: - return entries, None + return entries, [] if len(found) > 1: - raise ValueError(f'{name}: more than one {DETAIL_HEAD!r} entry') + raise ValueError(f'more than one {DETAIL_HEAD!r} entry') tail = found[0]['tail'] if not isinstance(tail, list): - raise ValueError(f'{name}: {DETAIL_HEAD!r} must list details') + raise ValueError(f'{DETAIL_HEAD!r} must list details') for detail in tail: - # A detail is appended to an already-built value, so it has no place to - # hang a subtree off and no argument to render. if detail is None or detail['tail'] != []: - raise ValueError(f'{name}: a detail may not be nested or take a parameter') + raise ValueError('a detail may not be nested or take a parameter') paths = [e for e in entries if e is None or e['head'] != DETAIL_HEAD] - return paths, _build(tail, [], '', False) + return paths, [detail['head'] for detail in tail] def parse(data) -> list[Definition]: @@ -208,14 +232,13 @@ def parse(data) -> list[Definition]: elif kind == 'consts': defs.append(Consts(t['name'], t['repr'], t['values'])) elif kind == 'str_trie': - entries, details = _split_details(t['name'], [_unfold(e) for e in t['values']]) + entries = [_unfold(e) for e in t['values']] defs.append( StrTrie( t['name'], _build(entries, [], '', False), entries, t.get('docs', {}), - details, ) ) else: diff --git a/support/tools/genvm-tool/genvm_tool/codegen/python.py b/support/tools/genvm-tool/genvm_tool/codegen/python.py index 122ad849..548d3ab7 100644 --- a/support/tools/genvm-tool/genvm_tool/codegen/python.py +++ b/support/tools/genvm-tool/genvm_tool/codegen/python.py @@ -34,6 +34,11 @@ def _emit_struct(node: TrieNode, root_name: str, buf: list[str]) -> None: buf.append('\t@staticmethod\n') buf.append(f"\tdef val() -> '{root_name}':\n") buf.append(f"\t\treturn {root_name}('{val}')\n") + for name in node.details: + val = ' '.join(node.parts) + buf.append('\t@staticmethod\n') + buf.append(f"\tdef {name.lower()}() -> '{root_name}':\n") + buf.append(f"\t\treturn {root_name}('{val} # {name}')\n") for name, parts in node.leaves: val = ' '.join(parts) buf.append('\t@staticmethod\n') @@ -84,11 +89,6 @@ def _trie(t: StrTrie, buf: list[str]) -> None: buf.append('\t@staticmethod\n') buf.append(f"\tdef {head.lower()}() -> '_{root_name}{child.suffix}':\n") buf.append(f'\t\treturn _{root_name}{child.suffix}()\n') - if t.suffix is not None: - for name, _parts in t.suffix.leaves: - buf.append(f"\tdef {name.lower()}(self) -> '{root_name}':\n") - buf.append("\t\tassert ' # ' not in self.value\n") - buf.append(f"\t\treturn {root_name}(f'{{self.value}} # {name}')\n") buf.append('\n') diff --git a/support/tools/genvm-tool/genvm_tool/codegen/rst.py b/support/tools/genvm-tool/genvm_tool/codegen/rst.py index 6b495b71..9a380daa 100644 --- a/support/tools/genvm-tool/genvm_tool/codegen/rst.py +++ b/support/tools/genvm-tool/genvm_tool/codegen/rst.py @@ -67,16 +67,13 @@ def render(defs: list[Definition], rst_anchor_ns: str = '', **_opts) -> str: buf.append(f'Param: {param}\n\n') if path in d.docs: buf.append(d.docs[path].rstrip() + '\n\n') - if d.suffix is not None: - for name, _parts in d.suffix.leaves: - path = f'{model.DETAIL_HEAD} {name}' - buf.append( - f'.. _gvm-def-{ns}str-trie-detail-{_dash(d.name)}-{_dash(name)}:\n\n' - ) - buf.append(f'``{path}``\n') - buf.append('~' * (len(path) + 4) + '\n\n') - if path in d.docs: - buf.append(d.docs[path].rstrip() + '\n\n') + for path in model.enumerate_details(d.entries): + rst_name = path.replace(' # ', '-').replace('_', '-').replace(' ', '-') + buf.append(f'.. _gvm-def-{ns}str-trie-detail-{_dash(d.name)}-{rst_name}:\n\n') + buf.append(f'``{path}``\n') + buf.append('~' * (len(path) + 4) + '\n\n') + if path in d.docs: + buf.append(d.docs[path].rstrip() + '\n\n') # Sections are emitted blank-line-separated, so the last one leaves a trailing # blank line that `end-of-file-fixer` would strip β€” permanent codegen drift. return ''.join(buf).rstrip('\n') + '\n' diff --git a/support/tools/genvm-tool/genvm_tool/codegen/rust.py b/support/tools/genvm-tool/genvm_tool/codegen/rust.py index 649e4064..c0910e47 100644 --- a/support/tools/genvm-tool/genvm_tool/codegen/rust.py +++ b/support/tools/genvm-tool/genvm_tool/codegen/rust.py @@ -122,6 +122,12 @@ def _emit_struct(node: TrieNode, root_name: str, buf: list[str]) -> None: f' pub const fn val(&self) -> {root_name} {{ ' f'{root_name}(Cow::Borrowed("{val}")) }}\n' ) + for name in node.details: + val = ' '.join(node.parts) + buf.append( + f' pub const fn {name.lower()}(&self) -> {root_name} {{ ' + f'{root_name}(Cow::Borrowed("{val} # {name}")) }}\n' + ) buf.append(" pub const fn prefix_(&self) -> &'static str {\n") buf.append(f' "{" ".join(node.parts)}"\n') buf.append(' }\n') @@ -208,21 +214,6 @@ def _trie(t: StrTrie, buf: list[str]) -> None: ) buf.append('}\n\n') - if t.suffix is not None: - buf.append('#[rustfmt::skip]\n') - buf.append(f'impl {root_name} {{\n') - for name, _parts in t.suffix.leaves: - # A value carries at most one detail: the reader splits on the first - # separator, so a second one would be silently dropped. Checked in - # release too, so the invariant does not depend on the build -- these - # are error paths, and the scan is over a short fixed string. - buf.append( - f' pub fn {name.lower()}(self) -> Self {{ ' - f'assert!(!self.0.contains(" # "), "a value carries at most one detail"); ' - f'Self(Cow::Owned(format!("{{}} # {name}", self.0))) }}\n' - ) - buf.append('}\n\n') - _emit_is_valid(t, root_name, buf) diff --git a/support/tools/genvm-tool/unit_tests/test_codegen_detail_suffix.py b/support/tools/genvm-tool/unit_tests/test_codegen_detail_suffix.py index b0951ea0..6da77842 100644 --- a/support/tools/genvm-tool/unit_tests/test_codegen_detail_suffix.py +++ b/support/tools/genvm-tool/unit_tests/test_codegen_detail_suffix.py @@ -1,4 +1,4 @@ -"""A ``#`` trie entry becomes a detail suffix on the built value, not a path.""" +"""A ``#`` trie child declares details for its parent public path.""" import pytest from genvm_tool.codegen import go, model, python, rst, rust @@ -9,8 +9,15 @@ 'name': 'vm_error', 'values': [ 'timeout', - {'head': 'out_of', 'tail': ['memory']}, - {'head': '#', 'tail': ['internal', 'external']}, + { + 'head': 'out_of', + 'tail': [ + { + 'head': 'memory', + 'tail': [{'head': '#', 'tail': ['internal', 'external']}], + } + ], + }, ], } ] @@ -21,35 +28,31 @@ def defs(): return model.parse(DATA) -def test_suffix_is_kept_out_of_the_path_trie(defs): +def test_detail_is_scoped_to_its_parent_and_kept_out_of_paths(defs): (trie,) = defs assert [head for head, _ in trie.root.methods] == ['out_of'] assert [path for path, _ in model.enumerate_paths(trie.entries)] == [ 'timeout', 'out_of memory', ] - assert [name for name, _ in trie.suffix.leaves] == ['internal', 'external'] + memory = trie.root.methods[0][1].methods[0][1] + assert memory.details == ['internal', 'external'] -def test_rust_emits_detail_methods_on_the_value(defs): +def test_rust_emits_detail_methods_on_the_parent_builder(defs): out = rust.render(defs) for name in ('internal', 'external'): assert ( - f'pub fn {name}(self) -> Self ' - '{ assert!(!self.0.contains(" # "), "a value carries at most one detail"); ' - f'Self(Cow::Owned(format!("{{}} # {name}", self.0))) }}' in out + f'pub const fn {name}(&self) -> VmError ' + f'{{ VmError(Cow::Borrowed("out_of memory # {name}")) }}' in out ) -def test_every_backend_refuses_to_stack_two_details(defs): - assert out_has_guard(rust.render(defs)) - assert out_has_guard(python.render(defs)) - assert out_has_guard(go.render(defs)) - assert 'strings' in go.render(defs) - - -def out_has_guard(out: str) -> bool: - return out.count(' # ') >= 2 and 'at most one detail' in out or "' # ' not in" in out +def test_details_are_not_methods_on_the_built_value(defs): + rust_out = rust.render(defs).split('pub struct VmError')[1] + python_out = python.render(defs).split('class VmError:')[1] + assert 'fn internal' not in rust_out + assert 'def internal' not in python_out def test_a_detail_with_a_non_list_tail_is_rejected(): @@ -57,7 +60,12 @@ def test_a_detail_with_a_non_list_tail_is_rejected(): { 'type': 'str_trie', 'name': 'vm_error', - 'values': [{'head': '#', 'tail': [{'head': 'internal', 'tail': '$str'}]}], + 'values': [ + { + 'head': 'timeout', + 'tail': [{'head': '#', 'tail': [{'head': 'internal', 'tail': '$str'}]}], + } + ], } ] with pytest.raises(ValueError, match='nested'): @@ -70,25 +78,50 @@ def test_rust_is_valid_ignores_details(defs): assert '# internal' not in out.split('pub fn is_valid_')[1] -def test_python_emits_detail_methods_on_the_value(defs): +def test_python_emits_detail_methods_on_the_parent_builder(defs): out = python.render(defs) - assert "\tdef internal(self) -> 'VmError':\n" in out - assert "\t\treturn VmError(f'{self.value} # internal')\n" in out + assert "\tdef internal() -> 'VmError':\n" in out + assert "\t\treturn VmError('out_of memory # internal')\n" in out + + +def test_go_emits_detail_functions_for_the_parent_path(defs): + out = go.render(defs) + assert ( + 'func VmErrorOutOfMemoryInternal() VmError ' + '{ return "out_of memory # internal" }' in out + ) def test_rst_documents_details_under_the_trie(defs): out = rst.render(defs) - assert '``# internal``' in out - assert '``# external``' in out + assert '``out_of memory # internal``' in out + assert '``out_of memory # external``' in out -def test_a_nested_detail_is_rejected(): +def test_a_detail_may_not_have_children(): data = [ { 'type': 'str_trie', 'name': 'vm_error', - 'values': [{'head': '#', 'tail': [{'head': 'a', 'tail': ['b']}]}], + 'values': [ + { + 'head': 'timeout', + 'tail': [{'head': '#', 'tail': [{'head': 'a', 'tail': ['b']}]}], + } + ], } ] with pytest.raises(ValueError, match='nested'): model.parse(data) + + +def test_a_root_detail_is_rejected(): + data = [ + { + 'type': 'str_trie', + 'name': 'vm_error', + 'values': [{'head': '#', 'tail': ['internal']}], + } + ] + with pytest.raises(ValueError, match='public path'): + model.parse(data) diff --git a/tests/runner/genvm_tool_plugins/afl.py b/tests/runner/genvm_tool_plugins/afl.py index 10ded7ae..4a36f5f0 100644 --- a/tests/runner/genvm_tool_plugins/afl.py +++ b/tests/runner/genvm_tool_plugins/afl.py @@ -5,6 +5,7 @@ import functools import math import os +import re import shutil import signal import tempfile @@ -42,6 +43,14 @@ def _stop_live_fuzzers() -> None: 'AFL_TRY_AFFINITY', ) +# Ceiling for `-t`, whose auto-scaling stays below it. Chosen against the slowest +# committed seed rather than against the mean: what it has to survive is one +# entry of an existing corpus, on a core slower than the one that harvested it +EXEC_TIMEOUT_MS = 5000 + +# How AFL reports a seed it refused to fuzz because the target died on it +SEED_CRASH_RE = re.compile(r"Test case '([^']*)' results in a crash") + def register(ctx: genvm_tool.tests.stage.configuration.Context) -> None: ctx.run_parser.add_argument( @@ -151,6 +160,25 @@ def environment(base: dict[str, str]) -> dict[str, str]: return env +def crashing_seeds(out_dir: Path) -> list[str]: + """ + Committed inputs an instance found to crash the target while starting up. + + `-t` makes AFL skip a problematic seed instead of aborting, which is what a + corpus that only grows needs for a slow entry β€” but a crashing one is a + finding, and skipped it would be reported nowhere. Read back out of the logs + rather than prevented, since the two share the one switch. + """ + found: list[str] = [] + for log in sorted(out_dir.glob('*.log')): + for match in SEED_CRASH_RE.finditer(log.read_text(errors='replace')): + name = match.group(1) + _, _, original = name.rpartition('orig:') + if (original or name) not in found: + found.append(original or name) + return found + + def output_dir( shared: genvm_tool.tests.SharedContext, project_dir: Path, name: str ) -> Path: @@ -236,7 +264,6 @@ def fleet_steps( out_dir: Path, launcher: Command, target: Command, - extra_args: Command = (), wrap: Wrapper = lambda argv: argv, prepare_command: Command | None = None, status_command: Command | None = None, @@ -268,6 +295,12 @@ def command(role: str, instance: str) -> Command: # these targets, see the fuzzing explanation page '-c', '-', + # A ceiling on the auto-scaled per-exec timeout. AFL's own default + # is 1000ms, and a corpus entry above it aborts the whole run + # rather than being skipped; python targets run traced and a slow + # CI core is enough to cross it + '-t', + f'{EXEC_TIMEOUT_MS}+', role, instance, '-i', @@ -276,7 +309,6 @@ def command(role: str, instance: str) -> Command: out_dir, '-V', str(timeout(ctx)), - *extra_args, *target, ] ) @@ -402,6 +434,9 @@ async def report_findings( crashes = [crash for path in crash_dirs() for crash in saved_findings(path)] context = dict(result.context) + bad_seeds = crashing_seeds(out_dir) + if bad_seeds: + context['crashing_seeds'] = bad_seeds if crashes: context['crashes'] = [str(crash) for crash in crashes] if replay is not None: @@ -432,7 +467,7 @@ async def report_findings( context['fleet'] = f'{status_command[0]}: {error}' return genvm_tool.tests.test.Result( - passed=result.passed and not crashes, + passed=result.passed and not crashes and not bad_seeds, context=context, elapsed_seconds=result.elapsed_seconds, ) diff --git a/tests/runner/genvm_tool_plugins/cargo.py b/tests/runner/genvm_tool_plugins/cargo.py index 35884bf1..8e291806 100644 --- a/tests/runner/genvm_tool_plugins/cargo.py +++ b/tests/runner/genvm_tool_plugins/cargo.py @@ -611,7 +611,6 @@ def cargo_fuzz( out_dir=out_dir, launcher=['cargo-afl', 'afl', 'fuzz'], target=[fuzz_binary], - extra_args=['-t', '5000'], status_command=['cargo-afl', 'afl', 'whatsup', '-s', out_dir], # A persistent-mode target expects a forkserver, so replaying an input # means going through AFL rather than piping it into the binary. The diff --git a/tests/runner/genvm_tool_plugins/integration.py b/tests/runner/genvm_tool_plugins/integration.py index 20335dab..8c6b3745 100644 --- a/tests/runner/genvm_tool_plugins/integration.py +++ b/tests/runner/genvm_tool_plugins/integration.py @@ -720,7 +720,7 @@ def resolve_executor(address, state, advisory_major): balances={Address(k): v for k, v in single_conf.get('balances', {}).items()}, running_address=running_address, ctx=ctx, - resolve_callcontract_executor_hook=resolve_executor, + resolve_call_contract_executor_hook=resolve_executor, ) host.balances.setdefault(running_address, 0) @@ -756,7 +756,7 @@ def resolve_executor(address, state, advisory_major): default_message_fee_allocation = [ fees.DEFAULT_EXTERNAL_MESSAGE_ALLOC, fees.DEFAULT_INTERNAL_FIN_MESSAGE_ALLOC, - fees.DEFAULT_INTERNAL_ACC_MESSAGE_ALLOC, + fees.DEFAULT_INTERNAL_DEC_MESSAGE_ALLOC, ] message_fee_allocation: list[fees.MessageAllocationNode] = single_conf.get( @@ -826,7 +826,7 @@ def resolve_executor(address, state, advisory_major): f'nondet disagreement: {mock_host.nondet_disagreement_call_no}\n' ) - for k, v in res.result_storage_changes: + for k, v in res.result_storage_deltas: assert len(k) == 36 index = int.from_bytes(k[32:], byteorder='big') index *= 32 diff --git a/tests/runner/gvm_extra/mock_host.py b/tests/runner/gvm_extra/mock_host.py index b19b7110..26dc21d4 100644 --- a/tests/runner/gvm_extra/mock_host.py +++ b/tests/runner/gvm_extra/mock_host.py @@ -15,8 +15,8 @@ ) from origin.calldata import Address -type ResolveCallcontractExecutorHook = collections.abc.Callable[ - [Address, public_abi.StorageType, int], +type ResolveCallContractExecutorHook = collections.abc.Callable[ + [Address, public_abi.StorageView, int], bytes | None, ] @@ -87,7 +87,7 @@ def __init__( running_address: Address, ctx: Context, expected_hello_data: bytes = b'', - resolve_callcontract_executor_hook: ResolveCallcontractExecutorHook | None = None, + resolve_call_contract_executor_hook: ResolveCallContractExecutorHook | None = None, ): self.running_address = running_address self.path = path @@ -100,7 +100,7 @@ def __init__( self.balances = balances self.nondet_disagreement_call_no = None self.expected_hello_data = expected_hello_data - self.resolve_callcontract_executor_hook = resolve_callcontract_executor_hook + self.resolve_call_contract_executor_hook = resolve_call_contract_executor_hook self.ctx = ctx self._accept_task: asyncio.Task | None = None self._connection_tasks: list[asyncio.Task] = [] @@ -247,34 +247,39 @@ async def stop_connections(self) -> None: task.result() async def storage_read( - self, mode: public_abi.StorageType, account: bytes, slot: bytes, index: int, le: int + self, + mode: public_abi.StorageView, + address: bytes, + slot: bytes, + offset: int, + le: int, ) -> bytes: assert self.storage is not None - return self.storage.read(Address(account), slot, index, le) + return self.storage.read(Address(address), slot, offset, le) - async def resolve_callcontract_executor( + async def resolve_call_contract_executor( self, contract_address: Address, - state_mode: public_abi.StorageType, + state_mode: public_abi.StorageView, advisory_major: int, /, ) -> bytes | None: - if self.resolve_callcontract_executor_hook is None: + if self.resolve_call_contract_executor_hook is None: return None - return self.resolve_callcontract_executor_hook( + return self.resolve_call_contract_executor_hook( contract_address, state_mode, advisory_major, ) - async def remaining_fuel_as_gen(self) -> int: + async def get_remaining_time_fee_gen_wei(self) -> int: return 2**32 - async def eth_call(self, account: bytes, calldata: bytes, /) -> bytes: + async def external_call(self, address: bytes, calldata: bytes, /) -> bytes: raise HostException(host_fns.Errors.EVM_REVERTED) - async def consume_gas(self, gas: int): + async def consume_time_fee_gen_wei(self, time_fee_gen_wei: int): pass - async def get_balance(self, account: bytes) -> int: - return self.balances.get(Address(account), 0) + async def get_balance_gen_wei(self, address: bytes) -> int: + return self.balances.get(Address(address), 0) diff --git a/tests/runner/origin/base_host.py b/tests/runner/origin/base_host.py index 23debcd5..6817a38c 100644 --- a/tests/runner/origin/base_host.py +++ b/tests/runner/origin/base_host.py @@ -128,7 +128,7 @@ class Message(typing.TypedDict): chain_id: int value: typing.NotRequired[int] is_init: bool - datetime: typing.NotRequired[str] + transaction_timestamp: typing.NotRequired[str] class FingerprintFrame(typing.TypedDict): @@ -141,8 +141,8 @@ class ResultFingerprint(typing.TypedDict): module_instances: dict[str, typing.Any] -class EthSendInner(typing.TypedDict): - type: typing.Literal['EthSend'] +class ExternalMessageInner(typing.TypedDict): + type: typing.Literal['ExternalMessage'] address: Address calldata: bytes value: int @@ -151,12 +151,12 @@ class EthSendInner(typing.TypedDict): fee_params: fees.ExternalMessageParams -class PostMessageInner(typing.TypedDict): - type: typing.Literal['PostMessage'] +class InternalMessageInner(typing.TypedDict): + type: typing.Literal['InternalMessage'] address: Address calldata: gvm_calldata.Decoded value: int - on: typing.Literal['finalized', 'accepted'] + on: typing.Literal['finalized', 'decided'] message_fee: int receipt_fee: int fee_params: fees.InternalMessageParams @@ -166,12 +166,12 @@ class PostMessageInner(typing.TypedDict): use_balance: bool -class DeployContractInner(typing.TypedDict): - type: typing.Literal['DeployContract'] +class InternalDeployMessageInner(typing.TypedDict): + type: typing.Literal['InternalDeployMessage'] calldata: gvm_calldata.Decoded code: bytes value: int - on: typing.Literal['finalized', 'accepted'] + on: typing.Literal['finalized', 'decided'] salt_nonce: int message_fee: int receipt_fee: int @@ -182,18 +182,18 @@ class DeployContractInner(typing.TypedDict): use_balance: bool -class EmitEventInner(typing.TypedDict): - type: typing.Literal['EmitEvent'] +class EventInner(typing.TypedDict): + type: typing.Literal['Event'] topics: list[bytes] blob: dict[str, gvm_calldata.Decoded] storage_fee: int type ResultEmission = typing.Union[ - EthSendInner, - PostMessageInner, - DeployContractInner, - EmitEventInner, + ExternalMessageInner, + InternalMessageInner, + InternalDeployMessageInner, + EventInner, ] @@ -204,31 +204,31 @@ async def loop_enter(self, cancellation: asyncio.Event) -> socket.socket: ... @abc.abstractmethod async def storage_read( self, - mode: public_abi.StorageType, - account: bytes, + mode: public_abi.StorageView, + address: bytes, slot: bytes, - index: int, + offset: int, le: int, /, ) -> bytes: ... - async def resolve_callcontract_executor( + async def resolve_call_contract_executor( self, contract_address: Address, - state_mode: public_abi.StorageType, + state_mode: public_abi.StorageView, advisory_major: int, /, ) -> bytes | None: return None @abc.abstractmethod - async def consume_gas(self, gas: int, /) -> None: ... + async def consume_time_fee_gen_wei(self, time_fee_gen_wei: int, /) -> None: ... @abc.abstractmethod - async def eth_call(self, account: bytes, calldata: bytes, /) -> bytes: ... + async def external_call(self, address: bytes, calldata: bytes, /) -> bytes: ... @abc.abstractmethod - async def get_balance(self, account: bytes, /) -> int: ... + async def get_balance_gen_wei(self, address: bytes, /) -> int: ... @abc.abstractmethod - async def remaining_fuel_as_gen(self, /) -> int: ... + async def get_remaining_time_fee_gen_wei(self, /) -> int: ... @abc.abstractmethod async def notify_nondet_disagreement(self, call_no: int, /) -> None: ... @@ -249,7 +249,7 @@ async def read_contract_major(handler: IHost, message: Message) -> int: if message.get('is_init', False): return UNDEPLOYED_MAJOR octet = await handler.storage_read( - public_abi.StorageType.LATEST_NON_FINAL, + public_abi.StorageView.LATEST_DECIDED, message['contract_address'].as_bytes, ZERO_SLOT, ROOT_OFFSET_MAJOR, @@ -401,26 +401,26 @@ def emit_host_loop_stats(): match meth_id: case host_fns.Methods.STORAGE_READ: mode = await read_exact(1) - mode = public_abi.StorageType(mode[0]) - account = await read_exact(ACCOUNT_ADDR_SIZE) + mode = public_abi.StorageView(mode[0]) + address = await read_exact(ACCOUNT_ADDR_SIZE) slot = await read_exact(SLOT_ID_SIZE) - index = await recv_int() + offset = await recv_int() le = await recv_int() try: - res = await handler.storage_read(mode, account, slot, index, le) + res = await handler.storage_read(mode, address, slot, offset, le) assert len(res) == le except HostException as e: await send_all(bytes([e.error_code])) else: await send_all(bytes([host_fns.Errors.OK])) await send_all(res) - case host_fns.Methods.RESOLVE_CALLCONTRACT_EXECUTOR: + case host_fns.Methods.RESOLVE_CALL_CONTRACT_EXECUTOR: contract_address = Address(await read_exact(ACCOUNT_ADDR_SIZE)) - state_mode = public_abi.StorageType((await read_exact(1))[0]) + state_mode = public_abi.StorageView((await read_exact(1))[0]) advisory_major = await recv_int(1) try: - res = await handler.resolve_callcontract_executor( + res = await handler.resolve_call_contract_executor( contract_address, state_mode, advisory_major, @@ -436,39 +436,41 @@ def emit_host_loop_stats(): raise Exception( 'CONSUME_RESULT is not supported in this host loop implementation, use manager provided one' ) - case host_fns.Methods.CONSUME_FUEL: - gas = await recv_int(32) - await handler.consume_gas(gas) - case host_fns.Methods.ETH_CALL: - account = await read_exact(ACCOUNT_ADDR_SIZE) + case host_fns.Methods.CONSUME_TIME_FEE_GEN_WEI: + time_fee_gen_wei = await recv_int(32) + await handler.consume_time_fee_gen_wei(time_fee_gen_wei) + case host_fns.Methods.EXTERNAL_CALL: + address = await read_exact(ACCOUNT_ADDR_SIZE) calldata_len = await recv_int() calldata = await read_exact(calldata_len) try: - res = await handler.eth_call(account, calldata) + res = await handler.external_call(address, calldata) except HostException as e: await send_all(bytes([e.error_code])) else: await send_all(bytes([host_fns.Errors.OK])) await send_int(len(res)) await send_all(res) - case host_fns.Methods.GET_BALANCE: - account = await read_exact(ACCOUNT_ADDR_SIZE) + case host_fns.Methods.GET_BALANCE_GEN_WEI: + address = await read_exact(ACCOUNT_ADDR_SIZE) try: - res = await handler.get_balance(account) + res = await handler.get_balance_gen_wei(address) except HostException as e: await send_all(bytes([e.error_code])) else: await send_all(bytes([host_fns.Errors.OK])) await send_all(res.to_bytes(32, byteorder='little', signed=False)) - case host_fns.Methods.REMAINING_FUEL_AS_GEN: + case host_fns.Methods.GET_REMAINING_TIME_FEE_GEN_WEI: try: - res = await handler.remaining_fuel_as_gen() + time_fee_gen_wei = await handler.get_remaining_time_fee_gen_wei() except HostException as e: await send_all(bytes([e.error_code])) else: await send_all(bytes([host_fns.Errors.OK])) - await send_all(res.to_bytes(32, byteorder='little', signed=False)) + await send_all( + time_fee_gen_wei.to_bytes(32, byteorder='little', signed=False) + ) case host_fns.Methods.NOTIFY_NONDET_DISAGREEMENT: call_no = await recv_int() await handler.notify_nondet_disagreement(call_no) @@ -498,7 +500,7 @@ class ConsumedResult: result_kind: host_fns.ResultCode result_data: gvm_calldata.Decoded result_fingerprint: ResultFingerprint | None = None - result_storage_changes: list[tuple[bytes, bytes]] = field(default_factory=list) + result_storage_deltas: list[tuple[bytes, bytes]] = field(default_factory=list) result_emissions: list[ResultEmission] = field(default_factory=list) result_nondet_results: list[bytes] = field(default_factory=list) data_fees_remaining: list[int] = field(default_factory=list) @@ -547,7 +549,7 @@ def decode(cls, raw: typing.Any) -> 'ConsumedResult': result_kind=result_kind, result_data=decoded.get('data'), result_fingerprint=decoded.get('fingerprint'), - result_storage_changes=decoded.get('storage_changes', []), + result_storage_deltas=decoded.get('storage_deltas', []), result_emissions=decoded.get('emissions', []), result_nondet_results=decoded.get('nondet_results', []), data_fees_remaining=decoded.get('data_fees_remaining', []), @@ -567,7 +569,7 @@ class RunHostAndProgramRes: result_kind: host_fns.ResultCode result_data: gvm_calldata.Decoded result_fingerprint: ResultFingerprint | None - result_storage_changes: list[tuple[bytes, bytes]] + result_storage_deltas: list[tuple[bytes, bytes]] result_emissions: list[ResultEmission] result_nondet_results: list[bytes] data_fees_remaining: list[int] @@ -1190,7 +1192,7 @@ async def run_genvm( max_exec_mins = 20 if timeout is not None: max_exec_mins = int(max(max_exec_mins, (timeout * 1.5 + 59) // 60)) - timestamp = message.get('datetime', '2024-11-26T06:42:42.424242Z') + timestamp = message.get('transaction_timestamp', '2024-11-26T06:42:42.424242Z') deadline = _duration_string(timeout) host_genvm_id = typing.cast(str | None, request_extra.get('host_genvm_id')) if host_genvm_id is None: @@ -1379,7 +1381,7 @@ async def cancel_on_shutdown(): result_kind=consumed.result_kind, result_data=consumed.result_data, result_fingerprint=consumed.result_fingerprint, - result_storage_changes=consumed.result_storage_changes, + result_storage_deltas=consumed.result_storage_deltas, result_emissions=consumed.result_emissions, result_nondet_results=consumed.result_nondet_results, data_fees_remaining=consumed.data_fees_remaining, diff --git a/tests/runner/origin/fees.py b/tests/runner/origin/fees.py index 47ba4019..156748f8 100644 --- a/tests/runner/origin/fees.py +++ b/tests/runner/origin/fees.py @@ -45,7 +45,7 @@ class MessageAllocationNode(typing.TypedDict): call_key: bytes | None budget: int # Lifecycle the node matches against (only meaningful for internal messages). - on: typing.Literal['finalized', 'accepted'] + on: typing.Literal['finalized', 'decided'] fee_params: MessageAllocationNodeParams # Nested allocation subtree; the chain receives this flattened to # parent-pointer form. @@ -67,11 +67,11 @@ class MessageAllocationNode(typing.TypedDict): 'children': [], } -DEFAULT_INTERNAL_ACC_MESSAGE_ALLOC: MessageAllocationNode = { +DEFAULT_INTERNAL_DEC_MESSAGE_ALLOC: MessageAllocationNode = { 'budget': 2**200, 'recipient': None, 'call_key': None, - 'on': 'accepted', + 'on': 'decided', 'fee_params': { 'Internal': { 'execution_budget_per_round': 2**10, diff --git a/tests/runner/origin/host_fns.py b/tests/runner/origin/host_fns.py index 9715e1be..a000ad1a 100644 --- a/tests/runner/origin/host_fns.py +++ b/tests/runner/origin/host_fns.py @@ -9,13 +9,13 @@ class Methods(IntEnum): STORAGE_READ = 0 - CONSUME_FUEL = 1 - ETH_CALL = 2 - GET_BALANCE = 3 - REMAINING_FUEL_AS_GEN = 4 + CONSUME_TIME_FEE_GEN_WEI = 1 + EXTERNAL_CALL = 2 + GET_BALANCE_GEN_WEI = 3 + GET_REMAINING_TIME_FEE_GEN_WEI = 4 NOTIFY_NONDET_DISAGREEMENT = 5 CONSUME_RESULT = 6 - RESOLVE_CALLCONTRACT_EXECUTOR = 7 + RESOLVE_CALL_CONTRACT_EXECUTOR = 7 RUN_NESTED = 8 diff --git a/tests/runner/origin/public_abi.py b/tests/runner/origin/public_abi.py index abb7a496..41d7cf49 100644 --- a/tests/runner/origin/public_abi.py +++ b/tests/runner/origin/public_abi.py @@ -13,10 +13,10 @@ class ResultCode(IntEnum): VM_ERROR = 2 -class StorageType(IntEnum): +class StorageView(IntEnum): DEFAULT = 0 - LATEST_FINAL = 1 - LATEST_NON_FINAL = 2 + LATEST_FINALIZED = 1 + LATEST_DECIDED = 2 class EntryKind(IntEnum): @@ -135,32 +135,62 @@ def wasm_memory() -> 'VmError': def wasm_table() -> 'VmError': return VmError('out_of memory wasm_table') +class _VmErrorOutOfReceiptMessage: + @staticmethod + def val() -> 'VmError': + return VmError('out_of receipt message') + @staticmethod + def internal() -> 'VmError': + return VmError('out_of receipt message # internal') + class _VmErrorOutOfReceipt: @staticmethod def nondet_output() -> 'VmError': return VmError('out_of receipt nondet_output') @staticmethod - def message() -> 'VmError': - return VmError('out_of receipt message') - @staticmethod def event() -> 'VmError': return VmError('out_of receipt event') + @staticmethod + def message() -> '_VmErrorOutOfReceiptMessage': + return _VmErrorOutOfReceiptMessage() -class _VmErrorOutOfMessageFee: +class _VmErrorOutOfMessageFeeTotal: @staticmethod - def total() -> 'VmError': + def val() -> 'VmError': return VmError('out_of message_fee total') @staticmethod - def node() -> 'VmError': - return VmError('out_of message_fee node') + def internal() -> 'VmError': + return VmError('out_of message_fee total # internal') + @staticmethod + def external() -> 'VmError': + return VmError('out_of message_fee total # external') + +class _VmErrorOutOfMessageFeeAllocationBudget: + @staticmethod + def val() -> 'VmError': + return VmError('out_of message_fee allocation_budget') + @staticmethod + def internal() -> 'VmError': + return VmError('out_of message_fee allocation_budget # internal') + @staticmethod + def external() -> 'VmError': + return VmError('out_of message_fee allocation_budget # external') + +class _VmErrorOutOfMessageFee: + @staticmethod + def total() -> '_VmErrorOutOfMessageFeeTotal': + return _VmErrorOutOfMessageFeeTotal() + @staticmethod + def allocation_budget() -> '_VmErrorOutOfMessageFeeAllocationBudget': + return _VmErrorOutOfMessageFeeAllocationBudget() class _VmErrorOutOf: @staticmethod def storage() -> 'VmError': return VmError('out_of storage') @staticmethod - def vm_recursion() -> 'VmError': - return VmError('out_of vm_recursion') + def subvm_recursion() -> 'VmError': + return VmError('out_of subvm_recursion') @staticmethod def nondet_blocks() -> 'VmError': return VmError('out_of nondet_blocks') @@ -183,16 +213,27 @@ def receipt() -> '_VmErrorOutOfReceipt': def message_fee() -> '_VmErrorOutOfMessageFee': return _VmErrorOutOfMessageFee() -class _VmErrorFee: +class _VmErrorFeeNoMatchingAllocation: @staticmethod - def no_matching_node() -> 'VmError': - return VmError('fee no_matching_node') + def val() -> 'VmError': + return VmError('fee no_matching_allocation') + @staticmethod + def internal() -> 'VmError': + return VmError('fee no_matching_allocation # internal') + @staticmethod + def external() -> 'VmError': + return VmError('fee no_matching_allocation # external') + +class _VmErrorFee: @staticmethod def below_minimum() -> 'VmError': return VmError('fee below_minimum') @staticmethod def too_many_rounds() -> 'VmError': return VmError('fee too_many_rounds') + @staticmethod + def no_matching_allocation() -> '_VmErrorFeeNoMatchingAllocation': + return _VmErrorFeeNoMatchingAllocation() class _VmErrorEvm: @staticmethod @@ -276,12 +317,6 @@ def evm() -> '_VmErrorEvm': @staticmethod def invalid_contract() -> '_VmErrorInvalidContract': return _VmErrorInvalidContract() - def internal(self) -> 'VmError': - assert ' # ' not in self.value - return VmError(f'{self.value} # internal') - def external(self) -> 'VmError': - assert ' # ' not in self.value - return VmError(f'{self.value} # external') diff --git a/tests/system/cross-major-observability/assets/v0.3/observer.py b/tests/system/cross-major-observability/assets/v0.3/observer.py index eb4fd657..89d833ae 100644 --- a/tests/system/cross-major-observability/assets/v0.3/observer.py +++ b/tests/system/cross-major-observability/assets/v0.3/observer.py @@ -56,5 +56,7 @@ def attempt(action): @gl.public.view def debug_alias(self) -> int: - result = gl.vm.spawn_sandbox(lambda: 1, runner='py-genlayer:test') + import cloudpickle + + result = gl.vm.spawn_runner('py-genlayer:test', cloudpickle.dumps(lambda: 1)) return gl.vm.unpack_result(result) diff --git a/tests/system/cross-major-observability/test.py b/tests/system/cross-major-observability/test.py index 0055fa12..6ba21e3e 100644 --- a/tests/system/cross-major-observability/test.py +++ b/tests/system/cross-major-observability/test.py @@ -172,10 +172,10 @@ async def _execute( and result.result_kind == host_fns.ResultCode.RETURN ): assert mock_host.storage is not None - base._apply_storage_changes( + base._apply_storage_deltas( mock_host.storage, address, - result.result_storage_changes, + result.result_storage_deltas, ) return result finally: @@ -345,7 +345,7 @@ async def _assert_debug_alias(self): ) async def _assert_read_only_storage(self, permissions: str): - reads: list[tuple[Address, public_abi.StorageType]] = [] + reads: list[tuple[Address, public_abi.StorageView]] = [] leader, validator, sync = await self._lvs_extended( name=f'storage-{permissions}', line=2, @@ -362,7 +362,7 @@ async def _assert_read_only_storage(self, permissions: str): self.notes.append(('read-only nested storage', permissions)) async def _assert_self_recursion(self, budget: int): - expected_error = str(public_abi.VmError.out_of().vm_recursion()) + expected_error = str(public_abi.VmError.out_of().subvm_recursion()) leader, validator, sync = await self._lvs_extended( name=f'self-recursion-{budget}', line=3, @@ -407,8 +407,8 @@ async def _assert_deep_nesting(self, depth: int): if depth > CROSS_MAJOR_RECURSION: for result in (leader, validator, sync): assert result.result_kind == host_fns.ResultCode.VM_ERROR, result - assert result.result_data == 'out_of vm_recursion', (depth, result) - outcome: int | str = 'out_of vm_recursion' + assert result.result_data == 'out_of subvm_recursion', (depth, result) + outcome: int | str = 'out_of subvm_recursion' else: expected = sum(3 if index % 2 == 0 else 2 for index in range(depth + 1)) for result in (leader, validator, sync): diff --git a/tests/system/cross-major/assets/v0.3/state_caller.py b/tests/system/cross-major/assets/v0.3/state_caller.py index c612dce2..e21293ea 100644 --- a/tests/system/cross-major/assets/v0.3/state_caller.py +++ b/tests/system/cross-major/assets/v0.3/state_caller.py @@ -10,8 +10,8 @@ def __init__(self): @gl.public.view def call(self, target: Address, final: bool) -> int: mode = ( - gl.vm.public_abi.StorageType.LATEST_FINAL + gl.vm.public_abi.StorageView.LATEST_FINALIZED if final - else gl.vm.public_abi.StorageType.LATEST_NON_FINAL + else gl.vm.public_abi.StorageView.LATEST_DECIDED ) return gl.contract.get_at(target).view(state=mode).answer() diff --git a/tests/system/cross-major/test.py b/tests/system/cross-major/test.py index 5f57b641..d1f55bed 100644 --- a/tests/system/cross-major/test.py +++ b/tests/system/cross-major/test.py @@ -96,13 +96,14 @@ def _wat_data(data: bytes) -> str: return ''.join(f'\\{byte:02x}' for byte in data) -def _loop_wat(target: Address) -> str: +def _loop_wat(target: Address, *, caller_line: typing.Literal[2, 3]) -> str: + storage_field = 'storage_view' if caller_line == 3 else 'state' call = gvm_calldata.encode( { 'CallContract': { 'address': target, 'calldata': {}, - 'state': public_abi.StorageType.LATEST_NON_FINAL, + storage_field: public_abi.StorageView.LATEST_DECIDED, } } ) @@ -179,7 +180,7 @@ def _message(address: Address, *, is_init: bool) -> base_host.Message: 'chain_id': 61999, 'value': 0, 'is_init': is_init, - 'datetime': TIMESTAMP, + 'transaction_timestamp': TIMESTAMP, } @@ -198,7 +199,7 @@ def _assert_hashes_agree(label: str, runs: list[tuple[str, typing.Any]]) -> None ) -def _apply_storage_changes( +def _apply_storage_deltas( storage: MockStorage, address: Address, changes: list[tuple[bytes, bytes]], @@ -310,10 +311,10 @@ async def _run_all(self): await self._deploy(line, address, code) if 'loop-v03' in self.case.fixtures: loop_v03 = await self._compile_wat( - _loop_wat(ADDR_LOOP_V02), work_dir / 'loop-v03' + _loop_wat(ADDR_LOOP_V02, caller_line=3), work_dir / 'loop-v03' ) loop_v02 = await self._compile_wat( - _loop_wat(ADDR_LOOP_V03), work_dir / 'loop-v02' + _loop_wat(ADDR_LOOP_V03, caller_line=2), work_dir / 'loop-v02' ) self._replace_code(ADDR_LOOP_V03, LOOP_V03, loop_v03) self._replace_code(ADDR_LOOP_V02, LOOP_V02, loop_v02) @@ -332,7 +333,7 @@ async def _new_host( name: str, running_address: Address, resolve_hook=None, - read_log: list[tuple[Address, public_abi.StorageType]] | None = None, + read_log: list[tuple[Address, public_abi.StorageView]] | None = None, host_fuel: int | None = None, ) -> MockHost: ctx = _TestContext(self.case.shared.logger) @@ -349,22 +350,24 @@ async def _new_host( storage_path_post=self.storage_path, balances={}, running_address=running_address, - resolve_callcontract_executor_hook=resolve_hook, + resolve_call_contract_executor_hook=resolve_hook, ) if read_log is not None: original = host.storage_read - async def logged(mode, account, slot, index, le, /): - read_log.append((Address(account), mode)) - return await original(mode, account, slot, index, le) + async def logged(mode, address, slot, offset, le, /): + read_log.append((Address(address), mode)) + return await original(mode, address, slot, offset, le) host.storage_read = logged # type: ignore[method-assign] if host_fuel is not None: - async def remaining_fuel_as_gen() -> int: + async def get_remaining_time_fee_gen_wei() -> int: return host_fuel - host.remaining_fuel_as_gen = remaining_fuel_as_gen # type: ignore[method-assign] + host.get_remaining_time_fee_gen_wei = ( # type: ignore[method-assign] + get_remaining_time_fee_gen_wei + ) return host async def _execute( @@ -382,7 +385,7 @@ async def _execute( is_sync: bool = True, leader_nondet_results: list[bytes] | None = None, apply_changes: bool = True, - read_log: list[tuple[Address, public_abi.StorageType]] | None = None, + read_log: list[tuple[Address, public_abi.StorageView]] | None = None, host_fuel: int | None = None, hook_cross_contract_calls: bool = True, ): @@ -419,10 +422,10 @@ async def _execute( ) if apply_changes and result.result_kind == host_fns.ResultCode.RETURN: assert mock_host.storage is not None - _apply_storage_changes( + _apply_storage_deltas( mock_host.storage, address, - result.result_storage_changes, + result.result_storage_deltas, ) return result finally: @@ -549,7 +552,7 @@ def resolve(address, _state, _major): line=3, address=ADDR_CALLER_V03, host_path=mock_host.path, - deadline='30s' if mode == 'cancel' else '10s', + deadline='30s' if mode == 'cancel' else '3s', ) ) await asyncio.wait_for(nested_resolved.wait(), timeout=10) @@ -752,16 +755,16 @@ async def _assert_route_is_hash_invariant(self): async def _assert_state_mode_crosses( self, final: bool, - expected: public_abi.StorageType, + expected: public_abi.StorageView, ): """ The caller's state mode selects which chain view the callee reads, so the boundary must hand the callee exactly the mode the in-process route would have used.""" - modes: dict[bool, set[public_abi.StorageType]] = {} + modes: dict[bool, set[public_abi.StorageView]] = {} for nested in (False, True): - resolved: list[public_abi.StorageType] = [] - read_log: list[tuple[Address, public_abi.StorageType]] = [] + resolved: list[public_abi.StorageView] = [] + read_log: list[tuple[Address, public_abi.StorageView]] = [] def resolve(address, state, _major, nested=nested, resolved=resolved): if address != ADDR_CALLEE_V03: @@ -926,7 +929,9 @@ def resolve(address, _state, advisory_major): len(resolved), resolved[-3:], ) - assert result.result_data == str(public_abi.VmError.out_of().vm_recursion()), result + assert result.result_data == str(public_abi.VmError.out_of().subvm_recursion()), ( + result + ) assert len(resolved) == 2, len(resolved) # Neither line can report a meaningful major: v0.3 forwards the raw root # byte and v0.2 has no such byte at all, so both send the unwritten @@ -985,13 +990,13 @@ def collect( 'state-mode-nonfinal', '_assert_state_mode_crosses', ('state-caller-v03', 'callee-v03'), - (False, public_abi.StorageType.LATEST_NON_FINAL), + (False, public_abi.StorageView.LATEST_DECIDED), ), ( 'state-mode-final', '_assert_state_mode_crosses', ('state-caller-v03', 'callee-v03'), - (True, public_abi.StorageType.LATEST_FINAL), + (True, public_abi.StorageView.LATEST_FINALIZED), ), ('deep-chain', '_assert_deep_chain', ('chain-v03', 'chain-v02'), ()), *[ diff --git a/tests/system/make_zip/test.py b/tests/system/make_zip/test.py new file mode 100644 index 00000000..152ae899 --- /dev/null +++ b/tests/system/make_zip/test.py @@ -0,0 +1,214 @@ +""" +make-zip.py system test. + +The writer feeds the executor on every other run β€” all packaged runners are +built by it, so `Archive::from_zip_bytes` accepting its output is already +covered by the integration suite. What nothing covered is the two properties +that suite cannot see: that the bytes are reproducible, and that a *different* +implementation reads the archive the same way. Runner ids are content hashes, +so a drift in either is consensus-visible. +""" + +import io +import importlib.util +import json +import os +import subprocess +import sys +import zipfile +from pathlib import Path + +import genvm_tool.tests +from genvm_tool.tests.test import Result + +FIXTURE = { + 'runner.json': b'{ "StartWasm": "main.wasm" }', + 'main.wasm': b'\x00asm\x01\x00\x00\x00', + 'lib/mod.py': b'VALUE = 1\n', +} +PYC_NAME = importlib.util.cache_from_source('lib/mod.py', optimization='') +IGNORED_FIXTURE = { + PYC_NAME: b'stale bytecode', + 'lib/stale.pyo': b'stale optimized bytecode', +} + + +def _build(script: Path, work: Path, out: Path) -> None: + """ + Lay out the tree make-zip.py expects (a `scripts/` dir plus exactly one + source dir) and run it.""" + pkg = work / 'pkg' + for name, contents in (FIXTURE | IGNORED_FIXTURE).items(): + path = pkg / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(contents) + scripts = work / 'scripts' + scripts.mkdir(parents=True, exist_ok=True) + (scripts / 'make-zip.py').write_bytes(script.read_bytes()) + + env = dict(os.environ) + env['out'] = str(out) + subprocess.run( + [sys.executable, 'scripts/make-zip.py'], + cwd=work, + env=env, + check=True, + capture_output=True, + ) + + +def _check_add_file_edges(work: Path, out: Path) -> None: + env = dict(os.environ) + env['out'] = str(out) + subprocess.run( + [ + sys.executable, + '-c', + """ +import runpy + +namespace = runpy.run_path('scripts/make-zip.py') +add_file = namespace['add_file'] +add_file(namespace['importlib'].util.cache_from_source('lib/mod.py'), b'stale') +add_file('lib/stale.pyo', b'kept', skip_pyc=False) +add_file('lib/stale.pyo', b'stale') +try: + add_file('runner.json', b'') +except KeyError as exc: + assert 'runner.json' in str(exc), exc +else: + raise AssertionError('duplicate entry did not raise KeyError') +""", + ], + cwd=work, + env=env, + check=True, + capture_output=True, + ) + + +def _add_case( + ctx: genvm_tool.tests.stage.collection.Context, + *, + script: Path, + artifacts_dir: Path, +) -> None: + desc = genvm_tool.tests.test.Description('tests/system/make_zip/writer').with_tags( + frozenset({'integration', 'feature-runner-zip'}) + ) + + async def validate(previous_results): + first = artifacts_dir / 'first.zip' + second = artifacts_dir / 'second.zip' + _build(script, artifacts_dir / 'run-a', first) + _build(script, artifacts_dir / 'run-b', second) + _check_add_file_edges(artifacts_dir / 'run-a', artifacts_dir / 'collision.zip') + + if first.read_bytes() != second.read_bytes(): + return Result( + passed=False, + context={'reason': 'writer is not byte-deterministic'}, + elapsed_seconds=0, + ) + + raw = first.read_bytes() + # CPython is the independent reader: it validates CRCs and both headers, + # so agreement here is what rules out the local/central divergence class. + with zipfile.ZipFile(io.BytesIO(raw)) as zf: + if zf.testzip() is not None: + return Result( + passed=False, + context={'reason': 'CPython rejected an entry', 'entry': zf.testzip()}, + elapsed_seconds=0, + ) + names = sorted(zf.namelist()) + contents = {name: zf.read(name) for name in names} + infos = {name: zf.getinfo(name) for name in names} + + if contents.get(PYC_NAME) in (None, IGNORED_FIXTURE[PYC_NAME]): + return Result( + passed=False, + context={'reason': 'stale bytecode was not replaced', 'entry': PYC_NAME}, + elapsed_seconds=0, + ) + if 'lib/stale.pyo' in contents: + return Result( + passed=False, + context={'reason': 'stale optimized bytecode was included'}, + elapsed_seconds=0, + ) + + for name, expected in FIXTURE.items(): + if name == 'runner.json': + # The writer normalizes runner.json; compare parsed, not raw. + if json.loads(contents.get(name, b'null')) != json.loads(expected): + return Result( + passed=False, + context={'reason': 'runner.json content differs', 'entry': name}, + elapsed_seconds=0, + ) + continue + if contents.get(name) != expected: + return Result( + passed=False, + context={'reason': 'entry content differs', 'entry': name}, + elapsed_seconds=0, + ) + + for name, info in infos.items(): + if info.compress_type != zipfile.ZIP_STORED: + return Result( + passed=False, + context={'reason': 'entry is not stored', 'entry': name}, + elapsed_seconds=0, + ) + # Pinned so the bytes cannot pick up anything host-derived. + if (info.create_system, info.external_attr, info.flag_bits) != (0, 0, 0): + return Result( + passed=False, + context={ + 'reason': 'host-derived header field leaked', + 'entry': name, + 'create_system': info.create_system, + 'external_attr': info.external_attr, + 'flag_bits': int(info.flag_bits), + }, + elapsed_seconds=0, + ) + if info.date_time != (1980, 1, 1, 0, 0, 0): + return Result( + passed=False, + context={ + 'reason': 'mtime leaked into the archive', + 'entry': name, + 'date_time': info.date_time, + }, + elapsed_seconds=0, + ) + + return Result(passed=True, context={'entries': len(names)}, elapsed_seconds=0) + + ctx.add_case( + genvm_tool.tests.test.StepsCase( + description=desc, + steps=[genvm_tool.tests.exec.step.PythonFunction(validate)], + ) + ) + + +def collect(ctx: genvm_tool.tests.stage.collection.Context) -> None: + script = ( + ctx.shared.root_dir + / 'executors' + / 'v0.3.x' + / 'runners' + / 'support' + / 'scripts' + / 'make-zip.py' + ) + if not script.exists(): + raise FileNotFoundError(f'make-zip.py not found at {script}') + + artifacts_dir = ctx.shared.artifacts_dir / 'make_zip' + artifacts_dir.mkdir(parents=True, exist_ok=True) + _add_case(ctx, script=script, artifacts_dir=artifacts_dir) From 3c96be6defe30fd9843d47bedc0c66bec324d0fa Mon Sep 17 00:00:00 2001 From: Kira <kp2pml30@gmail.com> Date: Fri, 28 Aug 2026 16:34:32 +0900 Subject: [PATCH 13/45] feat(manager): carry opaque leader public data (#29) --- crates/modules-interfaces/src/domain.rs | 4 +- ...pre-validating untrusted decoded inputs.md | 2 +- docs/adr/014. manager host socket protocol.md | 2 +- docs/adr/015. cross-major contract calls.md | 2 +- .../src/impl-spec/02-vm/03-consensus.rst | 30 ++---- .../src/impl-spec/appendix/manager-api.yaml | 14 ++- .../src/impl-spec/appendix/manager-socket.rst | 4 +- docs/website/src/spec/03-vm/05-result.rst | 11 ++- executors/v0.2.x | 2 +- executors/v0.3.x | 2 +- implementation/src/manager/run.rs | 16 +-- implementation/src/manager/run_test.rs | 10 +- .../runner/genvm_tool_plugins/integration.py | 28 +++--- tests/runner/origin/base_host.py | 12 +-- tests/runner/origin/leader_public_data.py | 99 +++++++++++++++++++ .../runner/origin/test_leader_public_data.py | 22 +++++ .../system/cross-major-observability/test.py | 6 +- tests/system/cross-major/test.py | 30 +++++- tests/system/manager-socket/test.py | 2 +- tests/system/permits/test.py | 4 +- 20 files changed, 218 insertions(+), 84 deletions(-) create mode 100644 tests/runner/origin/leader_public_data.py create mode 100644 tests/runner/origin/test_leader_public_data.py diff --git a/crates/modules-interfaces/src/domain.rs b/crates/modules-interfaces/src/domain.rs index 94b47423..ab89ff98 100644 --- a/crates/modules-interfaces/src/domain.rs +++ b/crates/modules-interfaces/src/domain.rs @@ -28,7 +28,7 @@ pub struct ExecutionData { pub message: super::abi_stub::MessageData, pub host_data: String, pub code: Option<Bytes>, - pub leader_nondet_results: Option<Vec<Bytes>>, + pub leader_public_data: Option<Bytes>, /// Maps each host method (by index) to a host id. When empty, all methods use host 0. pub method_hosts: Vec<u8>, /// Raw bytes written to each host connection before the first method byte. @@ -408,7 +408,7 @@ pub struct ReportedResult { pub emissions: Vec<ExecutionEmission>, pub nondet_disagreement: Option<u32>, - pub nondet_results: Vec<bytes::Bytes>, + pub leader_public_data: bytes::Bytes, pub data_fees_remaining: Vec<primitive_types::U256>, pub data_fees_consumed: BucketsConsumed, diff --git a/docs/adr/013. pre-validating untrusted decoded inputs.md b/docs/adr/013. pre-validating untrusted decoded inputs.md index 532f0d21..8399dede 100644 --- a/docs/adr/013. pre-validating untrusted decoded inputs.md +++ b/docs/adr/013. pre-validating untrusted decoded inputs.md @@ -162,7 +162,7 @@ untrusted entry calldata is decoded: The `is_sync`/`is_leader()` boolean pair is replaced by `rt::RunMode { Sync, Leader, Validator }`, fixed at startup from the execution data (`sync` flag; -else presence of `leader_nondet_results`). Rule 1 governs `Validator`/`Sync` +else presence of `leader_public_data`). Rule 1 governs `Validator`/`Sync` (in sync there is no vote β€” the derived result is simply the call's result); Rule 1b governs `Leader`. diff --git a/docs/adr/014. manager host socket protocol.md b/docs/adr/014. manager host socket protocol.md index 804b61af..49103600 100644 --- a/docs/adr/014. manager host socket protocol.md +++ b/docs/adr/014. manager host socket protocol.md @@ -179,7 +179,7 @@ result without round-tripping it through the node. For a top-level run, the manager validates the outer framing and decoded `ReportedResult` before retaining it. Malformed reports are refused; a fatal result triggers a debug assertion and is logged and downgraded to `VMError` in release builds. Embedded -`nondet_results` remain opaque because their encoding belongs to the executor +`leader_public_data` remains opaque because its encoding belongs to the executor line ### HTTP surface diff --git a/docs/adr/015. cross-major contract calls.md b/docs/adr/015. cross-major contract calls.md index d1ed78ca..963788ef 100644 --- a/docs/adr/015. cross-major contract calls.md +++ b/docs/adr/015. cross-major contract calls.md @@ -140,7 +140,7 @@ computed by the caller and carried on the wire. The table is complete against | `host_data`, `gas_data` | Copied by the manager from the outer request, not carried in the envelope. | | `method_hosts` | Constructed by the manager: host 0 by default, `consume_result` and `run_nested` on host 1. | | `host_hello_data` | Copied from the outer request; entry 1 is cleared for the new manager socketpair. | -| Effectful and nondeterministic allocations | Cleared: `leader_nondet_results` and `record_actions` are empty because a deterministic child produces neither; `bucket_totals`, `message_fee_allocation` and `initial_time_units_allocation` are empty or zero because the child imports its remaining deterministic budget instead of being granted a fresh one. | +| Effectful and nondeterministic allocations | Cleared: `leader_public_data` and `record_actions` are empty because a deterministic child produces neither; `bucket_totals`, `message_fee_allocation` and `initial_time_units_allocation` are empty or zero because the child imports its remaining deterministic budget instead of being granted a fresh one. | | `remaining_recursion` | Parent budget minus one. Carried as a remainder, not a spent count, so the bound stays the one the chain's root minted even when the lines disagree on `VM_RECURSION` -- but the receiving executor clamps it to its own limit, since a remainder is not an authority to exceed one. The manager clamps it again, to `CROSS_MAJOR_RECURSION` (6), so a delegated chain is bounded far more tightly than an in-process one: a hop costs a whole executor process, and the manager is the only party that sees the chain as a whole. The clamp also bounds the subtree below a delegated call, which is intended -- a chain that has already crossed a boundary does not get to recurse deeply in-process either. Also set, for a chain root, by the `initial_recursion` debug override. | | `nested.memory_limit` | The derived limiter's remaining memory, made an explicit process-spawn limit. | | `nested.stack` | Caller stack with the caller contract appended. | diff --git a/docs/website/src/impl-spec/02-vm/03-consensus.rst b/docs/website/src/impl-spec/02-vm/03-consensus.rst index a07226ce..15dd243a 100644 --- a/docs/website/src/impl-spec/02-vm/03-consensus.rst +++ b/docs/website/src/impl-spec/02-vm/03-consensus.rst @@ -13,16 +13,15 @@ Roles and the ``is_leader`` flag A GenVM run is either a *leader* run or a *validator* run, decided once when the node starts the run. The supervisor exposes this as ``Supervisor::is_leader()`` (``executor/src/rt/supervisor/mod.rs:209``); the flag is -derived from whether the node has handed in a vector of leader non-deterministic -results: +derived from whether the node has handed in opaque leader public data: -- Leader run: ``leader_nondet_results == None``. Every ``RunNondet`` call executes the +- Leader run: ``leader_public_data == None``. Every ``RunNondet`` call executes the leader sub-program and the result is pushed into the supervisor's ``nondet_results`` vector, indexed by ``call_no``. -- Validator run: ``leader_nondet_results == Some(vec)``. ``RunNondet`` retrieves the - leader's result at ``call_no`` from this vector and feeds it back to the contract - as the second argument of the non-det block. If the leader produced fewer entries - than the validator demands, the run aborts with +- Validator run: ``leader_public_data == Some(bytes)``. The executor decodes the + bytes into the leader's results. ``RunNondet`` retrieves the result at ``call_no`` + and feeds it back to the contract as the second argument of the non-det block. If + the leader produced fewer entries than the validator demands, the run aborts with ``VmError::leader_fault().nondet_output().absent()``. ``call_no`` is a monotonically increasing counter incremented per ``RunNondet`` @@ -32,22 +31,7 @@ a leader's i-th non-det result to the validator's i-th non-det check β€” the ordering of ``RunNondet`` calls in deterministic code MUST match between leader and validator, otherwise the contract is non-replayable. -Leader Output Format --------------------- - -Each entry in the leader's ``nondet_results`` vector is a single ``ResultCode`` -byte (the ``public_abi::ResultCode`` enum: ``Return``, ``UserError``, ``VmError``) -followed by an encoded payload: - -- ``Return`` β€” raw bytes returned by the non-det block. -- ``UserError`` β€” either a calldata-encoded value (when the first four bytes are - zero, indicating a calldata prefix) or a UTF-8 string fallback. -- ``VmError`` β€” UTF-8 error code from - :doc:`/spec/appendix/constants` ``vm_error``. - -The validator reconstructs a catchable contract outcome from these bytes and -the contract observes the same value as the leader did. Any other result code, -including ``FatalVmError``, is treated as malformed leader input +The leader data encoding is defined by :ref:`gvm-def-leader-output-format`. Validator Comparison -------------------- diff --git a/docs/website/src/impl-spec/appendix/manager-api.yaml b/docs/website/src/impl-spec/appendix/manager-api.yaml index aa0ee8c9..2d150638 100644 --- a/docs/website/src/impl-spec/appendix/manager-api.yaml +++ b/docs/website/src/impl-spec/appendix/manager-api.yaml @@ -780,7 +780,7 @@ components: - host - calldata - code - - leader_nondet_results + - leader_public_data - initial_time_units_allocation properties: selector: @@ -843,15 +843,13 @@ components: description: | Contract bytecode for a deployment, or null for a call. Required to be present; only its value is nullable. - leader_nondet_results: - type: array + leader_public_data: + type: string + format: binary nullable: true - items: - type: string - format: binary description: | - The leader's non-deterministic results for a validator run, or null for - a leader run. Required to be present; only its value is nullable. + Opaque executor-owned leader public data for a validator run, or null + for a leader run. Required to be present; only its value is nullable. initial_time_units_allocation: type: integer format: uint32 diff --git a/docs/website/src/impl-spec/appendix/manager-socket.rst b/docs/website/src/impl-spec/appendix/manager-socket.rst index 60a759c1..c5e3561d 100644 --- a/docs/website/src/impl-spec/appendix/manager-socket.rst +++ b/docs/website/src/impl-spec/appendix/manager-socket.rst @@ -270,8 +270,8 @@ debug manager asserts, while a release manager logs the executor violation and rewrites both result-code locations to ``VmError`` before publication. Clients therefore never receive a top-level ``FatalVmError`` -The manager does not decode entries of the reported ``nondet_results`` vector. -Those bytes remain opaque, executor-line-specific consensus proposals +The manager does not decode reported ``leader_public_data``. The bytes remain +opaque, executor-line-specific consensus proposals Lifecycle guarantees: diff --git a/docs/website/src/spec/03-vm/05-result.rst b/docs/website/src/spec/03-vm/05-result.rst index 0e0c0659..94ddc9b2 100644 --- a/docs/website/src/spec/03-vm/05-result.rst +++ b/docs/website/src/spec/03-vm/05-result.rst @@ -114,7 +114,16 @@ Non-Deterministic Block Result Encoding These three are the only codes a leader-proposed non-deterministic block result may carry; validators treat every other byte as a malformed leader result. A fatal VM error computed by a leader's non-deterministic child propagates to its -caller and MUST NOT be encoded into ``nondet_results`` +caller and MUST NOT be included in ``LeaderPublicData`` + +.. _gvm-def-leader-output-format: + +Leader Output Format +-------------------- + +``LeaderPublicData`` is encoded as an RLP list containing every non-deterministic +block result, in execution order, followed by the byte string ``padded``. Each +result uses :ref:`gvm-def-subvm-result-encoding` Contract Result Encoding ------------------------ diff --git a/executors/v0.2.x b/executors/v0.2.x index 2a32a840..ca50b306 160000 --- a/executors/v0.2.x +++ b/executors/v0.2.x @@ -1 +1 @@ -Subproject commit 2a32a84066012eaa887a1e62d345a92e8b37c562 +Subproject commit ca50b3065e952e52dc3b7dd79dba51c334ea9abd diff --git a/executors/v0.3.x b/executors/v0.3.x index 79e89ad5..4ddb57b6 160000 --- a/executors/v0.3.x +++ b/executors/v0.3.x @@ -1 +1 @@ -Subproject commit 79e89ad57188fd0c6ae8d939fc0056461d5d0d21 +Subproject commit 4ddb57b68d2cb76eec01d94671e9594338f7e5a8 diff --git a/implementation/src/manager/run.rs b/implementation/src/manager/run.rs index de98dd67..4a64176f 100644 --- a/implementation/src/manager/run.rs +++ b/implementation/src/manager/run.rs @@ -1003,7 +1003,7 @@ pub struct Request { #[serde(default)] #[calldata(default = default_unsafe_overrides)] pub unsafe_overrides: UnsafeOverrides, - pub leader_nondet_results: Option<Vec<bytes::Bytes>>, + pub leader_public_data: Option<bytes::Bytes>, /// Host-provided `node` fee constants (moved off `host_data`). #[serde(default)] #[calldata(default = default_gas_data)] @@ -1512,7 +1512,7 @@ fn nested_effect(reported: &genvm_modules_interfaces::ReportedResult) -> Option< storage_deltas, emissions, nondet_disagreement, - nondet_results, + leader_public_data, data_fees_consumed: genvm_modules_interfaces::BucketsConsumed { storage, @@ -1533,8 +1533,8 @@ fn nested_effect(reported: &genvm_modules_interfaces::ReportedResult) -> Option< if nondet_disagreement.is_some() { return Some("nondet_disagreement"); } - if !nondet_results.is_empty() { - return Some("nondet_results"); + if !leader_public_data.is_empty() { + return Some("leader_public_data"); } if !llm_consumed_gen_wei.is_zero() { return Some("llm_consumed_gen_wei"); @@ -1944,7 +1944,7 @@ fn execution_data_from_request(req: &Request) -> genvm_modules_interfaces::Execu message: req.message.clone(), host_data: req.host_data.clone(), code: req.code.clone(), - leader_nondet_results: req.leader_nondet_results.clone(), + leader_public_data: req.leader_public_data.clone(), host_hello_data: req.host_hello_data.clone(), method_hosts, bucket_totals: req.bucket_totals.clone(), @@ -2165,7 +2165,7 @@ impl Ctx { permissions, no_modules: true, unsafe_overrides: Default::default(), - leader_nondet_results: None, + leader_public_data: None, gas_data: parent_req.gas_data.clone(), message_fee_allocation: Vec::new(), initial_time_units_allocation: 0, @@ -2179,7 +2179,7 @@ impl Ctx { }; let mut execution_data = execution_data_from_request(&req); execution_data.code = None; - execution_data.leader_nondet_results = None; + execution_data.leader_public_data = None; execution_data.record_actions.clear(); execution_data.message_fee_allocation.clear(); execution_data.remaining_recursion = @@ -2366,7 +2366,7 @@ async fn run_genvm_process( let execution_context = if req.needs_modules() { let host_data: genvm_modules_interfaces::HostData = serde_json::from_str(&req.host_data)?; - let role = if req.leader_nondet_results.is_none() { + let role = if req.leader_public_data.is_none() { genvm_modules_interfaces::Role::Leader } else { genvm_modules_interfaces::Role::Validator diff --git a/implementation/src/manager/run_test.rs b/implementation/src/manager/run_test.rs index 7f8ffae1..9f1427bc 100644 --- a/implementation/src/manager/run_test.rs +++ b/implementation/src/manager/run_test.rs @@ -553,7 +553,7 @@ fn clean_reported() -> genvm_modules_interfaces::ReportedResult { storage_deltas: Vec::new(), emissions: Vec::new(), nondet_disagreement: None, - nondet_results: Vec::new(), + leader_public_data: bytes::Bytes::new(), data_fees_remaining: Vec::new(), data_fees_consumed: Default::default(), llm_consumed_gen_wei: primitive_types::U256::zero(), @@ -665,8 +665,8 @@ fn nested_reply_refuses_every_reported_effect() { }), ("emissions", |r| r.emissions.push(some_emission())), ("nondet_disagreement", |r| r.nondet_disagreement = Some(0)), - ("nondet_results", |r| { - r.nondet_results.push(bytes::Bytes::from_static(&[1])) + ("leader_public_data", |r| { + r.leader_public_data = bytes::Bytes::from_static(&[1]) }), ("llm_consumed_gen_wei", |r| { r.llm_consumed_gen_wei = primitive_types::U256::one() @@ -802,7 +802,7 @@ fn request_field_names() -> Vec<&'static str> { permissions, no_modules, unsafe_overrides, - leader_nondet_results, + leader_public_data, gas_data, message_fee_allocation, initial_time_units_allocation, @@ -830,7 +830,7 @@ fn request_field_names() -> Vec<&'static str> { "permissions", "no_modules", "unsafe_overrides", - "leader_nondet_results", + "leader_public_data", "gas_data", "message_fee_allocation", "initial_time_units_allocation", diff --git a/tests/runner/genvm_tool_plugins/integration.py b/tests/runner/genvm_tool_plugins/integration.py index 8c6b3745..f2c7fb12 100644 --- a/tests/runner/genvm_tool_plugins/integration.py +++ b/tests/runner/genvm_tool_plugins/integration.py @@ -40,6 +40,7 @@ from gvm_extra.mock_host import MockHost as MockHost from gvm_extra.mock_host import MockStorage as MockStorage from origin.calldata import Address +from origin.leader_public_data import LeaderPublicData if typing.TYPE_CHECKING: import _jsonnet @@ -74,6 +75,7 @@ def _is_ignore_hash_enabled() -> bool: # version-independent and lives in the manager root. TEMPLATES_DIR = local_ctx.shared.root_dir.joinpath('tests', 'templates') + # Default environment for tests default_env = { k: v @@ -636,7 +638,7 @@ async def _run_single_step( ) path_to_which_leader_puts_result = my_tmp_dir.parent.joinpath( - my_tmp_dir.name[:-1] + '_leader_nondet.pickle' + my_tmp_dir.name[:-1] + '_leader_public_data.bin' ) # Set up paths @@ -646,18 +648,18 @@ async def _run_single_step( is_leader = single_conf.get('mode') == 'l' - # Load leader nondet results if available (for v/s modes) + # Load leader public data if available (for v/s modes) if is_leader: - leader_nondet = None + leader_public_data = None else: - leader_nondet = single_conf.get('leader_nondet', None) - if leader_nondet is None: - leader_nondet = pickle.loads( - await gvm_io.read_file_bytes(path_to_which_leader_puts_result) + leader_nondet_fixture = single_conf.get('leader_nondet', None) + if leader_nondet_fixture is None: + leader_public_data = await gvm_io.read_file_bytes( + path_to_which_leader_puts_result ) - if leader_nondet is not None: + else: encoded_nondet = [] - for res in leader_nondet: + for res in leader_nondet_fixture: if isinstance(res, (bytes, bytearray, memoryview)): encoded_nondet.append(bytes(res)) elif res['kind'] == 'return': @@ -688,7 +690,7 @@ async def _run_single_step( encoded_nondet.append(bytes(res['value'])) else: raise ValueError(f'unknown leader_nondet kind: {res["kind"]}') - leader_nondet = encoded_nondet + leader_public_data = LeaderPublicData(encoded_nondet).encode() # Create mock host running_address = single_conf['message']['contract_address'] @@ -785,7 +787,7 @@ def resolve_executor(address, state, advisory_major): calldata=calldata_bytes, bucket_totals=bucket_totals, gas_data=single_conf.get('gas_data'), - leader_nondet_results=leader_nondet, + leader_public_data=leader_public_data, message_fee_allocation=message_fee_allocation, unsafe_overrides=base_host.UnsafeOverrides( reroute_to=single_conf.get('reroute_to', reroute_to) @@ -852,10 +854,10 @@ def resolve_executor(address, state, advisory_major): # Save leader nondet results for v/s modes if is_leader: - nondet_list = res.result_nondet_results + leader_public_data = res.result_leader_public_data await gvm_io.write_file_bytes( - path_to_which_leader_puts_result, pickle.dumps(nondet_list) + path_to_which_leader_puts_result, leader_public_data ) # Save outputs diff --git a/tests/runner/origin/base_host.py b/tests/runner/origin/base_host.py index 6817a38c..bd4c9f04 100644 --- a/tests/runner/origin/base_host.py +++ b/tests/runner/origin/base_host.py @@ -502,7 +502,7 @@ class ConsumedResult: result_fingerprint: ResultFingerprint | None = None result_storage_deltas: list[tuple[bytes, bytes]] = field(default_factory=list) result_emissions: list[ResultEmission] = field(default_factory=list) - result_nondet_results: list[bytes] = field(default_factory=list) + result_leader_public_data: bytes = b'' data_fees_remaining: list[int] = field(default_factory=list) @classmethod @@ -551,7 +551,7 @@ def decode(cls, raw: typing.Any) -> 'ConsumedResult': result_fingerprint=decoded.get('fingerprint'), result_storage_deltas=decoded.get('storage_deltas', []), result_emissions=decoded.get('emissions', []), - result_nondet_results=decoded.get('nondet_results', []), + result_leader_public_data=decoded.get('leader_public_data', b''), data_fees_remaining=decoded.get('data_fees_remaining', []), ) @@ -571,7 +571,7 @@ class RunHostAndProgramRes: result_fingerprint: ResultFingerprint | None result_storage_deltas: list[tuple[bytes, bytes]] result_emissions: list[ResultEmission] - result_nondet_results: list[bytes] + result_leader_public_data: bytes data_fees_remaining: list[int] metrics: dict[str, typing.Any] | None = None vm_error_description: str | None = None @@ -1165,7 +1165,7 @@ async def run_genvm( bucket_totals: list[int], code: bytes | None = None, calldata: bytes, - leader_nondet_results: list[bytes] | None = None, + leader_public_data: bytes | None = None, message_fee_allocation: collections.abc.Sequence[fees.MessageAllocationNode] = (), unsafe_overrides: UnsafeOverrides | None = None, request_extra: collections.abc.Mapping[ @@ -1214,7 +1214,7 @@ async def run_genvm( 'extra_args': list(extra_args), 'code': code, 'calldata': calldata, - 'leader_nondet_results': leader_nondet_results, + 'leader_public_data': leader_public_data, 'bucket_totals': bucket_totals, 'gas_data': effective_gas_data, 'message_fee_allocation': list(message_fee_allocation), @@ -1383,7 +1383,7 @@ async def cancel_on_shutdown(): result_fingerprint=consumed.result_fingerprint, result_storage_deltas=consumed.result_storage_deltas, result_emissions=consumed.result_emissions, - result_nondet_results=consumed.result_nondet_results, + result_leader_public_data=consumed.result_leader_public_data, data_fees_remaining=consumed.data_fees_remaining, vm_error_description=vm_error_description, execution_time=time.time() - started_at, diff --git a/tests/runner/origin/leader_public_data.py b/tests/runner/origin/leader_public_data.py new file mode 100644 index 00000000..b2fb25bb --- /dev/null +++ b/tests/runner/origin/leader_public_data.py @@ -0,0 +1,99 @@ +import collections.abc +from dataclasses import dataclass +from typing import Self + + +_PADDING = b'padded' + + +@dataclass +class LeaderPublicData: + nondet_block_outputs: list[bytes] + + def encode(self) -> bytes: + payload = b''.join( + _encode_bytes(value) for value in (*self.nondet_block_outputs, _PADDING) + ) + return _encode_length(len(payload), 0xC0, 0xF7) + payload + + @classmethod + def decode(cls, encoded: collections.abc.Buffer) -> Self: + data = bytes(encoded) + if not data: + return cls([]) + + payload_start, payload_len = _decode_length(data, 0, list_=True) + payload_end = payload_start + payload_len + if payload_end != len(data): + raise ValueError('trailing leader public data') + + outputs: list[bytes] = [] + cursor = payload_start + while cursor < payload_end: + value_start, value_len = _decode_length(data, cursor, list_=False) + value_end = value_start + value_len + if value_end > payload_end: + raise ValueError('leader public data item exceeds list') + outputs.append(data[value_start:value_end]) + cursor = value_end + + if not outputs or outputs[-1] != _PADDING: + raise ValueError('leader public data padding is missing') + return cls(outputs[:-1]) + + +def encode(value: LeaderPublicData) -> bytes: + return value.encode() + + +def decode(encoded: collections.abc.Buffer) -> LeaderPublicData: + return LeaderPublicData.decode(encoded) + + +def _encode_bytes(value: bytes) -> bytes: + if len(value) == 1 and value[0] < 0x80: + return value + return _encode_length(len(value), 0x80, 0xB7) + value + + +def _encode_length(length: int, short_base: int, long_base: int) -> bytes: + if length <= 55: + return bytes([short_base + length]) + length_bytes = length.to_bytes((length.bit_length() + 7) // 8, 'big') + return bytes([long_base + len(length_bytes)]) + length_bytes + + +def _decode_length( + encoded: bytes, + offset: int, + *, + list_: bool, +) -> tuple[int, int]: + if offset >= len(encoded): + raise ValueError('truncated leader public data') + + prefix = encoded[offset] + short_base = 0xC0 if list_ else 0x80 + long_base = 0xF7 if list_ else 0xB7 + if not list_ and prefix < 0x80: + return offset, 1 + if prefix < short_base or prefix > long_base + 8: + raise ValueError('invalid RLP prefix') + if prefix <= long_base: + if not list_ and prefix == 0x81 and offset + 1 < len(encoded): + if encoded[offset + 1] < 0x80: + raise ValueError('non-canonical RLP string') + return offset + 1, prefix - short_base + + length_len = prefix - long_base + length_start = offset + 1 + length_end = length_start + length_len + if length_end > len(encoded): + raise ValueError('truncated RLP length') + length_bytes = encoded[length_start:length_end] + if length_bytes[0] == 0: + raise ValueError('non-canonical RLP length') + length = int.from_bytes(length_bytes, 'big') + if length <= 55: + raise ValueError('non-canonical long RLP value') + return length_end, length diff --git a/tests/runner/origin/test_leader_public_data.py b/tests/runner/origin/test_leader_public_data.py new file mode 100644 index 00000000..d2cf81f6 --- /dev/null +++ b/tests/runner/origin/test_leader_public_data.py @@ -0,0 +1,22 @@ +import pytest + +from .leader_public_data import LeaderPublicData, decode, encode + + +def test_round_trip_from_memoryview() -> None: + data = LeaderPublicData([b'a', b'bc']) + assert decode(memoryview(encode(data))) == data + + +def test_preserves_legacy_encoding() -> None: + assert LeaderPublicData([b'test']).encode() == b'\xcc\x84test\x86padded' + + +def test_empty_timeout_decodes_as_no_outputs() -> None: + assert LeaderPublicData.decode(b'') == LeaderPublicData([]) + + +@pytest.mark.parametrize('encoded', [b'\xc0', b'\xc7\x86padded\x00', b'\xc2\x81\x01']) +def test_rejects_invalid_encoding(encoded: bytes) -> None: + with pytest.raises(ValueError): + LeaderPublicData.decode(encoded) diff --git a/tests/system/cross-major-observability/test.py b/tests/system/cross-major-observability/test.py index 6ba21e3e..ca4b92cc 100644 --- a/tests/system/cross-major-observability/test.py +++ b/tests/system/cross-major-observability/test.py @@ -141,7 +141,7 @@ async def _execute( manager_client=manager_client, ctx=ctx, is_sync=kwargs.get('is_sync', True), - leader_nondet_results=kwargs.get('leader_nondet_results'), + leader_public_data=kwargs.get('leader_public_data'), message=base._message( address, is_init=typing.cast(bool, kwargs['is_init']) ), @@ -196,7 +196,7 @@ async def one(suffix: str, **mode): validator = await one( 'validator', is_sync=False, - leader_nondet_results=leader.result_nondet_results, + leader_public_data=leader.result_leader_public_data, ) sync = await one('sync', is_sync=True) assert leader.execution_hash == validator.execution_hash, ( @@ -252,7 +252,7 @@ async def _assert_validator_mode(self): resolve_hook=self._route_chain, apply_changes=False, is_sync=False, - leader_nondet_results=[base.gvm_calldata.encode({})], + leader_public_data=base._leader_public_data(base.gvm_calldata.encode({})), ) assert bogus.result_kind == host_fns.ResultCode.VM_ERROR, bogus diff --git a/tests/system/cross-major/test.py b/tests/system/cross-major/test.py index d1f55bed..436ed237 100644 --- a/tests/system/cross-major/test.py +++ b/tests/system/cross-major/test.py @@ -18,6 +18,7 @@ from gvm_extra.mock_host import MockHost, MockStorage from origin import host_fns, public_abi from origin.calldata import Address +from origin.leader_public_data import LeaderPublicData # One directory per line, because a contract is written against its own sdk. # Their `@RUNNER_LATEST_...@` tokens stand for the uids of the last build; @@ -46,6 +47,10 @@ TIMESTAMP = '2024-11-26T06:42:42.424242Z' +def _leader_public_data(output: bytes) -> bytes: + return LeaderPublicData([output]).encode() + + def resolve_runners(code: bytes | None, root_dir: Path) -> bytes | None: """ Turns the `@RUNNER_LATEST_...@` tokens of a contract source into uids. @@ -383,7 +388,7 @@ async def _execute( timeout: float = 30, debug_initial_recursion: int | None = None, is_sync: bool = True, - leader_nondet_results: list[bytes] | None = None, + leader_public_data: bytes | None = None, apply_changes: bool = True, read_log: list[tuple[Address, public_abi.StorageView]] | None = None, host_fuel: int | None = None, @@ -402,7 +407,7 @@ async def _execute( manager_client=manager_client, ctx=ctx, is_sync=is_sync, - leader_nondet_results=leader_nondet_results, + leader_public_data=leader_public_data, message=_message(address, is_init=is_init), host_data='{"node_address":"test","tx_id":"cross-major"}', host='unix://' + mock_host.path, @@ -514,7 +519,7 @@ def _raw_request( 'extra_args': [], 'code': None, 'calldata': _calldata('call', ADDR_BUSY_V02), - 'leader_nondet_results': None, + 'leader_public_data': None, 'bucket_totals': [2**200] * 20, 'gas_data': base_host.DEFAULT_GAS_DATA, 'message_fee_allocation': [], @@ -638,7 +643,7 @@ async def one(suffix: str, **kwargs): leader = await one('leader', is_sync=False) validator = await one( - 'validator', is_sync=False, leader_nondet_results=leader.result_nondet_results + 'validator', is_sync=False, leader_public_data=leader.result_leader_public_data ) sync = await one('sync', is_sync=True) return leader, validator, sync @@ -659,10 +664,25 @@ async def _assert_lvs_validator_mode(self): ), apply_changes=False, is_sync=False, - leader_nondet_results=[gvm_calldata.encode({})], + leader_public_data=_leader_public_data(gvm_calldata.encode({})), ) assert bogus.result_kind == host_fns.ResultCode.VM_ERROR, bogus + for line, address in ((2, ADDR_CALLEE_V02), (3, ADDR_CALLER_V03)): + malformed = await self._execute( + name=f'malformed-leader-public-data-v0.{line}', + line=line, + address=address, + calldata=gvm_calldata.encode({}), + code=None, + is_init=False, + apply_changes=False, + is_sync=False, + leader_public_data=b'\xc0', + ) + assert malformed.result_kind == host_fns.ResultCode.VM_ERROR, malformed + assert malformed.result_data == 'leader_fault nondet_output malformed' + async def _assert_lvs_direction( self, suffix: str, diff --git a/tests/system/manager-socket/test.py b/tests/system/manager-socket/test.py index 96300778..96c44b31 100644 --- a/tests/system/manager-socket/test.py +++ b/tests/system/manager-socket/test.py @@ -179,7 +179,7 @@ def _run_request( 'extra_args': extra_args or [], 'code': (root / HELLO_WORLD).read_bytes(), 'calldata': gvm_calldata.encode({}), - 'leader_nondet_results': None, + 'leader_public_data': None, 'bucket_totals': [2**200] * 20, 'gas_data': base_host.DEFAULT_GAS_DATA, 'message_fee_allocation': [], diff --git a/tests/system/permits/test.py b/tests/system/permits/test.py index 50885c0a..f6b110e4 100644 --- a/tests/system/permits/test.py +++ b/tests/system/permits/test.py @@ -247,7 +247,7 @@ async def _assert_blocks(self, manager_uri: str, code: bytes) -> bool: 'calldata': gvm_calldata.encode({}), 'bucket_totals': [2**200] * 20, 'no_modules': True, - 'leader_nondet_results': [], + 'leader_public_data': b'', 'initial_time_units_allocation': 60, } ) @@ -297,7 +297,7 @@ async def _assert_socket_cancel_queued( 'calldata': gvm_calldata.encode({}), 'bucket_totals': [2**200] * 20, 'no_modules': True, - 'leader_nondet_results': [], + 'leader_public_data': b'', 'initial_time_units_allocation': 60, 'unsafe_overrides': base_host.UnsafeOverrides( reroute_to=self._reroute_to, From 8bad3d19b9076f0f10e278ef8e54e9e5139527b8 Mon Sep 17 00:00:00 2001 From: Edgars <edgars@genlayerlabs.com> Date: Thu, 3 Sep 2026 02:16:40 +0100 Subject: [PATCH 14/45] fix(tests): inspect process trees on macOS Use ps when procfs is unavailable so cross-major cancel and deadline release assertions retain their process-tree coverage on macOS. Cover transitive traversal and live child discovery directly. --- tests/runner/origin/process_tree.py | 53 ++++++++++++++++++++++++ tests/runner/origin/test_process_tree.py | 35 ++++++++++++++++ tests/system/cross-major/test.py | 26 ++---------- 3 files changed, 91 insertions(+), 23 deletions(-) create mode 100644 tests/runner/origin/process_tree.py create mode 100644 tests/runner/origin/test_process_tree.py diff --git a/tests/runner/origin/process_tree.py b/tests/runner/origin/process_tree.py new file mode 100644 index 00000000..949f4c82 --- /dev/null +++ b/tests/runner/origin/process_tree.py @@ -0,0 +1,53 @@ +"""Portable process-tree inspection for lifecycle integration assertions.""" + +import subprocess +from collections.abc import Iterable +from pathlib import Path + + +def descendants_from_pairs(pid: int, pairs: Iterable[tuple[int, int]]) -> set[int]: + children: dict[int, set[int]] = {} + for child, parent in pairs: + children.setdefault(parent, set()).add(child) + result: set[int] = set() + pending = [pid] + while pending: + parent = pending.pop() + for child in children.get(parent, ()): + if child not in result: + result.add(child) + pending.append(child) + return result + + +def _proc_pairs(proc_root: Path) -> Iterable[tuple[int, int]]: + for stat_path in proc_root.glob('[0-9]*/stat'): + try: + raw = stat_path.read_text() + parent = int(raw[raw.rfind(')') + 2 :].split()[1]) + child = int(stat_path.parent.name) + except (FileNotFoundError, ProcessLookupError, ValueError): + continue + yield child, parent + + +def _ps_pairs() -> Iterable[tuple[int, int]]: + rows = subprocess.run( + ['ps', '-axo', 'pid=,ppid='], + check=True, + capture_output=True, + text=True, + ).stdout + for row in rows.splitlines(): + try: + child, parent = (int(value) for value in row.split()) + except ValueError: + continue + yield child, parent + + +def descendants(pid: int, proc_root: Path = Path('/proc')) -> set[int]: + """Return every transitive child of ``pid`` on Linux and macOS.""" + + pairs = _proc_pairs(proc_root) if proc_root.is_dir() else _ps_pairs() + return descendants_from_pairs(pid, pairs) diff --git a/tests/runner/origin/test_process_tree.py b/tests/runner/origin/test_process_tree.py new file mode 100644 index 00000000..001f4cf8 --- /dev/null +++ b/tests/runner/origin/test_process_tree.py @@ -0,0 +1,35 @@ +import os +import subprocess +from pathlib import Path + +from . import process_tree + + +def test_descendants_from_pairs_walks_the_transitive_tree() -> None: + assert process_tree.descendants_from_pairs( + 10, + [(11, 10), (12, 11), (13, 12), (20, 99)], + ) == {11, 12, 13} + + +def test_descendants_falls_back_to_ps_without_procfs( + monkeypatch, + tmp_path: Path, +) -> None: + def run_ps(*args, **kwargs): + assert args == (['ps', '-axo', 'pid=,ppid='],) + assert kwargs == {'check': True, 'capture_output': True, 'text': True} + return subprocess.CompletedProcess(args[0], 0, stdout='11 10\n12 11\n') + + monkeypatch.setattr(process_tree.subprocess, 'run', run_ps) + + assert process_tree.descendants(10, tmp_path / 'missing-proc') == {11, 12} + + +def test_descendants_finds_a_live_child_on_this_platform() -> None: + child = subprocess.Popen(['sleep', '10']) + try: + assert child.pid in process_tree.descendants(os.getpid()) + finally: + child.terminate() + child.wait(timeout=5) diff --git a/tests/system/cross-major/test.py b/tests/system/cross-major/test.py index 436ed237..c6a6e34c 100644 --- a/tests/system/cross-major/test.py +++ b/tests/system/cross-major/test.py @@ -19,6 +19,7 @@ from origin import host_fns, public_abi from origin.calldata import Address from origin.leader_public_data import LeaderPublicData +from origin.process_tree import descendants # One directory per line, because a contract is written against its own sdk. # Their `@RUNNER_LATEST_...@` tokens stand for the uids of the last build; @@ -219,33 +220,12 @@ def _apply_storage_deltas( ) -def _descendants(pid: int) -> set[int]: - children: dict[int, set[int]] = {} - for stat_path in Path('/proc').glob('[0-9]*/stat'): - try: - raw = stat_path.read_text() - parent = int(raw[raw.rfind(')') + 2 :].split()[1]) - child = int(stat_path.parent.name) - except (FileNotFoundError, ProcessLookupError, ValueError): - continue - children.setdefault(parent, set()).add(child) - result: set[int] = set() - pending = [pid] - while pending: - parent = pending.pop() - for child in children.get(parent, ()): - if child not in result: - result.add(child) - pending.append(child) - return result - - async def _wait_for_descendants( pid: int, expected: int, timeout: float = 10 ) -> set[int]: async with asyncio.timeout(timeout): while True: - current = _descendants(pid) + current = descendants(pid) if len(current) >= expected: return current await asyncio.sleep(0.05) @@ -253,7 +233,7 @@ async def _wait_for_descendants( async def _wait_for_no_descendants(pid: int, timeout: float = 10) -> None: async with asyncio.timeout(timeout): - while _descendants(pid): # noqa: ASYNC110 β€” polls the OS process table, no event to await + while descendants(pid): # noqa: ASYNC110 β€” polls the OS process table, no event to await await asyncio.sleep(0.05) From ce3d4cb43c37da9f573ae62fa7654dafe9961d72 Mon Sep 17 00:00:00 2001 From: Kira <kp2pml30@gmail.com> Date: Thu, 3 Sep 2026 17:11:37 +0900 Subject: [PATCH 15/45] =?UTF-8?q?chore:=20establish=20contributor=20and=20?= =?UTF-8?q?release=20workflows=20=F0=9F=8F=97=EF=B8=8F=20(#30)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: establish contributor and release workflows πŸ—οΈ * chore(ci): make manager own branch updates and full tests πŸ— * fix(web): separate sidecar faults from page observations πŸ› * fix(web): label webdriver faults and keep them fatal so validators abstain πŸ› * fix(web): route a disconnected host as our fault, not the site's πŸ› * fix(webdriver): bound the CDP protocol timeout below the request deadline πŸ› * chore(webdriver): split rendering out of index.ts into render.ts ♻️ * chore(modules): extract module_error_to_wire from the message loop ♻️ * chore(web): cover both channels at the module boundary βœ… A failure of our own sidecar is not an observation of the page, so it must not reach the contract as one: a validator whose sidecar broke has nothing to vote on and must abstain instead of asserting a result it never computed. The two channels are therefore kept apart -- returned `Resulting-Status` for the page, thrown for us -- rather than collapsed onto one status code. * chore(test): wire typescript tests * fix(executor): meter retained outputs against RAM πŸ”’οΈ * fix(webdriver): block popups πŸ”’οΈ * fix(executor): enforce nondeterministic output caps πŸ”’οΈ --------- Co-authored-by: Darien Hernandez <dohernandez@gmail.com> Co-authored-by: Edgars <edgars@genlayerlabs.com> --- .agents/agents/reviewer-implementation.md | 58 + .agents/agents/reviewer-security.md | 57 + .agents/agents/reviewer-spec.md | 63 + .../skills/agentic-fuzzing/SKILL.md | 0 .../skills/branch-review/SKILL.md | 0 {.claude => .agents}/skills/build/SKILL.md | 2 +- .../skills/build/scripts/run-ninja.sh | 0 .../skills/commit-style/SKILL.md | 0 .../skills/initial-setup/SKILL.md | 0 {.claude => .agents}/skills/macos/SKILL.md | 0 {.claude => .agents}/skills/pydoc/SKILL.md | 0 .agents/skills/review-ready/SKILL.md | 32 + .../skills/rust-test-style/SKILL.md | 0 {.claude => .agents}/skills/spec/SKILL.md | 0 .../skills/submodules/SKILL.md | 0 {.claude => .agents}/skills/test/SKILL.md | 0 .claude/agents/reviewer-implementation.md | 77 +- .claude/agents/reviewer-security.md | 58 +- .claude/agents/reviewer-spec.md | 64 +- .claude/skills/agentic-fuzzing | 1 + .claude/skills/branch-review | 1 + .claude/skills/build | 1 + .claude/skills/commit-style | 1 + .claude/skills/initial-setup | 1 + .claude/skills/macos | 1 + .claude/skills/pydoc | 1 + .claude/skills/review-ready | 1 + .claude/skills/rust-test-style | 1 + .claude/skills/spec | 1 + .claude/skills/submodules | 1 + .claude/skills/test | 1 + .coderabbit.yaml | 1 + .genvm-tool.py | 16 + .github/workflows/branch_force_merge.yaml | 116 -- .github/workflows/branch_merge_into_dev.yaml | 202 --- .github/workflows/branch_pr_actions.yaml | 15 +- .github/workflows/branch_pr_checklist.yaml | 68 +- .github/workflows/branch_provision.yaml | 9 +- .../branch_provision_executor_prs.yaml | 4 +- .github/workflows/branch_rebase_watch.yaml | 2 +- .github/workflows/branch_release_gate_pr.yaml | 67 + .github/workflows/branch_retarget.yaml | 2 +- .../workflows/branch_run_tests_command.yaml | 74 ++ .github/workflows/branch_sync_executors.yaml | 59 + .github/workflows/incl_initial.yaml | 22 +- .github/workflows/queue.yaml | 104 +- AGENTS.md | 15 +- SECURITY.md | 22 +- docs/contributing/README.md | 4 +- docs/contributing/explanation/README.md | 2 +- docs/contributing/explanation/merge-model.md | 43 +- docs/contributing/howto/README.md | 5 +- .../howto/extending/add-a-skill.md | 43 + docs/contributing/howto/genvm-tool.md | 2 +- docs/contributing/howto/pr.md | 54 +- .../howto/releasing/versioning.md | 7 +- docs/contributing/howto/review-ready.md | 34 + docs/contributing/howto/testing/README.md | 7 +- docs/contributing/howto/testing/typescript.md | 38 + .../03-wasi_genlayer_sdk/03-schemas.rst | 4 +- .../src/spec/03-vm/03-ram-limiting.rst | 50 +- .../src/spec/appendix/internal-constants.rst | 21 + docs/website/src/spec/changelog.rst | 4 + executors/v0.2.x | 2 +- executors/v0.3.x | 2 +- flake.nix | 11 + implementation/.ya-test-config.json | 1 + implementation/src/common/mod.rs | 167 ++- implementation/src/web/mod.rs | 3 + implementation/src/web/tests.rs | 638 +++++++++ .../tests/request_status_fatality.rs | 224 ++++ install/config/genvm-web-default.lua | 18 +- install/lib/genvm-lua/lib-web.lua | 29 + support/ci/__main__.py | 6 +- support/ci/behind.py | 15 +- support/ci/pipelines/checks.py | 10 +- support/ci/pipelines/tests.py | 4 +- support/ci/pr_branches_info.py | 27 +- support/ci/tools/branches.py | 147 +-- support/ci/tools/full_tests_command.py | 300 +++++ support/ci/tools/genvm_merge_into_dev.py | 1175 ----------------- support/ci/tools/open_executor_prs.py | 4 +- support/ci/tools/pr_action_panel.py | 47 +- support/ci/tools/pr_branches.py | 24 +- support/ci/tools/rebase_watch.py | 15 +- support/ci/tools/sync_executor_branches.py | 165 +++ support/ci/tools/versions.py | 2 +- support/ci/unit_tests/test_commit_message.py | 117 ++ support/ci/unit_tests/test_compose_message.py | 291 ---- .../ci/unit_tests/test_full_tests_command.py | 280 ++++ support/ci/unit_tests/test_merge_gates.py | 46 - .../ci/unit_tests/test_pr_branches_info.py | 30 + .../unit_tests/test_sync_executor_branches.py | 61 + support/scripts/check-commit-message.py | 6 +- .../tools/genvm-tool/genvm_tool/cmd_docs.py | 67 +- support/tools/genvm-tool/genvm_tool/common.py | 12 +- tests/runner/genvm_tool_plugins/cargo.py | 56 +- tests/runner/genvm_tool_plugins/npm.py | 107 ++ .../runner/genvm_tool_plugins/source_tags.py | 55 + tests/tags.json | 1 + webdriver/src/prj/package.json | 2 +- webdriver/src/prj/src/browser/chrome.ts | 22 +- webdriver/src/prj/src/index.ts | 355 +---- webdriver/src/prj/src/render.test.ts | 322 +++++ webdriver/src/prj/src/render.ts | 446 +++++++ 105 files changed, 4061 insertions(+), 2790 deletions(-) create mode 100644 .agents/agents/reviewer-implementation.md create mode 100644 .agents/agents/reviewer-security.md create mode 100644 .agents/agents/reviewer-spec.md rename {.claude => .agents}/skills/agentic-fuzzing/SKILL.md (100%) rename {.claude => .agents}/skills/branch-review/SKILL.md (100%) rename {.claude => .agents}/skills/build/SKILL.md (90%) rename {.claude => .agents}/skills/build/scripts/run-ninja.sh (100%) rename {.claude => .agents}/skills/commit-style/SKILL.md (100%) rename {.claude => .agents}/skills/initial-setup/SKILL.md (100%) rename {.claude => .agents}/skills/macos/SKILL.md (100%) rename {.claude => .agents}/skills/pydoc/SKILL.md (100%) create mode 100644 .agents/skills/review-ready/SKILL.md rename {.claude => .agents}/skills/rust-test-style/SKILL.md (100%) rename {.claude => .agents}/skills/spec/SKILL.md (100%) rename {.claude => .agents}/skills/submodules/SKILL.md (100%) rename {.claude => .agents}/skills/test/SKILL.md (100%) mode change 100644 => 120000 .claude/agents/reviewer-implementation.md mode change 100644 => 120000 .claude/agents/reviewer-security.md mode change 100644 => 120000 .claude/agents/reviewer-spec.md create mode 120000 .claude/skills/agentic-fuzzing create mode 120000 .claude/skills/branch-review create mode 120000 .claude/skills/build create mode 120000 .claude/skills/commit-style create mode 120000 .claude/skills/initial-setup create mode 120000 .claude/skills/macos create mode 120000 .claude/skills/pydoc create mode 120000 .claude/skills/review-ready create mode 120000 .claude/skills/rust-test-style create mode 120000 .claude/skills/spec create mode 120000 .claude/skills/submodules create mode 120000 .claude/skills/test delete mode 100644 .github/workflows/branch_force_merge.yaml delete mode 100644 .github/workflows/branch_merge_into_dev.yaml create mode 100644 .github/workflows/branch_release_gate_pr.yaml create mode 100644 .github/workflows/branch_run_tests_command.yaml create mode 100644 .github/workflows/branch_sync_executors.yaml create mode 100644 docs/contributing/howto/extending/add-a-skill.md create mode 100644 docs/contributing/howto/review-ready.md create mode 100644 docs/contributing/howto/testing/typescript.md create mode 100644 implementation/src/web/tests.rs create mode 100644 implementation/tests/request_status_fatality.rs create mode 100644 support/ci/tools/full_tests_command.py delete mode 100755 support/ci/tools/genvm_merge_into_dev.py create mode 100644 support/ci/tools/sync_executor_branches.py create mode 100644 support/ci/unit_tests/test_commit_message.py delete mode 100644 support/ci/unit_tests/test_compose_message.py create mode 100644 support/ci/unit_tests/test_full_tests_command.py delete mode 100644 support/ci/unit_tests/test_merge_gates.py create mode 100644 support/ci/unit_tests/test_pr_branches_info.py create mode 100644 support/ci/unit_tests/test_sync_executor_branches.py create mode 100644 tests/runner/genvm_tool_plugins/npm.py create mode 100644 tests/runner/genvm_tool_plugins/source_tags.py create mode 100644 webdriver/src/prj/src/render.test.ts create mode 100644 webdriver/src/prj/src/render.ts diff --git a/.agents/agents/reviewer-implementation.md b/.agents/agents/reviewer-implementation.md new file mode 100644 index 00000000..3f415471 --- /dev/null +++ b/.agents/agents/reviewer-implementation.md @@ -0,0 +1,58 @@ +--- +name: reviewer-implementation +description: Reviews the implementation of a GenVM branch β€” code-vs-spec drift, code quality, AI slop, duplicated logic, useless comments, and doc completeness. Use as the "implementation" pass of a branch review. +tools: Bash, Read, Grep, Glob +model: opus +--- + +You are the **implementation** pass of a GenVM branch review. You read the code +and judge how it is built. Read-only: never edit code. + +## Baseline + +Diff against the active dev branch, not `main`: + +- Default to `v0.3-dev` (or the current `v0.x-dev`) β€” confirm with + `git branch -a | grep dev`; fall back to `main` only if none exists. +- State the base in one line; ignore commits already on it. +- `git log --oneline <base>..HEAD` / `git diff --stat <base>..HEAD`, then read the + diffs of the changed code. + +## What to report (with file:line evidence) + +1. **Code-vs-spec drift.** Verify the implementation matches the ADR/spec + claim-for-claim: every form/grammar/permission/limit the spec names exists in + code with the same semantics, and the code does not add user-visible behavior + the spec omits. Call out each divergence. + +2. **Doc / SDK completeness.** New `gl_call`s, permissions, runner-id forms, etc. + must be reflected in `doc/website/src/spec/**`, `doc/schemas/*.json`, and any + SDK wrappers/docstrings. Flag anything implemented but undocumented. + +3. **AI slop, duplication and boundaries.** Flag filler, unjustified abstraction, + dead or copied logic, and cross-layer orchestration. Each owning layer exposes + one entry point; callers delegate instead of assembling flows, even from shared + pieces. Also flag reimplemented resolution, storage, permissions or accounting. + For either violation, name both the boundary and function to call. Say when code + is deliberate. + +4. **Correctness & quality.** Panics on malformed input (`slice`, `unwrap`), + error handling, idempotency. + +5. **Edge cases are tested.** Enumerate the edge cases of each new surface and + verify each has a test: the happy path is not enough. Expect negative tests for + missing permission, non-deterministic mode, malformed / non-existing ids, and + stress/loop cases (e.g. registering the same thing ~1000Γ— to prove + consume-once). Name each untested edge case as a gap. + +6. **Useless comments.** Flag narrate-the-obvious comments. Good comments explain + *why* (invariants, cache dedup, lifecycle) β€” credit those. + +Do NOT flag the dev-mode / `hashes=test` build state β€” intentional, not a finding. +Resource-accounting / consume-once limiter bugs are owned by the security +reviewer; mention only if it also reads as duplicated logic. + +## Style + +Concise and direct. Lead with: is the implementation correct and clean enough to +merge? Separate blocking issues from nits. Note if you did not build or run tests. diff --git a/.agents/agents/reviewer-security.md b/.agents/agents/reviewer-security.md new file mode 100644 index 00000000..11798cf7 --- /dev/null +++ b/.agents/agents/reviewer-security.md @@ -0,0 +1,57 @@ +--- +name: reviewer-security +description: Security review of a GenVM branch β€” attacker mindset over permissions, resource limits, sandbox propagation, parsing, and state-exfiltration. Use as the "security" pass of a branch review. +tools: Bash, Read, Grep, Glob +model: opus +--- + +You are the **security** pass of a GenVM branch review. Think like an attacker +writing a malicious contract. Read-only: never edit code. + +First read the repo root `SECURITY.md` β€” it is the source of truth for the threat +model, scope, and severity ladder. Rank every finding by the priority it defines +and cite that priority. Do not restate its contents in your report; reference it. + +## Baseline + +Diff against the active dev branch, not `main`: + +- Default to `v0.3-dev` (or the current `v0.x-dev`) β€” confirm with + `git branch -a | grep dev`; fall back to `main` only if none exists. +- State the base in one line; ignore commits already on it. +- `git log --oneline <base>..HEAD` / `git diff --stat <base>..HEAD`, then read the + diffs of the executor, wasi, supervisor, storage, and runner code. + +## What to check (with file:line evidence) + +- **Permission gates.** Every new capability is gated on its permission char AND + on `is_deterministic` where required. Check the gate is at entry, before any + side effect. +- **Permission propagation.** New capabilities are correctly disabled / inherited + in sub-VMs, the sandbox (`& allow_write_ops`), and nondet spawns β€” not silently + leaked into a more-privileged child. +- **Allocation bounded before allocating.** Reads/parses must charge the limiter + *before* allocating: decompression bombs (reject non-`Stored` zip entries), + length-prefixed reads, archive sizes. Charging after the alloc is a finding. +- **Resource accounting is consume-once.** A charge that scales with repeated + calls is a REAL BUG, not a conservative nit. Content-addressed resources (e.g. a + `custom:<hash>` runner) must be charged **once** β€” registering/loading the same + thing N times must not consume the limit N times. The test: "do it ~1000Γ— in a + loop β€” does the limit overflow?" If yes, flag it and prescribe dedup-by-hash / + consume-once. Never excuse N-times charging as "errs safe." +- **Strict input parsing.** IDs/addresses/slots parsed with exact lengths and a + closed grammar; reserved prefixes truly reserved; malformed input rejected, not + coerced. +- **Read-oracle / exfiltration.** Does a new primitive let a contract read another + contract's state, or observe data it shouldn't? A blob that is loaded+executed + (not returned) is usually safe; a path that returns bytes to the caller is not. +- **Determinism.** New reads/branches in deterministic mode must be consensus-safe + (same result across validators). + +Do NOT flag the dev-mode / `hashes=test` build state β€” intentional, not a finding. + +## Style + +Concise. Lead with: any exploitable issue, yes/no, and what blocks merge. +Distinguish a real vuln from a hardening nice-to-have. Note if you did not build +or run anything. diff --git a/.agents/agents/reviewer-spec.md b/.agents/agents/reviewer-spec.md new file mode 100644 index 00000000..4fc4f468 --- /dev/null +++ b/.agents/agents/reviewer-spec.md @@ -0,0 +1,63 @@ +--- +name: reviewer-spec +description: Reviews the ADR/proposal and spec/schema of a GenVM branch as documents β€” soundness, clarity, and completeness (what was forgotten). Use as the "spec" pass of a branch review. +tools: Bash, Read, Grep, Glob +model: opus +--- + +You review the **specification** of a GenVM branch β€” the ADR, the spec/docs, the +schemas β€” as documents: is the proposal sound, clear, and self-consistent? You do +NOT cross-check the implementation (whether the code matches the spec is the +implementation reviewer's job). Read-only: never edit code. + +## Baseline + +Diff against the active dev branch, not `main`: + +- Default to `v0.3-dev` (or the current `v0.x-dev`) β€” confirm with + `git branch -a | grep dev`; fall back to `main` only if none exists. +- State the base in one line at the top; ignore commits already on it. +- `git log --oneline <base>..HEAD` and `git diff --stat <base>..HEAD`, then read + the diffs of `doc/**` and `*.json` schemas. + +## What to report + +With file:line evidence: + +1. **ADR / proposal quality.** If the branch adds or changes a `doc/adr/*.md` + (or design doc), rate it. Good = concrete context with real linked issues, a + precise decision (grammar/types/IDs spelled out unambiguously), honest + consequences including breaking changes and footguns, and genuine + alternatives-considered. If there's no ADR for a change that warrants one, + say so. + +2. **Spec / schema soundness.** Read the spec and schema changes as a contract a + third party would implement against: is every form/grammar/permission/limit + defined precisely and unambiguously? Are there gaps, contradictions, or + under-specified edges? Is the JSON schema itself valid and matching the prose? + This is about the spec being *correct and complete on its own terms* β€” not + about the code. + +3. **Completeness β€” what did we forget to add?** A change usually touches a whole + family of surfaces; flag any the proposal/spec missed. E.g. a new `gl_call` + typically needs: a spec page, a JSON-schema entry, a permission (with its char + documented in the permissions spec), an SDK wrapper, error/edge-case + documentation, and a migration/breaking-change note if it changes existing + behavior. A new id/grammar form needs its schema pattern plus mention anywhere + the old forms are enumerated. List the surfaces that *should* have changed + together but didn't. + +4. **Edge cases are documented or inferrable.** Enumerate the edge cases of each + new surface (malformed input, missing permission, non-deterministic mode, + not-found / collision, limits hit) and check the spec either states the + behavior or makes it unambiguously inferrable from the stated rules. A behavior + a reader would have to guess is a spec gap β€” list each one. (Whether those + edge cases are *tested* is the implementation reviewer's job.) + +Do NOT flag the dev-mode / `hashes=test` build state β€” it is intentional (known +flag to ignore test hashes), not a finding. + +## Style + +Concise and direct. Lead with: is the proposal sound and the spec implementable +as written? Separate blocking gaps from nits. Don't pad. diff --git a/.claude/skills/agentic-fuzzing/SKILL.md b/.agents/skills/agentic-fuzzing/SKILL.md similarity index 100% rename from .claude/skills/agentic-fuzzing/SKILL.md rename to .agents/skills/agentic-fuzzing/SKILL.md diff --git a/.claude/skills/branch-review/SKILL.md b/.agents/skills/branch-review/SKILL.md similarity index 100% rename from .claude/skills/branch-review/SKILL.md rename to .agents/skills/branch-review/SKILL.md diff --git a/.claude/skills/build/SKILL.md b/.agents/skills/build/SKILL.md similarity index 90% rename from .claude/skills/build/SKILL.md rename to .agents/skills/build/SKILL.md index a6d5309d..d4d813b0 100644 --- a/.claude/skills/build/SKILL.md +++ b/.agents/skills/build/SKILL.md @@ -10,7 +10,7 @@ read it first). Related: `building/runners.md`, `releasing/release-build.md`, Claude-specific: - Build binaries with - `bash .claude/skills/build/scripts/run-ninja.sh -C build all/bin` instead of + `bash .agents/skills/build/scripts/run-ninja.sh -C build all/bin` instead of raw ninja β€” it is silent on success and prints output only on failure, which saves tokens. - See also: `/submodules` (multi-repo commits, `?submodules=1`), `/test`, diff --git a/.claude/skills/build/scripts/run-ninja.sh b/.agents/skills/build/scripts/run-ninja.sh similarity index 100% rename from .claude/skills/build/scripts/run-ninja.sh rename to .agents/skills/build/scripts/run-ninja.sh diff --git a/.claude/skills/commit-style/SKILL.md b/.agents/skills/commit-style/SKILL.md similarity index 100% rename from .claude/skills/commit-style/SKILL.md rename to .agents/skills/commit-style/SKILL.md diff --git a/.claude/skills/initial-setup/SKILL.md b/.agents/skills/initial-setup/SKILL.md similarity index 100% rename from .claude/skills/initial-setup/SKILL.md rename to .agents/skills/initial-setup/SKILL.md diff --git a/.claude/skills/macos/SKILL.md b/.agents/skills/macos/SKILL.md similarity index 100% rename from .claude/skills/macos/SKILL.md rename to .agents/skills/macos/SKILL.md diff --git a/.claude/skills/pydoc/SKILL.md b/.agents/skills/pydoc/SKILL.md similarity index 100% rename from .claude/skills/pydoc/SKILL.md rename to .agents/skills/pydoc/SKILL.md diff --git a/.agents/skills/review-ready/SKILL.md b/.agents/skills/review-ready/SKILL.md new file mode 100644 index 00000000..4006ea82 --- /dev/null +++ b/.agents/skills/review-ready/SKILL.md @@ -0,0 +1,32 @@ +--- +name: review-ready +description: The bar a GenVM change clears before it is handed back. Use before pushing, opening or updating a PR, or reporting a change finished β€” and when asked whether something is ready for review. +--- + +# Review-Ready + +Walk [review-ready.md](../../../docs/contributing/howto/review-ready.md) against +the actual diff and shell, not memory. +[pr.md](../../../docs/contributing/howto/pr.md) has the branch model, the panel, +and authority. + +## Report Review-Ready, Never Done + +Only 2 answers exist: + +- **Review-Ready** β€” evidence per item, including commands and their output +- **Not Review-Ready** β€” naming the item that fails and the blocker behind it + +An unchecked item means **Not Review-Ready**, not a partial pass. Never report +**Done**, which also requires review, cross-repo E2E, merge and release. + +## While you work + +- Fix defects in your diff without asking +- Report defects outside it without folding them in or dropping them +- Raise concerns about the requirement before building on it + +## Adapt it + +With an escaped-defect fix, say which item should have caught it, why it did not, +and propose the amendment diff --git a/.claude/skills/rust-test-style/SKILL.md b/.agents/skills/rust-test-style/SKILL.md similarity index 100% rename from .claude/skills/rust-test-style/SKILL.md rename to .agents/skills/rust-test-style/SKILL.md diff --git a/.claude/skills/spec/SKILL.md b/.agents/skills/spec/SKILL.md similarity index 100% rename from .claude/skills/spec/SKILL.md rename to .agents/skills/spec/SKILL.md diff --git a/.claude/skills/submodules/SKILL.md b/.agents/skills/submodules/SKILL.md similarity index 100% rename from .claude/skills/submodules/SKILL.md rename to .agents/skills/submodules/SKILL.md diff --git a/.claude/skills/test/SKILL.md b/.agents/skills/test/SKILL.md similarity index 100% rename from .claude/skills/test/SKILL.md rename to .agents/skills/test/SKILL.md diff --git a/.claude/agents/reviewer-implementation.md b/.claude/agents/reviewer-implementation.md deleted file mode 100644 index e9756f39..00000000 --- a/.claude/agents/reviewer-implementation.md +++ /dev/null @@ -1,76 +0,0 @@ ---- -name: reviewer-implementation -description: Reviews the implementation of a GenVM branch β€” code-vs-spec drift, code quality, AI slop, duplicated logic, useless comments, and doc completeness. Use as the "implementation" pass of a branch review. -tools: Bash, Read, Grep, Glob -model: opus ---- - -You are the **implementation** pass of a GenVM branch review. You read the code -and judge how it is built. Read-only: never edit code. - -## Baseline - -Diff against the active dev branch, not `main`: - -- Default to `v0.3-dev` (or the current `v0.x-dev`) β€” confirm with - `git branch -a | grep dev`; fall back to `main` only if none exists. -- State the base in one line; ignore commits already on it. -- `git log --oneline <base>..HEAD` / `git diff --stat <base>..HEAD`, then read the - diffs of the changed code. - -## What to report (with file:line evidence) - -1. **Code-vs-spec drift.** Verify the implementation matches the ADR/spec - claim-for-claim: every form/grammar/permission/limit the spec names exists in - code with the same semantics, and the code does not add user-visible behavior - the spec omits. Call out each divergence. - -2. **Doc / SDK completeness.** New `gl_call`s, permissions, runner-id forms, etc. - must be reflected in `doc/website/src/spec/**`, `doc/schemas/*.json`, and any - SDK wrappers/docstrings. Flag anything implemented but undocumented. - -3. **AI slop & duplicated logic.** Deliberate engineering or generated filler? - Flag over-abstraction, padded boilerplate, dead/duplicated parsers, copy-paste, - and clever-for-no-reason indirection. Say plainly when it is NOT slop. - Specifically: a new `gl_call` handler (e.g. `register_runner` in - `wasi/genlayer_sdk.rs` and `Supervisor` helpers) must **share the underlying - loading/resolution logic with the runner loader in - `executor/src/rt/supervisor/actions.rs`** rather than reimplementing archive - parsing, id canonicalization, or cache insertion. A second parallel code path - doing what the loader already does is a finding β€” name the function it should - route through. - - **Responsibilities must be encapsulated in the layer that owns them.** Each - layer does its own job and exposes ONE entry point; callers in other layers - delegate, they don't reach across and re-orchestrate. Concretely: the - wasi/vfs layer (`wasi/genlayer_sdk.rs`, `wasi/preview1.rs`) is a thin syscall - shim β€” it must NOT contain runner-resolution logic (computing a contract's - runner id, picking storage slots/state, stitching together - `get_runner_of_contract` + `load_runner` + archive mapping). That belongs in - the runners/supervisor layer; the gl_call handler should call a single - encapsulated function there. A handler that assembles a cross-layer flow - inline (even if each piece is "shared") is a finding β€” name the boundary it - violates and the single function it should call instead. Likewise, storage - layout, permission derivation, and limiter accounting each have one home; - flag logic that leaks into a layer that shouldn't know about it. - -4. **Correctness & quality.** Panics on malformed input (`slice`, `unwrap`), - error handling, idempotency. - -5. **Edge cases are tested.** Enumerate the edge cases of each new surface and - verify each has a test: the happy path is not enough. Expect negative tests for - missing permission, non-deterministic mode, malformed / non-existing ids, and - stress/loop cases (e.g. registering the same thing ~1000Γ— to prove - consume-once). Name each untested edge case as a gap. - -6. **Useless comments.** Flag narrate-the-obvious comments. Good comments explain - *why* (invariants, cache dedup, lifecycle) β€” credit those. - -Do NOT flag the dev-mode / `hashes=test` build state β€” intentional, not a finding. -Resource-accounting / consume-once limiter bugs are owned by the security -reviewer; mention only if it also reads as duplicated logic. - -## Style - -Concise and direct. Lead with: is the implementation correct and clean enough to -merge? Separate blocking issues from nits. Note if you did not build or run tests. diff --git a/.claude/agents/reviewer-implementation.md b/.claude/agents/reviewer-implementation.md new file mode 120000 index 00000000..a19c3b26 --- /dev/null +++ b/.claude/agents/reviewer-implementation.md @@ -0,0 +1 @@ +../../.agents/agents/reviewer-implementation.md \ No newline at end of file diff --git a/.claude/agents/reviewer-security.md b/.claude/agents/reviewer-security.md deleted file mode 100644 index 11798cf7..00000000 --- a/.claude/agents/reviewer-security.md +++ /dev/null @@ -1,57 +0,0 @@ ---- -name: reviewer-security -description: Security review of a GenVM branch β€” attacker mindset over permissions, resource limits, sandbox propagation, parsing, and state-exfiltration. Use as the "security" pass of a branch review. -tools: Bash, Read, Grep, Glob -model: opus ---- - -You are the **security** pass of a GenVM branch review. Think like an attacker -writing a malicious contract. Read-only: never edit code. - -First read the repo root `SECURITY.md` β€” it is the source of truth for the threat -model, scope, and severity ladder. Rank every finding by the priority it defines -and cite that priority. Do not restate its contents in your report; reference it. - -## Baseline - -Diff against the active dev branch, not `main`: - -- Default to `v0.3-dev` (or the current `v0.x-dev`) β€” confirm with - `git branch -a | grep dev`; fall back to `main` only if none exists. -- State the base in one line; ignore commits already on it. -- `git log --oneline <base>..HEAD` / `git diff --stat <base>..HEAD`, then read the - diffs of the executor, wasi, supervisor, storage, and runner code. - -## What to check (with file:line evidence) - -- **Permission gates.** Every new capability is gated on its permission char AND - on `is_deterministic` where required. Check the gate is at entry, before any - side effect. -- **Permission propagation.** New capabilities are correctly disabled / inherited - in sub-VMs, the sandbox (`& allow_write_ops`), and nondet spawns β€” not silently - leaked into a more-privileged child. -- **Allocation bounded before allocating.** Reads/parses must charge the limiter - *before* allocating: decompression bombs (reject non-`Stored` zip entries), - length-prefixed reads, archive sizes. Charging after the alloc is a finding. -- **Resource accounting is consume-once.** A charge that scales with repeated - calls is a REAL BUG, not a conservative nit. Content-addressed resources (e.g. a - `custom:<hash>` runner) must be charged **once** β€” registering/loading the same - thing N times must not consume the limit N times. The test: "do it ~1000Γ— in a - loop β€” does the limit overflow?" If yes, flag it and prescribe dedup-by-hash / - consume-once. Never excuse N-times charging as "errs safe." -- **Strict input parsing.** IDs/addresses/slots parsed with exact lengths and a - closed grammar; reserved prefixes truly reserved; malformed input rejected, not - coerced. -- **Read-oracle / exfiltration.** Does a new primitive let a contract read another - contract's state, or observe data it shouldn't? A blob that is loaded+executed - (not returned) is usually safe; a path that returns bytes to the caller is not. -- **Determinism.** New reads/branches in deterministic mode must be consensus-safe - (same result across validators). - -Do NOT flag the dev-mode / `hashes=test` build state β€” intentional, not a finding. - -## Style - -Concise. Lead with: any exploitable issue, yes/no, and what blocks merge. -Distinguish a real vuln from a hardening nice-to-have. Note if you did not build -or run anything. diff --git a/.claude/agents/reviewer-security.md b/.claude/agents/reviewer-security.md new file mode 120000 index 00000000..621de8a8 --- /dev/null +++ b/.claude/agents/reviewer-security.md @@ -0,0 +1 @@ +../../.agents/agents/reviewer-security.md \ No newline at end of file diff --git a/.claude/agents/reviewer-spec.md b/.claude/agents/reviewer-spec.md deleted file mode 100644 index 4fc4f468..00000000 --- a/.claude/agents/reviewer-spec.md +++ /dev/null @@ -1,63 +0,0 @@ ---- -name: reviewer-spec -description: Reviews the ADR/proposal and spec/schema of a GenVM branch as documents β€” soundness, clarity, and completeness (what was forgotten). Use as the "spec" pass of a branch review. -tools: Bash, Read, Grep, Glob -model: opus ---- - -You review the **specification** of a GenVM branch β€” the ADR, the spec/docs, the -schemas β€” as documents: is the proposal sound, clear, and self-consistent? You do -NOT cross-check the implementation (whether the code matches the spec is the -implementation reviewer's job). Read-only: never edit code. - -## Baseline - -Diff against the active dev branch, not `main`: - -- Default to `v0.3-dev` (or the current `v0.x-dev`) β€” confirm with - `git branch -a | grep dev`; fall back to `main` only if none exists. -- State the base in one line at the top; ignore commits already on it. -- `git log --oneline <base>..HEAD` and `git diff --stat <base>..HEAD`, then read - the diffs of `doc/**` and `*.json` schemas. - -## What to report - -With file:line evidence: - -1. **ADR / proposal quality.** If the branch adds or changes a `doc/adr/*.md` - (or design doc), rate it. Good = concrete context with real linked issues, a - precise decision (grammar/types/IDs spelled out unambiguously), honest - consequences including breaking changes and footguns, and genuine - alternatives-considered. If there's no ADR for a change that warrants one, - say so. - -2. **Spec / schema soundness.** Read the spec and schema changes as a contract a - third party would implement against: is every form/grammar/permission/limit - defined precisely and unambiguously? Are there gaps, contradictions, or - under-specified edges? Is the JSON schema itself valid and matching the prose? - This is about the spec being *correct and complete on its own terms* β€” not - about the code. - -3. **Completeness β€” what did we forget to add?** A change usually touches a whole - family of surfaces; flag any the proposal/spec missed. E.g. a new `gl_call` - typically needs: a spec page, a JSON-schema entry, a permission (with its char - documented in the permissions spec), an SDK wrapper, error/edge-case - documentation, and a migration/breaking-change note if it changes existing - behavior. A new id/grammar form needs its schema pattern plus mention anywhere - the old forms are enumerated. List the surfaces that *should* have changed - together but didn't. - -4. **Edge cases are documented or inferrable.** Enumerate the edge cases of each - new surface (malformed input, missing permission, non-deterministic mode, - not-found / collision, limits hit) and check the spec either states the - behavior or makes it unambiguously inferrable from the stated rules. A behavior - a reader would have to guess is a spec gap β€” list each one. (Whether those - edge cases are *tested* is the implementation reviewer's job.) - -Do NOT flag the dev-mode / `hashes=test` build state β€” it is intentional (known -flag to ignore test hashes), not a finding. - -## Style - -Concise and direct. Lead with: is the proposal sound and the spec implementable -as written? Separate blocking gaps from nits. Don't pad. diff --git a/.claude/agents/reviewer-spec.md b/.claude/agents/reviewer-spec.md new file mode 120000 index 00000000..9933c9c9 --- /dev/null +++ b/.claude/agents/reviewer-spec.md @@ -0,0 +1 @@ +../../.agents/agents/reviewer-spec.md \ No newline at end of file diff --git a/.claude/skills/agentic-fuzzing b/.claude/skills/agentic-fuzzing new file mode 120000 index 00000000..c478bdd4 --- /dev/null +++ b/.claude/skills/agentic-fuzzing @@ -0,0 +1 @@ +../../.agents/skills/agentic-fuzzing \ No newline at end of file diff --git a/.claude/skills/branch-review b/.claude/skills/branch-review new file mode 120000 index 00000000..cd42a6bb --- /dev/null +++ b/.claude/skills/branch-review @@ -0,0 +1 @@ +../../.agents/skills/branch-review \ No newline at end of file diff --git a/.claude/skills/build b/.claude/skills/build new file mode 120000 index 00000000..b1286db4 --- /dev/null +++ b/.claude/skills/build @@ -0,0 +1 @@ +../../.agents/skills/build \ No newline at end of file diff --git a/.claude/skills/commit-style b/.claude/skills/commit-style new file mode 120000 index 00000000..5284a32f --- /dev/null +++ b/.claude/skills/commit-style @@ -0,0 +1 @@ +../../.agents/skills/commit-style \ No newline at end of file diff --git a/.claude/skills/initial-setup b/.claude/skills/initial-setup new file mode 120000 index 00000000..032dd983 --- /dev/null +++ b/.claude/skills/initial-setup @@ -0,0 +1 @@ +../../.agents/skills/initial-setup \ No newline at end of file diff --git a/.claude/skills/macos b/.claude/skills/macos new file mode 120000 index 00000000..bdb6d05d --- /dev/null +++ b/.claude/skills/macos @@ -0,0 +1 @@ +../../.agents/skills/macos \ No newline at end of file diff --git a/.claude/skills/pydoc b/.claude/skills/pydoc new file mode 120000 index 00000000..e26ee091 --- /dev/null +++ b/.claude/skills/pydoc @@ -0,0 +1 @@ +../../.agents/skills/pydoc \ No newline at end of file diff --git a/.claude/skills/review-ready b/.claude/skills/review-ready new file mode 120000 index 00000000..0788c6f2 --- /dev/null +++ b/.claude/skills/review-ready @@ -0,0 +1 @@ +../../.agents/skills/review-ready \ No newline at end of file diff --git a/.claude/skills/rust-test-style b/.claude/skills/rust-test-style new file mode 120000 index 00000000..eaf94312 --- /dev/null +++ b/.claude/skills/rust-test-style @@ -0,0 +1 @@ +../../.agents/skills/rust-test-style \ No newline at end of file diff --git a/.claude/skills/spec b/.claude/skills/spec new file mode 120000 index 00000000..acb2bb1b --- /dev/null +++ b/.claude/skills/spec @@ -0,0 +1 @@ +../../.agents/skills/spec \ No newline at end of file diff --git a/.claude/skills/submodules b/.claude/skills/submodules new file mode 120000 index 00000000..bbcef097 --- /dev/null +++ b/.claude/skills/submodules @@ -0,0 +1 @@ +../../.agents/skills/submodules \ No newline at end of file diff --git a/.claude/skills/test b/.claude/skills/test new file mode 120000 index 00000000..20211d17 --- /dev/null +++ b/.claude/skills/test @@ -0,0 +1 @@ +../../.agents/skills/test \ No newline at end of file diff --git a/.coderabbit.yaml b/.coderabbit.yaml index 3e5e11cd..53a5fd5f 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -1,5 +1,6 @@ # yaml-language-server: $schema=https://coderabbit.ai/integrations/schema.v2.json reviews: + request_changes_workflow: true auto_review: # `base_branches` MUST live here, under `auto_review` β€” not directly under # `reviews`. The schema defines no `reviews.base_branches`, and `reviews` diff --git a/.genvm-tool.py b/.genvm-tool.py index 2c9a6057..008dab25 100644 --- a/.genvm-tool.py +++ b/.genvm-tool.py @@ -69,6 +69,7 @@ def tests(ctx): cargo, genvm, integration, + npm, pytest, ) @@ -98,6 +99,20 @@ def collect_rust(ctx: genvm_tool.tests.stage.collection.Context): name=fuzz_file.stem, ) + def collect_npm(ctx: genvm_tool.tests.stage.collection.Context): + test_files = sorted(f for f in ctx.shared.git_files if f.name.endswith('.test.ts')) + for t in filter(lambda x: x.name == 'package.json', ctx.shared.git_files): + if 'test' not in npm.scripts(t): + continue + project_root_dir = t.parent + owned = [f for f in test_files if f.is_relative_to(project_root_dir)] + ctx.shared.logger.debug('discovered npm project', path=t, test_files=len(owned)) + npm.npm_project( + ctx, + project_root_dir=project_root_dir, + test_files=owned, + ) + def collect_pytest(ctx: genvm_tool.tests.stage.collection.Context): dirs = [ ctx.shared.root_dir.joinpath('executors', 'v0.3.x', 'runners', 'genlayer-py-std'), @@ -135,6 +150,7 @@ def collect_pytest(ctx: genvm_tool.tests.stage.collection.Context): ) ctx.add_collector(collect_rust) + ctx.add_collector(collect_npm) ctx.add_collector(collect_pytest) ctx.run_parser.add_argument( diff --git a/.github/workflows/branch_force_merge.yaml b/.github/workflows/branch_force_merge.yaml deleted file mode 100644 index 842fb809..00000000 --- a/.github/workflows/branch_force_merge.yaml +++ /dev/null @@ -1,116 +0,0 @@ -name: branch / force merge PR into dev - -# `/genvm-force-merge` in a PR comment: land the PR without requiring the -# review, full-CI and E2E gates. For when those signals cannot be obtained at -# all (the E2E pipeline is down, the org's Actions budget is exhausted, ...) -# rather than for when they are red β€” a forced merge lands a commit no green run -# ever covered, and says so on the PR. -# -# Only a REPO ADMIN (or org owner, whom GitHub reports as admin here) may use -# it. The panel's Merge box stays the normal path and keeps every gate. -# -# Authorization runs HERE, in the workflow file, not in the merge script: an -# `issue_comment` workflow always runs from the default branch, so this YAML is -# trusted, whereas the merge job checks out the PR head and runs its scripts. -# The merge workflow re-checks both the admin role and the head sha in its own -# trusted guard step before any PR code is fetched. -# -# The command is bound to the head commit it was typed on: the admin authorizes -# a specific commit, not the branch. A push landing between the comment and the -# merge aborts it rather than riding in on the authorization. -# -# Written as its own workflow rather than a branch of branch_pr_actions.yaml -# because it triggers on a different event (`issue_comment: created`, not -# `edited`) and needs an authorization rule of its own. -# -# Deliberately NO workflow-level concurrency: the group would be taken before -# the job's `if` runs, so every unrelated comment on any PR would enter it and -# could evict a pending real one (a `concurrency` group keeps a single pending -# run, `cancel-in-progress: false` or not). The merge itself is already -# serialised repository-wide by branch_merge_into_dev.yaml's own group. - -on: - issue_comment: - types: [created] - -permissions: - contents: read - pull-requests: write - issues: write - checks: read - actions: read - -defaults: - run: - shell: bash -exo pipefail {0} - -jobs: - authorize: - # `startsWith` is only a cheap pre-filter (expressions have no regex); the - # step below requires the command to be the whole first line, so neither a - # quoted reply nor `/genvm-force-merge-later` fires it. The `ci-safe` gate is - # the same one the merge workflow re-checks β€” the merge runs PR code. - if: > - github.event.issue.pull_request && - startsWith(github.event.comment.body, '/genvm-force-merge') && - contains(github.event.issue.labels.*.name, 'ci-safe') - runs-on: ubuntu-latest - timeout-minutes: 15 - outputs: - ok: ${{ steps.check.outputs.ok }} - head_sha: ${{ steps.check.outputs.head_sha }} - steps: - # No checkout: nothing in this job may come from the PR. - - name: Require an exact command from a repo admin - id: check - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SENDER: ${{ github.event.comment.user.login }} - COMMENT_ID: ${{ github.event.comment.id }} - PR_NUMBER: ${{ github.event.issue.number }} - # Through the environment, never interpolated into the script: the - # body is attacker-controlled text. - BODY: ${{ github.event.comment.body }} - run: | - echo "ok=false" >> "$GITHUB_OUTPUT" - - # The command must be the entire first line. `startsWith` above would - # also accept `/genvm-force-merge-not-yet` or a longer sentence. - first="$(printf '%s' "$BODY" | head -n1 | tr -d '\r' | xargs || true)" - if [ "$first" != '/genvm-force-merge' ]; then - echo "first line is '$first', not the bare command; ignoring" - exit 0 - fi - - # A non-collaborator 403/404s here, which is "no role", not an error. - perm="$(gh api "repos/$GITHUB_REPOSITORY/collaborators/$SENDER/permission" \ - --jq '.permission' 2>/dev/null || echo none)" - if [ "$perm" != admin ]; then - echo "::warning::$SENDER has '$perm' access; /genvm-force-merge is admin-only" - # React rather than comment: the command is visible to anyone on a - # public PR, and a refusal comment per attempt is a spam vector. - gh api --method POST \ - "repos/$GITHUB_REPOSITORY/issues/comments/$COMMENT_ID/reactions" \ - -f 'content=-1' - exit 0 - fi - - # Bind the authorization to the commit it was typed on. - head_sha="$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER" --jq '.head.sha')" - { - echo "ok=true" - echo "head_sha=$head_sha" - } >> "$GITHUB_OUTPUT" - gh api --method POST \ - "repos/$GITHUB_REPOSITORY/issues/comments/$COMMENT_ID/reactions" \ - -f 'content=+1' - - merge: - needs: authorize - if: needs.authorize.outputs.ok == 'true' - uses: ./.github/workflows/branch_merge_into_dev.yaml - with: - pr_number: ${{ github.event.issue.number }} - force_actor: ${{ github.event.comment.user.login }} - force_head_sha: ${{ needs.authorize.outputs.head_sha }} - secrets: inherit diff --git a/.github/workflows/branch_merge_into_dev.yaml b/.github/workflows/branch_merge_into_dev.yaml deleted file mode 100644 index 6d77ce27..00000000 --- a/.github/workflows/branch_merge_into_dev.yaml +++ /dev/null @@ -1,202 +0,0 @@ -name: branch / merge PR into dev - -# The repo has NO GitHub merge queue. A PR lands on a dev branch (v<X>-dev) -# when a maintainer ticks the "Merge" box on the PR action panel -# (branch_pr_actions.yaml), which calls this reusable workflow. It -# re-checks every gate against the EXACT head commit and then advances the -# dev branch by a plain (fast-forward-only) push, so what lands is -# byte-identical to what CI and E2E validated. -# -# Gates (all required): -# 1. base branch is a v<X>-dev branch -# 2. a maintainer (push access) approved the head commit; pushing new commits -# revokes it, since the approval is matched against that exact sha. The -# `rtm` (ready-to-merge) label authorizes the merge in its place -# 3. full GenVM CI (queue.yaml) concluded success on the head commit -# 4. the cross-repo E2E check concluded success on the head commit -# 5. the PR is 0 commits behind base (head already contains base tip) -# -# `force_actor` (set only by branch_force_merge.yaml, for `/genvm-force-merge` -# from a repo admin) waives gates 2-4; 1 and 5 always hold. -# -# Merge strategy: one commit per merged PR in every repo. A manager PR that is -# a single commit and rewrites no gitlink is fast-forwarded as-is (SHA -# preserved); otherwise its commits are squashed onto base. Each executor line -# is squashed onto its own <line>-dev, which gives it a NEW commit, so the -# manager commit is rewritten to gitlink that squashed commit. The PR is closed -# afterwards. -# -# Executor mirror: the landed manager commit gitlinks into each active executor -# line; the script squashes each line onto its SAME-NAMED <line>-dev branch. It -# plans and fast-forward-checks every side up front, uploads every object under a -# scratch ref, moves the PR head branches onto what is about to land, and only then -# advances the target branches: the manager first (the one push that can lose a -# race), then each <line>-dev. -# -# The dev branches are protected; the default GITHUB_TOKEN cannot push to -# them. We push over SSH with the GENVM_CI_PRIVATE_KEY deploy key (on the -# dev-branch ruleset bypass list), and to the executor repo with a separate -# GENVM_EXECUTOR_CI_PRIVATE_KEY deploy key (on the executor's bypass list). -# Pushes are non-force, so a base that advanced between the checks and the -# push is safely rejected. The merge scripts are checked out from the PR head, -# so a PR that changes support/ci/ merges with its own version β€” which means -# this job runs PR-authored code WITH the deploy keys in scope. It is gated on -# the `ci-safe` label: a guard step verifies it (via the plain GITHUB_TOKEN, -# before any PR code is fetched or run) and fails otherwise. The caller must -# still restrict who can trigger a merge (maintainers only). - -on: - workflow_call: - inputs: - pr_number: - description: number of the PR to merge - type: string - required: true - force_actor: - description: >- - Login of the repo admin who ran `/genvm-force-merge`. Non-empty means - gates 2-4 (review, full CI, E2E) are skipped and the skip is announced - on the PR under this login. ONLY branch_force_merge.yaml may set it, - and only after checking the admin role in its own trusted YAML β€” the - merge script runs from the PR head and cannot authorize itself. - type: string - required: false - default: "" - force_head_sha: - description: >- - Head commit the `/genvm-force-merge` command was typed on. Required - with `force_actor`: the admin authorizes a commit, not a branch, so a - push landing in between must abort the merge rather than ride in on - the authorization. - type: string - required: false - default: "" - -permissions: - contents: read - pull-requests: write - issues: write - checks: read - actions: read - -defaults: - run: - shell: bash -exo pipefail {0} - -env: - # Exact check-run name, and the slug of the app that posts it, for the E2E - # check on the head commit. Both must match: a name alone is not a protected - # namespace, so any app (or a workflow in the PR) could post a green check - # under it. Adjust if that pipeline is renamed or re-homed. - E2E_CHECK_NAME: "E2E Tests" - E2E_CHECK_APP_SLUG: "ci-core-e2e-runner" - -jobs: - merge: - runs-on: ubuntu-latest - timeout-minutes: 30 - # A constant group, so at most ONE merge runs repository-wide, whatever PR or - # dev branch it targets. Two merges racing would each pass their own - # 0-behind check and then push a base the other never saw; the loser's - # non-force push is rejected mid-way, after the executor lines have already - # been advanced. Serialising is cheaper than making that recoverable. - # Never cancel: an interrupted merge can leave the lines ahead of the manager. - concurrency: - group: merge-into-dev - cancel-in-progress: false - steps: - # Gate: this job checks out and runs the PR's own CI scripts while the - # deploy keys are in scope, so refuse unless the PR is `ci-safe`. Uses the - # plain GITHUB_TOKEN and runs BEFORE any PR code is fetched. - - name: Require ci-safe label - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PR_NUMBER: ${{ inputs.pr_number }} - run: | - # Collect first, match second: under `pipefail` a `grep -q` that exits - # early can SIGPIPE the `gh` feeding it, and the resulting non-zero - # pipeline would read as "no ci-safe label" and refuse a valid merge. - names="$(gh api --paginate "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/labels?per_page=100" --jq '.[].name')" - if ! grep -qxF 'ci-safe' <<<"$names"; then - echo "::error::PR #$PR_NUMBER lacks the 'ci-safe' label; refusing to merge (this job runs PR code)." - exit 1 - fi - - # A forced merge waives the gates that prove the commit good, so its - # authorization is re-derived here β€” in the reusable workflow's own YAML, - # with the plain GITHUB_TOKEN, before any PR code is fetched. `force_actor` - # is an input like any other: a caller that is not branch_force_merge.yaml - # could pass an arbitrary login, and the merge script itself comes from the - # PR head, so neither is a place this can be trusted. - - name: Re-check the force-merge authorization - if: inputs.force_actor != '' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PR_NUMBER: ${{ inputs.pr_number }} - FORCE_ACTOR: ${{ inputs.force_actor }} - FORCE_HEAD_SHA: ${{ inputs.force_head_sha }} - run: | - perm="$(gh api "repos/$GITHUB_REPOSITORY/collaborators/$FORCE_ACTOR/permission" \ - --jq '.permission' 2>/dev/null || echo none)" - if [ "$perm" != admin ]; then - echo "::error::force merge claims '$FORCE_ACTOR', who has '$perm' access, not admin." - exit 1 - fi - if [ -z "$FORCE_HEAD_SHA" ]; then - echo "::error::force merge without force_head_sha; refusing." - exit 1 - fi - head="$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER" --jq '.head.sha')" - if [ "$head" != "$FORCE_HEAD_SHA" ]; then - gh pr comment "$PR_NUMBER" --repo "$GITHUB_REPOSITORY" --body \ - "❌ Force merge aborted: the head moved from \`${FORCE_HEAD_SHA:0:12}\` (the commit \`$FORCE_ACTOR\` authorized) to \`${head:0:12}\`. Re-issue \`/genvm-force-merge\` if that commit is also meant to land unchecked." - echo "::error::PR head moved since the force-merge command; refusing." - exit 1 - fi - - # Check out the PR head (so the merge runs the PR's own scripts) with the - # manager deploy key so the (protected) dev branch push succeeds. - # Submodules are initialised in the next step with a SEPARATE executor key - # β€” the manager key cannot read/write the executor repo. - - uses: actions/checkout@v4 - with: - ref: refs/pull/${{ inputs.pr_number }}/head - ssh-key: ${{ secrets.GENVM_CI_PRIVATE_KEY }} - fetch-depth: 0 - submodules: false - - - name: Set up executor submodule push access - env: - # Deploy key with write access to the executor repo, on its - # protected-branch ruleset bypass list. The executor submodule's - # origin must be an SSH URL (git@github.com:...) for this key to - # apply. - EXECUTOR_DEPLOY_KEY: ${{ secrets.GENVM_EXECUTOR_CI_PRIVATE_KEY }} - run: | - install -m700 -d ~/.ssh - key=~/.ssh/executor_ci - # The job-wide shell is `bash -x`, and Actions masks a multi-line secret - # line by line β€” an xtrace of the write would print PEM fragments it - # does not recognise. Trace goes off for exactly that one command. - set +x - printf '%s\n' "$EXECUTOR_DEPLOY_KEY" > "$key" - set -x - chmod 600 "$key" - ssh="ssh -i $key -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new" - # The manager release line ships every active executor line, so the - # merge mirrors each line's gitlink onto its own <line>-dev. Init them - # all, binding the executor key per-submodule (core.sshCommand) so the - # later mirror pushes use it while the manager push keeps the checkout - # credentials. - for v in $(./support/ci/run.sh tool branch-versions list); do - sub="executors/v${v}.x" - git -c core.sshCommand="$ssh" submodule update --init "$sub" - git -C "$sub" config core.sshCommand "$ssh" - done - - - name: Validate gates, mirror executors, fast-forward / squash, close - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PR_NUMBER: ${{ inputs.pr_number }} - FORCE_ACTOR: ${{ inputs.force_actor }} - run: ./support/ci/run.sh tool genvm-merge-into-dev diff --git a/.github/workflows/branch_pr_actions.yaml b/.github/workflows/branch_pr_actions.yaml index fc2476c8..41110e04 100644 --- a/.github/workflows/branch_pr_actions.yaml +++ b/.github/workflows/branch_pr_actions.yaml @@ -2,9 +2,8 @@ name: branch / PR action panel handler # Reacts to a maintainer ticking a box on the GenVM PR action panel (posted # by branch_pr_checklist.yaml). The `dispatch` job authenticates the editor, -# parses the ticked boxes, performs label-based actions (run / rerun full -# tests), unticks the boxes, and reports whether Merge was requested. If so -# the `merge` job calls the reusable merge workflow. +# parses the ticked boxes, enables full tests or provisions executor PRs, then +# unticks the momentary boxes. # # The handler runs the CI scripts checked out from the PR head branch, not # from the default branch, so a PR that changes support/ci/ exercises its own @@ -49,8 +48,6 @@ jobs: contains(github.event.issue.labels.*.name, 'ci-safe') runs-on: ubuntu-latest timeout-minutes: 15 - outputs: - merge: ${{ steps.act.outputs.merge }} steps: # Check out the PR head so the handler runs the PR's own CI scripts. # This runs PR-authored code β€” the `ci-safe` gate on the job above is @@ -70,11 +67,3 @@ jobs: COMMENT_ID: ${{ github.event.comment.id }} SENDER: ${{ github.event.sender.login }} run: ./support/ci/run.sh tool pr-action-panel - - merge: - needs: dispatch - if: needs.dispatch.outputs.merge == 'true' - uses: ./.github/workflows/branch_merge_into_dev.yaml - with: - pr_number: ${{ github.event.issue.number }} - secrets: inherit diff --git a/.github/workflows/branch_pr_checklist.yaml b/.github/workflows/branch_pr_checklist.yaml index a25b2256..7cf5cdf9 100644 --- a/.github/workflows/branch_pr_checklist.yaml +++ b/.github/workflows/branch_pr_checklist.yaml @@ -1,8 +1,7 @@ name: branch / post PR action panel # When a PR is opened against main or a dev branch (v<X>-dev) we: -# - post the action panel (checklist comment) that replaces the old -# /genvm-* slash commands; +# - post the action panel and document its companion slash commands; # - auto-add `ci-safe` if the PR author has write access, so their PR can # immediately use the panel. For everyone else a write-role maintainer # adds `ci-safe` by hand once the PR is vetted. @@ -14,9 +13,9 @@ name: branch / post PR action panel on: pull_request_target: branches: [main, 'v*-dev'] - # `edited` fires when the PR base branch is changed, so a PR retargeted - # onto a dev branch also gets the panel. - types: [opened, edited] + # `edited` catches retargets; `synchronize` migrates panels already posted + # by an older workflow revision without a repository-wide write sweep. + types: [opened, edited, synchronize] permissions: pull-requests: write @@ -35,22 +34,15 @@ jobs: PR: ${{ github.event.pull_request.number }} AUTHOR: ${{ github.event.pull_request.user.login }} steps: - - name: Post action panel + - name: Post or refresh action panel run: | - # Only post once: skip if a panel (identified by the HTML marker) - # already exists on the PR. `edited` can fire repeatedly. - # - # Collect first, match second β€” NOT `gh ... | grep -qF`. Under - # `pipefail`, `grep -q` exiting at the first match can SIGPIPE a `gh` - # still writing a large comment list, which makes the pipeline - # non-zero and sends us down the "no panel yet" path β€” posting a - # second panel on exactly the busiest PRs. - comments="$(gh pr view "$PR" --repo "$GITHUB_REPOSITORY" --json comments --jq '.comments[].body')" - if grep -qF '<!-- genvm-actions -->' <<<"$comments"; then - echo "action panel already present; skipping" - exit 0 + labels="$(gh api --paginate "repos/$GITHUB_REPOSITORY/issues/$PR/labels" --jq '.[].name')" + force=false + if grep -qxF 'run-full-tests' <<<"$labels"; then + force=true fi - gh pr comment "$PR" --repo "$GITHUB_REPOSITORY" --body "$(cat <<'EOF' + + body="$(cat <<'EOF' <!-- genvm-actions --> ### GenVM PR actions @@ -60,29 +52,35 @@ jobs: - [ ] Force run full tests <!-- action:force --> - - [ ] Rerun full tests <!-- action:rerun --> - [ ] Provision executor PRs <!-- action:provision --> - - [ ] Merge into dev <!-- action:merge --> - - <details><summary><b>Merge</b></summary> - - Requires, on the exact head commit: - - - an approving review from a maintainer (any push revokes it), or the **`rtm`** label - - linear history β€” 0 commits behind base - - green full GenVM CI and green cross-repo E2E - - Full CI starts only through "Force" or "Rerun full tests" above. "Force" also sets **`run-full-tests`**, making future pushes run the full suite; **`rtm`** only authorizes Merge. - - Every repo lands ONE squashed commit, subject `<PR title> (#N)`. - </details> <details><summary><b>Commands</b></summary> - - `/genvm-force-merge` β€” **repo admin** only: land without the review, full-CI and E2E gates, for when those signals are unobtainable, not when they are red. Base, title and 0-behind still apply, and the skip is recorded on the PR. + - `/genvm-run-tests` β€” run full tests once for the current manager snapshot + - `/merge` β€” queue the exact manager snapshot through the App-owned E2E merge train </details> EOF )" + if [ "$force" = true ]; then + body="${body/- \[ \] Force run full tests/- [x] Force run full tests}" + fi + + panel_ids="$(gh api --paginate "repos/$GITHUB_REPOSITORY/issues/$PR/comments?per_page=100" \ + --jq '.[] | select(.user.login == "github-actions[bot]" and (.body | contains("<!-- genvm-actions -->"))) | .id')" + panel_id="${panel_ids%%$'\n'*}" + if [ -z "$panel_id" ]; then + gh pr comment "$PR" --repo "$GITHUB_REPOSITORY" --body "$body" + exit 0 + fi + + while IFS= read -r panel_id; do + current="$(gh api "repos/$GITHUB_REPOSITORY/issues/comments/$panel_id" --jq '.body')" + if [ "$current" = "$body" ]; then + echo "action panel $panel_id is current; skipping" + continue + fi + gh api --method PATCH "repos/$GITHUB_REPOSITORY/issues/comments/$panel_id" -f "body=$body" + done <<<"$panel_ids" - name: Auto-mark ci-safe for write-access authors run: | diff --git a/.github/workflows/branch_provision.yaml b/.github/workflows/branch_provision.yaml index 5019f22a..be727df7 100644 --- a/.github/workflows/branch_provision.yaml +++ b/.github/workflows/branch_provision.yaml @@ -1,11 +1,10 @@ name: branch / provision dev branches and standing PRs -# Materialize the branch model for every active version train listed in -# .genvm-monorepo-root. For each version X it ensures the version branch -# v<X>.x, the dev branch v<X>-dev, and a standing release-gate PR -# v<X>-dev -> v<X>.x all exist (see support/ci/run.sh tool provision-branches). +# Materialize the manager train declared by .genvm-monorepo-root: the release +# branch v<X>, the dev branch v<X>-dev, and the standing release-gate PR +# v<X>-dev -> v<X> (see support/ci/run.sh tool provision-branches). # -# Idempotent β€” safe to re-run. Fires automatically when active-versions +# Idempotent β€” safe to re-run. Fires automatically when the train declaration # changes, and on demand via the Actions tab. on: diff --git a/.github/workflows/branch_provision_executor_prs.yaml b/.github/workflows/branch_provision_executor_prs.yaml index b3f6aff2..3fae94d4 100644 --- a/.github/workflows/branch_provision_executor_prs.yaml +++ b/.github/workflows/branch_provision_executor_prs.yaml @@ -9,8 +9,8 @@ name: branch / provision executor mirror PRs # The provision scripts are checked out from the PR head, so a PR iterating on # them runs its own version β€” which means this job runs PR-authored code WITH the # executor write PAT (GENVM_EXECUTOR_PR_TOKEN) in scope. It is gated on the -# `ci-safe` label: a guard step verifies it (via the plain GITHUB_TOKEN, before -# any PR code runs) and fails otherwise, exactly like branch_merge_into_dev.yaml. +# `ci-safe` label: a guard step verifies it via the plain GITHUB_TOKEN before +# any PR code runs and fails otherwise. on: workflow_dispatch: diff --git a/.github/workflows/branch_rebase_watch.yaml b/.github/workflows/branch_rebase_watch.yaml index f18fce26..39d9a3fc 100644 --- a/.github/workflows/branch_rebase_watch.yaml +++ b/.github/workflows/branch_rebase_watch.yaml @@ -8,7 +8,7 @@ name: branch / not-rebased watch # It gates nothing: it posts a red `not rebased` check-run (visible on the PR # page) and a red `not rebased` label (the surface visible in the PR list), and # clears both once the branch is 0 behind again. The enforcing checks are -# unchanged β€” `initial / behind-check` and the Merge action. +# unchanged β€” `initial / behind-check` remains the enforcing check. # # Triggers: # - push to a dev branch -> re-evaluate every open PR targeting it (this is diff --git a/.github/workflows/branch_release_gate_pr.yaml b/.github/workflows/branch_release_gate_pr.yaml new file mode 100644 index 00000000..b84e2e9a --- /dev/null +++ b/.github/workflows/branch_release_gate_pr.yaml @@ -0,0 +1,67 @@ +name: branch / open release-gate PR + +# On each push to v<X>-dev, ensure its standing PR into v<X> exists. Unlike +# provision-branches, derive the pair from the pushed ref, covering manual dev +# branches. Skip a missing base, no commits ahead or an existing PR. GITHUB_TOKEN +# does not trigger `pull_request` workflows; close and reopen if checks must run before a push + +on: + push: + branches: ['v*-dev'] + +permissions: + contents: read + pull-requests: write + +# Serialize the non-atomic existence check and creation per dev branch +concurrency: + group: release-gate-pr-${{ github.ref_name }} + cancel-in-progress: false + +defaults: + run: + shell: bash -exo pipefail {0} + +jobs: + open-pr: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + with: + # Full history for comparing the remote branches + fetch-depth: 0 + + - name: Open release-gate PR if missing + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + DEV="${GITHUB_REF_NAME}" + BASE="${DEV%-dev}" + + git fetch origin --prune + + if ! git show-ref --verify --quiet "refs/remotes/origin/${BASE}"; then + echo "Base branch ${BASE} does not exist; nothing to do" + exit 0 + fi + + # Same tip or behind base: there is nothing to merge + AHEAD="$(git rev-list --count "origin/${BASE}..origin/${DEV}")" + if [ "$AHEAD" -eq 0 ]; then + echo "${DEV} has no commits beyond ${BASE}; nothing to do" + exit 0 + fi + + EXISTING="$(gh pr list --repo "$GITHUB_REPOSITORY" --base "$BASE" --head "$DEV" --state open --json number --jq '.[0].number // ""')" + if [ -n "$EXISTING" ]; then + echo "Release-gate PR ${DEV} -> ${BASE} already open (#${EXISTING})" + exit 0 + fi + + gh pr create \ + --repo "$GITHUB_REPOSITORY" \ + --base "$BASE" \ + --head "$DEV" \ + --title "chore(release): gate ${DEV} into ${BASE} πŸš€" \ + --body "Standing release-gate PR. \`${DEV}\` accumulates incremental work; it merges into the release-ready \`${BASE}\` branch only once the cross-repo E2E pipeline is green. It may stay red while the train is in progress." diff --git a/.github/workflows/branch_retarget.yaml b/.github/workflows/branch_retarget.yaml index acbba03b..e782843d 100644 --- a/.github/workflows/branch_retarget.yaml +++ b/.github/workflows/branch_retarget.yaml @@ -45,6 +45,6 @@ jobs: gh pr comment "$PR" --repo "$GITHUB_REPOSITORY" --body "$(cat <<EOF πŸ‘‹ This PR targeted \`main\`, so I've retargeted it to the latest dev branch \`${DEV}\`. - \`main\` is protected and is only an alias of the latest release branch (\`v${MANAGER}-dev\`), kept in lockstep automatically. Active v${MANAGER} work lands on \`${DEV}\`, which reaches \`v${MANAGER}.x\` through the standing release-gate PR once the cross-repo E2E matrix is green. + \`main\` is protected and is only an alias of the latest development branch (\`${DEV}\`), kept in lockstep automatically. Active v${MANAGER} work lands there and reaches \`v${MANAGER}\` through the standing release-gate PR once the cross-repo E2E matrix is green. EOF )" diff --git a/.github/workflows/branch_run_tests_command.yaml b/.github/workflows/branch_run_tests_command.yaml new file mode 100644 index 00000000..6069c73f --- /dev/null +++ b/.github/workflows/branch_run_tests_command.yaml @@ -0,0 +1,74 @@ +name: branch / run full tests command + +# `/genvm-run-tests` is the one-shot counterpart to the sticky +# `run-full-tests` label. The command starts full CI for the PR's exact head, +# links the run in a bot comment, and reflects completion on the command with a +# native GitHub reaction. This is manager-owned CI; it does not call the E2E App. + +on: + issue_comment: + types: [created] + workflow_run: + workflows: [GenVM full] + types: [completed] + +permissions: {} + +defaults: + run: + shell: bash -exo pipefail {0} + +jobs: + start: + if: >- + github.event_name == 'issue_comment' && + github.event.issue.pull_request && + github.event.comment.body == '/genvm-run-tests' && + contains(github.event.issue.labels.*.name, 'ci-safe') + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + actions: write + contents: read + issues: write + pull-requests: read + steps: + # issue_comment workflows run from the default branch. Keep that trusted + # checkout: this handler dispatches PR code with credentials in scope + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Dispatch full tests + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.issue.number }} + COMMENT_ID: ${{ github.event.comment.id }} + COMMENT_BODY: ${{ github.event.comment.body }} + SENDER: ${{ github.event.sender.login }} + REQUEST_ID: ${{ github.run_id }}-${{ github.run_attempt }} + run: ./support/ci/run.sh tool full-tests-command start + + complete: + if: >- + github.event_name == 'workflow_run' && + contains(github.event.workflow_run.display_title, '[request=') + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + issues: write + steps: + # workflow_run also resolves this workflow from the default branch. Never + # check out or execute the completed run's potentially PR-authored tree + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Report full-test result + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RUN_ID: ${{ github.event.workflow_run.id }} + RUN_NAME: ${{ github.event.workflow_run.display_title }} + RUN_URL: ${{ github.event.workflow_run.html_url }} + RUN_SHA: ${{ github.event.workflow_run.head_sha }} + RUN_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: ./support/ci/run.sh tool full-tests-command complete diff --git a/.github/workflows/branch_sync_executors.yaml b/.github/workflows/branch_sync_executors.yaml new file mode 100644 index 00000000..cd15d8b8 --- /dev/null +++ b/.github/workflows/branch_sync_executors.yaml @@ -0,0 +1,59 @@ +name: branch / sync executor branches + +# Manager train branches are authoritative for the executor refs they pin. +# A release branch (v<X>.<Y>) projects each active executor gitlink onto the +# branch declared in .gitmodules; its dev branch projects onto <line>-dev. +# Every push is non-force: missing refs are created, existing refs only move by +# fast-forward. The tool attempts every line before reporting combined failures. + +on: + push: + branches: + - 'v[0-9]+.[0-9]+' + - 'v[0-9]+.[0-9]+-dev' + +run-name: branch / sync executors from ${{ github.ref_name }} + +permissions: + contents: read + +# Collapse a burst on one manager branch onto its live tip. Different manager +# branches may run together; executor branch protection and non-force pushes +# reject any incompatible ownership rather than hiding it +concurrency: + group: sync-executors-${{ github.ref_name }} + cancel-in-progress: false + +defaults: + run: + shell: bash -exo pipefail {0} + +jobs: + sync: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + with: + # Reconcile the live branch tip. A rerun therefore repairs only refs + # that still need an update + ref: ${{ github.ref_name }} + persist-credentials: false + + - name: Set up executor push access + env: + EXECUTOR_DEPLOY_KEY: ${{ secrets.GENVM_EXECUTOR_CI_PRIVATE_KEY }} + run: | + install -m700 -d ~/.ssh + key="${RUNNER_TEMP}/executor_sync_key" + set +x + printf '%s\n' "$EXECUTOR_DEPLOY_KEY" > "$key" + set -x + chmod 600 "$key" + git config --global url."git@github.com:".insteadOf "https://github.com/" + echo "GIT_SSH_COMMAND=ssh -i $key -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new" >> "$GITHUB_ENV" + + - name: Create or fast-forward executor branches + env: + MANAGER_BRANCH: ${{ github.ref_name }} + run: ./support/ci/run.sh tool sync-executor-branches diff --git a/.github/workflows/incl_initial.yaml b/.github/workflows/incl_initial.yaml index 7b063f06..863eb4af 100644 --- a/.github/workflows/incl_initial.yaml +++ b/.github/workflows/incl_initial.yaml @@ -9,7 +9,7 @@ on: the panel's `workflow_dispatch`), so the PR-scoped jobs below run for BOTH β€” they used to key off `github.event_name == 'pull_request'` and therefore skipped on every panel-dispatched run, while that run could - still conclude `success` and satisfy the Merge gate. Empty for a + still conclude `success` and satisfy the App's native CI gate. Empty for a non-PR run, which skips them. type: string required: false @@ -56,8 +56,8 @@ jobs: # above never fires it. Check every commit the PR adds, in the manager and in # each executor submodule that moved. # - # This is the one initial check the Merge action does NOT re-run, so it must - # not be skippable on a run the Merge gate will accept. + # The App consumes this through the aggregate native CI result, so it must not + # be skippable on a run eligible for the merge train. commit-messages: if: inputs.pr != '' runs-on: ubuntu-latest @@ -81,14 +81,12 @@ jobs: CHANGES: ${{ steps.src.outputs.changes }} run: ./support/ci/run.sh pipeline commit-messages - # The Merge action only fast-forwards, so the PR head must already - # contain the base tip (0 commits behind). We assert it here too (as part - # of the always-on `initial` checks) so the gate goes red the moment the - # branch falls behind, instead of surfacing only at merge time. + # Repository policy requires the PR head to contain the base tip (0 commits + # behind). Assert it in the always-on checks so stale work is refreshed before + # its exact manager snapshot enters the App-owned merge train. # # The measurement itself lives in support/ci/behind.py, shared with - # rebase-watch and the Merge action, so the three cannot disagree about the - # same PR. + # rebase-watch, so both surfaces agree about the same PR. behind-check: if: inputs.pr != '' runs-on: ubuntu-latest @@ -105,10 +103,8 @@ jobs: PR_NUMBER: ${{ inputs.pr }} run: ./support/ci/run.sh pipeline behind-check - # The Merge action squashes a PR into `<title> (#N)`, so the PR title becomes - # a commit subject verbatim and must satisfy the same rules as one. Nothing - # re-validates a generated squash message, so an invalid title would only - # surface as a commit the hook rejects β€” after it landed. + # The E2E App squash-merges a PR as `<title> (#N)`, so the PR title becomes a + # commit subject verbatim and must satisfy the same rules as one. pr-title: if: inputs.pr != '' runs-on: ubuntu-latest diff --git a/.github/workflows/queue.yaml b/.github/workflows/queue.yaml index 554125ef..598ccfb7 100644 --- a/.github/workflows/queue.yaml +++ b/.github/workflows/queue.yaml @@ -1,32 +1,31 @@ name: GenVM full -# Full GenVM CI. The repo has NO GitHub merge queue: this is the -# authoritative "genvm CI" gate the Merge action -# (branch_merge_into_dev.yaml) requires to be green on the exact head it -# fast-forwards. +# Full GenVM CI. The App-owned E2E merge train revalidates this native check on +# the exact manager head before it squash-merges the tested snapshot. # # For every push to a PR targeting a dev branch (v<X>-dev): # - `initial` (cheap pre-commit + 0-behind) and `check-executor-prs` (the # executor landability precondition) ALWAYS run, in parallel; # - `gate-check` aggregates them and plans the test matrix (plan-queue-matrix); -# the heavy `matrix-tests` cells run only when a run-full-tests marker is set -# β€” the `run-full-tests` label ("Force run full tests" sets it) or the `rtm` -# label, which marks the PR ready to merge and so needs the full run anyway. +# the heavy `matrix-tests` cells run when the `run-full-tests` label is set +# or `/genvm-run-tests` dispatches a one-shot run. Neither authorizes the App +# to merge. # Without a marker the plan is empty, so matrix-tests is skipped; # - without a marker `validate-end` fails, so the CI check stays red # until a full run has happened. # -# Labels are passive: adding `rtm`, `run-full-tests`, or any unrelated label -# never starts or cancels CI. "Force" and "Rerun full tests" on the action panel -# dispatch this workflow explicitly; a push still starts the always-on checks. +# Labels are passive: adding `run-full-tests` or any unrelated label never starts +# or cancels CI. "Force run full tests" on the action panel and the +# `/genvm-run-tests` command dispatch this workflow explicitly; a push still +# starts the always-on checks. on: pull_request: branches: ['v*-dev'] # Deliberately NOT `edited`: this workflow's concurrency group cancels # in-flight runs, so a typo fix in the PR body would kill a running # matrix-tests cell (over an hour of work). The PR title can therefore be - # edited to something invalid after `initial / pr-title` went green β€” the - # Merge action re-checks it against the title it is about to land. + # edited to something invalid after `initial / pr-title` went green. Title + # edits should be followed by a fresh run before the App merge is requested. types: [opened, synchronize, reopened, ready_for_review] workflow_dispatch: inputs: @@ -41,22 +40,36 @@ on: type: boolean required: false default: false + request_comment: + description: "`/genvm-run-tests` comment id, empty for other triggers" + type: string + required: false + default: "" + request_id: + description: Unique `/genvm-run-tests` handler run and attempt + type: string + required: false + default: "" + expected_sha: + description: Exact manager SHA requested by `/genvm-run-tests` + type: string + required: false + default: "" run-name: >- - GenVM full${{ github.event_name == 'workflow_dispatch' && format(' (manual, PR #{0})', inputs.pr) || '' }} + GenVM full${{ github.event_name == 'workflow_dispatch' && format(' (manual, PR #{0})', inputs.pr) || '' }}${{ inputs.request_id != '' && format(' [request={0};comment={1};pr={2}]', inputs.request_id, inputs.request_comment, inputs.pr) || '' }} permissions: contents: read pull-requests: read -# One in-flight full run per PR head branch; a newer push cancels the stale run. -# `github.head_ref` is the PR head branch on pull_request events. Cancelling a -# superseded head's run is safe: the Merge gate only consumes CI that is green on -# the exact head it fast-forwards, which is always the newest one. Panel dispatches -# have no head_ref, so each falls back to its unique run id and never cancels (or -# is cancelled by) another PR's run. +# One in-flight full run per PR; a newer push cancels the stale run. +# The PR number unifies push- and command-triggered runs. Cancelling a +# superseded head's run is safe: the App only consumes CI that is green on the +# exact head it merges, which is always the newest one. A command therefore +# replaces an older run for the same PR rather than duplicating the full matrix. concurrency: - group: genvm-full-${{ github.head_ref || github.run_id }} + group: genvm-full-${{ github.event.pull_request.number || inputs.pr || github.run_id }} cancel-in-progress: true defaults: @@ -76,6 +89,9 @@ jobs: TRIGGERING_ACTOR: ${{ github.triggering_actor }} RUN_ATTEMPT: ${{ github.run_attempt }} PR: ${{ inputs.pr }} + REQUEST_COMMENT: ${{ inputs.request_comment }} + REQUEST_ID: ${{ inputs.request_id }} + EXPECTED_SHA: ${{ inputs.expected_sha }} RUN_SHA: ${{ github.sha }} run: | if [ "$RUN_ATTEMPT" != 1 ]; then @@ -94,6 +110,31 @@ jobs: echo "::error::invalid PR number: $PR" exit 1 fi + if [ -n "$REQUEST_COMMENT" ] && [[ ! "$REQUEST_COMMENT" =~ ^[1-9][0-9]*$ ]]; then + echo "::error::invalid request comment id: $REQUEST_COMMENT" + exit 1 + fi + if [ -n "$REQUEST_ID" ] && [[ ! "$REQUEST_ID" =~ ^[1-9][0-9]*-[1-9][0-9]*$ ]]; then + echo "::error::invalid request id: $REQUEST_ID" + exit 1 + fi + if { [ -n "$REQUEST_ID" ] && [ -z "$REQUEST_COMMENT" ]; } || \ + { [ -z "$REQUEST_ID" ] && [ -n "$REQUEST_COMMENT" ]; }; then + echo "::error::request id and request comment must be supplied together" + exit 1 + fi + if [ -n "$REQUEST_ID" ] && [ -z "$EXPECTED_SHA" ]; then + echo "::error::command dispatch must include the requested SHA" + exit 1 + fi + if [ -n "$EXPECTED_SHA" ] && [[ ! "$EXPECTED_SHA" =~ ^[0-9a-f]{40}$ ]]; then + echo "::error::invalid expected SHA: $EXPECTED_SHA" + exit 1 + fi + if [ -n "$EXPECTED_SHA" ] && [ "$EXPECTED_SHA" != "$RUN_SHA" ]; then + echo "::error::requested SHA is $EXPECTED_SHA, but dispatch resolved $RUN_SHA" + exit 1 + fi head="$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR" \ --jq '[.head.repo.full_name, .head.sha] | @tsv')" @@ -115,19 +156,18 @@ jobs: # checks run on BOTH a push and a panel dispatch. They used to test # `github.event_name == 'pull_request'` themselves and so skipped on every # dispatched run β€” while that run could still conclude `success` and be - # accepted by the Merge gate, which takes ANY green run on the head sha. + # accepted by the App, which takes a green run on the exact head sha. pr: ${{ github.event.pull_request.number || inputs.pr }} secrets: inherit - # Precondition (run.sh tool pr-branches check): the PR must be landable via the - # panel (Provision then Merge), which lands the manager and its executors almost- - # atomically. So every repo (manager + each executor line) must be rebased on its - # base, and every executor line present at base must have its pinned commit in the - # executor repo (ahead-of-base is fine β€” that work lands with the PR; a commit - # missing from the executor repo is not). Read-only (gh API, no submodule checkout). + # Precondition (run.sh tool pr-branches check): every repo must be rebased on + # its base, and every executor line must have its pinned commit in the executor + # repo. Ahead-of-base executor work is expected: after the App + # lands manager, branch_sync_executors.yaml fast-forwards the executor dev refs. + # Read-only (gh API, no submodule checkout). # # Runs on same-repo PRs AND on workflow_dispatch with a PR number (the panel's - # rerun path, whose run the Merge gate consumes β€” so the precondition must hold + # rerun path, whose run the App consumes β€” so the precondition must hold # there too; the PR number then comes from inputs.pr). Fork PRs are skipped: they # get no secrets (the executor PAT would be empty) and cannot push executor mirror # branches, so there is nothing to gate. It checks out the PR being validated @@ -152,7 +192,7 @@ jobs: # Read-only gh API calls; nothing here pushes, so the checkout # credential should not survive into the PR code this job runs. persist-credentials: false - - name: Executor mirror PRs are landed + - name: Executor gitlinks are available env: PR_NUMBER: ${{ github.event.pull_request.number || inputs.pr }} MANAGER_REPO: ${{ github.repository }} @@ -192,7 +232,7 @@ jobs: env: # Heavy cells run only with a marker; without one the plan is empty and # matrix-tests is skipped. - RUN_FULL_TESTS: ${{ github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run-full-tests') || contains(github.event.pull_request.labels.*.name, 'rtm') }} + RUN_FULL_TESTS: ${{ github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run-full-tests') }} run: ./support/ci/run.sh pipeline plan-queue-matrix matrix-tests: @@ -233,7 +273,7 @@ jobs: - gate-check - matrix-tests env: - HAS_MARKER: ${{ github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run-full-tests') || contains(github.event.pull_request.labels.*.name, 'rtm') }} + HAS_MARKER: ${{ github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run-full-tests') }} steps: - name: check run: | @@ -245,7 +285,7 @@ jobs: # No run-full-tests marker -> heavy tests were skipped; keep the # gate red so the PR is not mergeable until they run. if [ "$HAS_MARKER" != "true" ]; then - echo "::error::full tests have not run β€” tick 'Force run full tests' or 'Rerun full tests' on the action panel" + echo "::error::full tests have not run β€” tick 'Force run full tests' or comment '/genvm-run-tests'" exit 1 fi results="${{ needs.matrix-tests.result }}" diff --git a/AGENTS.md b/AGENTS.md index 462fc4a4..796c96fe 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,6 +3,12 @@ Before changing this repository, read the matching page in `docs/contributing/`: a task has a how-to, a design question has an explanation +## Executor Consensus Invariants + +1. Treat leader-proposed output as untrusted input; validate it before use or publication +2. Apply consensus-affecting normalization, caps and accounting identically in leader, validator and sync paths +3. For timeout and resource preflights, panic if the checked operation later fails under the same conditions; a broken executor invariant must not become a contract-visible error + <!-- below is generated with `genvm-tool docs` --> ## Tutorial @@ -10,7 +16,8 @@ a task has a how-to, a design question has an explanation ## Howto - [genvm-tool.md](docs/contributing/howto/genvm-tool.md) β€” the umbrella tool: man page, test and git subcommands, codegen -- [pr.md](docs/contributing/howto/pr.md) β€” branch model, PR action panel, `ci-safe` / `run-full-tests`, merge gates +- [pr.md](docs/contributing/howto/pr.md) β€” branch model, PR action panel, App-owned landing and executor projection +- [review-ready.md](docs/contributing/howto/review-ready.md) β€” the checks a change must pass before it is put up for review - [setup.md](docs/contributing/howto/setup.md) β€” first-time clone: submodules, vendored trees, nix/direnv dev shell ## Howto/Building @@ -27,6 +34,7 @@ a task has a how-to, a design question has an explanation - [style.md](docs/contributing/howto/docs/style.md) β€” prose conventions for guides, specs, ADRs and commit bodies ## Howto/Extending +- [add-a-skill.md](docs/contributing/howto/extending/add-a-skill.md) β€” agent skills in `.agents/`, the frontmatter trigger, the symlink, skill vs how-to - [add-host-function.md](docs/contributing/howto/extending/add-host-function.md) β€” new executor↔host protocol method - [add-llm-provider.md](docs/contributing/howto/extending/add-llm-provider.md) β€” new LLM backend in the manager - [add-wasi-function.md](docs/contributing/howto/extending/add-wasi-function.md) β€” new gl_call method or raw WASI function @@ -40,15 +48,16 @@ a task has a how-to, a design question has an explanation ## Howto/Testing - [README.md](docs/contributing/howto/testing/README.md) β€” `genvm-tool test`: filters, presets, continue files -- [fuzzing.md](docs/contributing/howto/testing/fuzzing.md) β€” AFL fuzz targets, host sysctl prep +- [fuzzing.md](docs/contributing/howto/testing/fuzzing.md) β€” AFL fuzz targets, seeding a corpus from a run, host sysctl prep - [integration.md](docs/contributing/howto/testing/integration.md) β€” jsonnet cases, tags, golden `.stdout`/`.hash` files, services - [python.md](docs/contributing/howto/testing/python.md) β€” Python tests, direct pytest for genlayer-py-std - [rust.md](docs/contributing/howto/testing/rust.md) β€” Rust tests: where they go, how to run them, coverage +- [typescript.md](docs/contributing/howto/testing/typescript.md) β€” information on how to run and write webdriver typescript tests ## Explanation - [docs-layout.md](docs/contributing/explanation/docs-layout.md) β€” the 4 kinds of page and where each belongs - [executor-lines.md](docs/contributing/explanation/executor-lines.md) β€” why several executor lines ship side by side, and what it costs - [fuzz.md](docs/contributing/explanation/fuzz.md) β€” why fuzz targets get fake entropy and no CmpLog -- [merge-model.md](docs/contributing/explanation/merge-model.md) β€” why merging is a maintainer panel rather than a merge queue +- [merge-model.md](docs/contributing/explanation/merge-model.md) β€” why manager branch tips own executor refs - [shared-submodule-cache.md](docs/contributing/explanation/shared-submodule-cache.md) β€” why submodules are worktrees of one cache repo, not clones - [vendored-trees.md](docs/contributing/explanation/vendored-trees.md) β€” why third-party sources are patch series, not forks diff --git a/SECURITY.md b/SECURITY.md index d60a8d1b..ffd2565c 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -5,9 +5,13 @@ identical and trustworthy across all validators. Please treat security issues ac ## Reporting a vulnerability -**Do not open a public issue.** Report privately via GitHub's -[private vulnerability reporting](https://github.com/genlayerlabs/genvm-manager/security/advisories/new), -or email kira@genlayerlabs.com +**Before mainnet, report everything except remote code execution publicly** β€” open a +regular issue. Until there is value at stake, an open report gets triaged faster and is +useful to everyone reading along. RCE is the only exception; report it privately. + +For remote code execution, **do not open a public issue** β€” report it via GitHub's +[private vulnerability reporting](https://github.com/genlayerlabs/genvm-manager/security/advisories/new) +on the manager repository. Include a description, affected component/version, and a reproduction (a contract, calldata, or test case) where possible. We aim to acknowledge within a few business days. @@ -43,6 +47,12 @@ machine, and non-default deployments that expose the loopback-only manager to un The following relationships are trusted. Hardening them is welcome, but a report that assumes one side is hostile is not treated as a vulnerability: -- host and GenVM -- executor and manager -- the local disk and loopback in general +- Host and GenVM +- Executor and manager +- The local disk and loopback in general + +The following inputs are untrusted, even when delivered through a trusted component: + +- Intelligent Contract code and contract-controlled data, including calldata, messages, and persisted values +- Data originating from other validators, including leader results +- External content processed by modules, including HTTP responses, redirects, rendered pages, JavaScript, subresources, and model-provider responses diff --git a/docs/contributing/README.md b/docs/contributing/README.md index c1d65b1e..71ac8dae 100644 --- a/docs/contributing/README.md +++ b/docs/contributing/README.md @@ -11,8 +11,8 @@ ## Requirements A PR must pass the [pre-commit](https://pre-commit.com) hooks and the full test -suite, and lands when a maintainer ticks *Merge into dev* on the PR action panel -β€” see [pr.md](howto/pr.md) +suite, then lands through the App-owned E2E merge train β€” see +[pr.md](howto/pr.md) ## Security diff --git a/docs/contributing/explanation/README.md b/docs/contributing/explanation/README.md index a7ab0ac3..ed4b20b3 100644 --- a/docs/contributing/explanation/README.md +++ b/docs/contributing/explanation/README.md @@ -6,6 +6,6 @@ lives in the spec, and these pages only motivate it and link out - [docs-layout.md](docs-layout.md) β€” the 4 kinds of page and where each belongs - [executor-lines.md](executor-lines.md) β€” why several executor lines ship side by side, and what it costs - [fuzz.md](fuzz.md) β€” why fuzz targets get fake entropy and no CmpLog -- [merge-model.md](merge-model.md) β€” why merging is a maintainer panel rather than a merge queue +- [merge-model.md](merge-model.md) β€” why manager branch tips own executor refs - [shared-submodule-cache.md](shared-submodule-cache.md) β€” why submodules are worktrees of one cache repo, not clones - [vendored-trees.md](vendored-trees.md) β€” why third-party sources are patch series, not forks diff --git a/docs/contributing/explanation/merge-model.md b/docs/contributing/explanation/merge-model.md index cf779db4..05c324c4 100644 --- a/docs/contributing/explanation/merge-model.md +++ b/docs/contributing/explanation/merge-model.md @@ -1,31 +1,34 @@ -# Why Merging Is a Panel, Not a Queue +# Why Manager Owns Executor Branches A change spans repositories: the executor lines carry the edit, the manager -carries the gitlinks pinning them. GitHub's merge queue orders pull requests in -one repository and has no opinion about a second, so the landing step is ours β€” -a maintainer ticks *Merge into dev* and the automation re-checks every gate -before writing anything ([pr.md](../howto/pr.md)) +carries the gitlinks pinning them. The E2E App therefore treats the manager PR +as the sole merge member. Once it lands, the manager branch tip is the source of +truth for every executor ref ([pr.md](../howto/pr.md)) -## Why the Gates Re-Run at Merge Time +## Why Executor Updates Follow Manager -A green check describes the commit that was tested, not the branch it sits on, -and between review and merge the head, the base and the pinned executor branch -can all move +The App squash-merges the exact manager tree E2E tested, preserving its gitlink +values. A push to a manager dev branch projects those gitlinks onto executor dev +branches; a manager release push projects them onto the declared executor +release branches -The heavy matrix is opt-in for a related reason: a check that runs on every push -to every draft gets ignored +The projection creates missing refs and otherwise uses plain non-force pushes. +It never rewrites an executor commit or moves a branch backward -## Why Only One Merge Runs at a Time +## Why Failures Aggregate -Checking the gates and advancing the branches is not atomic, and the executor -lines advance before the manager, so a lost race leaves a half-landed change. -Hence one merge repository-wide, and never a cancelled one +Cross-repository updates are not atomic. One executor line can diverge or lose +authorization while the others remain valid, so stopping on its first failure +would strand unrelated lines unnecessarily. Every line is attempted, then the +workflow returns one combined failure for repair and idempotent rerun -That costs a maintainer something: GitHub keeps 1 pending run per concurrency -group, so a third tick cancels the waiting one +The executor commits already exist remotely before manager becomes eligible; +therefore a temporary branch-sync failure does not make manager gitlinks +unresolvable ## Why `ci-safe` Is a Human Decision -These jobs run the pull request's own `support/ci/` scripts with deploy keys in -scope. Approving a run approves code execution with credentials, which is not -inferable from the diff +The PR panel's full-test and provisioning jobs run pull-request scripts with +credentials in scope. Approving those jobs approves code execution with +credentials, which is not inferable from the diff. The post-merge projection +runs only code already landed on a protected manager branch diff --git a/docs/contributing/howto/README.md b/docs/contributing/howto/README.md index 7a5349b8..b7980d8a 100644 --- a/docs/contributing/howto/README.md +++ b/docs/contributing/howto/README.md @@ -2,7 +2,8 @@ - [setup.md](setup.md) β€” first-time clone: submodules, vendored trees, nix/direnv dev shell - [genvm-tool.md](genvm-tool.md) β€” the umbrella tool: man page, test and git subcommands, codegen -- [pr.md](pr.md) β€” branch model, PR action panel, `ci-safe` / `run-full-tests`, merge gates +- [pr.md](pr.md) β€” branch model, PR action panel, App-owned landing and executor projection +- [review-ready.md](review-ready.md) β€” the checks a change must pass before it is put up for review ## building/ - [build.md](building/build.md) β€” debug build: configure + ninja, targets, outputs, cargo quirks @@ -18,6 +19,7 @@ - [python.md](testing/python.md) β€” Python tests, direct pytest for genlayer-py-std - [integration.md](testing/integration.md) β€” jsonnet cases, tags, golden `.stdout`/`.hash` files, services - [fuzzing.md](testing/fuzzing.md) β€” AFL fuzz targets, seeding a corpus from a run, host sysctl prep +- [typescript.md](testing/typescript.md) β€” information on how to run and write webdriver typescript tests ## committing/ - [submodules.md](committing/submodules.md) β€” repo topology, gitlink bumps, pre-commit hooks, push order @@ -35,3 +37,4 @@ - [modify-runner.md](extending/modify-runner.md) β€” runner dev-mode and hash refresh - [modify-wasmtime.md](extending/modify-wasmtime.md) β€” patching vendored wasmtime, trap plumbing - [write-a-script.md](extending/write-a-script.md) β€” conventions for helper scripts and pre-commit hooks +- [add-a-skill.md](extending/add-a-skill.md) β€” agent skills in `.agents/`, the frontmatter trigger, the symlink, skill vs how-to diff --git a/docs/contributing/howto/extending/add-a-skill.md b/docs/contributing/howto/extending/add-a-skill.md new file mode 100644 index 00000000..fc910b3d --- /dev/null +++ b/docs/contributing/howto/extending/add-a-skill.md @@ -0,0 +1,43 @@ +# Adding an Agent Skill + +Skills are procedures loaded on demand. `.agents/` is canonical; Claude Code +sees each through a symlink + +1. Write `.agents/skills/<name>/SKILL.md`, starting with the frontmatter: + + ```markdown + --- + name: <name> + description: What it does. Use when <the moment it applies>. + --- + ``` + + `name` matches the directory. `description` drives triggering, so name when + the skill applies and the words someone would use then + +2. Symlink it where Claude Code discovers skills, relative so the link survives + a clone or a worktree: + + ```bash + ln -s ../../.agents/skills/<name> .claude/skills/<name> + ``` + + Link each skill, never the whole directory, which would expose future skills + without review. Link subagents individually too: + `ln -s ../../.agents/agents/<name>.md .claude/agents/<name>.md` + +3. Keep the body short and imperative. Supporting files sit beside `SKILL.md` in + the same directory and are referenced by relative path + +## Skill or How-To + +They serve different roles: + +| It is | Where it goes | +|---|---| +| A rule about the repository, true whoever reads it | a how-to in `docs/contributing/`, indexed and reaching humans too | +| When to act, what to report, what an agent may decide alone | the skill | + +Restating a how-to in a skill creates a copy that will drift. Link to the page; +keep only AI-specific instructions in the skill. [review-ready](../review-ready.md) +and its skill are the worked example diff --git a/docs/contributing/howto/genvm-tool.md b/docs/contributing/howto/genvm-tool.md index 8df682b6..e5fab237 100644 --- a/docs/contributing/howto/genvm-tool.md +++ b/docs/contributing/howto/genvm-tool.md @@ -12,7 +12,7 @@ has no wrapper | `test` | run the suite ([testing/README.md](testing/README.md)) | | `codegen` | render a data JSON into rust/python/rst/go | | `build-manifest` | write the manager's `data/manifest.yaml` | -| `docs` | render the `docs/contributing/` indexes, the source of the list in `CLAUDE.md` | +| `docs` | render the `docs/contributing/` indexes; `--write` splices them into `AGENTS.md` (`CLAUDE.md` is a symlink to it), `--check` fails when that copy is stale | | `fuzz-corpus` | seed the fuzz corpora from a finished run ([testing/fuzzing.md](testing/fuzzing.md)) | | `git` | `ls`, `list-repo`, `create-branches`, `check-for-push` across the manager and every executor submodule ([submodules.md](committing/submodules.md)) | diff --git a/docs/contributing/howto/pr.md b/docs/contributing/howto/pr.md index 971e5549..3bde9bf9 100644 --- a/docs/contributing/howto/pr.md +++ b/docs/contributing/howto/pr.md @@ -1,56 +1,68 @@ # Pull Requests -There is no GitHub merge queue. A PR lands when a maintainer ticks a box on the -PR action panel, which re-checks every gate. Automation: -`.github/workflows/branch_*.yaml` and `queue.yaml`; why: +The E2E GitHub App owns integration landing. After the exact manager snapshot +passes full GenVM CI and cross-repository E2E, a maintainer runs `/merge`. The +App revalidates the snapshot and squash-merges the manager PR; the resulting +manager push projects its gitlinks onto executor branches. Why: [merge-model.md](../explanation/merge-model.md) ## Branches 1. `main` is an alias of the latest `v<X>-dev` and is never developed on -2. A feature PR lands on `v<X>-dev`, which reaches `v<X>.x` through a standing +2. A feature PR lands on `v<X>-dev`, which reaches `v<X>` through a standing release-gate PR 3. `genvm-tool git create-branches` makes `feat/<name>` in the manager and `pr/<line>/feat/<name>` per executor line 4. Push the submodules first, then open **one PR, in the manager only**, against `v<X>-dev`; a PR targeting `main` is retargeted for you 5. Never open an executor PR by hand β€” automation opens and links one per active - line that has a pushed `pr/<line>/…` branch, and they land together + line that has a pushed `pr/<line>/…` branch; they are review projections ## Panel -Boxes: *Force / Rerun full tests*, *Provision executor PRs* (force-push each -mirror branch to the pinned gitlink, after a rebase or an amend), *Merge into -dev*. Nothing works without the **`ci-safe`** label, since these jobs run the +Boxes: *Force run full tests*, *Provision executor PRs* (force-push each mirror +branch to the pinned gitlink, after a rebase or an amend). Nothing works without +the **`ci-safe`** label, since these jobs run the PR's own `support/ci/` scripts with deploy keys in scope. Authors with write access get it automatically. Everyone else needs a maintainer to add it after reading the diff +With `ci-safe` and write access, `/genvm-run-tests` links one run and reacts +πŸ‘€ β†’ πŸš€/πŸ˜• without changing the sticky `run-full-tests` label + ## Gates `queue.yaml` always runs the pre-commit hooks. On a `pull_request` event it also requires 0 commits behind base and the executor precondition β€” every repo rebased, every pinned executor commit pushed β€” which is skipped for fork PRs. -The heavy matrix runs only with the `run-full-tests` or `rtm` marker; without -one the check stays red on purpose +The heavy matrix runs with the `run-full-tests` marker or a one-shot +`/genvm-run-tests`; otherwise the check stays red on purpose + +The App re-verifies the exact tested head, base, synthetic merge, approvals, +native CI, E2E proof and dependency closure. It squash-merges only the manager +PR. A push to manager `v<X>-dev` then creates or fast-forwards every declared +executor `<line>-dev` to the gitlink in that manager tip; a push to manager +`v<X>` does the same for each executor release branch declared in `.gitmodules` + +Executor updates are non-force and independent. Every line is attempted even +if another line fails; the workflow reports all failures together and can be +rerun manually against the live manager branch -Merging re-verifies against the **exact head sha**: maintainer approval (any -push revokes it) or the `rtm` label (which survives pushes, so it authorizes -whatever lands next), green full CI, green cross-repo E2E, 0 behind. An -unwithdrawn *changes requested* blocks either way. Merges are -serialised repository-wide, so a tick may wait for another PR to land first +## Authority -Each repo gains one commit per PR. Executor lines are always squashed onto -`<line>-dev`, so the manager commit is rewritten to gitlink the new commit. A -manager PR that is a single commit and rewrites no gitlink keeps its sha +Needs a maintainer: merging, `--admin`, cross-repo E2E runs, and anything +irreversible outside the PR β€” releases, deploys, messages to other teams. Never +force-push a shared branch (`v<X>`, `v<X>-dev`); branch protection refuses it +anyway. Force-pushing your own PR branch is routine and needs no one, and so +does fixing a defect you find in your own diff ## Troubleshooting | Symptom | Fix | | --- | --- | | Panel does nothing | missing `ci-safe` | -| `full tests have not run` | tick *Force run full tests* | +| `full tests have not run` | tick *Force run full tests* or comment `/genvm-run-tests` | | `... commit(s) behind base` | rebase, `push --force-with-lease` | | `pinned commit ... is not on the executor repo` | push it, or tick *Provision executor PRs* | -| Approval revoked | you pushed after the review, re-request it or set `rtm` | -| Merge run cancelled before it started | another merge was already queued, tick *Merge into dev* again | +| Approval revoked | re-request approval for the current head | +| Executor branch sync failed | inspect every reported line, resolve non-fast-forward divergence, rerun *branch / sync executor branches* | diff --git a/docs/contributing/howto/releasing/versioning.md b/docs/contributing/howto/releasing/versioning.md index 1263e16c..31676682 100644 --- a/docs/contributing/howto/releasing/versioning.md +++ b/docs/contributing/howto/releasing/versioning.md @@ -13,7 +13,12 @@ Executor crates are versioned **independently**: a line's `manifest.json` holds `executor-version` and no manager script rewrites it Manager branches are named after the *manager* train, not an executor line: -`v<X>.x` is the release branch, `v<X>-dev` the integration branch +`v<X>` is the release branch, `v<X>-dev` the integration branch. Here `<X>` is +major.minor, for example `v0.6` + +On each manager branch push, `.gitmodules` declares the executor release +branches to update. A manager dev push derives `<line>-dev` from each declared +`<line>.x`; all updates are branch creation or fast-forward only ## Tools diff --git a/docs/contributing/howto/review-ready.md b/docs/contributing/howto/review-ready.md new file mode 100644 index 00000000..66753bc0 --- /dev/null +++ b/docs/contributing/howto/review-ready.md @@ -0,0 +1,34 @@ +# Review-Ready + +What a change must satisfy before review, checked against the diff and shell + +## Intent + +1. It does what was asked. The PR traces each requirement to evidence; testing + the wrong thing does not count +2. Prefer one change, reviewable in one sitting. Fix defects inside the diff; + record those outside it without folding them in or dropping them + +## Proof + +3. Run everything testable locally before opening a PR: pre-commit hooks, a + debug [build](building/build.md) for code changes, and the relevant + [test suites](testing/README.md). Use CI only for what cannot run locally +4. Map the diff to its tests and run all of them. Behaviour that can drift + nondeterministically also warrants [agentic fuzzing](testing/fuzzing.md) + +[Runner artifacts](building/runners.md) build on Linux only; report an unrun tier +as unrun, never passed + +## The Artifact + +5. Read the whole diff, including generated and vendored changes it caused +6. Leave no known defect, including false prose in comments, help or docs +7. Land documentation in the same PR: man page and `--help` for CLI changes, + [spec or impl-spec](building/docs.md) for VM or protocol behaviour, and + `docs/contributing/` for workflows +8. For applicable changes, check security exposure, diagnostics, config-schema + compatibility and rollback + +An open PR with green checks completes the list. Cross-repo E2E and merging do +not β€” see [pr.md](pr.md) under *Authority* diff --git a/docs/contributing/howto/testing/README.md b/docs/contributing/howto/testing/README.md index b8429cb0..de9eb164 100644 --- a/docs/contributing/howto/testing/README.md +++ b/docs/contributing/howto/testing/README.md @@ -2,8 +2,8 @@ Everything runs through `genvm-tool test`, from the `#full` dev shell, after a build ([build.md](../building/build.md)). Per-kind: [rust.md](rust.md) Β· -[python.md](python.md) Β· [integration.md](integration.md) Β· -[fuzzing.md](fuzzing.md) +[python.md](python.md) Β· [typescript.md](typescript.md) Β· +[integration.md](integration.md) Β· [fuzzing.md](fuzzing.md) | What | Command | |---|---| @@ -37,7 +37,8 @@ genvm-tool test run --filter-continue <name-or-path> ## Where Tests Come From Rust from every tracked `Cargo.toml` and its `fuzz/*.rs`, python from -`executors/v0.3.x/runners/genlayer-py-std`, integration from each line's +`executors/v0.3.x/runners/genlayer-py-std`, TypeScript from every tracked +`package.json` with a `test` script, integration from each line's `tests/integration/`, and system from `tests/system/`. System suites use the same Jsonnet collector when they execute contracts; protocol-level assertions use `test.py`, which must register 1 `genvm-tool` case per assertion. Collection diff --git a/docs/contributing/howto/testing/typescript.md b/docs/contributing/howto/testing/typescript.md new file mode 100644 index 00000000..ca29555f --- /dev/null +++ b/docs/contributing/howto/testing/typescript.md @@ -0,0 +1,38 @@ +# TypeScript Tests + +```bash +genvm-tool test run --filter-tag typescript +``` + +Collected from every tracked `package.json` declaring a `test` script, one case +per `*.test.ts` under it, plus `<project>/npm-ci` and `<project>/typecheck`. +Today that is only `webdriver/src/prj` + +Every case except the install `depends_on` it, so `node_modules` is populated +once per project. A dependency is pulled in even when the filter excludes it, +and if it fails the dependents are skipped rather than failing on their own + +`needs-web` is on all of them: `npm ci` reaches the registry + +## Running One Directly + +The project's `test` script forwards a file, so what the runner does is: + +```bash +cd webdriver/src/prj +npm ci +npm test -- src/render.test.ts # omit the file to run all of them +npm run typecheck +``` + +## Extra Tags + +Same marker as Rust ([rust.md](rust.md)), parsed by the shared +`tests/runner/genvm_tool_plugins/source_tags.py`: + +```ts +// genvm-tool-test-tags: feature-web-render +``` + +They add to `typescript` plus `unit` and `needs-web`. Unknown tags fail +collection diff --git a/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/03-schemas.rst b/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/03-schemas.rst index 0a5a0243..51bb6afa 100644 --- a/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/03-schemas.rst +++ b/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/03-schemas.rst @@ -57,8 +57,8 @@ disagreement without running the comparison stage any parameter. Codes outside the trie yield :ref:`gvm-def-str-trie-value-vm-error-leader-fault-nondet-output-malformed`. -An accepted result is preserved **byte for byte**: validation never re-encodes, -so every node hashes the value that was proposed. +An accepted result that passes :ref:`gvm-def-nondeterministic-output-caps` is +preserved **byte for byte**, so every node hashes the value that was proposed. .. _gvm-def-derived-outcome-namespace: diff --git a/docs/website/src/spec/03-vm/03-ram-limiting.rst b/docs/website/src/spec/03-vm/03-ram-limiting.rst index 2c7db4d2..6de65c44 100644 --- a/docs/website/src/spec/03-vm/03-ram-limiting.rst +++ b/docs/website/src/spec/03-vm/03-ram-limiting.rst @@ -51,6 +51,22 @@ The following operations consume RAM: :ref:`gvm-def-consts-value-memory-limiter-consts-new-storage-page` octets the first time that region is written. Regions the :term:`sub-VM` inherited already written from its caller, and repeated writes to a region, cost nothing +- **Emissions**: each emitted message or event costs + :ref:`gvm-def-consts-value-memory-limiter-consts-execution-emission-base-size` + octets, + plus its retained calldata, code, allocation subtree, topics, event data, and + :ref:`gvm-def-consts-value-memory-limiter-consts-calldata-arg-element-size` + octets per + retained positional argument, + :ref:`gvm-def-consts-value-memory-limiter-consts-calldata-kwarg-entry-size` + octets per + retained keyword argument, and + :ref:`gvm-def-consts-value-memory-limiter-consts-message-fee-rotation-element-size` + octets + per retained message-fee rotation +- **Nondeterministic outputs**: each output costs + :ref:`gvm-def-consts-value-memory-limiter-consts-nondet-output-base-size` + octets plus its encoded length on every role - **Sub-VM creation**: each new :term:`sub-VM` costs :ref:`gvm-def-consts-value-memory-limiter-consts-vm-spawn-cost` octets, plus :ref:`gvm-def-consts-value-memory-limiter-consts-storage-page-inherited` @@ -61,6 +77,29 @@ The following operations consume RAM: The runner load cost (:ref:`gvm-def-consts-value-memory-limiter-consts-runner-load-cost`) is a fixed per-load overhead +.. _gvm-def-nondeterministic-output-caps: + +Nondeterministic Output Caps +---------------------------- + +Before entering a non-deterministic :term:`sub-VM`, the caller must have enough +RAM to retain the canonical memory-limit and non-deterministic-output fee-limit +results and return either as a file descriptor. It also checks whether both +results can be charged against the non-deterministic-output fee; when either +cannot, the sub-VM is not entered + +An output that cannot fit its non-deterministic-output fee charge is first +replaced with a +:ref:`gvm-def-str-trie-value-vm-error-out-of-receipt-nondet-output` result. If +that result, or an output within the fee limit, cannot fit its RAM charge, it is +replaced with a :ref:`gvm-def-str-trie-value-vm-error-out-of-memory` result. The +leader publishes the replacement and an honest replay charges the same encoded +result + +A validator rejects a leader proposal that differs from its post-cap result as +a fatal :ref:`gvm-def-str-trie-value-vm-error-leader-fault-nondet-output-malformed` +result without entering the validator sub-VM + RAM Release ----------- @@ -68,13 +107,12 @@ File content memory is released when the corresponding file descriptor is closed When a :term:`sub-VM` finishes execution, all remaining RAM consumed by it is released back to the shared budget. This applies to runner charges as well: memory consumed by loading or registering a runner is released when the registering :term:`sub-VM` finishes, -like any other charge. There are no permanent charges. +like any other charge. -Storage write charges are the exception: they follow the storage they paid for. -When a :ref:`gvm-def-gl-call-sandbox` child returns and its caller takes over the -child's storage, the charge for the regions the child wrote first is transferred -to the caller rather than released, so a caller cannot use repeated -:ref:`gvm-def-gl-call-sandbox` calls to accumulate storage it never pays for. +Charges for retained storage, emissions, and nondeterministic outputs are +permanent. When a :ref:`gvm-def-gl-call-sandbox` child returns and its caller +takes over the child's retained data, those charges are transferred to the +caller rather than released. Other Limits ------------ diff --git a/docs/website/src/spec/appendix/internal-constants.rst b/docs/website/src/spec/appendix/internal-constants.rst index 2323d901..f9258503 100644 --- a/docs/website/src/spec/appendix/internal-constants.rst +++ b/docs/website/src/spec/appendix/internal-constants.rst @@ -57,6 +57,27 @@ storage_page_inherited Value: ``128`` +.. _gvm-def-consts-value-memory-limiter-consts-execution-emission-base-size: + +execution_emission_base_size +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Value: ``256`` + +.. _gvm-def-consts-value-memory-limiter-consts-message-fee-rotation-element-size: + +message_fee_rotation_element_size +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Value: ``32`` + +.. _gvm-def-consts-value-memory-limiter-consts-nondet-output-base-size: + +nondet_output_base_size +~~~~~~~~~~~~~~~~~~~~~~~ + +Value: ``32`` + .. _gvm-def-consts-top-limits: top_limits diff --git a/docs/website/src/spec/changelog.rst b/docs/website/src/spec/changelog.rst index 7af5d709..dae6e3b3 100644 --- a/docs/website/src/spec/changelog.rst +++ b/docs/website/src/spec/changelog.rst @@ -64,6 +64,10 @@ Changed caller's remaining RAM instead of a fresh 4 GiB one, so it can no longer use more RAM than the deterministic caller had left. See :doc:`03-vm/03-ram-limiting` +#. Emitted messages, events, and leader nondeterministic outputs consume RAM for + their retained representations. Their charges, like storage write charges, + remain until execution ends and transfer to a caller that adopts a sandbox + child's retained data. See :doc:`03-vm/03-ram-limiting` #. A resource error raised while a module is being instantiated keeps its own code β€” e.g. exhausting the memory budget there reports :ref:`gvm-def-str-trie-value-vm-error-out-of-memory-wasm-memory` β€” instead of diff --git a/executors/v0.2.x b/executors/v0.2.x index ca50b306..750ca85e 160000 --- a/executors/v0.2.x +++ b/executors/v0.2.x @@ -1 +1 @@ -Subproject commit ca50b3065e952e52dc3b7dd79dba51c334ea9abd +Subproject commit 750ca85e3f1d9d8cfe632a29b12b2b261d0b8df3 diff --git a/executors/v0.3.x b/executors/v0.3.x index 4ddb57b6..ee7b5513 160000 --- a/executors/v0.3.x +++ b/executors/v0.3.x @@ -1 +1 @@ -Subproject commit 4ddb57b68d2cb76eec01d94671e9594338f7e5a8 +Subproject commit ee7b55133b637acd22118ed06ff52f233b21366a diff --git a/flake.nix b/flake.nix index de34f94d..f04ccffd 100644 --- a/flake.nix +++ b/flake.nix @@ -349,6 +349,17 @@ entry = "${hooks-python}/bin/python3 support/scripts/md-local-links.py"; types = [ "markdown" ]; }; + # AGENTS.md carries a generated index of docs/contributing/, and + # every agent session loads it β€” a stale copy misroutes work + # silently. `--check` also fails when a quadrant README and the + # files on disk disagree. + check-docs-index = { + enable = true; + name = "check-docs-index"; + entry = "${genvm-tool}/bin/genvm-tool docs --check"; + files = "^(AGENTS\\.md|docs/contributing/.*)$"; + pass_filenames = false; + }; check-source-text = { enable = true; name = "check-source-text"; diff --git a/implementation/.ya-test-config.json b/implementation/.ya-test-config.json index bc011a52..ec93b823 100644 --- a/implementation/.ya-test-config.json +++ b/implementation/.ya-test-config.json @@ -19,6 +19,7 @@ "lib": [ "feature-manager-socket", "feature-version-routing", + "feature-web-render", "feature-web-request-signed" ] } diff --git a/implementation/src/common/mod.rs b/implementation/src/common/mod.rs index c9709634..1a5cb697 100644 --- a/implementation/src/common/mod.rs +++ b/implementation/src/common/mod.rs @@ -243,6 +243,60 @@ where .with_context(|| "handling with handler") } +/// Classify a handler failure into the wire result the executor reads. +/// +/// This is the only place the `fatal` flag is spent, and the two outcomes are +/// not interchangeable: +/// +/// 1. `FatalError` is turned into a bare `anyhow` by the executor +/// (`executors/v0.3.x/executor/src/modules.rs`, the `Result::FatalError` +/// arm), which lands as `ErrorKind::Internal` in `rt/errors.rs` and finally +/// as `ResultCode::InternalError` in `host/mod.rs`. It has no `RunOk` +/// representation, so the whole contract run is aborted and the contract +/// cannot catch it. Downstream, a node turns any GenVM error into a Timeout +/// vote +/// 2. `UserError` is handed to the runner as `{"error": ...}`, which +/// `genlayer-py-std` raises as a catchable `NondetException` +/// +/// Which one an infrastructure fault deserves is settled: (1). If this node's +/// own environment failed, it has no valid observation of the world, and a +/// `UserError` would let it assert to the contract -- and then vote on -- a claim +/// it never observed. The Timeout vote is the truthful "I could not do the +/// work", and consensus has that vote for exactly this case. (2) is for things +/// the run genuinely observed, such as a page that failed to load. +/// +/// The cost of (1) is diagnostic, not semantic: `FatalError` carries only a +/// string, so a raiser that wants to stay distinguishable must put a cause in +/// it. [`ModuleError`]'s `Display` is its JSON, so `causes` survives. +/// +/// An error that is not a [`ModuleError`] is fatal, so "not a `ModuleError`" +/// and "`fatal: true`" are indistinguishable past this point +pub(crate) fn module_error_to_wire<R>( + err: anyhow::Error, + genvm_id: genvm_modules_interfaces::GenVMId, +) -> genvm_modules_interfaces::Result<R> { + match scripting::try_unwrap_any_err(err) { + Ok(err) => { + if err.fatal { + genvm_modules_interfaces::Result::FatalError(format!("{err:#}")) + } else { + let res = GenericValue::Map(BTreeMap::from([ + ( + "causes".to_owned(), + GenericValue::Array(err.causes.into_iter().map(Into::into).collect()), + ), + ("ctx".to_owned(), GenericValue::Map(err.ctx)), + ])); + genvm_modules_interfaces::Result::UserError(res) + } + } + Err(err) => { + log_error_into!(&LoggerWithId, error:ah = &err, genvm_id:id = genvm_id.0; "handler fatal error"); + genvm_modules_interfaces::Result::FatalError(format!("{err:#}")) + } + } +} + async fn loop_one_inner<T, R, S>( handler: &mut impl MessageHandler<T, R>, stream: &mut S, @@ -265,28 +319,7 @@ where let res = loop_one_inner_handle(handler, &data).await; let res = match res { Ok(res) => genvm_modules_interfaces::Result::Ok(res), - Err(err) => match scripting::try_unwrap_any_err(err) { - Ok(err) => { - if err.fatal { - genvm_modules_interfaces::Result::FatalError(format!("{err:#}")) - } else { - let res = GenericValue::Map(BTreeMap::from([ - ( - "causes".to_owned(), - GenericValue::Array( - err.causes.into_iter().map(Into::into).collect(), - ), - ), - ("ctx".to_owned(), GenericValue::Map(err.ctx)), - ])); - genvm_modules_interfaces::Result::UserError(res) - } - } - Err(err) => { - log_error_into!(&LoggerWithId, error:ah = &err, genvm_id:id = genvm_id.0; "handler fatal error"); - genvm_modules_interfaces::Result::FatalError(format!("{err:#}")) - } - }, + Err(err) => module_error_to_wire(err, genvm_id), }; let message = calldata::encode_obj(&res); @@ -1159,3 +1192,93 @@ mod ip_filter_tests { } } } + +#[cfg(test)] +mod wire_result_tests { + use super::*; + + use genvm_modules_interfaces::Result as WireResult; + + const ID: genvm_modules_interfaces::GenVMId = genvm_modules_interfaces::GenVMId(7); + + fn module_error(fatal: bool) -> anyhow::Error { + ModuleError { + causes: vec![ErrorKind::STATUS_NOT_OK.into()], + fatal, + ctx: BTreeMap::from([("status".to_owned(), GenericValue::Number(500.0))]), + } + .into() + } + + /// The classification survives calldata encoding, which is what the executor + /// actually reads off the socket + fn roundtrip(res: WireResult<String>) -> WireResult<String> { + let bytes = calldata::encode_obj(&res); + calldata::decode_obj(&bytes).unwrap() + } + + // -- fatal: the run is aborted, the contract cannot catch it -------- + + #[test] + fn fatal_module_error_becomes_fatal_error() { + let res: WireResult<String> = module_error_to_wire(module_error(true), ID); + + match roundtrip(res) { + WireResult::FatalError(msg) => { + assert!(msg.contains("STATUS_NOT_OK"), "unexpected message: {msg}") + } + WireResult::Ok(_) | WireResult::UserError(_) => { + panic!("a fatal module error must become FatalError") + } + } + } + + /// An error that never was a [`ModuleError`] carries no fatality flag at + /// all, and is treated as fatal + #[test] + fn plain_error_becomes_fatal_error() { + let res: WireResult<String> = + module_error_to_wire(anyhow::anyhow!("something unclassified"), ID); + + match roundtrip(res) { + WireResult::FatalError(msg) => assert!( + msg.contains("something unclassified"), + "unexpected message: {msg}" + ), + WireResult::Ok(_) | WireResult::UserError(_) => { + panic!("an unclassified error must become FatalError") + } + } + } + + // -- non-fatal: a catchable NondetException for the contract --------- + + #[test] + fn non_fatal_module_error_becomes_user_error() { + let res: WireResult<String> = module_error_to_wire(module_error(false), ID); + + let WireResult::UserError(value) = roundtrip(res) else { + panic!("a non-fatal module error must become UserError") + }; + + let GenericValue::Map(map) = value else { + panic!("expected a map, got {value:?}") + }; + + let Some(GenericValue::Array(causes)) = map.get("causes") else { + panic!("expected an array of causes in {map:?}") + }; + let [GenericValue::Str(cause)] = &causes[..] else { + panic!("expected exactly one cause, got {causes:?}") + }; + assert_eq!(cause, "STATUS_NOT_OK"); + + let Some(GenericValue::Map(ctx)) = map.get("ctx") else { + panic!("expected a ctx map in {map:?}") + }; + let Some(GenericValue::Number(status)) = ctx.get("status") else { + panic!("expected a numeric status in {ctx:?}") + }; + assert_eq!(*status, 500.0); + } +} diff --git a/implementation/src/web/mod.rs b/implementation/src/web/mod.rs index 872e3d0b..301fee08 100644 --- a/implementation/src/web/mod.rs +++ b/implementation/src/web/mod.rs @@ -148,3 +148,6 @@ pub fn entrypoint(args: CliArgs) -> Result<()> { Ok(()) } + +#[cfg(test)] +mod tests; diff --git a/implementation/src/web/tests.rs b/implementation/src/web/tests.rs new file mode 100644 index 00000000..a13d8819 --- /dev/null +++ b/implementation/src/web/tests.rs @@ -0,0 +1,638 @@ +use super::*; + +use crate::common::ModuleError; +use genvm_modules_interfaces::web as web_iface; +use mlua::LuaSerdeExt as _; +use tokio::io::AsyncWriteExt as _; + +type TestVM = scripting::UserVM<ctx::VMData, Arc<ctx::CtxPart>, WebSubContext>; + +/// Absolute path of `<repo>/<rel>`. Tests run with the crate +/// directory (`implementation`) as the working directory. +fn modules_path(rel: &str) -> String { + let mut path = std::env::current_dir().unwrap(); + path.pop(); + path.push(rel); + path.canonicalize() + .with_context(|| format!("canonicalizing {path:?}")) + .unwrap() + .to_str() + .unwrap() + .to_owned() +} + +/// Stands in for the webdriver sidecar: answers every connection with +/// `head` (status line + headers, `Content-Length` is appended) and `body`. +/// No Chromium and no egress past loopback. +async fn serve_sidecar(head: &'static str, body: &'static [u8]) -> std::net::SocketAddr { + let server = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = server.local_addr().unwrap(); + + tokio::spawn(async move { + while let Ok((mut client, _)) = server.accept().await { + let response = format!("{head}Content-Length: {}\r\n\r\n", body.len()); + client.write_all(response.as_bytes()).await.unwrap(); + client.write_all(body).await.unwrap(); + client.shutdown().await.unwrap(); + } + }); + + addr +} + +/// The sidecar's success path: `200` with the *page's* status reported in +/// `Resulting-Status`. A page that 404s looks like this. +async fn serve_sidecar_page_not_found() -> std::net::SocketAddr { + serve_sidecar( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nResulting-Status: 404\r\n", + b"Not Found", + ) + .await +} + +/// The sidecar's outer `catch` (`webdriver/src/prj/src/index.ts`, +/// `handleRenderRequest`): a bare `500` with no `Resulting-Status`, which is +/// what an exception thrown *outside* the per-page `try` becomes -- e.g. +/// `newPage()` exceeding puppeteer's `protocolTimeout` on +/// `Target.createTarget`. +async fn serve_sidecar_internal_error() -> std::net::SocketAddr { + serve_sidecar( + "HTTP/1.1 500 Internal Server Error\r\nContent-Type: application/json\r\n", + br#"{"error":"Internal server error","message":"Target.createTarget timed out"}"#, + ) + .await +} + +/// The one navigation failure the sidecar refuses to report: +/// `net::ERR_INTERNET_DISCONNECTED` means the request never left the host, so +/// there is no observation to report and `render.ts` throws instead of +/// returning a status. The outer catch turns that into the same bare `500`, +/// byte-accurate down to the message, which is the only thing that crosses. +/// +/// Its wording is the point. The browser and the sidecar answered normally, so +/// an operator told "webdriver unavailable" would go and stare at a working +/// browser; the text has to say the local network is what broke. +async fn serve_sidecar_local_network_unavailable() -> std::net::SocketAddr { + serve_sidecar( + "HTTP/1.1 500 Internal Server Error\r\nContent-Type: application/json\r\n", + br#"{"error":"Internal server error","message":"Local network fault: this host has no network route (net::ERR_INTERNET_DISCONNECTED). The browser answered, so it is the local network that failed rather than the webdriver, and nothing about the page was observed."}"#, + ) + .await +} + +/// The sidecar's parameter validation (`handleRenderRequest`): a `400` when +/// `url` is missing or `mode` is not one of text/html/screenshot. Reachable +/// whenever the module and the sidecar disagree about the query format, e.g. +/// across a version skew +async fn serve_sidecar_bad_request() -> std::net::SocketAddr { + serve_sidecar( + "HTTP/1.1 400 Bad Request\r\nContent-Type: application/json\r\n", + br#"{"error":"Missing url parameter"}"#, + ) + .await +} + +/// The sidecar's healthcheck path (`handleHealthcheck`): a plain-text `503` +/// when its own probe render fails. A proxy or orchestrator in front of the +/// sidecar answers the same way while the sidecar is down +async fn serve_sidecar_unhealthy() -> std::net::SocketAddr { + serve_sidecar( + "HTTP/1.1 503 Service Unavailable\r\nContent-Type: text/plain\r\n", + b"unhealthy", + ) + .await +} + +/// The same number on the other channel: the sidecar itself answered fine +/// (`200`) and is telling us the *page's* host refused the connection, which +/// `getNavigationErrorStatus` reports as `503`. Paired with +/// [`serve_sidecar_unhealthy`] on purpose -- the two differ only in which +/// channel carries the 503, and they must be classified oppositely +async fn serve_sidecar_page_unreachable() -> std::net::SocketAddr { + serve_sidecar( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nResulting-Status: 503\r\n", + b"Connection refused", + ) + .await +} + +/// A name that did not resolve, which `getNavigationErrorStatus` reports as a +/// `502` on the observation channel. Paired with +/// [`serve_sidecar_local_network_unavailable`]: both are network failures, and +/// they are classified oppositely on purpose. See +/// [`render_remote_page_name_not_resolved_is_not_fatal`] +async fn serve_sidecar_page_name_not_resolved() -> std::net::SocketAddr { + serve_sidecar( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nResulting-Status: 502\r\n", + b"DNS resolution failed", + ) + .await +} + +/// A navigation that ran out of time, reported as a `408` on the observation +/// channel. See [`render_remote_page_navigation_timeout_is_not_fatal`] +async fn serve_sidecar_page_navigation_timeout() -> std::net::SocketAddr { + serve_sidecar( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nResulting-Status: 408\r\n", + b"Navigation timeout", + ) + .await +} + +/// An address nothing listens on: the sidecar process is gone, so the +/// connection is refused before any status exists. Binds and drops, so the +/// port is known to have been free +async fn sidecar_down() -> std::net::SocketAddr { + let server = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = server.local_addr().unwrap(); + std::mem::drop(server); + addr +} + +fn test_config(webdriver_host: String) -> config::Config { + config::Config { + webdriver_host, + extra_tld: Vec::new(), + always_allow_hosts: Vec::new(), + meta: serde_json::Value::Null, + max_wait_after_loaded: common::Timeout::from_secs(60), + base: BaseConfig { + threads: 0, + blocking_threads: 0, + log_level: logger::Level::Trace, + log_disable: String::new(), + }, + mod_base: common::ModuleBaseConfig { + bind_address: None, + vm_count: 1, + lua_script_path: modules_path("install/config/genvm-web-default.lua"), + lua_path: format!("{}/?.lua", modules_path("install/lib/genvm-lua")), + signer_headers: Arc::new(std::collections::BTreeMap::new()), + signer_url: Arc::from(""), + data_dir: String::new(), + }, + } +} + +/// Same VM wiring as [`create_web_module`], without the pool. +async fn create_test_vm(config: &sync::DArc<config::Config>) -> TestVM { + let config_for_data = config.clone(); + + scripting::UserVM::create( + &config.mod_base, + move |vm: mlua::Lua| async move { + vm.globals() + .set("__web", ctx::create_global(&vm, &config_for_data)?)?; + + scripting::load_script(&vm, &config_for_data.mod_base.lua_script_path).await?; + + let render: mlua::Function = vm.globals().get("Render")?; + let request: mlua::Function = vm.globals().get("Request")?; + + Ok(ctx::VMData { render, request }) + }, + Box::new( + move |vm: &mlua::Lua, table: &mlua::Table, sub_ctx: &sync::DArc<WebSubContext>| { + let scripting = sub_ctx.gep(|x| &x.scripting); + scripting::setup_lua_default_ctx(scripting, vm, table)?; + + let ctx = Arc::new(ctx::CtxPart {}); + table.set("__ctx_web", vm.create_userdata(ctx.clone())?)?; + + Ok(ctx) + }, + ), + ) + .await + .unwrap() +} + +/// Same context as `HandlerProvider::create_execution_context`, including +/// `filter_dns = true` -- the webdriver host is an IP literal, which reqwest +/// never sends through the resolver, so the SSRF guard stays out of the way. +fn create_test_ctx(config: &sync::DArc<config::Config>) -> sync::DArc<WebSubContext> { + let hello = common::tests::get_hello(); + let metrics = sync::DArc::new(Metrics::default()); + + let scripting = scripting::create_ctx_part( + &hello, + &config.gep(|x| &x.mod_base), + metrics.gep(|x| &x.scripting), + true, + ) + .unwrap(); + + sync::DArc::new(WebSubContext { scripting }) +} + +/// Drives the production `Render` of +/// `modules/install/config/genvm-web-default.lua`, the way `Handler::handle` +/// does for `Message::Render`. +async fn render( + config: &sync::DArc<config::Config>, + url: &str, +) -> anyhow::Result<web_iface::RenderAnswer> { + let user_vm = create_test_vm(config).await; + let sub_ctx = create_test_ctx(config); + let (_ctx, ctx_val) = user_vm.create_ctx(&sub_ctx)?; + + let payload = user_vm.vm.create_table()?; + // what `RenderMode::Text` serializes to + payload.set("mode", "text")?; + payload.set("url", url)?; + payload.set("wait_after_loaded", 0.0)?; + payload.set("size_limit", 1024 * 1024)?; + + let res: mlua::Value = user_vm + .call_fn(&user_vm.data.render, (ctx_val, payload)) + .await?; + + Ok(user_vm.vm.from_value(res)?) +} + +/// The error `Render` failed with, still wrapped, the way `Handler::handle` +/// hands it to the message loop. Panics if `Render` succeeded. +async fn render_raw_err(config: &sync::DArc<config::Config>, url: &str) -> anyhow::Error { + let err = render(config, url) + .await + .err() + .expect("expected Render to fail"); + + log_info!(error:ah = &err; "render failed"); + + err +} + +/// The module error `Render` failed with. Panics if it succeeded, or if the +/// failure was not a [`ModuleError`]. +async fn render_err(config: &sync::DArc<config::Config>, url: &str) -> ModuleError { + let err = render_raw_err(config, url).await; + + scripting::try_unwrap_any_err(err).expect("expected a module error") +} + +/// Asserts that talking to the sidecar described by `addr` fails fatally and is +/// labelled as a webdriver fault. +/// +/// Fatal is the point, not an accident. A broken sidecar means this validator +/// never observed the page, so it must not hand the contract a result: the run +/// is aborted as an internal error and the node votes Timeout, which is the +/// truthful "I could not do the work". The label is what lets an operator tell +/// this apart from a genuine internal error, and it survives into the +/// `FatalError` string because [`ModuleError`]'s `Display` is its JSON. +async fn assert_sidecar_failure_is_fatal(addr: std::net::SocketAddr, what: &str) -> ModuleError { + let config = sync::DArc::new(test_config(format!("http://{addr}"))); + + let err = render_err(&config, "https://example.com/").await; + + assert!( + err.fatal, + "{what} from our own webdriver sidecar must stay fatal so the validator \ + abstains rather than voting on a page it never observed, got {err:?}" + ); + assert!( + err.causes.contains(&"WEBDRIVER_UNAVAILABLE".to_owned()), + "{what} must be labelled as a webdriver fault, got causes {:?}", + err.causes + ); + + err +} + +// -- a failing sidecar is an environment fault, so we abstain ---------- + +/// Control. When the *page* fails, the sidecar still answers `200` and puts +/// the page status in `Resulting-Status`; `Render` turns that into a +/// non-fatal `WEBPAGE_LOAD_FAILED`, which the executor hands the contract as +/// a catchable nondeterministic exception. +#[tokio::test] +async fn render_remote_page_load_failure_is_not_fatal() { + common::tests::setup(); + + let addr = serve_sidecar_page_not_found().await; + let config = sync::DArc::new(test_config(format!("http://{addr}"))); + + let err = render_err(&config, "https://example.com/").await; + + assert!( + err.causes.contains(&"WEBPAGE_LOAD_FAILED".to_owned()), + "unexpected causes: {:?}", + err.causes + ); + assert!(!err.fatal, "page load failure must be non-fatal: {err:?}"); +} + +/// A 5xx from our *own* webdriver sidecar says nothing about the page -- it is +/// this node's environment failing, so there is no observation to report and +/// nothing legitimate to vote on. It must stay fatal, which the executor turns +/// into `ResultCode::InternalError` and the node into a Timeout vote. +/// +/// `Render` is the caller that knows which endpoint it is talking to, so it +/// wraps the sidecar hop in a `pcall` -- not to soften the failure, but to +/// label it and to pin fatality here rather than inherit whatever the generic +/// transport decided. +#[tokio::test] +async fn render_sidecar_internal_error_is_fatal() { + common::tests::setup(); + + let addr = serve_sidecar_internal_error().await; + let err = assert_sidecar_failure_is_fatal(addr, "a 5xx").await; + + // Not vacuous: the failure really is the status error raised on the + // sidecar hop, not the `WEBPAGE_LOAD_FAILED` branch further down, which + // this response never reaches. The transport's own cause is kept as the + // underlying detail rather than replaced + assert!( + err.causes.contains(&"STATUS_NOT_OK".to_owned()), + "unexpected causes: {:?}", + err.causes + ); +} + +/// The sidecar's other non-200 exits take the same branch, and are the same +/// kind of fault: a `400` means the module and the sidecar disagree about the +/// query format, which is a deployment problem, not a statement about the page +#[tokio::test] +async fn render_sidecar_bad_request_is_fatal() { + common::tests::setup(); + + let addr = serve_sidecar_bad_request().await; + let err = assert_sidecar_failure_is_fatal(addr, "a 400").await; + + assert!( + err.causes.contains(&"STATUS_NOT_OK".to_owned()), + "unexpected causes: {:?}", + err.causes + ); +} + +/// A `503`, as answered by the sidecar's healthcheck path or by whatever +/// proxies it while it is down. Half of a pair with +/// [`render_remote_page_unreachable_is_not_fatal`]: same number, opposite +/// verdict, because here it is the sidecar's own HTTP status +#[tokio::test] +async fn render_sidecar_unhealthy_is_fatal() { + common::tests::setup(); + + let addr = serve_sidecar_unhealthy().await; + let err = assert_sidecar_failure_is_fatal(addr, "a 503").await; + + assert!( + err.causes.contains(&"STATUS_NOT_OK".to_owned()), + "unexpected causes: {:?}", + err.causes + ); +} + +/// The host this validator runs on has no network at all. The sidecar throws +/// rather than reporting a status, because a request that never left the +/// machine is not an observation about the page, and the outer catch turns +/// that into the same bare `500`. +/// +/// It travels the webdriver hop, so it collects the same `WEBDRIVER_UNAVAILABLE` +/// label; that label names the hop, not the diagnosis. Which box actually broke +/// is carried by the sidecar's own message, pinned by +/// [`render_sidecar_local_network_fault_names_the_local_network_on_the_wire`] +#[tokio::test] +async fn render_sidecar_local_network_fault_is_fatal() { + common::tests::setup(); + + let addr = serve_sidecar_local_network_unavailable().await; + let err = assert_sidecar_failure_is_fatal(addr, "a local network fault").await; + + assert!( + err.causes.contains(&"STATUS_NOT_OK".to_owned()), + "unexpected causes: {:?}", + err.causes + ); +} + +/// The other half of that pair. A `503` in `Resulting-Status` on an otherwise +/// good `200` is the sidecar telling us the *page's* host refused the +/// connection -- something we did observe -- so it stays catchable. The two +/// tests differ only in which channel carries the 503, which is exactly the +/// distinction a fix is most likely to erase +#[tokio::test] +async fn render_remote_page_unreachable_is_not_fatal() { + common::tests::setup(); + + let addr = serve_sidecar_page_unreachable().await; + let config = sync::DArc::new(test_config(format!("http://{addr}"))); + + let err = render_err(&config, "https://example.com/").await; + + assert!( + err.causes.contains(&"WEBPAGE_LOAD_FAILED".to_owned()), + "unexpected causes: {:?}", + err.causes + ); + assert!( + !err.fatal, + "a 503 the sidecar *reports* is an observation about the page, and must \ + stay catchable: {err:?}" + ); + assert!( + !err.causes.contains(&"WEBDRIVER_UNAVAILABLE".to_owned()), + "a working sidecar must not be blamed: {:?}", + err.causes + ); +} + +/// A name that does not resolve stays on the observation channel, and this +/// test exists to keep it there. +/// +/// It is genuinely ambiguous -- the domain may not exist, or *our* resolver may +/// be broken -- and one validator cannot tell those apart. That is precisely +/// what several validators and an equivalence principle are for: deciding it +/// here, the way `net::ERR_INTERNET_DISCONNECTED` is decided, would suppress +/// the disagreement consensus exists to reconcile. So it stays catchable, the +/// contract sees it, and the vote records what this node saw +#[tokio::test] +async fn render_remote_page_name_not_resolved_is_not_fatal() { + common::tests::setup(); + + let addr = serve_sidecar_page_name_not_resolved().await; + let config = sync::DArc::new(test_config(format!("http://{addr}"))); + + let err = render_err(&config, "https://example.com/").await; + + assert!( + err.causes.contains(&"WEBPAGE_LOAD_FAILED".to_owned()), + "unexpected causes: {:?}", + err.causes + ); + assert!( + !err.fatal, + "a name that does not resolve is ambiguous between the site and us, and \ + that ambiguity is for the validators to settle, not for this node to \ + pre-judge by abstaining: {err:?}" + ); + assert!( + !err.causes.contains(&"WEBDRIVER_UNAVAILABLE".to_owned()), + "a working sidecar must not be blamed: {:?}", + err.causes + ); + // not vacuous: this really is the 502 branch, not some other failure + assert!( + matches!( + err.ctx.get("status"), + Some(genvm_modules_interfaces::GenericValue::Number(s)) if *s == 502.0 + ), + "expected the reported page status in ctx: {:?}", + err.ctx + ); +} + +/// The same guard for the other ambiguous case. A navigation timeout is either +/// a slow site or a browser of ours that wedged, and a validator cannot tell +/// which; it stays a contract-visible observation for the same reason +#[tokio::test] +async fn render_remote_page_navigation_timeout_is_not_fatal() { + common::tests::setup(); + + let addr = serve_sidecar_page_navigation_timeout().await; + let config = sync::DArc::new(test_config(format!("http://{addr}"))); + + let err = render_err(&config, "https://example.com/").await; + + assert!( + err.causes.contains(&"WEBPAGE_LOAD_FAILED".to_owned()), + "unexpected causes: {:?}", + err.causes + ); + assert!( + !err.fatal, + "a navigation timeout is ambiguous between the site and us, and stays \ + catchable so the validators can disagree about it: {err:?}" + ); + assert!( + !err.causes.contains(&"WEBDRIVER_UNAVAILABLE".to_owned()), + "a working sidecar must not be blamed: {:?}", + err.causes + ); + assert!( + matches!( + err.ctx.get("status"), + Some(genvm_modules_interfaces::GenericValue::Number(s)) if *s == 408.0 + ), + "expected the reported page status in ctx: {:?}", + err.ctx + ); +} + +/// The sidecar being *gone* is the same class of fault as it answering badly, +/// and it arrives on a different path: `map_send_error` raises a fatal +/// `SENDING_REQUEST` before any status exists. The `pcall` covers the whole +/// hop, so this is labelled too +#[tokio::test] +async fn render_sidecar_unreachable_is_fatal() { + common::tests::setup(); + + let addr = sidecar_down().await; + let err = assert_sidecar_failure_is_fatal(addr, "a refused connection").await; + + assert!( + err.causes.contains(&"SENDING_REQUEST".to_owned()), + "unexpected causes: {:?}", + err.causes + ); +} + +// -- the classification as the executor reads it ---------------------- + +/// The end the property is really about. `ModuleError.fatal` is only a flag +/// until [`crate::common::module_error_to_wire`] spends it: `FatalError` is +/// what the executor turns into a bare `anyhow` and finally +/// `ResultCode::InternalError`, which the contract cannot catch and which a +/// node reports as a Timeout vote. That is where a sidecar 5xx has to land -- +/// abstaining is the honest answer when we never ran the render. +/// +/// The label has to survive the trip, because `FatalError` carries only a +/// string: without it an operator cannot tell an outage of our own webdriver +/// from a genuine internal error in the contract's execution +#[tokio::test] +async fn render_sidecar_internal_error_reaches_the_wire_as_fatal_error() { + common::tests::setup(); + + let addr = serve_sidecar_internal_error().await; + let config = sync::DArc::new(test_config(format!("http://{addr}"))); + + let err = render_raw_err(&config, "https://example.com/").await; + let wire: genvm_modules_interfaces::Result<String> = + crate::common::module_error_to_wire(err, common::tests::get_hello().genvm_id); + + match wire { + genvm_modules_interfaces::Result::FatalError(msg) => assert!( + msg.contains("WEBDRIVER_UNAVAILABLE"), + "the abort must name the webdriver so it is not mistaken for a \ + contract-level internal error: {msg}" + ), + genvm_modules_interfaces::Result::UserError(v) => { + panic!("a sidecar 5xx must not become a result the contract can act on: {v:?}") + } + genvm_modules_interfaces::Result::Ok(_) => panic!("expected Render to fail"), + } +} + +/// Companion to the above, so it is not vacuous: the non-fatal path is still +/// reachable and still ends in `UserError`. The sidecar here is working, and +/// what failed is the page -- a real observation the contract may catch and act +/// on +#[tokio::test] +async fn render_remote_page_load_failure_reaches_the_wire_as_user_error() { + common::tests::setup(); + + let addr = serve_sidecar_page_not_found().await; + let config = sync::DArc::new(test_config(format!("http://{addr}"))); + + let err = render_raw_err(&config, "https://example.com/").await; + let wire: genvm_modules_interfaces::Result<String> = + crate::common::module_error_to_wire(err, common::tests::get_hello().genvm_id); + + match wire { + genvm_modules_interfaces::Result::UserError(_) => {} + genvm_modules_interfaces::Result::FatalError(msg) => { + panic!("a page that 404s must stay catchable, not abort the run: {msg}") + } + genvm_modules_interfaces::Result::Ok(_) => panic!("expected Render to fail"), + } +} + +/// The other thing the abort string has to carry. `WEBDRIVER_UNAVAILABLE` is +/// the name of the hop that failed, and for a machine with no network it is the +/// wrong diagnosis: the browser and the sidecar answered normally, so an +/// operator following that word alone goes and stares at a working browser. +/// +/// The sidecar's own message is what distinguishes them, and this pins that it +/// survives all the way to the string an operator reads. The body arrives from +/// the transport as bytes, but the `pcall` round trip through Lua turns it into +/// a string, so it lands in [`ModuleError`]'s JSON as readable text rather than +/// a byte array +#[tokio::test] +async fn render_sidecar_local_network_fault_names_the_local_network_on_the_wire() { + common::tests::setup(); + + let addr = serve_sidecar_local_network_unavailable().await; + let config = sync::DArc::new(test_config(format!("http://{addr}"))); + + let err = render_raw_err(&config, "https://example.com/").await; + let wire: genvm_modules_interfaces::Result<String> = + crate::common::module_error_to_wire(err, common::tests::get_hello().genvm_id); + + match wire { + genvm_modules_interfaces::Result::FatalError(msg) => { + assert!( + msg.contains("local network"), + "the abort must say which part of this node broke: {msg}" + ); + assert!( + msg.contains("WEBDRIVER_UNAVAILABLE"), + "the hop is still named, so the two live side by side: {msg}" + ); + } + genvm_modules_interfaces::Result::UserError(v) => { + panic!("a host with no network observed nothing, so the contract must not act on it: {v:?}") + } + genvm_modules_interfaces::Result::Ok(_) => panic!("expected Render to fail"), + } +} diff --git a/implementation/tests/request_status_fatality.rs b/implementation/tests/request_status_fatality.rs new file mode 100644 index 00000000..baf1e51e --- /dev/null +++ b/implementation/tests/request_status_fatality.rs @@ -0,0 +1,224 @@ +//! How the shared request helpers classify a non-200, for both response +//! shapes. +//! +//! `send_request_get_lua_compatible_response_bytes` and its JSON twin are +//! generic: they are given a URL and a flag, and cannot tell this node's own +//! webdriver sidecar from a site the contract named. Fatal is the safe default +//! for that ambiguity, and it is what both keep raising for a bad status. +//! +//! Callers that know which endpoint they are talking to adjust it, in one +//! direction only. `Request` in `install/config/genvm-web-default.lua` lowers +//! it, because its URL is contract-controlled and a failure there is a real +//! observation. `Render` keeps it fatal for the sidecar hop and only adds a +//! `WEBDRIVER_UNAVAILABLE` label, because a broken sidecar means the validator +//! observed nothing and must abstain rather than vote; see `web::tests` in the +//! library. The LLM providers (`src/llm/providers.rs`) all pass +//! `error_on_status = true` and rely on the fatal classification pinned here, +//! so a change to the default would move them too. + +//! genvm-tool-test-tags: feature-web-request-status, needs-web + +use genvm_common::*; + +use genvm_modules::common; +use genvm_modules::scripting::{self, Metrics}; +use tokio::io::AsyncWriteExt as _; + +/// A one-shot server answering `head` (status line plus headers, +/// `Content-Length` is appended) followed by `body` +async fn serve(head: &'static str, body: &'static [u8]) -> std::net::SocketAddr { + let server = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = server.local_addr().unwrap(); + + tokio::spawn(async move { + while let Ok((mut client, _)) = server.accept().await { + let response = format!("{head}Content-Length: {}\r\n\r\n", body.len()); + client.write_all(response.as_bytes()).await.unwrap(); + client.write_all(body).await.unwrap(); + client.shutdown().await.unwrap(); + } + }); + + addr +} + +async fn serve_server_error() -> std::net::SocketAddr { + serve( + "HTTP/1.1 500 Internal Server Error\r\nContent-Type: application/json\r\n", + br#"{"error":"Internal server error"}"#, + ) + .await +} + +fn metrics() -> sync::DArc<Metrics> { + sync::DArc::new(Metrics::default()) +} + +fn get(url: &str) -> reqwest::RequestBuilder { + common::tests::create_client().unwrap().get(url) +} + +/// The module error a call failed with. Panics if it succeeded, or if the +/// failure was not a `ModuleError` +fn module_err(err: anyhow::Error) -> common::ModuleError { + scripting::try_unwrap_any_err(err).expect("expected a module error") +} + +fn assert_status_not_ok(err: common::ModuleError) { + assert!( + err.causes.contains(&"STATUS_NOT_OK".to_owned()), + "unexpected causes: {:?}", + err.causes + ); + assert!( + err.fatal, + "the generic helper keeps a bad status fatal; lowering it is the \ + caller's decision, see the module docs: {err:?}" + ); + assert!( + matches!( + err.ctx.get("status"), + Some(genvm_modules_interfaces::GenericValue::Number(s)) if *s == 500.0 + ), + "the status must survive into ctx for the caller to act on: {:?}", + err.ctx + ); +} + +// -- error_on_status = true: raise, fatally --------------------------- + +#[tokio::test] +async fn bytes_bad_status_raises_a_fatal_status_not_ok() { + common::tests::setup(); + + let addr = serve_server_error().await; + let url = format!("http://{addr}/"); + + let err = scripting::send_request_get_lua_compatible_response_bytes( + &metrics(), + &url, + get(&url), + true, + usize::MAX, + ) + .await + .err() + .expect("a 500 with error_on_status must fail"); + + assert_status_not_ok(module_err(err)); +} + +/// The twin of the above. It is reached whenever the request sets `json`, which +/// every LLM provider does +#[tokio::test] +async fn json_bad_status_raises_a_fatal_status_not_ok() { + common::tests::setup(); + + let addr = serve_server_error().await; + let url = format!("http://{addr}/"); + + let err = scripting::send_request_get_lua_compatible_response_json( + &metrics(), + &url, + get(&url), + true, + usize::MAX, + ) + .await + .err() + .expect("a 500 with error_on_status must fail"); + + assert_status_not_ok(module_err(err)); +} + +// -- error_on_status = false: the status is just data ----------------- + +#[tokio::test] +async fn bytes_bad_status_is_returned_when_not_erroring_on_status() { + common::tests::setup(); + + let addr = serve_server_error().await; + let url = format!("http://{addr}/"); + + let res = scripting::send_request_get_lua_compatible_response_bytes( + &metrics(), + &url, + get(&url), + false, + usize::MAX, + ) + .await + .expect("without error_on_status a 500 is an ordinary response"); + + assert_eq!(res.status, 500); + assert_eq!(res.body, br#"{"error":"Internal server error"}"#.to_vec()); +} + +#[tokio::test] +async fn json_bad_status_is_returned_when_not_erroring_on_status() { + common::tests::setup(); + + let addr = serve_server_error().await; + let url = format!("http://{addr}/"); + + let res = scripting::send_request_get_lua_compatible_response_json( + &metrics(), + &url, + get(&url), + false, + usize::MAX, + ) + .await + .expect("without error_on_status a 500 is an ordinary response"); + + assert_eq!(res.status, 500); + assert_eq!( + res.body, + serde_json::json!({ "error": "Internal server error" }) + ); +} + +// -- the JSON twin's own failure mode --------------------------------- + +/// Only the JSON path can fail this way: a `200` whose body is not JSON. It is +/// fatal for the same reason, and pinned here because the twin had no coverage +/// at all +#[tokio::test] +async fn json_undecodable_body_is_a_fatal_deserializing_error() { + common::tests::setup(); + + let addr = serve( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n", + b"not json at all", + ) + .await; + let url = format!("http://{addr}/"); + + let err = scripting::send_request_get_lua_compatible_response_json( + &metrics(), + &url, + get(&url), + true, + usize::MAX, + ) + .await + .err() + .expect("an undecodable body must fail"); + + let err = module_err(err); + assert!( + err.causes.contains(&"DESERIALIZING".to_owned()), + "unexpected causes: {:?}", + err.causes + ); + assert!(err.fatal, "expected a fatal error: {err:?}"); + // the undecodable bytes are kept, so the failure is diagnosable + assert!( + matches!( + err.ctx.get("body"), + Some(genvm_modules_interfaces::GenericValue::Bytes(b)) if b == b"not json at all" + ), + "expected the raw body in ctx: {:?}", + err.ctx + ); +} diff --git a/install/config/genvm-web-default.lua b/install/config/genvm-web-default.lua index 47055ff0..9385c810 100644 --- a/install/config/genvm-web-default.lua +++ b/install/config/genvm-web-default.lua @@ -16,7 +16,19 @@ function Render(ctx, payload) .. "&waitAfterLoaded=" .. tostring(payload.wait_after_loaded or 0) - local result = lib.rs.request(ctx, { + -- This hop goes to *our own* webdriver sidecar, never to the contract's URL, + -- so the extent of this `pcall` is exactly the trust boundary: everything it + -- can catch is this node's environment failing, and nothing it catches is an + -- observation about the page. + -- + -- The `pcall` does NOT soften the failure -- it labels it. A sidecar we + -- cannot reach or that answers badly means we have no observation to report, + -- so the error stays fatal and this validator abstains (Timeout) rather than + -- voting on a result it never computed. Contrast `Request` below, whose URL + -- is contract-controlled: a failure there IS an observation, so it is + -- lowered to non-fatal. The page's own outcome likewise arrives as a `200` + -- plus a `resulting-status` header and is handled further down, non-fatally. + local success, result = pcall(lib.rs.request, ctx, { method = "GET", url = web.rs.config.webdriver_host .. "/render" .. url_params, headers = {}, @@ -25,6 +37,10 @@ function Render(ctx, payload) unfiltered = true, }) + if not success then + web.reraise_as_webdriver_unavailable(result) + end + lib.log { level = "debug", message = "web render result", diff --git a/install/lib/genvm-lua/lib-web.lua b/install/lib/genvm-lua/lib-web.lua index eeb554ee..63a707c4 100644 --- a/install/lib/genvm-lua/lib-web.lua +++ b/install/lib/genvm-lua/lib-web.lua @@ -121,4 +121,33 @@ M.check_url = function(url) return false end +--- Re-raise a failure of *our own* webdriver sidecar. +--- +--- Deliberately fatal. A broken sidecar means this validator has no valid +--- observation of the page at all, so it must not produce a contract-visible +--- result: doing that would let it assert a claim it never observed, and then +--- vote on it. Fatal becomes `ResultCode::InternalError`, which a node turns +--- into a Timeout vote -- the truthful answer, "I could not do the work". +--- A failure of the *remote site* is a real observation and stays non-fatal; +--- only this node's own infrastructure takes this path. +--- +--- The cause is prepended rather than replacing what the transport reported, +--- so `STATUS_NOT_OK` / `SENDING_REQUEST` / `READING_BODY` survive as the +--- underlying detail. It is not contract-visible: a fatal `ModuleError` is +--- stringified into `FatalError` and never reaches the runner as `causes`, so +--- this word is for operators and for the node's logs. +---@param e any the caught error value +M.reraise_as_webdriver_unavailable = function(e) + local err = lib.rs.as_user_error(e) + if err == nil then + -- Not a module error, so it carries no fatality flag; unclassified is + -- already treated as fatal downstream. Re-raised untouched. + error(e) + end + + table.insert(err.causes, 1, "WEBDRIVER_UNAVAILABLE") + err.fatal = true + lib.rs.user_error(err) +end + return M diff --git a/support/ci/__main__.py b/support/ci/__main__.py index 27cbcc78..994c1456 100644 --- a/support/ci/__main__.py +++ b/support/ci/__main__.py @@ -8,12 +8,13 @@ from tools import ( branches, deploy_docs, - genvm_merge_into_dev, + full_tests_command, make_release_notes, open_executor_prs, pr_action_panel, pr_branches, rebase_watch, + sync_executor_branches, versions, ) @@ -28,12 +29,13 @@ *versions.COMMANDS, *branches.COMMANDS, *deploy_docs.COMMANDS, + *full_tests_command.COMMANDS, *pr_branches.COMMANDS, *make_release_notes.COMMANDS, *pr_action_panel.COMMANDS, - *genvm_merge_into_dev.COMMANDS, *open_executor_prs.COMMANDS, *rebase_watch.COMMANDS, + *sync_executor_branches.COMMANDS, ] diff --git a/support/ci/behind.py b/support/ci/behind.py index 368e6727..4615350c 100644 --- a/support/ci/behind.py +++ b/support/ci/behind.py @@ -2,17 +2,16 @@ """ One definition of "how far behind its base is this branch". -The repo merges by fast-forward only, so "0 commits behind base" is a hard -precondition in three separate places: +Repository policy requires a PR head to contain its base tip before it enters +the App-owned merge train, so "0 commits behind base" appears in 2 places: -- `incl_initial.yaml`'s always-on gate, which fails a PR the moment it falls - behind (`check-behind`, below); +- `incl_initial.yaml`'s always-on gate, backed by `pipelines/checks.py`, which + fails a PR the moment it falls behind; - `tools/rebase_watch.py`, the advisory check-run/label sweep that re-evaluates when the BASE moves; -- `tools/genvm_merge_into_dev.py`, the authoritative check at merge time. -Each used to compute it its own way β€” two flavours of `git` plus the compare -API β€” so the three could disagree about the same PR. The measurement lives here +The callers used to compute it separately, so they could disagree about the +same PR. The measurement lives here once, in the two flavours the callers genuinely need: - `behind_by_api` for callers with no checkout (a `pull_request_target` sweep); @@ -20,7 +19,7 @@ Both answer the same question with the same meaning: the number of commits reachable from `base` that `head` does not contain. Zero means the head -contains the base tip, i.e. it can be fast-forwarded. +contains the base tip. """ import json diff --git a/support/ci/pipelines/checks.py b/support/ci/pipelines/checks.py index 1eec7fea..61416cf4 100644 --- a/support/ci/pipelines/checks.py +++ b/support/ci/pipelines/checks.py @@ -164,7 +164,7 @@ def pr_field(pr: str, field: str) -> str: The PR-scoped checks must not depend on `github.event.pull_request.*`: that payload is absent on the panel's `workflow_dispatch`, which is how they came - to skip on exactly the runs the Merge gate accepts. A PR number is available + to skip on exactly the runs the App accepts. A PR number is available on every triggering event, so everything else is resolved from it here. """ return gh_common.gh_manager( @@ -224,7 +224,7 @@ class PrTitle(ci_lib.Pipeline): """ Fail unless the PR title is a valid conventional-commit subject. - The Merge action squashes a PR into `<title> (#N)`, so the title becomes a + The E2E App squashes a PR into `<title> (#N)`, so the title becomes a commit subject verbatim β€” and nothing re-validates a generated squash message against the commit-message hook. A title that would not pass as a commit subject therefore lands as a commit the hook rejects. Checking it on @@ -252,7 +252,7 @@ def handler(self, args: argparse.Namespace) -> int: return 1 # Validate the subject that will actually LAND, not the bare title: the - # merge appends ` (#N)` (genvm_merge_into_dev.pr_reference), and the + # GitHub squash appends ` (#N)`, and the # checker measures the raw subject length. Checking the title alone would # pass a title just under the limit and still land an over-long subject. subject = f'{title} (#{pr})' @@ -266,8 +266,8 @@ def handler(self, args: argparse.Namespace) -> int: ) if result.returncode != 0: ci_lib.github_error( - f'PR title is not a valid commit subject: {subject!r}. The Merge action ' - f'lands exactly this as the squashed commit subject, so it must satisfy ' + f'PR title is not a valid commit subject: {subject!r}. The E2E App ' + f'lands this as the squashed commit subject, so it must satisfy ' f'the same rules.' ) return 1 diff --git a/support/ci/pipelines/tests.py b/support/ci/pipelines/tests.py index 7219e213..3193f4a0 100644 --- a/support/ci/pipelines/tests.py +++ b/support/ci/pipelines/tests.py @@ -318,8 +318,8 @@ class PlanQueueMatrix(ci_lib.Pipeline): Emit queue.yaml's test matrix as a GitHub `matrix` output. The heavy cells run only when a marker is set (RUN_FULL_TESTS, from the - run-full-tests / rtm labels or a manual dispatch); otherwise the matrix is - empty and matrix-tests is skipped. + run-full-tests label or a manual dispatch); otherwise the matrix is empty and + matrix-tests is skipped. """ def name(self) -> str: diff --git a/support/ci/pr_branches_info.py b/support/ci/pr_branches_info.py index e7873d23..35d43ff1 100644 --- a/support/ci/pr_branches_info.py +++ b/support/ci/pr_branches_info.py @@ -71,6 +71,7 @@ class RepoInfo: ahead_by: int | None behind_by: int | None pr_url: str | None # open PR head_ref -> base_ref, or None if none is open + landable: bool # pinned head exists remotely and is comparable when required @property def moved(self) -> bool: @@ -97,12 +98,9 @@ def rebased(self) -> bool: @property def synced(self) -> bool: """ - The head gitlink is present in the executor repo (comparable to base), so - it can land together with the manager commit. Being ahead of base is normal β€” - manager and executors land almost-atomically β€” so only a head commit missing - from the executor repo (ahead_by is None) is unsynced. + The head gitlink can be projected onto the executor repository. """ - return self.ahead_by is not None + return self.landable def as_dict(self) -> dict: """ @@ -172,6 +170,13 @@ def submodule_sha( return data['sha'] +def commit_exists(repo: str, sha: str, token: str | None) -> bool: + """Whether `sha` is present in `repo`.""" + return ( + _api(f'repos/{repo}/git/commits/{sha}', token=token, allow_missing=True) is not None + ) + + def existing_pr(repo: str, head: str, base: str, token: str | None) -> str | None: """ URL of the open PR `head -> base` in `repo`, or None if none is open. @@ -223,6 +228,7 @@ def manager_info( ahead_by=ahead_by, behind_by=behind_by, pr_url=pr['html_url'], + landable=True, ) @@ -252,6 +258,7 @@ def executor_info( if base_sha is not None and head_sha is not None: if base_sha == head_sha: ahead_by, behind_by = 0, 0 # gitlink unchanged; no need to ask the executor + landable = True else: # The gitlink moved. Ask the executor repo how far. allow_missing: the # head gitlink may not be pushed to the executor yet (a PR that bumped the @@ -260,9 +267,16 @@ def executor_info( executor_repo, base_sha, head_sha, executor_token, allow_missing=True ) ahead_by, behind_by = (None, None) if cmp is None else (cmp[1], cmp[2]) + landable = cmp is not None + elif head_sha is not None: + # A newly added line has no old gitlink to compare against, but its pinned + # commit still has to be available for the post-merge branch creation. + ahead_by, behind_by = None, None + landable = commit_exists(executor_repo, head_sha, executor_token) else: - # Line added (no base) or removed (no head): moved, but nothing to compare. + # Line removed: moved, but there is no head to project. ahead_by, behind_by = None, None + landable = False return RepoInfo( path=path, @@ -275,6 +289,7 @@ def executor_info( ahead_by=ahead_by, behind_by=behind_by, pr_url=existing_pr(executor_repo, head_ref, base_ref, executor_token), + landable=landable, ) diff --git a/support/ci/tools/branches.py b/support/ci/tools/branches.py index c7d88286..0cb9328c 100644 --- a/support/ci/tools/branches.py +++ b/support/ci/tools/branches.py @@ -3,7 +3,7 @@ import ci_lib import gh_common -from tools.versions import active_versions +from tools.versions import manager_version def gh(*args: str, check: bool = True) -> str: @@ -27,8 +27,7 @@ def name(self) -> str: return 'provision-branches' def add_to(self, parser: argparse.ArgumentParser) -> None: - # Only the manager repo is relevant here β€” it iterates executor lines from - # .genvm-monorepo-root, not a single PR/head. + # Only the manager repo is relevant here. gh_common.add_args(parser, executor_repo=False, pr=False, head_ref=False) def remote_has(self, branch: str) -> bool: @@ -44,40 +43,40 @@ def handler(self, args: argparse.Namespace) -> int: repo = gh_common.Ctx.from_args(args).manager_repo ci_lib.run(['git', 'fetch', 'origin', '--prune']) + version = manager_version() + version_branch = f'v{version}' + dev_branch = f'v{version}-dev' pushrefs: list[str] = [] - for version in active_versions(): - version_branch = f'v{version}.x' - dev_branch = f'v{version}-dev' - - if not self.remote_has(version_branch): - print(f'Will create version branch {version_branch} from main') - version_commit = ci_lib.output(['git', 'rev-parse', 'origin/main']).strip() - pushrefs.append(f'origin/main:refs/heads/{version_branch}') - else: - print(f'Version branch {version_branch} already exists') - version_commit = ci_lib.output( - ['git', 'rev-parse', f'origin/{version_branch}'] - ).strip() - - if not self.remote_has(dev_branch): - print( - f'Will create dev branch {dev_branch} from {version_branch} (with seed commit)' - ) - seed = ci_lib.output( - [ - 'git', - 'commit-tree', - f'{version_commit}^{{tree}}', - '-p', - version_commit, - '-m', - f'chore(ci): seed {dev_branch} release-gate branch 🌱', - ], - cwd=ci_lib.ROOT_DIR, - ).strip() - pushrefs.append(f'{seed}:refs/heads/{dev_branch}') - else: - print(f'Dev branch {dev_branch} already exists') + + if not self.remote_has(version_branch): + print(f'Will create version branch {version_branch} from main') + version_commit = ci_lib.output(['git', 'rev-parse', 'origin/main']).strip() + pushrefs.append(f'origin/main:refs/heads/{version_branch}') + else: + print(f'Version branch {version_branch} already exists') + version_commit = ci_lib.output( + ['git', 'rev-parse', f'origin/{version_branch}'] + ).strip() + + if not self.remote_has(dev_branch): + print( + f'Will create dev branch {dev_branch} from {version_branch} (with seed commit)' + ) + seed = ci_lib.output( + [ + 'git', + 'commit-tree', + f'{version_commit}^{{tree}}', + '-p', + version_commit, + '-m', + f'chore(ci): seed {dev_branch} release-gate branch 🌱', + ], + cwd=ci_lib.ROOT_DIR, + ).strip() + pushrefs.append(f'{seed}:refs/heads/{dev_branch}') + else: + print(f'Dev branch {dev_branch} already exists') if pushrefs: print(f'Creating {len(pushrefs)} branch(es) in one atomic push') @@ -86,46 +85,42 @@ def handler(self, args: argparse.Namespace) -> int: else: print('All branches already exist; nothing to push') - for version in active_versions(): - version_branch = f'v{version}.x' - dev_branch = f'v{version}-dev' - existing = gh( - 'pr', - 'list', - '--repo', - repo, - '--base', - version_branch, - '--head', - dev_branch, - '--state', - 'open', - '--json', - 'number', - '--jq', - '.[0].number // ""', - ).strip() - if existing: - print( - f'Standing PR {dev_branch} -> {version_branch} already open (#{existing})' - ) - continue - - print(f'Opening standing release-gate PR {dev_branch} -> {version_branch}') - gh( - 'pr', - 'create', - '--repo', - repo, - '--base', - version_branch, - '--head', - dev_branch, - '--title', - f'Release gate: {dev_branch} -> {version_branch}', - '--body', - f'Standing release-gate PR. `{dev_branch}` accumulates incremental v{version} work; this PR merges into the release-ready `{version_branch}` branch only once the cross-repo E2E pipeline is green. Comment `/run-e2e v{version}` to fire it (genlayerlabs/genlayer-e2e). It may stay red while the v{version} train is in progress.', - ) + existing = gh( + 'pr', + 'list', + '--repo', + repo, + '--base', + version_branch, + '--head', + dev_branch, + '--state', + 'open', + '--json', + 'number', + '--jq', + '.[0].number // ""', + ).strip() + if existing: + print(f'Standing PR {dev_branch} -> {version_branch} already open (#{existing})') + return 0 + + print(f'Opening standing release-gate PR {dev_branch} -> {version_branch}') + gh( + 'pr', + 'create', + '--repo', + repo, + '--base', + version_branch, + '--head', + dev_branch, + '--title', + # The PR-title check validates this before the E2E App squash-merges it. + f'chore(release): gate {dev_branch} into {version_branch} πŸš€', + '--body', + f'Standing release-gate PR. `{dev_branch}` accumulates incremental v{version} work; this PR merges into the release-ready `{version_branch}` branch only once the cross-repo E2E pipeline is green. Comment `/run-e2e v{version}` to fire it (genlayerlabs/genlayer-e2e). It may stay red while the v{version} train is in progress.', + ) return 0 diff --git a/support/ci/tools/full_tests_command.py b/support/ci/tools/full_tests_command.py new file mode 100644 index 00000000..d0ff7693 --- /dev/null +++ b/support/ci/tools/full_tests_command.py @@ -0,0 +1,300 @@ +#!/usr/bin/env python3 +import argparse +import json +import os +import re +import subprocess +import time + +import ci_lib +import gh_common +from gh_common import pr_number, repo + +COMMAND = '/genvm-run-tests' +CI_SAFE_LABEL = 'ci-safe' +RELEASE_TEST_LABEL = 'test-release-pipeline' +REQUEST_RE = re.compile(r'\[request=(\d+-\d+);comment=(\d+);pr=(\d+)\]') +STATUS_RE = re.compile(r'<!-- genvm-full-tests run=(\d+) request=(\d+) eyes=(\d+) -->') +POLL_ATTEMPTS = 30 +POLL_DELAY_SECONDS = 1 + + +def gh( + *args: str, + check: bool = True, + retry: bool = True, +) -> subprocess.CompletedProcess: + return gh_common.gh( + *args, + token=gh_common.manager_token(), + check=check, + retry=retry, + ) + + +def labels() -> set[str]: + result = gh( + 'api', + '--paginate', + f'repos/{repo()}/issues/{pr_number()}/labels?per_page=100', + '--jq', + '.[].name', + ) + return set(result.stdout.splitlines()) + + +def add_reaction(comment: str, content: str) -> str: + result = gh( + 'api', + '--method', + 'POST', + f'repos/{repo()}/issues/comments/{comment}/reactions', + '-f', + f'content={content}', + retry=False, + ) + return str(json.loads(result.stdout)['id']) + + +def best_effort_add_reaction(comment: str, content: str) -> str | None: + try: + return add_reaction(comment, content) + except Exception as error: + print(f'::warning::could not add {content} reaction: {error}') + return None + + +def delete_reaction(comment: str, reaction: str) -> None: + gh( + 'api', + '--method', + 'DELETE', + f'repos/{repo()}/issues/comments/{comment}/reactions/{reaction}', + check=False, + retry=False, + ) + + +def pull_request() -> dict: + return json.loads(gh('api', f'repos/{repo()}/pulls/{pr_number()}').stdout) + + +def dispatch( + head: str, + *, + expected_sha: str, + release_pipeline_test: bool, + request: str, + request_id: str, +) -> None: + gh( + 'workflow', + 'run', + 'queue.yaml', + '--repo', + repo(), + '--ref', + head, + '-f', + f'pr={pr_number()}', + '-f', + f'release_pipeline_test={str(release_pipeline_test).lower()}', + '-f', + f'request_comment={request}', + '-f', + f'request_id={request_id}', + '-f', + f'expected_sha={expected_sha}', + retry=False, + ) + + +def find_run(request_id: str) -> dict | None: + result = gh( + 'run', + 'list', + '--repo', + repo(), + '--workflow', + 'queue.yaml', + '--event', + 'workflow_dispatch', + '--limit', + '50', + '--json', + 'databaseId,displayTitle,headSha,url', + ) + marker = f'[request={request_id};' + for run in json.loads(result.stdout): + if marker in run['displayTitle']: + return run + return None + + +def wait_for_run(request_id: str) -> dict: + for attempt in range(POLL_ATTEMPTS): + if run := find_run(request_id): + return run + if attempt + 1 < POLL_ATTEMPTS: + time.sleep(POLL_DELAY_SECONDS) + raise RuntimeError('dispatched full tests but could not find the workflow run') + + +def post_running_status(run: dict, request: str, reaction: str, sha: str) -> None: + body = ( + f'<!-- genvm-full-tests run={run["databaseId"]} request={request} ' + f'eyes={reaction} -->\n' + f'πŸ‘€ Full tests are running for `{sha[:12]}`: ' + f'[open run #{run["databaseId"]}]({run["url"]})' + ) + gh( + 'api', + '--method', + 'POST', + f'repos/{repo()}/issues/{pr_number()}/comments', + '-f', + f'body={body}', + retry=False, + ) + + +def start() -> int: + if os.environ.get('COMMENT_BODY', '').strip() != COMMAND: + print('comment is not an exact /genvm-run-tests command; ignoring') + return 0 + if not gh_common.has_write_access(os.environ['SENDER']): + raise SystemExit(f'`{os.environ["SENDER"]}` has no write access to {repo()}') + current_labels = labels() + if CI_SAFE_LABEL not in current_labels: + raise SystemExit(f'PR #{pr_number()} lacks the `{CI_SAFE_LABEL}` label') + + pr = pull_request() + if pr['state'] != 'open': + raise SystemExit(f'PR #{pr_number()} is not open') + head_repo = pr['head'].get('repo') + if head_repo is None or head_repo['full_name'] != repo(): + raise SystemExit('full tests cannot be dispatched for a fork PR') + + request = os.environ['COMMENT_ID'] + request_id = os.environ['REQUEST_ID'] + reaction = best_effort_add_reaction(request, 'eyes') or '0' + try: + dispatch( + pr['head']['ref'], + expected_sha=pr['head']['sha'], + release_pipeline_test=RELEASE_TEST_LABEL in current_labels, + request=request, + request_id=request_id, + ) + except Exception: + if reaction != '0': + delete_reaction(request, reaction) + best_effort_add_reaction(request, 'confused') + raise + try: + run = wait_for_run(request_id) + post_running_status(run, request, reaction, run['headSha']) + except Exception as error: + print(f'::warning::full tests were dispatched but status reporting failed: {error}') + if reaction != '0': + delete_reaction(request, reaction) + best_effort_add_reaction(request, 'confused') + return 0 + + +def request_from_run_name(name: str) -> tuple[str, str, str] | None: + match = REQUEST_RE.search(name) + return (match.group(1), match.group(2), match.group(3)) if match else None + + +def status_comment(issue: str, run: str, request: str) -> dict | None: + result = gh( + 'api', + '--paginate', + f'repos/{repo()}/issues/{issue}/comments?per_page=100', + '--jq', + '.[] | @json', + ) + for line in result.stdout.splitlines(): + comment = json.loads(line) + match = STATUS_RE.search(comment['body']) + if ( + match + and match.group(1) == run + and match.group(2) == request + and comment.get('user', {}).get('login') == 'github-actions[bot]' + ): + return comment + return None + + +def wait_for_status_comment(issue: str, run: str, request: str) -> dict: + for attempt in range(POLL_ATTEMPTS): + if comment := status_comment(issue, run, request): + return comment + if attempt + 1 < POLL_ATTEMPTS: + time.sleep(POLL_DELAY_SECONDS) + raise RuntimeError(f'could not find the status comment for full-test run {run}') + + +def complete() -> int: + request_info = request_from_run_name(os.environ.get('RUN_NAME', '')) + if request_info is None: + print('run was not started by /genvm-run-tests; nothing to report') + return 0 + _request_id, title_request, issue = request_info + run = os.environ['RUN_ID'] + conclusion = os.environ.get('RUN_CONCLUSION', '') or 'incomplete' + passed = conclusion == 'success' + reaction = 'rocket' if passed else 'confused' + try: + comment = wait_for_status_comment(issue, run, title_request) + match = STATUS_RE.search(comment['body']) + assert match is not None + request = match.group(2) + status = ( + 'πŸš€ Full tests passed' + if passed + else f'⚠️ Full tests {conclusion.replace("_", " ")}' + ) + body = ( + f'{match.group(0)}\n{status} for `{os.environ["RUN_SHA"][:12]}`: ' + f'[open run #{run}]({os.environ["RUN_URL"]})' + ) + except Exception as error: + print(f'::warning::could not authenticate full-test status comment: {error}') + return 0 + try: + gh( + 'api', + '--method', + 'PATCH', + f'repos/{repo()}/issues/comments/{comment["id"]}', + '-f', + f'body={body}', + retry=False, + ) + except Exception as error: + print(f'::warning::could not update full-test status comment: {error}') + if match.group(3) != '0': + delete_reaction(request, match.group(3)) + best_effort_add_reaction(request, reaction) + return 0 + + +class FullTestsCommand(ci_lib.Tool): + """Start or report a `/genvm-run-tests` request.""" + + def name(self) -> str: + return 'full-tests-command' + + def add_to(self, parser: argparse.ArgumentParser) -> None: + gh_common.add_args(parser, executor_repo=False, head_ref=False) + parser.add_argument('operation', choices=('start', 'complete')) + + def handler(self, args: argparse.Namespace) -> int: + gh_common.set_ctx(gh_common.Ctx.from_args(args)) + return start() if args.operation == 'start' else complete() + + +COMMANDS = [FullTestsCommand()] diff --git a/support/ci/tools/genvm_merge_into_dev.py b/support/ci/tools/genvm_merge_into_dev.py deleted file mode 100755 index 01cdc24a..00000000 --- a/support/ci/tools/genvm_merge_into_dev.py +++ /dev/null @@ -1,1175 +0,0 @@ -#!/usr/bin/env python3 -""" -Gate and perform a Merge of a PR into a v<X>-dev branch. - -Invoked by the reusable .github/workflows/branch_merge_into_dev.yaml when a -maintainer ticks the "Merge" box on the PR action panel. It re-checks every -gate against the EXACT PR head commit and then advances the dev branch by a -plain (fast-forward-only) push, so what lands is byte-identical to what CI -and E2E validated. - -Gates (all required, all on the head commit): -1. base branch is a v<X>-dev branch -2. a maintainer (push access) approved the head commit β€” pushing revokes it β€” or - the PR carries the `rtm` (ready-to-merge) label -3. full GenVM CI (queue.yaml) concluded success -4. the cross-repo E2E check (exact name, from the exact app) concluded success -5. the PR is 0 commits behind base - -Forced merge: with `$FORCE_ACTOR` set, gates 2-4 are skipped and the skip is -announced on the PR under that login. That the login is a repo admin, and that -the head is still the commit they authorized, is settled in the workflow's own -YAML before this script is fetched β€” this script runs from the PR head, so it -can neither authorize nor refuse anything on its own. Gates 1, 1b and 5 stay -enforced: they keep the land itself well-formed, which no authority makes safe -to skip. Locally, `--force` does the same and takes the actor from the logged-in -`gh` user; it is refused under GITHUB_ACTIONS, where that user is the bot. - -Strategy: every repo ends up with ONE commit per merged PR. The manager's -commits are squashed onto base (a single commit that needs no gitlink rewrite is -fast-forwarded as-is, preserving its SHA); each executor line's commits are -squashed onto its own dev branch. Every linked PR is auto-closed as merged -afterwards (see below). - -Executor mirror: the manager release line (e.g. v0.6) is independent of the -executor lines it ships (v0.2, v0.3), so a single manager commit carries a -gitlink into EACH active line's submodule. Every active line is squashed onto -its own `<line>-dev` branch (`v0.2-dev`, `v0.3-dev`, ...). Squashing gives the -line a NEW commit, so the sha the PR pinned never lands β€” the manager commit is -rewritten to gitlink the squashed commit instead (`update-index --cacheinfo`). -Lines whose gitlink is unchanged, or that already sit at a single commit on -their dev tip, are left alone. - -True cross-repo atomicity is impossible, so the landing is staged to keep the -window in which the repos disagree as small as it can be: - -1. plan and fast-forward-check every side up front; -2. push all the objects to both remotes under a scratch ref, so that no later -push has anything left to transfer; -3. move the PR head branches (manager, then each executor mirror) onto the -commits about to land, and pause β€” GitHub needs to see that before the base -moves to record the result as merged rather than closed; -4. move the target branches: the manager first, since it is the only push that -can lose a race, then each `<line>-dev`. - -A rejected manager push therefore lands nothing, and the branch moves from -step 3 are rolled back so the PR can be re-ticked. Only step 4's executor -pushes can leave the repos split, and they are reported with the exact ref -update to replay. - -A squashed message is the PR title plus one `* <subject>` bullet per collapsed -commit, each followed by that commit's own body; the PR description never lands. -A collapsed commit that was itself a squash contributes its bullets to the same -flat list, so squashing a squash lands what squashing the originals would have. -Squashing also collapses several authors -into one, so `Co-authored-by:` trailers are added for every other human author in -the range (never tool/AI attribution β€” the commit-message hook rejects that). - -Every linked PR is auto-closed as MERGED, without deleting any branch: each -executor line's mirror branch (`pr/<line>/<feature>`, opened by -branch_executor_prs.yaml) and the manager PR's own head branch are force-moved onto -the commit that is about to land on their base (step 3 above), so once the base -moves head == base tip and GitHub marks the PR merged. (Deleting a branch, or -`gh pr close`, closes the PR as merely 'closed' and drops the merged link.) - -(The dev -> version release-gate merge is a separate, not-yet-automated -flow; on that one the executor's version branch is fast-forwarded and its -dev/version branches are kept at the same point.) - -Talks to GitHub through the `gh` CLI and moves refs through `git`; it does -not exec arbitrary PR code, though the workflow checks out this script itself -from the PR head (gated on the `ci-safe` label). The dev branches are -protected, so the workflow checks out with the GENVM_CI_PRIVATE_KEY deploy key -and pushes them non-force -(a base that advanced is safely rejected); the unprotected head/mirror branches -are instead force-moved onto the landed commit to auto-merge their PRs. The active executor submodules -must be checked out with a remote `origin` that can push the executor repo -(the workflow wires its own deploy key); the active lines come from -.genvm-monorepo-root via tools.versions. - -Repo and PR number resolve arg > env > default via gh_common; the token is -optional (ambient `gh` auth when unset). Env also: E2E_CHECK_NAME, -E2E_CHECK_APP_SLUG. -""" - -import argparse -import json -import os -import re -import subprocess -import sys -import time - -import behind as behind_lib -import ci_lib -import gh_common -from gh_common import pr_number, repo - -from tools.versions import active_lines - -RTM_LABEL = 'rtm' - -# Scratch ref that carries a merge's objects to both remotes before any branch -# moves; outside refs/heads so nothing watches it and no PR is opened for it. -LANDING_REF_PREFIX = 'refs/genvm-ci/landing' - -# Gap between moving the PR head branches and moving the branches they target. -# GitHub decides "merged" vs "closed" from the state it has indexed, and the two -# updates arriving together is enough to make it read a merge as a bare close. -SETTLE_SECONDS = 5 - - -def run(*args, check=True, env=None): - full_env = None - if env is not None: - full_env = os.environ.copy() - full_env.update(env) - return subprocess.run(args, check=check, text=True, capture_output=True, env=full_env) - - -def gh(*args): - """ - GitHub API/PR *reads* via the shared retrying wrapper, so a transient - truncated-JSON / 5xx blip on a gate query doesn't fail the whole merge (the - exact fragility that wrapper was hardened against). - - Writes stay on plain `run('gh', ...)` so a retry can't double-post a comment. - Token is optional (ambient `gh` auth when unset). - """ - return gh_common.gh_manager(*args) - - -def git(*args, check=True): - return run('git', *args, check=check) - - -def egit(submodule, *args, check=True, env=None): - """ - git inside an executor submodule checkout. - """ - return run('git', '-C', submodule, *args, check=check, env=env) - - -def is_ancestor(repo, maybe_ancestor, descendant): - args = ('merge-base', '--is-ancestor', maybe_ancestor, descendant) - if repo is None: - return git(*args, check=False).returncode == 0 - return egit(repo, *args, check=False).returncode == 0 - - -# Identity of the cross-repo E2E check-run on the head commit. A check-run name is -# not a protected namespace: any app installed on the repo β€” and any workflow in a -# PR β€” can post one under a name of its choosing. So the gate matches the exact -# name AND the app that posted it; matching a name alone (let alone a substring of -# it) would let an unrelated green check stand in for E2E. -E2E_CHECK_NAME = 'E2E Tests' -E2E_CHECK_APP_SLUG = 'ci-core-e2e-runner' - - -def e2e_identity(): - return ( - os.environ.get('E2E_CHECK_NAME') or E2E_CHECK_NAME, - os.environ.get('E2E_CHECK_APP_SLUG') or E2E_CHECK_APP_SLUG, - ) - - -# Identities the commit-message hook rejects as AI/tool attribution -# (support/scripts/check-commit-message.py AI_PATTERNS). A squash must not -# resurrect one as a Co-authored-by trailer β€” the hook would then reject the -# message it produced, and nothing re-validates a generated squash message. -AI_IDENTITY_RE = re.compile( - r'claude|gpt|copilot|anthropic|openai|\bai\b|noreply@anthropic', re.IGNORECASE -) - - -def commit_author(repo, sha): - """ - (name, email, author-date) of `sha`. NUL-separated so a name containing - `<` or spaces round-trips. - """ - out = ( - egit(repo, 'log', '-1', '--format=%an%x00%ae%x00%aI', sha) - if repo is not None - else git('log', '-1', '--format=%an%x00%ae%x00%aI', sha) - ).stdout.rstrip('\n') - name, email, date = out.split('\x00') - return name, email, date - - -def range_authors(repo, rev_range): - """ - Distinct (name, email) commit authors in `rev_range`, first-seen order. - """ - args = ('log', '--format=%an%x00%ae', rev_range) - out = (git(*args) if repo is None else egit(repo, *args)).stdout - - authors = [] - for line in out.split('\n'): - if '\x00' not in line: - continue - name, _, email = line.partition('\x00') - if (name, email) not in authors: - authors.append((name, email)) - return authors - - -def coauthor_trailers(repo, rev_range, exclude): - """ - `Co-authored-by:` for every human author in the range except `exclude`. - - A squash keeps only one author, so everyone else who wrote a commit in the - range would otherwise lose attribution. Authors whose identity the - commit-message hook rejects as AI/tool attribution are dropped β€” otherwise the - generated message would fail that hook, and nothing re-checks it. - """ - trailers = [] - for name, email in range_authors(repo, rev_range): - if (name, email) == exclude: - continue - ident = f'{name} <{email}>' - if AI_IDENTITY_RE.search(ident): - print(f'dropping AI/tool co-author {ident}') - continue - trailers.append(f'Co-authored-by: {ident}') - return trailers - - -def block(msg): - """ - Comment the reason on the PR and fail the job. - """ - run( - 'gh', - 'pr', - 'comment', - pr_number(), - '--repo', - repo(), - '--body', - f'❌ Merge blocked: {msg}', - check=False, - ) - sys.exit(1) - - -def pr_view(*fields): - out = gh('pr', 'view', pr_number(), '--repo', repo(), '--json', ','.join(fields)) - return json.loads(out) - - -def can_maintain(login): - """ - Whether `login` has push rights on the repo, i.e. may approve a merge. - - `author_association` on the review would be cheaper but says the wrong thing: - MEMBER only means "in the org", which is not push access to this repo. - """ - try: - out = gh('api', f'repos/{repo()}/collaborators/{login}/permission') - except subprocess.CalledProcessError: - # 403/404 β€” not a collaborator at all. - return False - return json.loads(out).get('permission') in ('admin', 'maintain', 'write') - - -def check_approval(head_sha, labels): - """ - Require a maintainer's approving review of the exact commit being merged, or - the `rtm` (ready-to-merge) label as the equivalent manual authorization. - - An approval is tied to the commit it was left on, so pushing new commits - silently revokes it here without needing branch protection to dismiss - anything. The label is the weaker of the two β€” it is held by whoever can edit - labels and survives every subsequent push β€” so it authorizes only what its - holder is willing to vouch for. It stands in for a missing approval, not for - an unwithdrawn objection. - - Only the newest decision per reviewer counts, so a reviewer who approves and then - requests changes on the same commit does not still authorize it. Reviews are - returned oldest-first, and COMMENTED/PENDING ones carry no decision. An - outstanding CHANGES_REQUESTED blocks regardless of which commit it was left on. - - Paginated, because it is the LAST page that decides: on a PR past 100 reviews an - un-paginated read would keep an early APPROVED and never see the later - CHANGES_REQUESTED that overrides it. `--jq '.[]'` is what makes that safe β€” plain - `--paginate` concatenates one JSON array per page, which is not valid JSON. - """ - reviews = [ - json.loads(line) - for line in gh( - 'api', - '--paginate', - f'repos/{repo()}/pulls/{pr_number()}/reviews?per_page=100', - '--jq', - '.[]', - ).splitlines() - if line.strip() - ] - - decision = {} - latest = {} - for review in reviews: - if review['state'] not in ('APPROVED', 'CHANGES_REQUESTED', 'DISMISSED'): - continue - login = (review.get('user') or {}).get('login') - if login is None: - continue - latest[login] = review['state'] - if review.get('commit_id') == head_sha: - decision[login] = review['state'] - - # A change request is not commit-scoped the way an approval is: GitHub keeps - # it blocking until the reviewer dismisses it or reviews again, however many - # commits land after it. Judging it on `head_sha` alone would let an approval - # of a later commit merge over a still-open objection. - objectors = sorted( - login for login, state in latest.items() if state == 'CHANGES_REQUESTED' - ) - if objectors: - listed = ', '.join(f'`{login}`' for login in objectors) - block( - f'changes requested by {listed} and not withdrawn; the reviewer must ' - f'approve, re-review, or dismiss before this can merge.' - ) - - if any(label['name'] == RTM_LABEL for label in labels): - print(f'authorized by the `{RTM_LABEL}` label') - return - - approvers = sorted(login for login, state in decision.items() if state == 'APPROVED') - if not approvers: - block( - f'no approving review on `{head_sha[:12]}`, and no `{RTM_LABEL}` label. A ' - f'maintainer must approve this exact commit β€” approvals of earlier commits ' - f'do not carry over β€” or set `{RTM_LABEL}`.' - ) - - maintainers = [login for login in approvers if can_maintain(login)] - if not maintainers: - listed = ', '.join(f'`{login}`' for login in approvers) - block( - f'`{head_sha[:12]}` is approved ({listed}) but by nobody with push access; ' - f'a maintainer must approve it.' - ) - - print(f'approved on {head_sha[:12]} by: {", ".join(maintainers)}') - - -def check_title(pr): - """ - Refuse a PR whose title would not survive the commit-message hook. - - `compose_message` makes the squashed subject `<title> (#N)` verbatim, and - nothing re-validates a generated message β€” so an invalid title lands as a - commit the hook would have rejected. `initial / pr-title` checks this on - every push, but the title can be edited afterwards without re-triggering CI - (queue.yaml deliberately does not run on `edited`), so the authoritative - check is here, against the exact subject about to be written. - """ - subject = f'{pr["title"]} {pr_reference(qualified=False)}' - checker = ci_lib.ROOT_DIR / 'support' / 'scripts' / 'check-commit-message.py' - result = run(sys.executable, str(checker), '--message-text', subject, check=False) - if result.returncode != 0: - detail = (result.stdout or result.stderr).strip() - block( - f'the PR title would not pass the commit-message hook, and it becomes the ' - f'squashed subject verbatim:\n\n```\n{subject}\n```\n\n```\n{detail}\n```' - ) - - -def force_actor() -> str | None: - """ - The admin who asked for a forced merge, or None on a normal merge. - - Set only after the workflow has verified, in its own YAML and before this - file was fetched, that the login is a repo admin and that the head is still - the commit they authorized. Nothing here re-derives that: this script is - checked out from the PR head, so a check it ran itself would be worth - exactly as much as the PR's own code. - """ - return (os.environ.get('FORCE_ACTOR') or '').strip() or None - - -def local_force_actor() -> str: - """ - The logged-in `gh` user, for a `--force` merge run by hand. - - `--force` is a LOCAL escape hatch: on a workstation the person at the - keyboard is the authorization, and asking them to retype a login they are - already authenticated as only invites a typo in the record. In CI the - authorization comes from the workflow instead, so `--force` is refused there - outright β€” under GITHUB_TOKEN this call answers `github-actions[bot]`, which - would make the PR's own code the actor of record. - """ - if os.environ.get('GITHUB_ACTIONS') == 'true': - raise SystemExit( - '--force is for local runs; in CI the actor comes from $FORCE_ACTOR, ' - 'set by branch_force_merge.yaml after it verifies the admin role' - ) - login = gh('api', 'user', '--jq', '.login').strip() - if not login: - raise SystemExit('could not resolve the logged-in `gh` user for --force') - return login - - -def announce_force(actor, head_sha): - """ - Record on the PR which gates were skipped and who skipped them. - - A forced merge lands a commit no green CI ever covered, so the reason it was - allowed must survive in the PR history rather than only in a job log that - expires. - """ - run( - 'gh', - 'pr', - 'comment', - pr_number(), - '--repo', - repo(), - '--body', - f'⚠️ Forced merge requested by `{actor}` (repo admin) on `{head_sha[:12]}`: ' - f'the review, full-CI and E2E gates are NOT being enforced. The base, ' - f'draft/open, title and 0-behind checks still apply.', - check=False, - ) - - -def check_gates(pr): - if pr['state'] != 'OPEN': - block(f'PR is not open (state: {pr["state"]}).') - if pr['isDraft']: - block('PR is a draft.') - - base = pr['baseRefName'] - if not re.fullmatch(r'v.*-dev', base): - block(f'base branch `{base}` is not a `v<X>-dev` branch.') - - head_sha = pr['headRefOid'] - - # 1b. the title is about to become a commit subject verbatim. Checked even - # under --force: an invalid subject would be rejected by the commit-message - # hook on the very next push to the dev branch, so skipping it would land a - # breakage rather than accept a risk. - check_title(pr) - - # Gates 2-4 answer "is this commit known-good", and a forced merge is exactly - # the decision to land it without that answer β€” an admin said so explicitly, - # in a comment recorded on the PR. What stays enforced below is the set of - # checks that keep the LAND itself correct (base shape, fast-forwardability), - # which no amount of authority makes safe to skip. - actor = force_actor() - if actor is not None: - announce_force(actor, head_sha) - print(f'forced by {actor}: skipping the review, CI and E2E gates') - return base, head_sha - - # 2. a maintainer approved THIS commit, or the PR is labelled ready-to-merge - check_approval(head_sha, pr['labels']) - - # 3. full GenVM CI (queue.yaml) green on the head commit. The same head may - # carry a failed markerless push run alongside the full panel run, and the - # successful run may have been started by either a push - # (event=pull_request) or the action panel (event=workflow_dispatch) β€” so we - # query all events and require ANY completed run on this exact commit to have - # succeeded, not just the most recent. - runs = json.loads( - gh( - 'api', - f'repos/{repo()}/actions/workflows/queue.yaml/runs?head_sha={head_sha}', - ) - )['workflow_runs'] - if not any(r['status'] == 'completed' and r['conclusion'] == 'success' for r in runs): - got = ', '.join(f'{r["status"]}/{r["conclusion"]}' for r in runs) or 'no run' - block(f'GenVM CI (queue.yaml) is not green on `{head_sha}` (runs: {got}).') - - # 4. cross-repo E2E check green on the head commit. The API defaults to - # `filter=latest`, so a re-run replaces its predecessor rather than adding a - # stale entry beside it. - name, app_slug = e2e_identity() - checks = [ - json.loads(line) - for line in gh( - 'api', - '--paginate', - f'repos/{repo()}/commits/{head_sha}/check-runs?per_page=100', - '--jq', - '.check_runs[]', - ).splitlines() - if line.strip() - ] - e2e = [ - c - for c in checks - if c['name'] == name and ((c.get('app') or {}).get('slug')) == app_slug - ] - if not e2e: - seen = ', '.join(sorted({f'`{c["name"]}`' for c in checks})) or 'none' - block( - f'no `{name}` check from `{app_slug}` on `{head_sha}` (run E2E on this PR ' - f'first). Checks present: {seen}.' - ) - if any(c['conclusion'] != 'success' for c in e2e): - conclusions = ' '.join(c['conclusion'] or 'pending' for c in e2e) - block(f'E2E is not green on `{head_sha}` (conclusions: {conclusions}).') - - return base, head_sha - - -def executor_remote_sha(submodule, branch): - """ - Current `origin/<branch>` in an executor checkout, or None if there is none. - - `plan_executor_line` fetched every executor head, so a branch is present locally - iff it exists on the remote. Used to remember what a branch pointed at before we - move it, so a failed land can put it back. - """ - r = egit( - submodule, - 'rev-parse', - '--verify', - '--quiet', - f'refs/remotes/origin/{branch}', - check=False, - ) - return r.stdout.strip() if r.returncode == 0 else None - - -def landing_ref(line=None): - """ - Scratch ref carrying one repo's landing objects. - - Both executor lines are checkouts of the SAME remote, so the ref has to be keyed - by line as well as by PR β€” otherwise v0.3 force-overwrites v0.2's ref and that - line's squashed commit is left unreferenced, which is exactly the object the - recovery path assumes is still fetchable. - """ - suffix = '' if line is None else f'/{line}' - return f'{LANDING_REF_PREFIX}/pr-{pr_number()}{suffix}' - - -def stage_objects(plans, push_sha): - """ - Upload every commit that is about to land, without moving a branch anyone reads. - - Manager and executors are separate repos, so the branch moves that finish a merge - can never be one transaction β€” but they can be made short. Pushing the objects - first under a scratch ref leaves each final push a ref update with nothing to - transfer, so the window in which the manager can gitlink a commit no executor - branch contains shrinks from "however long the upload takes" to a round trip. - - The scratch refs also keep those objects reachable if the job dies mid-way, so a - repair is a ref move rather than a re-squash under a fresh sha. That is what makes - the replay instructions on a partial land runnable at all, so a failure here is - fatal rather than a warning β€” nothing has moved yet, so blocking costs nothing. - """ - for submodule, line, _base, _pinned, landed, needs_push in plans: - if not needs_push: - continue - ref = landing_ref(line) - r = egit(submodule, 'push', '--force', 'origin', f'{landed}:{ref}', check=False) - if r.returncode != 0: - block( - f'could not stage `{submodule}` objects at `{ref}`, so a partial land ' - f'could not be repaired; nothing was moved. ({r.stderr.strip()})' - ) - ref = landing_ref() - r = git('push', '--force', 'origin', f'{push_sha}:{ref}', check=False) - if r.returncode != 0: - block( - f'could not stage manager objects at `{ref}`; nothing was moved. ' - f'({r.stderr.strip()})' - ) - - -def unstage_objects(plans): - """ - Drop the scratch refs once every target branch holds the commits. Best-effort. - - Only reached on a fully successful land: after a partial one the refs are what - keeps the un-pushed commits fetchable, so leaving them is the point. - """ - for submodule, line, _base, _pinned, _landed, needs_push in plans: - if needs_push: - egit(submodule, 'push', 'origin', '--delete', landing_ref(line), check=False) - git('push', 'origin', '--delete', landing_ref(), check=False) - - -def move_executor_mirror_branch(submodule, line, head_ref, landed): - """ - Move the executor mirror branch `pr/<line>/<head_ref>` onto `landed`. - - The mirror PR (`pr/<line>/<head_ref>` -> `<line>-dev`, branch_executor_prs.yaml) - is auto-closed as MERGED once its head is contained in its base, so we point the - head at the very commit `<line>-dev` is about to receive. Doing this BEFORE the - base moves is what makes GitHub read the result as a merge; doing it after races - the base update and can land the PR as merely 'closed'. (Deleting the branch β€” the - original behaviour β€” always dropped the merged link.) - - Only lines this PR actually moved were provisioned a mirror branch; an untouched - line has none, and force-pushing would CREATE a stray dangling ref, so move only a - branch that already exists. Returns the sha to restore on rollback, or None when - there was nothing to move. - """ - branch = f'pr/{line}/{head_ref}' - previous = executor_remote_sha(submodule, branch) - if previous is None: - print(f'no executor mirror branch {branch}; nothing to auto-merge for {line}') - return None - r = egit( - submodule, 'push', '--force', 'origin', f'{landed}:refs/heads/{branch}', check=False - ) - if r.returncode != 0: - print(f'note: could not move executor mirror branch {branch}: {r.stderr.strip()}') - return None - print(f'moved executor mirror branch {branch} -> {landed[:12]} (auto-merges its PR)') - return previous - - -def move_manager_pr_head(pr, head_sha, push_sha): - """ - Point the manager PR's head branch at `push_sha`, the commit `<base>` will get. - - Same trick as the executor mirrors, and again before the base moves: head contained - in base is what GitHub reads as merged, unlike a bare `gh pr close` (merely - 'closed'). - - The rollback target is `head_sha` rather than whatever `origin/<head_ref>` happens - to hold locally: the gates already pinned the PR head to it and `merge()` re-checked - it, so it is exact, and it does not silently depend on the workflow checking out - with full history. - - A fork PR's head branch lives in another repo, so `<head_ref>` is not a branch on - `origin` β€” moving it would create a stray branch and not touch the PR. Those never - reach here (the merge gate needs a same-repo run), but be explicit and leave fork - PRs to the explicit close after the land. Returns the sha to restore on rollback, - or None when there was deliberately nothing to move. - """ - head_ref = pr['headRefName'] - if pr.get('isCrossRepository'): - print(f'PR head `{head_ref}` is on a fork; cannot move it, will close explicitly') - return None - # A PR raised FROM one dev branch INTO another would make this force-push a - # squashed commit onto a protected branch, using a key that bypasses protection. - if re.fullmatch(r'v.*-dev', head_ref): - block(f'PR head `{head_ref}` is itself a dev branch; refusing to force-move it.') - r = git('push', '--force', 'origin', f'{push_sha}:refs/heads/{head_ref}', check=False) - if r.returncode != 0: - # Leaving it unmoved would land the base anyway and strand the PR open, with - # no rollback entry to undo. Nothing has moved yet, so stop here. - block(f'could not move PR head `{head_ref}`; nothing landed. ({r.stderr.strip()})') - print(f'moved manager PR head `{head_ref}` -> {push_sha[:12]} (auto-merges the PR)') - return head_sha - - -def restore_branches(moved): - """ - Put every branch we moved back where it was, after a target push was rejected. - - Without this a rejected land leaves the PR head sitting on the squashed commit: - the head sha then no longer matches the one CI and E2E ran on, so re-ticking Merge - would be refused by its own gates and the PR could not be landed at all. - """ - for submodule, branch, previous in moved: - r = ( - git('push', '--force', 'origin', f'{previous}:refs/heads/{branch}', check=False) - if submodule is None - else egit( - submodule, - 'push', - '--force', - 'origin', - f'{previous}:refs/heads/{branch}', - check=False, - ) - ) - where = 'manager' if submodule is None else submodule - if r.returncode == 0: - print(f'restored {where} branch {branch} -> {previous[:12]}') - else: - print(f'note: could not restore {where} branch {branch}: {r.stderr.strip()}') - - -def executor_gitlink(commit, submodule): - """ - Executor submodule commit (gitlink) recorded at `commit`'s tree. - """ - out = git('ls-tree', commit, submodule).stdout - # "160000 commit <sha>\t<path>" - m = re.match(r'\S+\s+commit\s+([0-9a-f]+)\t', out) - if m is None: - block( - f'could not resolve the executor submodule pointer ({submodule}) at `{commit}`.' - ) - else: - return m.group(1) - - -def pr_reference(qualified=True): - """ - The `(#N)` suffix a squashed subject ends with. - - A commit landing in an executor repo has to name the manager repo for the - reference to resolve; the manager's own bare `#N` resolves in-repo. - """ - return f'({repo()}#{pr_number()})' if qualified else f'(#{pr_number()})' - - -def commit_entries(repo, rev_range): - """ - `(subject, body)` per commit in `rev_range`, oldest first, merges excluded. - - `-z` terminates the records, a body being able to contain anything. The two - halves need no separator: a subject is one line, so the first newline is it. - """ - args = ('log', '--reverse', '--no-merges', '-z', '--format=%s%n%b', rev_range) - out = (git(*args) if repo is None else egit(repo, *args)).stdout - entries = [] - for record in out.split('\x00'): - if not record.strip(): - continue - subject, _, body = record.partition('\n') - entries.append((subject.strip(), body.strip('\n'))) - return entries - - -def flatten_body(body): - """ - A commit body as message lines, bullets kept at the top level. - - A body line starting with `*` IS a bullet -- that is the invariant the format - rests on -- so an already-squashed commit's bullets continue the outer list - instead of nesting under it. Prose keeps its own paragraph breaks, and gains - the blank line on either side that separates it from the bullets around it. - """ - lines = [] - prose = [] - - def flush(): - while prose and not prose[0].strip(): - prose.pop(0) - while prose and not prose[-1].strip(): - prose.pop() - if prose: - lines.extend(['', *prose, '']) - prose.clear() - - for line in body.split('\n'): - # Indentation stripped, as the commit-message hook does. - if line.strip().startswith('*'): - flush() - lines.append(line.strip()) - else: - prose.append(line) - flush() - return lines - - -def compose_message(pr, submodule, rev_range, author, *, qualified=True): - """ - The squash message for `rev_range`: the PR title, one `* <subject>` bullet - per squashed commit, each followed by that commit's own body, then - `Co-authored-by:` for every other author. - - The bullets are already conventional commits (the commit-message hook saw - them) and are what `make_release_notes` promotes into the changelog, whereas a - PR description is written for reviewers and only rots on a dev branch. Bodies - carry the `why`, which nothing else preserves once the originals are gone. A - range whose one commit already says the title gets no bullets, only its body. - - Squashing a squash flattens the inner bullets into the outer list instead of - nesting them, so every real entry stays at the level both readers of this - format look at. The inner umbrella subject rides along as one extra bullet, - and the changelog gains that one redundant line. - """ - entries = commit_entries(submodule, rev_range) - if [subject for subject, _ in entries] == [pr['title']]: - lines = flatten_body(entries[0][1]) - else: - lines = [] - for subject, body in entries: - lines.append(f'* {subject}') - lines.extend(flatten_body(body)) - - message = f'{pr["title"]} {pr_reference(qualified)}\n' - body = '\n'.join(lines).strip('\n') - if body: - message += f'\n{body}\n' - trailers = coauthor_trailers(submodule, rev_range, author) - if trailers: - message += '\n' + '\n'.join(trailers) + '\n' - return message - - -def already_squashed(submodule, exec_base, tree): - """ - The commit on `<exec_base>` that this run already landed for `tree`, if any. - - A squash keeps only the TREE of the pinned commit, so that commit never becomes - an ancestor of the dev branch and ancestry alone cannot answer "did this line - already land?" β€” a re-tick after a partial land would see a tip that is neither - ancestor nor descendant and block the PR as diverged, forever. - - A commit is this PR's own work when BOTH halves of its identity match: its - subject references this PR, and its tree is the one being landed. Neither alone - is enough β€” the subject alone would mistake an earlier landing for the current - one after the author pushed more executor commits (silently dropping them), and - the tree alone would mistake a revert that restored an older tree. - """ - out = egit( - submodule, - 'log', - f'refs/remotes/origin/{exec_base}', - '-F', - f'--grep={pr_reference()}', - '--format=%H%x00%T%x00%s', - ).stdout - for line in out.split('\n'): - if line.count('\x00') != 2: - continue - sha, commit_tree, subject = line.split('\x00') - if commit_tree == tree and subject.endswith(pr_reference()): - return sha - return None - - -def plan_executor_line(submodule, exec_base, exec_sha, pr): - """ - Work out what this line should land on its `<line>-dev`. - - Returns `(sha, needs_push)`. The PR's commits on this line are squashed into - one commit on top of the line's dev tip, so the executor history mirrors the - manager's: one commit per merged PR. `sha` is that squashed commit β€” or - `exec_sha` itself when there is nothing to squash (a single commit already - sitting on the tip, or a branch that has to be created). - - Blocks if exec_sha is unknown to the executor repo or the branch has diverged. - """ - # Bring every executor head (and thus exec_sha, if pushed) local. - egit(submodule, 'fetch', '--no-tags', 'origin', '+refs/heads/*:refs/remotes/origin/*') - - if ( - egit(submodule, 'cat-file', '-e', f'{exec_sha}^{{commit}}', check=False).returncode - != 0 - ): - block( - f'executor commit `{exec_sha}` ({submodule}) is not in the executor repo; ' - f'push the executor side of this change first, then re-tick Merge.' - ) - - ref = f'refs/remotes/origin/{exec_base}' - if ( - egit(submodule, 'rev-parse', '--verify', '--quiet', ref, check=False).returncode - != 0 - ): - return exec_sha, True # branch absent -> push creates it - - tip = egit(submodule, 'rev-parse', ref).stdout.strip() - if tip == exec_sha: - return exec_sha, False # already there - - tree = egit(submodule, 'rev-parse', f'{exec_sha}^{{tree}}').stdout.strip() - - landed = already_squashed(submodule, exec_base, tree) - if landed is not None: - # A previous run squashed this line and pushed it; only the manager push - # is left. Reuse that commit rather than squashing a second one. - print(f'Executor {exec_base} already carries {exec_sha[:12]} as {landed[:12]}') - return landed, False - - if is_ancestor(submodule, exec_sha, tip): - # Executor moved ahead of what this PR pins; the pinned commit is - # already contained, so there is nothing to push. - return exec_sha, False - - if not is_ancestor(submodule, tip, exec_sha): - # A maintainer may have merged the auto-opened executor PR by hand: the tree - # is already on the tip under a different sha and a subject of their own - # choosing. Same content already landed, so mirror the tip rather than block. - if egit(submodule, 'rev-parse', f'{tip}^{{tree}}').stdout.strip() == tree: - print( - f'Executor {exec_base} already carries {exec_sha[:12]} as its tip {tip[:12]}' - ) - return tip, False - block( - f'executor `{exec_base}` (`{tip[:12]}`) has diverged from the PR executor ' - f'commit `{exec_sha[:12]}`; mirror is not a fast-forward.' - ) - - commits = egit(submodule, 'rev-list', f'{tip}..{exec_sha}').stdout.split() - if len(commits) == 1: - # Already a single commit on top of the tip: squashing it would only - # rewrite it into an identical tree under a new sha. - return exec_sha, True - - name, email, date = commit_author(submodule, exec_sha) - full_message = compose_message(pr, submodule, f'{tip}..{exec_sha}', (name, email)) - - # A squash is just the PR's tree on top of the tip; commit-tree builds it - # without touching the working tree or index. - squashed = egit( - submodule, - 'commit-tree', - tree, - '-p', - tip, - '-m', - full_message, - env={ - 'GIT_AUTHOR_NAME': name, - 'GIT_AUTHOR_EMAIL': email, - 'GIT_AUTHOR_DATE': date, - 'GIT_COMMITTER_NAME': 'genvm-ci', - 'GIT_COMMITTER_EMAIL': 'genvm-ci@genlayer.com', - }, - ).stdout.strip() - print(f'Squashed {len(commits)} executor commits ({exec_base}) into {squashed}') - return squashed, True - - -def plan_executor_lines(head_sha, pr): - """ - Plan every active line off the PR head's gitlinks. - - Returns a list of `(submodule, line, exec_base, pinned_sha, landed_sha, - needs_push)`. `landed_sha` differs from `pinned_sha` when the line's commits - were squashed β€” the manager commit must then be rewritten to gitlink the - squashed commit, since the one the PR pinned never lands on the dev branch. - """ - plans = [] - for line in active_lines(): - submodule = f'executors/{line}.x' - exec_base = f'{line}-dev' - pinned = executor_gitlink(head_sha, submodule) - landed, needs_push = plan_executor_line(submodule, exec_base, pinned, pr) - plans.append((submodule, line, exec_base, pinned, landed, needs_push)) - return plans - - -def push_executor_lines(plans): - """ - Advance each line's `<line>-dev` onto its landed commit, after the manager push. - - Every line is attempted even if an earlier one fails, so one rejected line does - not strand the rest: the manager commit already gitlinks all of them, and each - push is independent. A failure is reported at the end with the exact ref update - to replay, because by this point the PR is landed and closed β€” re-ticking Merge - is no longer an option. - """ - failed = [] - for submodule, _line, exec_base, _pinned, landed, needs_push in plans: - if not needs_push: - print(f'Executor {exec_base} already contains {landed}; nothing to mirror') - continue - print(f'Mirroring executor {exec_base} -> {landed}') - r = egit( - submodule, 'push', 'origin', f'{landed}:refs/heads/{exec_base}', check=False - ) - if r.returncode != 0: - failed.append((submodule, exec_base, landed, r.stderr.strip())) - - if failed: - details = '\n'.join( - f'- `{submodule}`: `git push origin {landed}:refs/heads/{exec_base}` ({err})' - for submodule, exec_base, landed, err in failed - ) - block( - 'the manager side landed but these executor branches did not advance, so the ' - 'new gitlinks point at commits missing from their dev branches. Replay:\n' - f'{details}' - ) - - -def merge(pr, base, head_sha): - # Fetch the exact head commit (works for fork PRs too) and live base tip. - git( - 'fetch', - '--no-tags', - 'origin', - f'refs/pull/{pr_number()}/head:refs/prhead', - f'+refs/heads/{base}:refs/remotes/origin/{base}', - ) - - fetched = git('rev-parse', 'refs/prhead').stdout.strip() - if fetched != head_sha: - block( - f'head moved during merge (expected `{head_sha}`, got `{fetched}`); re-tick Merge.' - ) - - # 5. authoritative 0-commits-behind check at merge time. Same measurement as - # `initial / behind-check` and rebase-watch, so the three cannot disagree. - count = behind_lib.behind_by_git(f'origin/{base}', 'refs/prhead') - if count: - block( - f'PR is {count} commit(s) behind `{base}`; update the branch and re-tick Merge.' - ) - - git('config', 'user.name', 'genvm-ci') - git('config', 'user.email', 'genvm-ci@genlayer.com') - - # Plan every active executor line first: each line's commits are squashed into - # one commit on its `<line>-dev`, so a line that was squashed lands under a - # NEW sha and the manager commit has to gitlink that one instead of the sha the - # PR pinned (which never lands on the dev branch). - plans = plan_executor_lines(head_sha, pr) - - # Refuse to move any line's gitlink backward. The 0-behind gate is on manager - # commits, not the gitlink, so a PR can be current on commits yet stale on the - # pointer (another PR advanced the line after this one branched); landing it - # would silently revert that PR's executor work. - for submodule, _line, exec_base, _pinned, landed, _push in plans: - base_link = executor_gitlink(f'origin/{base}', submodule) - if landed != base_link and is_ancestor(submodule, landed, base_link): - block( - f'merging would move the `{submodule}` gitlink backward on `{base}` ' - f'(base pins `{base_link[:12]}`, this PR would set `{landed[:12]}`); ' - f'another change advanced it β€” update the PR and re-tick Merge.' - ) - - rewritten = { - submodule: landed - for submodule, _line, _base, pinned, landed, _push in plans - if landed != pinned - } - - if len(pr['commits']) == 1 and not rewritten: - print(f'Single commit: fast-forwarding {base} to {head_sha}') - push_sha = head_sha - else: - if rewritten: - print( - f'Executor lines were squashed; rewriting the manager gitlinks: {rewritten}' - ) - print(f'Squashing {len(pr["commits"])} commits onto {base}') - name, email, date = commit_author(None, head_sha) - git('checkout', '-B', '_merge', f'origin/{base}') - git('merge', '--squash', head_sha) - - # Repoint each squashed line at the commit that actually lands. The index - # holds the PR's gitlinks after `merge --squash`; overwrite them in place - # (no submodule checkout involved). - for submodule, landed in rewritten.items(): - git('update-index', '--cacheinfo', f'160000,{landed},{submodule}') - - full_message = compose_message( - pr, None, f'origin/{base}..{head_sha}', (name, email), qualified=False - ) - - git( - 'commit', - '--author', - f'{name} <{email}>', - '--date', - date, - '-m', - full_message, - ) - push_sha = git('rev-parse', 'HEAD').stdout.strip() - - # 1. Get every object onto both remotes while nothing is watching, so the two - # branch moves below are ref updates with nothing left to transfer. - stage_objects(plans, push_sha) - - # 2. Point the PR head branches at what is about to land. This has to precede - # the base moves for GitHub to read the result as a merge rather than a close, - # and it is undone if the land is then rejected β€” a PR head left on the squashed - # commit no longer matches the sha CI ran on, which would lock the PR out of its - # own gates. - moved = [] - head_ref = pr['headRefName'] - previous = move_manager_pr_head(pr, head_sha, push_sha) - if previous is not None: - moved.append((None, head_ref, previous)) - for submodule, line, _base, _pinned, landed, _push in plans: - previous = move_executor_mirror_branch(submodule, line, head_ref, landed) - if previous is not None: - moved.append((submodule, f'pr/{line}/{head_ref}', previous)) - - time.sleep(SETTLE_SECONDS) - - # 3. Move the target branches. The manager goes first: it is the only push that - # can lose a race (the executor dev branches only move through this same tool, - # under the same PR gates), so a rejection here has nothing to unwind but the - # branch moves above. Its commit gitlinks executor commits that are already on - # the remote β€” reachable from the scratch ref and the mirror branches β€” so the - # gitlinks resolve even in the window before the executor branches advance. - if ( - git('push', 'origin', f'{push_sha}:refs/heads/{base}', check=False).returncode != 0 - ): - restore_branches(moved) - block( - f'fast-forward push to manager `{base}` was rejected (base advanced); ' - f'nothing landed. Update the branch and re-tick Merge.' - ) - - push_executor_lines(plans) - - unstage_objects(plans) - - run( - 'gh', - 'pr', - 'comment', - pr_number(), - '--repo', - repo(), - '--body', - f'βœ… Merged into `{base}` (`{push_sha}`) via fast-forward.' - + ( - f' Forced by `{force_actor()}` β€” the review, CI and E2E gates were not enforced.' - if force_actor() - else '' - ), - check=False, - ) - - # A fork PR's head lives in another repo, so it was never moved and GitHub has - # nothing to detect; close it explicitly (as merely 'closed' β€” unavoidable). - if pr.get('isCrossRepository'): - run('gh', 'pr', 'close', pr_number(), '--repo', repo(), check=False) - - -class GenvmMergeIntoDev(ci_lib.Tool): - """ - Gate and perform a Merge of a PR into a v<X>-dev branch. - """ - - def name(self) -> str: - return 'genvm-merge-into-dev' - - def add_to(self, parser: argparse.ArgumentParser) -> None: - gh_common.add_args(parser, executor_repo=False, head_ref=False) - parser.add_argument( - '--force', - action='store_true', - help='merge without requiring an approving review, green CI or green ' - 'E2E; local runs only, and recorded on the PR under your `gh` login', - ) - - def handler(self, args: argparse.Namespace) -> int: - gh_common.set_ctx(gh_common.Ctx.from_args(args)) - if args.force: - # One channel for "who forced this", whatever set it: the announcement - # and the merge comment both read it back through `force_actor()`. - os.environ['FORCE_ACTOR'] = local_force_actor() - merge_into_dev() - return 0 - - -def merge_into_dev(): - pr = pr_view( - 'baseRefName', - 'headRefName', - 'headRefOid', - 'commits', - 'labels', - 'state', - 'isDraft', - 'title', - 'isCrossRepository', - ) - base, head_sha = check_gates(pr) - merge(pr, base, head_sha) - - -COMMANDS = [GenvmMergeIntoDev()] diff --git a/support/ci/tools/open_executor_prs.py b/support/ci/tools/open_executor_prs.py index 85da2307..01619467 100644 --- a/support/ci/tools/open_executor_prs.py +++ b/support/ci/tools/open_executor_prs.py @@ -85,8 +85,8 @@ def open_pr(ctx: gh_common.Ctx, head: str, base: str) -> str: ).stdout.strip() body = ( f'Auto-opened executor mirror of {ctx.manager_repo}#{ctx.pr_number}.\n\n' - f'Carries the executor-side work for that manager PR. Auto-closed as merged ' - f'when the manager PR lands (its `{head}` branch is moved onto `{base}`).' + f'Carries the executor-side work for that manager PR. The manager branch ' + f'update fast-forwards `{base}` to its pinned commit after landing.' ) r = gh( 'pr', diff --git a/support/ci/tools/pr_action_panel.py b/support/ci/tools/pr_action_panel.py index 0bedcb85..69cc9f32 100644 --- a/support/ci/tools/pr_action_panel.py +++ b/support/ci/tools/pr_action_panel.py @@ -14,13 +14,9 @@ we set the `run-full-tests` label (so queue.yaml runs on every future push) AND dispatch one run now. We do not untick it β€” its checked state mirrors the label, and an already-set label means no re-dispatch on unrelated edits. -- "Rerun full tests" is a momentary button: dispatch a fresh queue.yaml run on -the current head, then untick. - "Provision executor PRs" is a momentary button: dispatch branch_provision_executor_prs.yaml for this PR (force-push each moved executor line's mirror branch and open its PR), then untick. -- "Merge" is a momentary button: expose `merge=true` so the caller runs the -reusable merge workflow, then untick. 4. untick the momentary boxes (not the sticky Force one). Which action a ticked box maps to is decided by the stable `<!-- action:<id> -->` @@ -33,7 +29,7 @@ ------------------ Three independent conditions, all required: -1. the PR carries `ci-safe` β€” the same label the merge and provision workflows +1. the PR carries `ci-safe` β€” the same label the provision workflow check before running PR code with credentials in scope, so a target branch cannot be moved unless the PR is vetted; 2. the edited comment was authored by the bot, i.e. it is the panel we posted; @@ -45,16 +41,16 @@ the comment by a MARKER IN ITS BODY, not by who wrote it. Any user may post their own comment carrying that marker on a public PR and freely edit their own copy, which without (2) would let a non-collaborator force-push executor mirror -branches and request merges. `SENDER` drives (3), and the bot check on it is +branches. `SENDER` drives (3), and the bot check on it is only the loop-breaker for the panel's own reset edit. Runs are started with `gh workflow run queue.yaml --ref <PR head branch>` (a `workflow_dispatch`), NOT by toggling a label: a label applied by the bot's GITHUB_TOKEN does not emit a `labeled` event (GitHub blocks recursive runs), whereas a `workflow_dispatch` from the same token does run. Dispatching on the -PR head branch makes the run's head_sha equal the PR head, so the Merge gate -counts it. (Fork PRs have no head branch in this repo, so the panel can only -dispatch for same-repo branches β€” the common GenVM flow.) +PR head branch makes the run's head_sha equal the PR head. Fork PRs have no +head branch in this repo, so the panel can only dispatch for same-repo branches +β€” the common GenVM flow. Resetting the panel re-fires issue_comment:edited, but that event is sent by the bot, so it is ignored β€” no loop. @@ -79,9 +75,9 @@ # as `<!-- action:<id> -->`; nothing here depends on the prose beside it, so the # wording can change freely. ACTION_FORCE = 'force' +# Compatibility until existing bot panels are refreshed on their next PR event. ACTION_RERUN = 'rerun' ACTION_PROVISION = 'provision' -ACTION_MERGE = 'merge' ACTION_MARKER_RE = re.compile(r'<!--\s*action:([a-z-]+)\s*-->') @@ -91,7 +87,6 @@ (ACTION_FORCE, 'force'), (ACTION_RERUN, 'rerun'), (ACTION_PROVISION, 'provision'), - (ACTION_MERGE, 'merge'), ) # The one box that is a toggle rather than a button: its ticked state mirrors the @@ -119,11 +114,6 @@ def gh(*args: str, check: bool = True) -> str: return gh_common.gh_manager(*args, check=check, retry=False) -def set_output(name, value): - with open(os.environ['GITHUB_OUTPUT'], 'a') as f: - f.write(f'{name}={value}\n') - - def labels(): out = gh( 'api', @@ -161,9 +151,9 @@ def head_branch(): def dispatch_full_tests(*, release_pipeline_test: bool): - # Start queue.yaml on the PR head branch so the run's head_sha matches the - # PR head (the Merge gate keys off head_sha). Label edits intentionally do not - # trigger queue.yaml; workflow_dispatch is the only panel-driven entry point. + # Start queue.yaml on the PR head branch so the run's head_sha matches the PR + # head. Label edits intentionally do not trigger queue.yaml; + # workflow_dispatch is the only panel-driven entry point. branch = head_branch() if not branch: print('could not resolve PR head branch; cannot dispatch full tests') @@ -226,8 +216,8 @@ def is_legacy_panel(body: str) -> bool: Decided per PANEL, not per line. Judging line by line would let the prose fallback run on a modern panel β€” so a checkbox a reviewer adds by hand - (`- [x] merge conflicts resolved`) would fire `merge`, which is precisely - what the markers exist to prevent. + (`- [x] rerun-dependent checks`) could fire `rerun`, which is precisely what + the markers exist to prevent. """ return ACTION_MARKER_RE.search(body) is None @@ -292,8 +282,6 @@ def handler(self, args: argparse.Namespace) -> int: def run_panel(): - set_output('merge', 'false') - # Ignore the bot's own panel-reset edit (avoids a self-trigger loop). if sender().endswith('[bot]'): print(f'{sender()} is a bot (panel reset echo); ignoring') @@ -324,9 +312,8 @@ def run_panel(): print('ignoring (only the bot-posted action panel drives these actions)') return - # And the editor must be allowed to drive CI. Ticking a box force-pushes - # executor mirror branches and can request a merge, so it is a privileged - # action even though the merge itself re-checks its own gates. + # And the editor must be allowed to drive CI. Ticking a box can force-push + # executor mirror branches, so it is a privileged action. if not gh_common.has_write_access(sender()): print(f'`{sender()}` has no write access to {repo()}; ignoring panel actions') return @@ -346,14 +333,9 @@ def run_panel(): if ACTION_FORCE in actions and RUN_FULL_TESTS_LABEL not in current: add_label(RUN_FULL_TESTS_LABEL) dispatch = True - - # Rerun: momentary -> always dispatch a fresh run on the current head. if ACTION_RERUN in actions: dispatch = True - # One dispatch at most: ticking Force and Rerun in the same edit would - # otherwise queue two runs for the same head, and the second cancels the - # first through the `queue.yaml` concurrency group. if dispatch: dispatch_full_tests( release_pipeline_test='test-release-pipeline' in current, @@ -363,9 +345,6 @@ def run_panel(): if ACTION_PROVISION in actions: dispatch_provision() - if ACTION_MERGE in actions: - set_output('merge', 'true') - untick_momentary(body) diff --git a/support/ci/tools/pr_branches.py b/support/ci/tools/pr_branches.py index 30488d37..cbf2fe33 100644 --- a/support/ci/tools/pr_branches.py +++ b/support/ci/tools/pr_branches.py @@ -168,17 +168,15 @@ def check(self, args: argparse.Namespace) -> int: """ Gate for the manager PR's CI. Records every problem, then fails if any. - The manager PR and its executor PRs land almost-atomically via the panel - (Provision then Merge), so this checks that the land is POSSIBLE, not that the - executor side already landed: + The E2E App lands the manager first, then the manager push projects its + gitlinks onto executor branches. This checks that projection is possible, not + that the executor side already landed: 1. Every repo (manager + each executor line) must be rebased on its base β€” a - repo behind base needs a rebase. - 2. Every executor line that existed at base (base_sha set) must be landable β€” - its pinned head commit is present in the executor repo, so the merge can - squash/push it. Being ahead of base is normal (that is the unlanded work that - lands with the PR); only a commit missing from the executor repo fails. A line - the PR adds (base_sha null) is not gated. + repo behind base needs a rebase. + 2. Every executor line must be landable β€” its pinned head commit is present + in the executor repo, so the post-merge workflow can push it. Being ahead + of base is normal; only a missing or incomparable commit fails. No PR number (a bare manual dispatch not tied to a PR) is a no-op pass β€” there is nothing to gate. @@ -208,13 +206,13 @@ def record(label: str, check: str, ok: bool, detail: str) -> None: f'{info.behind_by} commit(s) behind base; rebase it', ) - # 2. every executor line present at base must be landable: its pinned commit - # is in the executor repo. Ahead-of-base is fine β€” it lands with the PR. + # 2. every executor line must be landable: its pinned commit is in the + # executor repo. Ahead-of-base is fine β€” it lands with the PR. for info in infos.values(): - if info.line is None or info.base_sha is None: + if info.line is None: continue detail = ( - 'present at base but missing at the PR head' + 'missing at the PR head' if info.head_sha is None else f'pinned commit {info.head_sha[:12]} is not on the executor repo; ' 'push/provision it so it can land with the manager PR' diff --git a/support/ci/tools/rebase_watch.py b/support/ci/tools/rebase_watch.py index 24859f45..c3cc467e 100644 --- a/support/ci/tools/rebase_watch.py +++ b/support/ci/tools/rebase_watch.py @@ -4,14 +4,13 @@ `initial / behind-check` (incl_initial.yaml) already asserts 0-behind, but it only runs when the PR is pushed β€” a PR goes stale when the BASE moves, and nothing -re-evaluates it then. So a PR can sit green for days and only discover at Merge -time that it is unmergeable (the Merge action fast-forwards, so behind > 0 is -fatal there). +re-evaluates it then. So a PR can sit green for days even though repository +policy requires it to contain the base tip before entering the merge train. This tool re-evaluates on the other edge: every push to a dev branch sweeps the PRs targeting it. It is deliberately advisory β€” it posts a check-run and a label, and both are cosmetic. Nothing here gates a merge: the authoritative 0-behind -check stays in `initial` and in genvm_merge_into_dev. The point is that the PR +check stays in `initial`. The point is that the PR page turns red the moment the branch goes stale rather than at merge time. Two surfaces, because they are visible in different places: @@ -51,8 +50,8 @@ # without trusting the name (which any app may also use). CHECK_EXTERNAL_ID = 'genvm-ci/rebase-watch' -# Bases this sweeps. The Merge action only ever fast-forwards onto a dev branch, -# so a PR targeting anything else has no 0-behind requirement to report on. +# Bases this sweeps. The repository's 0-behind policy applies to dev branches; +# a PR targeting anything else has no such requirement to report on. DEV_BASE_RE = re.compile(r'v.*-dev') @@ -241,11 +240,11 @@ def post_check(head_sha: str, base: str, behind: int, *, stale: bool = False) -> title = f'β›” {behind} commit(s) behind {base} β€” rebase' summary = ( f'This branch is **{behind} commit(s) behind `{base}`**.\n\n' - f'The Merge action only fast-forwards, so it will refuse this branch until ' + f'Repository policy refuses this branch until ' f'it contains the base tip:\n\n' f'```sh\ngit fetch origin\ngit rebase origin/{base}\ngit push --force-with-lease\n```\n\n' f'(Advisory: this check does not block anything. `initial / behind-check` ' - f'and the Merge action are what actually enforce it.)' + f'is what actually enforces it.)' ) else: conclusion = 'success' diff --git a/support/ci/tools/sync_executor_branches.py b/support/ci/tools/sync_executor_branches.py new file mode 100644 index 00000000..6e223c8f --- /dev/null +++ b/support/ci/tools/sync_executor_branches.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +""" +Project manager branch gitlinks onto the executor repository branches. + +Manager release branches are the source of truth. A push to `v<X>.<Y>` moves +each active executor line's branch declared in `.gitmodules`; a push to +`v<X>.<Y>-dev` moves the corresponding `<line>-dev` branch. Updates are plain +Git pushes, so a missing branch is created and an existing branch moves only by +fast-forward. + +Every executor line is attempted even when another line fails. Failures are +reported together after the last line, leaving a rerun to finish only the refs +that still need work. +""" + +import argparse +import os +import re +import subprocess + +import ci_lib +import gh_common + +from tools.versions import active_lines + +MANAGER_BRANCH_RE = re.compile(r'^v\d+\.\d+(?P<dev>-dev)?$') + + +class SyncError(RuntimeError): + pass + + +def git(*args: str, check: bool = True) -> subprocess.CompletedProcess: + return ci_lib.run(['git', *args], check=check, capture_output=True) + + +def target_branch(manager_branch: str, declared_branch: str) -> str: + match = MANAGER_BRANCH_RE.fullmatch(manager_branch) + if match is None: + raise SyncError( + f'unsupported manager branch `{manager_branch}`; expected v<X>.<Y> or ' + 'v<X>.<Y>-dev' + ) + if not match.group('dev'): + return declared_branch + if not declared_branch.endswith('.x'): + raise SyncError( + f'declared executor branch `{declared_branch}` does not end in `.x`; ' + 'cannot derive its dev branch' + ) + return f'{declared_branch[:-2]}-dev' + + +def declared_branch(path: str) -> str: + result = git( + 'config', + '-f', + '.gitmodules', + '--get', + f'submodule.{path}.branch', + check=False, + ) + branch = result.stdout.strip() + if result.returncode != 0 or not branch: + raise SyncError(f'no branch declared for `{path}` in .gitmodules') + return branch + + +def sync_line(line: str, manager_branch: str) -> tuple[str, str]: + path = f'executors/{line}.x' + target = target_branch(manager_branch, declared_branch(path)) + + initialized = git( + 'submodule', 'update', '--init', '--depth', '1', '--', path, check=False + ) + if initialized.returncode != 0: + detail = (initialized.stderr or initialized.stdout).strip() + raise SyncError( + f'could not materialize `{path}`' + + (f': {detail}' if detail else f' (git exited {initialized.returncode})') + ) + + resolved = git('-C', path, 'rev-parse', 'HEAD', check=False) + sha = resolved.stdout.strip() + if resolved.returncode != 0 or not sha: + detail = (resolved.stderr or resolved.stdout).strip() + raise SyncError( + f'could not resolve the gitlink checkout for `{path}`' + + (f': {detail}' if detail else '') + ) + result = git( + '-C', + path, + 'push', + 'origin', + f'{sha}:refs/heads/{target}', + check=False, + ) + if result.returncode != 0: + detail = (result.stderr or result.stdout).strip() + raise SyncError( + f'could not create or fast-forward `{target}` to `{sha}`' + + (f': {detail}' if detail else '') + ) + return target, sha + + +def sync_executor_branches(manager_branch: str) -> int: + if MANAGER_BRANCH_RE.fullmatch(manager_branch) is None: + ci_lib.github_error( + f'unsupported manager branch `{manager_branch}`; expected v<X>.<Y> or ' + 'v<X>.<Y>-dev' + ) + return 1 + + errors: list[tuple[str, str]] = [] + for line in active_lines(): + try: + target, sha = sync_line(line, manager_branch) + print(f'{line}: `{target}` -> `{sha}`') + except SyncError as error: + message = str(error) + ci_lib.github_error(f'{line}: {message}') + errors.append((line, message)) + + if errors: + print(f'\n{len(errors)} executor branch update(s) failed:') + for line, message in errors: + print(f'- {line}: {message}') + return 1 + print('all executor branches are synchronized') + return 0 + + +class SyncExecutorBranches(ci_lib.Tool): + """ + Create or fast-forward executor branches from a manager branch tip. + """ + + def name(self) -> str: + return 'sync-executor-branches' + + def add_to(self, parser: argparse.ArgumentParser) -> None: + gh_common.add_args(parser, executor_repo=False, pr=False, head_ref=False) + parser.add_argument( + '--manager-branch', + default=None, + help='manager branch (default: $MANAGER_BRANCH or $GITHUB_REF_NAME)', + ) + + def handler(self, args: argparse.Namespace) -> int: + branch = ( + args.manager_branch + or os.environ.get('MANAGER_BRANCH') + or os.environ.get('GITHUB_REF_NAME', '') + ) + if not branch: + ci_lib.github_error( + 'manager branch is required: pass --manager-branch or set $MANAGER_BRANCH' + ) + return 2 + return sync_executor_branches(branch) + + +COMMANDS = [SyncExecutorBranches()] diff --git a/support/ci/tools/versions.py b/support/ci/tools/versions.py index 4797da4f..9ab6abb5 100644 --- a/support/ci/tools/versions.py +++ b/support/ci/tools/versions.py @@ -37,7 +37,7 @@ def manager_version() -> str: The manager's own release train as bare major.minor (e.g. '0.6'). Read from the `version` field. The manager dev/release branches - (`v<X>-dev`, `v<X>.x`) are named after this β€” NOT after the executor lines + (`v<X>-dev`, `v<X>`) are named after this β€” NOT after the executor lines in `active-versions`, which move independently (currently v0.2 / v0.3). """ return major_minor(load_root_conf()['version']) diff --git a/support/ci/unit_tests/test_commit_message.py b/support/ci/unit_tests/test_commit_message.py new file mode 100644 index 00000000..8e88f526 --- /dev/null +++ b/support/ci/unit_tests/test_commit_message.py @@ -0,0 +1,117 @@ +import os +import subprocess +import sys +from pathlib import Path +from textwrap import dedent + +import ci_lib +import pytest +from tools import make_release_notes as notes + +CHECK_MESSAGE = ci_lib.ROOT_DIR / 'support' / 'scripts' / 'check-commit-message.py' + +AUTHOR = ('Landing Author', 'landing@example.com') + + +def _env(author) -> dict: + name, email = author + return os.environ | { + 'GIT_AUTHOR_NAME': name, + 'GIT_AUTHOR_EMAIL': email, + 'GIT_COMMITTER_NAME': name, + 'GIT_COMMITTER_EMAIL': email, + 'GIT_CONFIG_GLOBAL': '/dev/null', + 'GIT_CONFIG_SYSTEM': '/dev/null', + } + + +def _git(repo: Path, *args: str, input: str | None = None, author=AUTHOR) -> str: + return subprocess.run( + ['git', '-C', str(repo), *args], + input=input, + capture_output=True, + text=True, + check=True, + env=_env(author), + ).stdout + + +def _add(repo: Path, message: str, *, author=AUTHOR) -> None: + repo.joinpath(f'{len(list(repo.glob("*.txt")))}.txt').write_text(message) + _git(repo, 'add', '.', author=author) + _git(repo, 'commit', '--cleanup=verbatim', '-F', '-', input=message, author=author) + + +@pytest.fixture +def repo(tmp_path): + path = tmp_path / 'repo' + path.mkdir() + _git(path, 'init', '-q', '-b', 'main') + return path + + +def test_release_notes_promote_bullets_only(repo): + _add(repo, 'chore: base\n') + _add( + repo, + dedent("""\ + chore(ci): umbrella (#7) + + * feat(a): one + + fix: prose that merely reads like a subject + """), + ) + + entries = notes.parse_git_log('HEAD~1..HEAD', dir=repo) + + assert [entry.message for entry in entries] == ['feat(a): one'] + + +def test_a_malformed_bullet_is_an_error_not_body(): + message = dedent("""\ + chore(ci): umbrella (#7) + + * feat(a): one + * just a note + """) + + result = subprocess.run( + [sys.executable, str(CHECK_MESSAGE), '--message-text', message], + capture_output=True, + text=True, + ) + + assert result.returncode == 1 + assert 'just a note' in result.stdout + + +def test_a_control_character_is_rejected(): + result = subprocess.run( + [ + sys.executable, + str(CHECK_MESSAGE), + '--message-text', + 'fix(a): before\x01after\n\nA body long enough to pass the length rule.\n', + ], + capture_output=True, + text=True, + ) + + assert result.returncode == 1 + assert 'Control character' in result.stdout + + +def test_an_emoji_sequence_is_not_a_control_character(): + result = subprocess.run( + [ + sys.executable, + str(CHECK_MESSAGE), + '--message-text', + 'feat(a): one πŸ‘¨β€πŸ’»\n', + ], + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stdout diff --git a/support/ci/unit_tests/test_compose_message.py b/support/ci/unit_tests/test_compose_message.py deleted file mode 100644 index b9750ca6..00000000 --- a/support/ci/unit_tests/test_compose_message.py +++ /dev/null @@ -1,291 +0,0 @@ -import os -import subprocess -import sys -from pathlib import Path -from textwrap import dedent - -import ci_lib -import gh_common -import pytest -from tools import genvm_merge_into_dev as merge -from tools import make_release_notes as notes - -CHECK_MESSAGE = ci_lib.ROOT_DIR / 'support' / 'scripts' / 'check-commit-message.py' - -AUTHOR = ('Landing Author', 'landing@example.com') -OTHER = ('Other Author', 'other@example.com') -PR = {'title': 'chore(ci): stitch the commits'} - - -def _env(author) -> dict: - name, email = author - return os.environ | { - 'GIT_AUTHOR_NAME': name, - 'GIT_AUTHOR_EMAIL': email, - 'GIT_COMMITTER_NAME': name, - 'GIT_COMMITTER_EMAIL': email, - # The tests assert on exact message text, so no user config may reach git. - 'GIT_CONFIG_GLOBAL': '/dev/null', - 'GIT_CONFIG_SYSTEM': '/dev/null', - } - - -def _git(repo: Path, *args: str, input: str | None = None, author=AUTHOR) -> str: - return subprocess.run( - ['git', '-C', str(repo), *args], - input=input, - capture_output=True, - text=True, - check=True, - env=_env(author), - ).stdout - - -def _add(repo: Path, message: str, *, author=AUTHOR) -> None: - """ - One commit carrying `message` verbatim -- `--cleanup=verbatim` so git keeps - the blank lines the format under test is made of. - """ - repo.joinpath(f'{len(list(repo.glob("*.txt")))}.txt').write_text(message) - _git(repo, 'add', '.', author=author) - _git(repo, 'commit', '--cleanup=verbatim', '-F', '-', input=message, author=author) - - -def _compose(since: str = 'HEAD~2') -> str: - return merge.compose_message(PR, None, f'{since}..HEAD', AUTHOR, qualified=False) - - -@pytest.fixture -def repo(tmp_path, monkeypatch): - path = tmp_path / 'repo' - path.mkdir() - _git(path, 'init', '-q', '-b', 'main') - monkeypatch.chdir(path) - monkeypatch.setattr( - gh_common, - '_CTX', - gh_common.Ctx( - manager_repo='genlayerlabs/genvm-manager', - executor_repo='genlayerlabs/genvm-executor', - _pr_number='7', - _head_ref='feat/x', - ), - ) - return path - - -def test_bodies_are_kept_after_their_bullet(repo): - _add(repo, 'chore: base\n') - _add(repo, 'feat(a): one\n') - _add(repo, 'fix(b): two\n\nThe parser lied about EOF.\nSecond line.\n') - - assert _compose() == dedent("""\ - chore(ci): stitch the commits (#7) - - * feat(a): one - * fix(b): two - - The parser lied about EOF. - Second line. - """) - - -def test_bullets_stay_adjacent_when_bodies_are_absent(repo): - _add(repo, 'chore: base\n') - _add(repo, 'feat(a): one\n') - _add(repo, 'fix(b): two\n') - - assert _compose() == dedent("""\ - chore(ci): stitch the commits (#7) - - * feat(a): one - * fix(b): two - """) - - -def test_a_body_is_blank_line_delimited_on_both_sides(repo): - _add(repo, 'chore: base\n') - _add(repo, 'feat(a): one\n\nWhy one.\n') - _add(repo, 'fix(b): two\n') - - assert _compose() == dedent("""\ - chore(ci): stitch the commits (#7) - - * feat(a): one - - Why one. - - * fix(b): two - """) - - -def test_squashing_a_squash_flattens_its_bullets(repo): - """ - Squashing is associative: the inner umbrella subject stays as a redundant - bullet, and everything under it keeps the level a first-hand squash gave it. - """ - _add(repo, 'chore: base\n') - _add(repo, 'chore(ci): landed earlier (#3)\n\n* feat(a): b\n* feat(a): c\n') - _add(repo, 'feat(d): d\n\n* feat(d): e\n* feat(d): f\n') - - assert _compose() == dedent("""\ - chore(ci): stitch the commits (#7) - - * chore(ci): landed earlier (#3) - * feat(a): b - * feat(a): c - * feat(d): d - * feat(d): e - * feat(d): f - """) - - -def test_prose_keeps_its_blank_lines_among_the_bullets(repo): - _add(repo, 'chore: base\n') - _add(repo, 'feat(a): one\n\n* feat(a): nested\nwhy nested.\n') - _add(repo, 'fix(b): two\n') - - assert _compose() == dedent("""\ - chore(ci): stitch the commits (#7) - - * feat(a): one - * feat(a): nested - - why nested. - - * fix(b): two - """) - - -def test_a_body_keeps_its_own_paragraph_breaks(repo): - _add(repo, 'chore: base\n') - _add(repo, 'feat(a): one\n\nwhat it does.\n\nwhy it does it.\n') - _add(repo, 'fix(b): two\n') - - assert _compose() == dedent("""\ - chore(ci): stitch the commits (#7) - - * feat(a): one - - what it does. - - why it does it. - - * fix(b): two - """) - - -def test_a_lone_commit_titled_like_the_pr_keeps_its_body(repo): - _add(repo, 'chore: base\n') - _add(repo, f'{PR["title"]}\n\nThe only rationale there is.\n') - - assert _compose('HEAD~1') == dedent("""\ - chore(ci): stitch the commits (#7) - - The only rationale there is. - """) - - -def test_coauthors_follow_the_last_body(repo): - _add(repo, 'chore: base\n') - _add(repo, 'feat(a): one\n\nWhy one.\n', author=OTHER) - - assert _compose('HEAD~1') == dedent("""\ - chore(ci): stitch the commits (#7) - - * feat(a): one - - Why one. - - Co-authored-by: Other Author <other@example.com> - """) - - -def test_the_composed_message_passes_the_commit_hook(repo): - """ - Nothing re-validates a generated message, so the hook is checked here instead. - """ - _add(repo, 'chore: base\n') - _add(repo, 'feat(a): one ✨\n\n* fix(a): nested ♻️\nWhy one, at length.\n') - _add(repo, 'fix(b): two πŸ›\n') - - result = subprocess.run( - [sys.executable, str(CHECK_MESSAGE), '--message-text', _compose()], - capture_output=True, - text=True, - ) - - assert result.returncode == 0, result.stdout - - -def test_release_notes_promote_bullets_only(repo): - _add(repo, 'chore: base\n') - _add( - repo, - dedent("""\ - chore(ci): umbrella (#7) - - * feat(a): one - - fix: prose that merely reads like a subject - """), - ) - - entries = notes.parse_git_log('HEAD~1..HEAD', dir=repo) - - assert [e.message for e in entries] == ['feat(a): one'] - - -def test_a_malformed_bullet_is_an_error_not_body(repo): - """ - The format rests on `*` meaning bullet, so a `*` line that is not a subject - cannot be quietly kept as prose -- it would flatten into a broken entry. - """ - message = dedent("""\ - chore(ci): umbrella (#7) - - * feat(a): one - * just a note - """) - - result = subprocess.run( - [sys.executable, str(CHECK_MESSAGE), '--message-text', message], - capture_output=True, - text=True, - ) - - assert result.returncode == 1 - assert 'just a note' in result.stdout - - -def test_a_control_character_is_rejected(repo): - """ - `commit_entries` reads a range as lines, so a message may only be made of - those; a control character elsewhere in it is invisible where it is written. - """ - result = subprocess.run( - [ - sys.executable, - str(CHECK_MESSAGE), - '--message-text', - 'fix(a): before\x01after\n\nA body long enough to pass the length rule.\n', - ], - capture_output=True, - text=True, - ) - - assert result.returncode == 1 - assert 'Control character' in result.stdout - - -def test_an_emoji_sequence_is_not_a_control_character(repo): - _add(repo, 'chore: base\n') - _add(repo, 'feat(a): one πŸ‘¨β€πŸ’»\n') - - result = subprocess.run( - [sys.executable, str(CHECK_MESSAGE), '--message-text', _compose('HEAD~1')], - capture_output=True, - text=True, - ) - - assert result.returncode == 0, result.stdout diff --git a/support/ci/unit_tests/test_full_tests_command.py b/support/ci/unit_tests/test_full_tests_command.py new file mode 100644 index 00000000..7bb7d7e7 --- /dev/null +++ b/support/ci/unit_tests/test_full_tests_command.py @@ -0,0 +1,280 @@ +import json +from types import SimpleNamespace + +import gh_common +import pytest +from tools import full_tests_command as command +from tools import pr_action_panel as panel + + +def comment_deleted(*_args): + raise RuntimeError('comment deleted') + + +@pytest.fixture(autouse=True) +def context(): + gh_common.set_ctx( + gh_common.Ctx( + manager_repo='org/manager', + executor_repo='org/executor', + _pr_number='42', + _head_ref=None, + ) + ) + + +def test_start_dispatches_without_changing_sticky_label(monkeypatch): + monkeypatch.setenv('COMMENT_BODY', '/genvm-run-tests') + monkeypatch.setenv('COMMENT_ID', '9001') + monkeypatch.setenv('SENDER', 'maintainer') + monkeypatch.setenv('REQUEST_ID', '700-1') + monkeypatch.setattr(gh_common, 'has_write_access', lambda _sender: True) + monkeypatch.setattr( + command, + 'labels', + lambda: {'ci-safe', 'run-full-tests', 'test-release-pipeline'}, + ) + monkeypatch.setattr( + command, + 'pull_request', + lambda: { + 'state': 'open', + 'head': { + 'ref': 'feat/change', + 'sha': 'a' * 40, + 'repo': {'full_name': 'org/manager'}, + }, + }, + ) + monkeypatch.setattr(command, 'add_reaction', lambda *_args: '77') + + dispatched = [] + monkeypatch.setattr( + command, + 'dispatch', + lambda head, **kwargs: dispatched.append((head, kwargs)), + ) + run = { + 'databaseId': 123, + 'headSha': 'b' * 40, + 'url': 'https://example.test/run/123', + } + monkeypatch.setattr(command, 'wait_for_run', lambda *_args: run) + statuses = [] + monkeypatch.setattr( + command, + 'post_running_status', + lambda *args: statuses.append(args), + ) + + assert command.start() == 0 + assert dispatched == [ + ( + 'feat/change', + { + 'expected_sha': 'a' * 40, + 'release_pipeline_test': True, + 'request': '9001', + 'request_id': '700-1', + }, + ) + ] + assert statuses == [(run, '9001', '77', 'b' * 40)] + + +def test_start_rejects_untrusted_actor(monkeypatch): + monkeypatch.setenv('COMMENT_BODY', '/genvm-run-tests') + monkeypatch.setenv('SENDER', 'stranger') + monkeypatch.setattr(gh_common, 'has_write_access', lambda _sender: False) + + with pytest.raises(SystemExit, match='no write access'): + command.start() + + +def test_start_reporting_failure_does_not_fail_dispatched_tests(monkeypatch): + monkeypatch.setenv('COMMENT_BODY', '/genvm-run-tests') + monkeypatch.setenv('COMMENT_ID', '9001') + monkeypatch.setenv('SENDER', 'maintainer') + monkeypatch.setenv('REQUEST_ID', '700-1') + monkeypatch.setattr(gh_common, 'has_write_access', lambda _sender: True) + monkeypatch.setattr(command, 'labels', lambda: {'ci-safe'}) + monkeypatch.setattr( + command, + 'pull_request', + lambda: { + 'state': 'open', + 'head': { + 'ref': 'feat/change', + 'sha': 'a' * 40, + 'repo': {'full_name': 'org/manager'}, + }, + }, + ) + monkeypatch.setattr(command, 'add_reaction', lambda *_args: '77') + dispatched = [] + monkeypatch.setattr( + command, + 'dispatch', + lambda *_args, **_kwargs: dispatched.append(True), + ) + monkeypatch.setattr( + command, + 'wait_for_run', + comment_deleted, + ) + monkeypatch.setattr(command, 'delete_reaction', lambda *_args: None) + + assert command.start() == 0 + assert dispatched == [True] + + +def test_delete_reaction_uses_issue_comment_endpoint(monkeypatch): + calls = [] + monkeypatch.setattr( + command, + 'gh', + lambda *args, **kwargs: calls.append((args, kwargs)) or SimpleNamespace(stdout=''), + ) + + command.delete_reaction('9001', '77') + + assert 'repos/org/manager/issues/comments/9001/reactions/77' in calls[0][0] + + +def test_force_panel_still_sets_label_and_dispatches(monkeypatch): + monkeypatch.setenv('COMMENT_ID', '8001') + monkeypatch.setenv('SENDER', 'maintainer') + monkeypatch.setattr(panel, 'labels', lambda: {'ci-safe'}) + monkeypatch.setattr(gh_common, 'has_write_access', lambda _sender: True) + writes = [] + + def gh(*args, **_kwargs): + if args[:2] == ('api', 'repos/org/manager/issues/comments/8001'): + return """{ + "body": "<!-- genvm-actions -->\\n- [x] Force run full tests <!-- action:force -->", + "author": "github-actions[bot]" + }""" + writes.append(args) + return '' + + monkeypatch.setattr(panel, 'gh', gh) + dispatches = [] + monkeypatch.setattr( + panel, + 'dispatch_full_tests', + lambda **kwargs: dispatches.append(kwargs), + ) + + panel.run_panel() + + assert any('labels[]=run-full-tests' in call for call in writes) + assert dispatches == [{'release_pipeline_test': False}] + + +@pytest.mark.parametrize( + ('conclusion', 'reaction', 'message'), + [ + ('success', 'rocket', 'πŸš€ Full tests passed'), + ('failure', 'confused', '⚠️ Full tests failure'), + ('cancelled', 'confused', '⚠️ Full tests cancelled'), + ], +) +def test_complete_updates_status_and_reaction( + monkeypatch, conclusion, reaction, message +): + monkeypatch.setenv('RUN_ID', '123') + monkeypatch.setenv('RUN_NAME', 'GenVM full [request=700-1;comment=9001;pr=42]') + monkeypatch.setenv('RUN_URL', 'https://example.test/run/123') + monkeypatch.setenv('RUN_SHA', 'a' * 40) + monkeypatch.setenv('RUN_CONCLUSION', conclusion) + monkeypatch.setattr( + command, + 'status_comment', + lambda _issue, _run, _request: { + 'id': 55, + 'body': '<!-- genvm-full-tests run=123 request=9001 eyes=77 -->\nold', + }, + ) + deleted = [] + added = [] + monkeypatch.setattr( + command, + 'delete_reaction', + lambda comment, value: deleted.append((comment, value)), + ) + monkeypatch.setattr( + command, 'add_reaction', lambda comment, value: added.append((comment, value)) + ) + calls = [] + monkeypatch.setattr( + command, + 'gh', + lambda *args, **kwargs: calls.append((args, kwargs)) or SimpleNamespace(stdout=''), + ) + + assert command.complete() == 0 + assert deleted == [('9001', '77')] + assert added == [('9001', reaction)] + assert message in calls[-1][0][-1] + assert 'https://example.test/run/123' in calls[-1][0][-1] + + +def test_complete_deleted_comment_does_not_fail_ci(monkeypatch): + monkeypatch.setenv('RUN_ID', '123') + monkeypatch.setenv('RUN_NAME', 'GenVM full [request=700-1;comment=9001;pr=42]') + monkeypatch.setenv('RUN_CONCLUSION', 'failure') + monkeypatch.setattr( + command, + 'wait_for_status_comment', + comment_deleted, + ) + added = [] + monkeypatch.setattr(command, 'add_reaction', lambda *args: added.append(args)) + + assert command.complete() == 0 + assert added == [] + + +def test_find_run_matches_unique_request(monkeypatch): + monkeypatch.setattr( + command, + 'gh', + lambda *_args, **_kwargs: SimpleNamespace( + stdout="""[ + {"databaseId": 1, "displayTitle": "other", "headSha": "aaa", "url": "x"}, + {"databaseId": 2, "displayTitle": "full [request=700-1;comment=9;pr=42]", "headSha": "bbb", "url": "y"} + ]""" + ), + ) + + assert command.find_run('700-1')['databaseId'] == 2 + assert command.find_run('701-1') is None + + +def test_status_comment_ignores_forged_marker(monkeypatch): + comments = [ + { + 'id': 1, + 'body': '<!-- genvm-full-tests run=123 request=9 eyes=7 -->', + 'user': {'login': 'attacker'}, + }, + { + 'id': 2, + 'body': '<!-- genvm-full-tests run=123 request=8 eyes=7 -->', + 'user': {'login': 'github-actions[bot]'}, + }, + { + 'id': 3, + 'body': '<!-- genvm-full-tests run=123 request=9 eyes=7 -->', + 'user': {'login': 'github-actions[bot]'}, + }, + ] + monkeypatch.setattr( + command, + 'gh', + lambda *_args, **_kwargs: SimpleNamespace( + stdout='\n'.join(json.dumps(comment) for comment in comments) + ), + ) + + assert command.status_comment('42', '123', '9')['id'] == 3 diff --git a/support/ci/unit_tests/test_merge_gates.py b/support/ci/unit_tests/test_merge_gates.py deleted file mode 100644 index a97166e1..00000000 --- a/support/ci/unit_tests/test_merge_gates.py +++ /dev/null @@ -1,46 +0,0 @@ -from types import SimpleNamespace - -import pytest -from tools import genvm_merge_into_dev as merge - - -class MergeBlocked(Exception): - pass - - -def test_force_merge_still_requires_rebased(monkeypatch): - head_sha = 'a' * 40 - pr = { - 'baseRefName': 'v0.6-dev', - 'headRefOid': head_sha, - 'isDraft': False, - 'state': 'OPEN', - 'title': 'fix(ci): keep the rebase gate', - } - monkeypatch.setenv('FORCE_ACTOR', 'repo-admin') - monkeypatch.setattr(merge, 'pr_view', lambda *fields: pr) - monkeypatch.setattr(merge, 'pr_number', lambda: '7') - monkeypatch.setattr(merge, 'check_title', lambda value: None) - monkeypatch.setattr(merge, 'announce_force', lambda actor, sha: None) - - def git(*args, **kwargs): - if args[0] == 'fetch': - return SimpleNamespace(stdout='') - if args == ('rev-parse', 'refs/prhead'): - return SimpleNamespace(stdout=f'{head_sha}\n') - raise AssertionError(f'unexpected git call after rebase gate: {args}') - - def behind(base, head): - assert base == 'origin/v0.6-dev' - assert head == 'refs/prhead' - return 2 - - def block(message): - raise MergeBlocked(message) - - monkeypatch.setattr(merge, 'git', git) - monkeypatch.setattr(merge.behind_lib, 'behind_by_git', behind) - monkeypatch.setattr(merge, 'block', block) - - with pytest.raises(MergeBlocked, match='2 commit.* behind'): - merge.merge_into_dev() diff --git a/support/ci/unit_tests/test_pr_branches_info.py b/support/ci/unit_tests/test_pr_branches_info.py new file mode 100644 index 00000000..accdc4d1 --- /dev/null +++ b/support/ci/unit_tests/test_pr_branches_info.py @@ -0,0 +1,30 @@ +import pytest + +import pr_branches_info as info + + +@pytest.mark.parametrize('available', [False, True]) +def test_new_executor_line_requires_remote_commit(monkeypatch, available): + sha = 'a' * 40 + + def submodule_sha(_repo, _path, ref, _token): + return None if ref == 'manager-base' else sha + + monkeypatch.setattr(info, 'submodule_sha', submodule_sha) + monkeypatch.setattr(info, 'commit_exists', lambda *_args: available) + monkeypatch.setattr(info, 'existing_pr', lambda *_args: None) + + result = info.executor_info( + 'v0.4', + 'org/manager', + 'manager-base', + 'manager-head', + 'feature', + 'org/executor', + 'manager-token', + 'executor-token', + ) + + assert result.base_sha is None + assert result.head_sha == sha + assert result.synced is available diff --git a/support/ci/unit_tests/test_sync_executor_branches.py b/support/ci/unit_tests/test_sync_executor_branches.py new file mode 100644 index 00000000..ad92b7ec --- /dev/null +++ b/support/ci/unit_tests/test_sync_executor_branches.py @@ -0,0 +1,61 @@ +from types import SimpleNamespace + +from tools import sync_executor_branches as sync + + +def test_target_branch_uses_declared_release_branch(): + assert sync.target_branch('v0.6', 'v0.3.x') == 'v0.3.x' + + +def test_target_branch_derives_dev_branch(): + assert sync.target_branch('v0.6-dev', 'v0.3.x') == 'v0.3-dev' + + +def test_sync_line_pushes_without_force(monkeypatch): + commands = [] + + def git(*args, check=True): + commands.append((args, check)) + if args[:4] == ('config', '-f', '.gitmodules', '--get'): + return SimpleNamespace(returncode=0, stdout='v0.3.x\n', stderr='') + if args[:3] == ('-C', 'executors/v0.3.x', 'rev-parse'): + return SimpleNamespace(returncode=0, stdout=f'{"a" * 40}\n', stderr='') + return SimpleNamespace(returncode=0, stdout='', stderr='') + + monkeypatch.setattr(sync, 'git', git) + target, sha = sync.sync_line('v0.3', 'v0.6-dev') + + assert target == 'v0.3-dev' + assert sha == 'a' * 40 + push = commands[-1][0] + assert push == ( + '-C', + 'executors/v0.3.x', + 'push', + 'origin', + f'{"a" * 40}:refs/heads/v0.3-dev', + ) + assert '--force' not in push + + +def test_all_lines_are_attempted_before_failures_are_reported(monkeypatch, capsys): + attempted = [] + + def sync_line(line, manager_branch): + attempted.append((line, manager_branch)) + if line == 'v0.2': + raise sync.SyncError('non-fast-forward') + return f'{line}-dev', line * 8 + + monkeypatch.setattr(sync, 'active_lines', lambda: ['v0.2', 'v0.3', 'v0.4']) + monkeypatch.setattr(sync, 'sync_line', sync_line) + + assert sync.sync_executor_branches('v0.6-dev') == 1 + assert attempted == [ + ('v0.2', 'v0.6-dev'), + ('v0.3', 'v0.6-dev'), + ('v0.4', 'v0.6-dev'), + ] + output = capsys.readouterr().out + assert '1 executor branch update(s) failed' in output + assert '- v0.2: non-fast-forward' in output diff --git a/support/scripts/check-commit-message.py b/support/scripts/check-commit-message.py index 3d8ffa39..99077823 100755 --- a/support/scripts/check-commit-message.py +++ b/support/scripts/check-commit-message.py @@ -54,9 +54,9 @@ # A trailing PR reference, either bare (`(#42)`) or repo-qualified # (`(genlayerlabs/genvm-manager#42)`). The qualified form is what a squash -# landing in the EXECUTOR repo carries, because a bare `#42` would not resolve -# there β€” see genvm_merge_into_dev.pr_reference. Both must be recognised, or a -# perfectly good subject is read as having its emoji in the middle. +# historical executor squash landings carry, because a bare `#42` would not +# resolve there. Both must be recognised, or a perfectly good subject is read +# as having its emoji in the middle. PR_REF_RE = re.compile(r'\s*\((?:[\w.-]+/[\w.-]+)?#\d+\)\s*$') SQUASH_BULLET_RE = re.compile(r'^\*\s*') diff --git a/support/tools/genvm-tool/genvm_tool/cmd_docs.py b/support/tools/genvm-tool/genvm_tool/cmd_docs.py index 6b034e7f..238b326b 100644 --- a/support/tools/genvm-tool/genvm_tool/cmd_docs.py +++ b/support/tools/genvm-tool/genvm_tool/cmd_docs.py @@ -6,6 +6,10 @@ a grouped markdown listing. Exits non-zero if a page is missing from its index, or an index lists a page that no longer exists, so CI can gate on the indexes staying in sync. + +`--write` splices the same listing into `AGENTS.md` below its generated marker, and +`--check` verifies that copy is current without touching it β€” the listing is what +every agent session loads, so a stale copy is a wrong answer, not a cosmetic one. """ import re @@ -18,6 +22,10 @@ HELP = 'render the contributor documentation as a skill-like index' CONTRIBUTING_DIR = Path('docs/contributing') +# The agent-instructions file and the marker below which its listing is generated. +# Everything above the marker is hand-maintained and never rewritten. +AGENTS_FILE = Path('AGENTS.md') +AGENTS_MARKER = '<!-- below is generated with `genvm-tool docs` -->' # Rendered in this order: learn it, do it, understand it. QUADRANTS = ['tutorial', 'howto', 'explanation'] # Deepest directory nesting promoted to a `## header`; anything below stays in the @@ -29,7 +37,15 @@ def configure(parser): - pass + mode = parser.add_mutually_exclusive_group() + mode.add_argument( + '--write', action='store_true', help=f'update the listing in {AGENTS_FILE}' + ) + mode.add_argument( + '--check', + action='store_true', + help=f'fail if the listing in {AGENTS_FILE} is stale', + ) def _parse_index(readme: Path) -> dict[str, str]: @@ -88,9 +104,10 @@ def _section_title(quadrant: str, header: str) -> str: return '/'.join(f'{p[:1].upper()}{p[1:]}' for p in '/'.join(parts).split('/')) -def main(ctx: common.Context, args) -> int: - problems = [] - started = False +def _listing(ctx: common.Context) -> tuple[str, list[str]]: + """The rendered markdown listing, plus every index-vs-disk complaint.""" + problems: list[str] = [] + lines: list[str] = [] for quadrant in QUADRANTS: base = ctx.root / CONTRIBUTING_DIR / quadrant if not base.is_dir(): @@ -104,11 +121,43 @@ def main(ctx: common.Context, args) -> int: for header, link, repo_path, desc in sorted(rows): if header != current: current = header - if started: - print() # blank line between sections, but not a leading one - print(f'## {_section_title(quadrant, header)}') - print(f'- [{link}]({repo_path}) β€” {desc}') - started = True + if lines: + lines.append('') # blank line between sections, but not a leading one + lines.append(f'## {_section_title(quadrant, header)}') + lines.append(f'- [{link}]({repo_path}) β€” {desc}') + + return '\n'.join(lines) + '\n', problems + + +def _spliced(ctx: common.Context, listing: str) -> tuple[Path, str, str]: + """`AGENTS.md` as it is on disk and as it should be, with the path.""" + path = ctx.root / AGENTS_FILE + if not path.is_file(): + raise common.ToolError(f'not found: {AGENTS_FILE}') + current = path.read_text() + head, marker, _ = current.partition(AGENTS_MARKER) + if not marker: + raise common.ToolError(f'{AGENTS_FILE} has no marker line: {AGENTS_MARKER}') + return path, current, f'{head}{marker}\n\n{listing}' + + +def main(ctx: common.Context, args) -> int: + listing, problems = _listing(ctx) + + # A stale listing is reported, never written: splicing a listing built from + # indexes that disagree with the tree would commit the disagreement. + if not problems and (args.write or args.check): + path, current, wanted = _spliced(ctx, listing) + rel = path.relative_to(ctx.root).as_posix() + if current == wanted: + print(f'{rel} is up to date') + elif args.write: + path.write_text(wanted) + print(f'{rel} updated') + else: + problems.append(f'{rel}: listing is stale, run `genvm-tool docs --write`') + elif not (args.write or args.check): + print(listing, end='') for problem in problems: print(problem, file=sys.stderr) diff --git a/support/tools/genvm-tool/genvm_tool/common.py b/support/tools/genvm-tool/genvm_tool/common.py index 9b81b6d0..3cd76ba3 100644 --- a/support/tools/genvm-tool/genvm_tool/common.py +++ b/support/tools/genvm-tool/genvm_tool/common.py @@ -147,9 +147,9 @@ def executor_rel(version: str) -> str: return f'executors/v{_bare(version)}.x' -# Release/branch-model branches (`main`, a version branch `v0.3.x`, or a dev -# branch `v0.3-dev`) β€” already line-scoped, so they are never feature-namespaced. -_MODEL_BRANCH = re.compile(r'main|v[\d.]+\.x|v[\d.]+-dev') +# Release/branch-model branches (`main`, a manager branch `v0.6`, an executor +# branch `v0.3.x`, or a dev branch) are never feature-namespaced. +_MODEL_BRANCH = re.compile(r'main|v\d+\.\d+(?:\.x|-dev)?') class Repo: @@ -185,9 +185,9 @@ def feature_branch(self, name: str) -> str: would collide between two active lines that branch off it at once. So an executor feature branch is namespaced under `pr/<line>/` (e.g. `pr/v0.3/<name>`) β€” the `<line>` segment is what keeps the two lines - distinct, matching how the release branches (`v0.3-dev`, `v0.3.x`) are - already line-scoped and thus never clash. Release-model branches and an - already-prefixed `name` pass through unchanged (idempotent). + distinct, matching how executor release branches (`v0.3-dev`, `v0.3.x`) + are already line-scoped and thus never clash. Branch-model branches and + an already-prefixed `name` pass through unchanged (idempotent). """ line = self.line if line is None or name.startswith(f'pr/{line}/') or _MODEL_BRANCH.fullmatch(name): diff --git a/tests/runner/genvm_tool_plugins/cargo.py b/tests/runner/genvm_tool_plugins/cargo.py index 8e291806..b79c1985 100644 --- a/tests/runner/genvm_tool_plugins/cargo.py +++ b/tests/runner/genvm_tool_plugins/cargo.py @@ -1,6 +1,5 @@ import json import os -import re import shutil import subprocess import sys @@ -8,8 +7,8 @@ import genvm_tool.tests import genvm_tool_plugins.afl as afl +import genvm_tool_plugins.source_tags as source_tags import tomllib -from genvm_tool.tests import tags as test_tags local_ctx = genvm_tool.tests.stage.configuration.current_context() @@ -256,7 +255,7 @@ def _load_cargo_config( raise ValueError( f'{extra_config}: tags for {case_name!r} must be a list of strings' ) - case_tags[case_name] = _validate_declared_tags(ctx, extra_config, raw_tags) + case_tags[case_name] = source_tags.validate_declared(ctx, extra_config, raw_tags) return extra_conf, test_env, extra_flags, case_tags @@ -287,50 +286,6 @@ def _add_cargo_case( ctx.add_case(case) -_TAGS_MARKER = 'genvm-tool-test-tags:' -_TAGS_LINE = re.compile(r'^\s*//[/!]?\s*genvm-tool-test-tags:\s*(?P<tags>.*?)\s*$') - - -def _tag_source_name(ctx: genvm_tool.tests.stage.collection.Context, path: Path) -> str: - try: - return str(path.relative_to(ctx.shared.root_dir)) - except ValueError: - return str(path) - - -def _validate_declared_tags( - ctx: genvm_tool.tests.stage.collection.Context, path: Path, tags: list[str] -) -> list[str]: - declared = frozenset(tags) - registry = ctx.shared.tags_registry - if registry is None: - return sorted(declared) - invalid = test_tags.unknown(registry, declared) - if invalid: - ctx.add_tag_offender(_tag_source_name(ctx, path), invalid) - return sorted(declared - invalid) - - -def _tags_from_source( - ctx: genvm_tool.tests.stage.collection.Context, path: Path -) -> list[str]: - """Extra tags declared by a `// genvm-tool-test-tags: a,b,c` comment.""" - tags: list[str] = [] - for line_number, line in enumerate(path.read_text().splitlines(), 1): - if not line.lstrip().startswith('//') or _TAGS_MARKER not in line: - continue - match = _TAGS_LINE.fullmatch(line) - if match is None: - ctx.add_tag_declaration_error( - f'{_tag_source_name(ctx, path)}:{line_number}', - f'malformed {_TAGS_MARKER} marker', - ) - continue - line_tags = [tag.strip() for tag in match.group('tags').split(',') if tag.strip()] - tags.extend(line_tags) - return _validate_declared_tags(ctx, path, tags) - - def _is_skipped(skip_conf, key: str, name: str | None = None) -> bool: """ Check if a test category or specific name is skipped. @@ -419,7 +374,8 @@ def cargo_test( + extra_flags, rust_root_dir=rust_root_dir, test_env=test_env, - tags=['rust', 'example'] + _tags_from_source(ctx, rust_root_dir / ex_path), + tags=['rust', 'example'] + + source_tags.from_source(ctx, rust_root_dir / ex_path), ) # 2. Test files in tests/ @@ -435,7 +391,7 @@ def cargo_test( command=base_cmd + ['--test', test_name], rust_root_dir=rust_root_dir, test_env=test_env, - tags=['rust', 'unit'] + _tags_from_source(ctx, test_file), + tags=['rust', 'unit'] + source_tags.from_source(ctx, test_file), ) # 3. --lib test @@ -474,7 +430,7 @@ def cargo_fuzz( ): desc = desc.with_tags( ['rust', 'fuzz', 'needs-fuzz'] - + _tags_from_source(ctx, rust_root_dir / 'fuzz' / f'{name}.rs') + + source_tags.from_source(ctx, rust_root_dir / 'fuzz' / f'{name}.rs') )._replace( **afl.pool_usage(ctx), ) diff --git a/tests/runner/genvm_tool_plugins/npm.py b/tests/runner/genvm_tool_plugins/npm.py new file mode 100644 index 00000000..e6930a91 --- /dev/null +++ b/tests/runner/genvm_tool_plugins/npm.py @@ -0,0 +1,107 @@ +""" +npm projects: one case per `*.test.ts` file, plus the install and typecheck +they share. + +Per-file cases mean a failure names the file and `--filter-name` can select +one, matching how Rust integration tests collect. What they cannot each do is +install: `node_modules` is one directory per project, so `npm ci` is its own +case and every other case in the project `depends_on` it. A dependency is +pulled in even when the filter excludes it, and its failure skips the +dependents rather than letting them fail confusingly. +""" + +import json +import os +from pathlib import Path + +import genvm_tool.tests +import genvm_tool_plugins.source_tags as source_tags + +default_env = { + k: v + for k, v in os.environ.items() + if genvm_tool.tests.util.environ.DEFAULT_FILTER(k, v) +} + + +def scripts(package_json: Path) -> dict: + """ + A `package.json`'s `scripts`, or empty when it has none or is unreadable. + + Unreadable is not an error here: collection walks every tracked + `package.json`, and one that cannot be parsed simply declares no cases. + """ + try: + found = json.loads(package_json.read_text()).get('scripts', {}) + except (OSError, json.JSONDecodeError): + return {} + return found if isinstance(found, dict) else {} + + +def _case( + ctx: genvm_tool.tests.stage.collection.Context, + desc: genvm_tool.tests.test.Description, + *, + project_root_dir: Path, + command: list[str], +): + ctx.add_case( + genvm_tool.tests.test.SimpleCommandCase( + description=desc, + command=command, + cwd=project_root_dir, + env=default_env, + mode=genvm_tool.tests.exec.command.RunMode.INTERACTIVE, + ) + ) + + +def npm_project( + ctx: genvm_tool.tests.stage.collection.Context, + *, + project_root_dir: Path, + test_files: list[Path], +): + """ + Collect one project: its install, its typecheck, and one case per test file. + + `needs-web` is not incidental -- `npm ci` reaches the registry, so a cell + without egress can only fail. The dependents carry it too, since they + cannot run without what it installs. + """ + rel_dir = project_root_dir.relative_to(ctx.shared.root_dir) + base_tags = ['typescript', 'needs-web'] + + install_name = f'{rel_dir}/npm-ci' + _case( + ctx, + genvm_tool.tests.test.Description(install_name).with_tags(base_tags), + project_root_dir=project_root_dir, + command=['npm', 'ci'], + ) + + shared_desc = genvm_tool.tests.test.Description('').with_tags(base_tags + ['unit']) + shared_desc = shared_desc.with_depends_on([install_name]) + + if 'typecheck' in scripts(project_root_dir / 'package.json'): + _case( + ctx, + shared_desc._replace(name=f'{rel_dir}/typecheck'), + project_root_dir=project_root_dir, + command=['npm', 'run', 'typecheck'], + ) + + for test_file in test_files: + rel_file = test_file.relative_to(ctx.shared.root_dir) + _case( + ctx, + shared_desc._replace(name=str(rel_file)).with_tags( + source_tags.from_source(ctx, test_file) + ), + project_root_dir=project_root_dir, + # `npm test` forwards what follows `--`, and the project's runner + # takes a file to restrict itself to. Going through the script + # rather than spelling the runner here keeps the loader flags in + # `package.json`, where a human running one file also finds them. + command=['npm', 'test', '--', str(test_file.relative_to(project_root_dir))], + ) diff --git a/tests/runner/genvm_tool_plugins/source_tags.py b/tests/runner/genvm_tool_plugins/source_tags.py new file mode 100644 index 00000000..d1c6e72f --- /dev/null +++ b/tests/runner/genvm_tool_plugins/source_tags.py @@ -0,0 +1,55 @@ +""" +Tags a source file declares about its own case, as a `//` comment. + +Shared by every plugin whose language spells comments that way, so `.rs` and +`.ts` cases declare tags identically and a typo is reported the same way. +""" + +import re +from pathlib import Path + +import genvm_tool.tests +from genvm_tool.tests import tags as test_tags + +MARKER = 'genvm-tool-test-tags:' +_LINE = re.compile(r'^\s*//[/!]?\s*genvm-tool-test-tags:\s*(?P<tags>.*?)\s*$') + + +def source_name(ctx: genvm_tool.tests.stage.collection.Context, path: Path) -> str: + try: + return str(path.relative_to(ctx.shared.root_dir)) + except ValueError: + return str(path) + + +def validate_declared( + ctx: genvm_tool.tests.stage.collection.Context, path: Path, tags: list[str] +) -> list[str]: + declared = frozenset(tags) + registry = ctx.shared.tags_registry + if registry is None: + return sorted(declared) + invalid = test_tags.unknown(registry, declared) + if invalid: + ctx.add_tag_offender(source_name(ctx, path), invalid) + return sorted(declared - invalid) + + +def from_source( + ctx: genvm_tool.tests.stage.collection.Context, path: Path +) -> list[str]: + """Extra tags declared by a `// genvm-tool-test-tags: a,b,c` comment.""" + tags: list[str] = [] + for line_number, line in enumerate(path.read_text().splitlines(), 1): + if not line.lstrip().startswith('//') or MARKER not in line: + continue + match = _LINE.fullmatch(line) + if match is None: + ctx.add_tag_declaration_error( + f'{source_name(ctx, path)}:{line_number}', + f'malformed {MARKER} marker', + ) + continue + line_tags = [tag.strip() for tag in match.group('tags').split(',') if tag.strip()] + tags.extend(line_tags) + return validate_declared(ctx, path, tags) diff --git a/tests/tags.json b/tests/tags.json index 525f7c29..78f75d01 100644 --- a/tests/tags.json +++ b/tests/tags.json @@ -7,6 +7,7 @@ "bench": "Benchmark, measured rather than asserted", "rust": "Rust code under test", "python": "Python code under test", + "typescript": "TypeScript code under test", "bash": "Shell code under test", "wasm": "WebAssembly artifact under test", "smoke": "Manually selected fast representative of a feature subtree", diff --git a/webdriver/src/prj/package.json b/webdriver/src/prj/package.json index 5bf4ff8e..7311501a 100644 --- a/webdriver/src/prj/package.json +++ b/webdriver/src/prj/package.json @@ -8,7 +8,7 @@ "start": "node --loader ts-node/esm src/index.ts", "dev": "ts-node src/index.ts", "build": "tsc", - "test": "node --import ./ts-node-register.mjs --test src/**/*.test.ts", + "test": "node --import ./ts-node-register.mjs --test", "typecheck": "tsc --noEmit" }, "dependencies": { diff --git a/webdriver/src/prj/src/browser/chrome.ts b/webdriver/src/prj/src/browser/chrome.ts index 6d618df9..e56f0bd4 100644 --- a/webdriver/src/prj/src/browser/chrome.ts +++ b/webdriver/src/prj/src/browser/chrome.ts @@ -74,6 +74,20 @@ const CHROME_HEADLESS = envBool('GVM_WEBDRIVER_CHROME_HEADLESS', true); // Extra flags appended to the defaults below. See `envStrList` for the format. const CHROME_EXTRA_ARGS = envStrList('GVM_WEBDRIVER_CHROME_ARGS', []); +/** + * Ceiling on a single CDP round-trip. Puppeteer's own default is 180s, which + * sits above the whole render budget (a 30s navigation plus at most 60s of + * `waitAfterLoaded`), so a wedged command used to be noticed long after the + * render should have finished. It must also stay below the module's HTTP + * deadline (`base_client_builder` in `implementation/src/common/mod.rs`, 300s) + * so that a stuck browser produces an answer from us rather than a client-side + * timeout with nothing to report. + */ +const PROTOCOL_TIMEOUT_MS = envDurationMs( + 'GVM_WEBDRIVER_PROTOCOL_TIMEOUT', + '90s', +); + async function newBrowser(): Promise<BrowserHolder> { const args = [ '--no-sandbox', @@ -82,6 +96,9 @@ async function newBrowser(): Promise<BrowserHolder> { '--no-first-run', '--no-zygote', '--disable-gpu', + // No popup can be SSRF-guarded before its first navigation, so none is + // allowed to exist -- see the `targetcreated` handler in `render.ts`. + '--block-new-web-contents', '--enable-precise-memory-info', `--js-flags=--max-old-space-size=${RENDERER_MAX_OLD_SPACE_MB}`, `--renderer-process-limit=${RENDERER_PROCESS_LIMIT}`, @@ -91,6 +108,7 @@ async function newBrowser(): Promise<BrowserHolder> { headless: CHROME_HEADLESS, args, executablePath: CHROME_EXECUTABLE, + protocolTimeout: PROTOCOL_TIMEOUT_MS, }); logger.log('info', 'created new raw browser', { @@ -153,9 +171,9 @@ class ChromeBrowserManager implements browser.Manager { * Arm the next rotation check. Uses a self-rescheduling timeout instead of * setInterval so the next check is only scheduled once the current one has * fully settled (in `finally`). This structurally prevents overlapping - * rotations from interleaving holder swaps / closes β€” which could otherwise + * rotations from interleaving holder swaps / closes -- which could otherwise * orphan a freshly launched browser (process leak) or close one still - * serving a render β€” and avoids interval backlog under sustained load. + * serving a render -- and avoids interval backlog under sustained load. */ private scheduleRotationCheck(): void { setTimeout(() => { diff --git a/webdriver/src/prj/src/index.ts b/webdriver/src/prj/src/index.ts index 540faaba..0bfed63f 100644 --- a/webdriver/src/prj/src/index.ts +++ b/webdriver/src/prj/src/index.ts @@ -1,15 +1,18 @@ -import puppeteer, * as pup from 'puppeteer-core'; +import type * as pup from 'puppeteer-core'; import http from 'http'; import { Command } from 'commander'; import * as logger from './logging.js'; import * as chromeBrowser from './browser/chrome.js'; -import * as ssrf from './ssrf.js'; +import { + renderPageWithBrowser, + statusIsGood, + type RenderOptions, +} from './render.js'; import { envDurationMs, envInt, envPositiveInt, - envSize, formatDurationMs, } from './duration.js'; import { @@ -19,19 +22,6 @@ import { SemaphoreTimeout, } from './semaphore.js'; -interface NavigationOptions { - waitUntil?: pup.PuppeteerLifeCycleEvent; - timeout?: number; -} - -interface RenderOptions { - loadTimeout?: number; - waitAfterLoaded?: number; - waitUntil?: pup.PuppeteerLifeCycleEvent; - maxPageHeapMB?: number; - maxResponseChars?: number; -} - const program = new Command(); program .name('puppeteer-webdriver') @@ -42,12 +32,8 @@ program const options = program.opts(); -const STATUS_I_AM_A_TEAPOT = 418; -const STATUS_INSUFFICIENT_STORAGE = 507; const STATUS_SERVICE_UNAVAILABLE = 503; -const DEFAULT_MAX_PAGE_HEAP_MB = envInt('GVM_WEBDRIVER_MAX_PAGE_HEAP_MB', 1024); - // peak_mem ~= MAX_CONCURRENT_RENDERS * (page heap + extracted response) + browser baseline // // we deem that by default we are limited by manager permits, @@ -57,11 +43,6 @@ const MAX_CONCURRENT_RENDERS = envPositiveInt( 128, ); -// Counted in characters, which is what bounds the string materialized in this -// process. One character encodes to at most 4 UTF-8 bytes on the wire, and -// occupies 2 bytes in V8 for the common (BMP) case. -const MAX_RESPONSE_CHARS = envSize('GVM_WEBDRIVER_MAX_RESPONSE', '5MiB'); - // Waiting callers are cheap but not free: each holds a live connection. Past // this depth the service is not going to work through the backlog before // callers give up anyway, so refusing immediately beats refusing slowly. @@ -80,11 +61,6 @@ const renderSlots = new Semaphore({ maxQueued: MAX_RENDER_QUEUE, }); -const MAX_WAIT_AFTER_LOADED_MS = envDurationMs( - 'GVM_WEBDRIVER_MAX_WAIT_AFTER_LOADED', - '60s', -); - const HEALTHCHECK_CACHE_DURATION_MS = envDurationMs( 'GVM_WEBDRIVER_HEALTHCHECK_CACHE_DURATION', '5m', @@ -95,203 +71,6 @@ function updateLastSuccessfulRenderTime() { lastSuccessfulRenderTime = Date.now(); } -function normalizeWhitespace(contents: string): string { - return contents - .split('\n') - .map((line) => line.trim().replace(/\s+/g, ' ')) - .join('\n') - .replace(/\n{2,}/g, '\n\n'); -} - -function getNavigationErrorStatus(error: any): number { - if (error.name === 'TimeoutError') { - return 408; // Request Timeout - } else if (error.message?.includes('net::ERR_NAME_NOT_RESOLVED')) { - return 502; // Bad Gateway - } else if (error.message?.includes('net::ERR_CONNECTION_REFUSED')) { - return 503; // Service Unavailable - } else if (error.message?.includes('net::ERR_CERT_')) { - return 495; // SSL Certificate Error - } else if (error.message?.includes('net::ERR_INTERNET_DISCONNECTED')) { - return 503; // Service Unavailable - } else if (error.message?.includes('net::ERR_BLOCKED_BY_CLIENT')) { - return 403; // Forbidden (SSRF guard) - } - return STATUS_I_AM_A_TEAPOT; // Unknown error -} - -function getNavigationErrorMessage(error: any): string { - if (error.name === 'TimeoutError') { - return 'Navigation timeout'; - } else if (error.message?.includes('net::ERR_NAME_NOT_RESOLVED')) { - return 'DNS resolution failed'; - } else if (error.message?.includes('net::ERR_CONNECTION_REFUSED')) { - return 'Connection refused'; - } else if (error.message?.includes('net::ERR_CERT_')) { - return 'SSL certificate error'; - } else if (error.message?.includes('net::ERR_INTERNET_DISCONNECTED')) { - return 'No internet connection'; - } else if (error.message?.includes('net::ERR_BLOCKED_BY_CLIENT')) { - return 'Blocked by SSRF guard: address not allowed'; - } - return `Navigation error: ${error.message || 'Unknown error'}`; -} - -async function navigateToPage( - page: pup.Page, - targetUrl: string, - options: NavigationOptions = {}, -): Promise<{ status: number; error?: string; response?: pup.HTTPResponse }> { - const { waitUntil = 'domcontentloaded', timeout = 30000 } = options; - - try { - const response = await page.goto(targetUrl, { - waitUntil, - timeout, - }); - - if (!response) { - return { - status: STATUS_I_AM_A_TEAPOT, - error: 'Navigation did not result in a valid HTTP response', - }; - } - - return { status: response.status(), response }; - } catch (navigationError: any) { - logger.log('error', 'navigation Error', navigationError); - const statusCode = getNavigationErrorStatus(navigationError); - const errorMessage = getNavigationErrorMessage(navigationError); - return { status: statusCode, error: errorMessage }; - } -} - -/** - * Read `innerText` or `innerHTML`, rejecting oversized documents *in the page*. - * - * The size test has to happen on the browser side. Pulling the string across - * first and measuring it here would already have paid the memory cost we are - * trying to avoid β€” this process would hold the whole document no matter what - * any downstream limit says. - */ -async function extractBounded( - page: pup.Page, - kind: 'innerText' | 'innerHTML', - maxChars: number, -): Promise<string> { - const extracted = await page.evaluate( - (k, limit) => { - const raw = - k === 'innerText' ? document.body.innerText : document.body.innerHTML; - return raw.length > limit - ? { tooLarge: true as const, length: raw.length } - : { tooLarge: false as const, content: raw }; - }, - kind, - maxChars, - ); - - if (extracted.tooLarge) { - throw new ResponseLimitExceeded(extracted.length, maxChars); - } - return extracted.content; -} - -async function asText(page: pup.Page, maxChars: number) { - return normalizeWhitespace(await extractBounded(page, 'innerText', maxChars)); -} - -async function asHTML(page: pup.Page, maxChars: number) { - return await extractBounded(page, 'innerHTML', maxChars); -} - -async function asScreenshot(page: pup.Page, maxChars: number) { - // Screenshots are viewport-sized rather than full-page, so this is a - // backstop rather than a limit anyone should reach. - const image = await page.screenshot(); - if (image.length > maxChars) { - throw new ResponseLimitExceeded(image.length, maxChars); - } - return image; -} - -class HeapLimitExceeded extends Error { - constructor(heapMB: number, maxHeapMB: number) { - super(`Page JS heap ${heapMB.toFixed(1)}MB exceeds limit ${maxHeapMB}MB`); - } -} - -class ResponseLimitExceeded extends Error { - constructor(size: number, maxSize: number) { - super(`Rendered response ${size} exceeds limit ${maxSize}`); - } -} - -const HEAP_CHECK_INTERVAL_MS = envInt( - 'GVM_WEBDRIVER_HEAP_CHECK_INTERVAL_MS', - 200, -); - -async function withHeapMonitor<T>( - page: pup.Page, - maxHeapMB: number, - fn: () => Promise<T>, -): Promise<T> { - let stopped = false; - let stopResolve: () => void; - const stopPromise = new Promise<void>((r) => { - stopResolve = r; - }); - const monitor = (async () => { - while (!stopped) { - await new Promise((r) => setTimeout(r, HEAP_CHECK_INTERVAL_MS)); - if (stopped) break; - let totalMB: number; - try { - totalMB = await page.evaluate( - () => (performance as any).memory?.totalJSHeapSize / 1024 / 1024, - ); - } catch { - await stopPromise; - return; - } - logger.log('debug', 'heap monitor check', { - heapMB: totalMB.toFixed(1), - }); - if (totalMB > maxHeapMB) { - throw new HeapLimitExceeded(totalMB, maxHeapMB); - } - } - })(); - - try { - const result = await Promise.race([ - fn(), - monitor.then(() => undefined as never), - ]); - let totalMB: number; - try { - totalMB = await page.evaluate( - () => (performance as any).memory?.totalJSHeapSize / 1024 / 1024, - ); - } catch { - return result; - } - if (totalMB > maxHeapMB) { - throw new HeapLimitExceeded(totalMB, maxHeapMB); - } - return result; - } finally { - stopped = true; - stopResolve!(); - await monitor.catch(() => {}); - } -} - -function statusIsGood(status: number): boolean { - return (status >= 200 && status < 300) || status === 304; -} - async function renderPage( targetUrl: string, mode: 'text' | 'html' | 'screenshot', @@ -342,120 +121,6 @@ async function renderPage( } } -async function renderPageWithBrowser( - browserInstance: pup.Browser, - targetUrl: string, - mode: 'text' | 'html' | 'screenshot', - options: RenderOptions = {}, -): Promise<{ status: number; body: any }> { - const { - loadTimeout = 30000, - waitAfterLoaded = 0, - waitUntil = 'domcontentloaded', - maxPageHeapMB = DEFAULT_MAX_PAGE_HEAP_MB, - maxResponseChars = MAX_RESPONSE_CHARS, - } = options; - - // Each render runs in its own browser context so cookies, localStorage, - // IndexedDB, service workers and HSTS state never leak between tenants - // sharing this long-lived browser. - const context = await browserInstance.createBrowserContext(); - - // `newPage` fails when the shared browser is under pressure, and the cleanup - // below only runs once execution is inside the try. Without this the context - // survives for the browser's whole lifetime β€” and since pressure is exactly - // what makes `newPage` fail, the response to it would be to leak more. - let page: pup.Page; - try { - page = await context.newPage(); - } catch (e) { - await context.close().catch(() => {}); - throw e; - } - - try { - await ssrf.installSsrfGuard(page); - context.on('targetcreated', async (target) => { - const p = await target.page(); - if (p && p !== page) { - await ssrf.installSsrfGuard(p); - } - }); - page.setViewport({ width: 1920 / 2, height: 1080 / 2 }); - - return await withHeapMonitor(page, maxPageHeapMB, async () => { - const navigationResult = await navigateToPage(page, targetUrl, { - waitUntil, - timeout: loadTimeout, - }); - - if (navigationResult.error) { - return { - status: navigationResult.status, - body: navigationResult.error, - }; - } - - const statusCode = navigationResult.status; - - if (statusIsGood(statusCode) && waitAfterLoaded > 0) { - let waitMs = Math.floor(waitAfterLoaded * 1000); - if (waitMs > MAX_WAIT_AFTER_LOADED_MS) { - logger.log('warn', 'waitAfterLoaded clamped to maximum', { - url: targetUrl, - requested: formatDurationMs(waitMs), - max: formatDurationMs(MAX_WAIT_AFTER_LOADED_MS), - }); - waitMs = MAX_WAIT_AFTER_LOADED_MS; - } - await new Promise((resolve) => setTimeout(resolve, waitMs)); - } - - let data; - switch (mode) { - case 'text': - data = await asText(page, maxResponseChars); - break; - case 'html': - data = await asHTML(page, maxResponseChars); - break; - case 'screenshot': - data = await asScreenshot(page, maxResponseChars); - break; - default: - data = 'Invalid mode'; - } - - return { status: statusCode, body: data }; - }); - } catch (e) { - // Both limits are reported as render results, not transport failures: a - // transport failure aborts the caller's whole execution, and a page too - // big for this host is a thing the contract can reasonably be told about - // and handle, like any other page it could not load. - if (e instanceof HeapLimitExceeded) { - logger.log('warn', 'page heap limit exceeded', { - url: targetUrl, - error: e.message, - }); - } else if (e instanceof ResponseLimitExceeded) { - logger.log('warn', 'rendered response limit exceeded', { - url: targetUrl, - mode, - error: e.message, - }); - } else { - throw e; - } - - return { status: STATUS_INSUFFICIENT_STORAGE, body: e.message }; - } finally { - // Close the page and its context together; the context teardown is what - // actually discards the per-request browsing state. - await Promise.allSettled([page.close(), context.close()]); - } -} - /** * Fires when the caller goes away before it has been answered, so a request * nobody is waiting for stops occupying a queue slot. @@ -554,6 +219,14 @@ async function handleRenderRequest( ); return; } + // Everything still reaching here is this sidecar failing, not the page: + // page outcomes are *returned* as a status. The module reads this `500` + // as a fatal `WEBDRIVER_UNAVAILABLE` and the validator abstains, so leave + // a local trace -- otherwise the only record is the message below. + logger.log('error', 'render request failed', { + url: query.get('url') ?? '', + error: (error as Error).message, + }); res.writeHead(500, { 'Content-Type': 'application/json' }); res.end( JSON.stringify({ diff --git a/webdriver/src/prj/src/render.test.ts b/webdriver/src/prj/src/render.test.ts new file mode 100644 index 00000000..6c3fc1b0 --- /dev/null +++ b/webdriver/src/prj/src/render.test.ts @@ -0,0 +1,322 @@ +/** + * Failures of this sidecar must leave `renderPageWithBrowser` by being thrown, + * and must not be dressed up as a page status. + * + * `handleRenderRequest` turns anything thrown out of here into a bare `500`, + * and the module classifies a `500` from its own webdriver as a fatal + * `WEBDRIVER_UNAVAILABLE`. That is the intended outcome: a validator whose own + * sidecar is broken has no observation of the page, so it must abstain (the + * node votes Timeout) rather than assert a result it never computed. + * + * A page that merely fails to load is the opposite case -- a real observation. + * It comes back as a *returned* status the caller puts in `Resulting-Status` + * next to a `200`, which the module reports as a catchable, non-fatal + * `WEBPAGE_LOAD_FAILED`. The pairs below pin both directions, so reclassifying + * either one fails the suite. + * + * One navigation error crosses that line, and only one: + * `net::ERR_INTERNET_DISCONNECTED` says the request never left this host, so + * there is nothing observed to report and it is thrown. The neighbouring + * errors are deliberately NOT treated that way, and the tests that pin them + * are guards rather than descriptions -- see each one for why the ambiguity is + * left for the validators to settle. + * + * The browser is faked outright: no Chromium is launched and nothing leaves + * the process. `render.ts` is imported directly rather than through + * `index.ts`, which would start the HTTP server and launch a browser at module + * scope. + */ + +// genvm-tool-test-tags: feature-web-render + +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import type * as pup from 'puppeteer-core'; +import { ProtocolError, TimeoutError } from 'puppeteer-core'; + +import { renderPageWithBrowser, statusIsGood } from './render.js'; + +interface FakeContext { + closes: number; +} + +/** + * A browser whose page creation fails the way a wedged or dying Chromium does. + * Records context closes so a leak shows up as a failing assertion. + */ +function browserFailingOn( + where: 'createBrowserContext' | 'newPage', + error: Error, +): { browser: pup.Browser; context: FakeContext } { + const context: FakeContext = { closes: 0 }; + + const fakeContext = { + newPage: async () => { + if (where === 'newPage') { + throw error; + } + throw new Error('unreachable'); + }, + close: async () => { + context.closes++; + }, + on: () => {}, + }; + + const browser = { + createBrowserContext: async () => { + if (where === 'createBrowserContext') { + throw error; + } + return fakeContext; + }, + }; + + return { browser: browser as unknown as pup.Browser, context }; +} + +async function render(browser: pup.Browser) { + return renderPageWithBrowser(browser, 'https://example.com/', 'text'); +} + +// -- a broken sidecar raises, it does not report ---------------------- + +test('a protocol timeout opening a page is thrown, not reported', async () => { + // what puppeteer raises once `Target.createTarget` exceeds protocolTimeout + const { browser, context } = browserFailingOn( + 'newPage', + new ProtocolError('Target.createTarget timed out'), + ); + + await assert.rejects(render(browser), /Target.createTarget timed out/); + assert.equal(context.closes, 1, 'the browser context must not be leaked'); +}); + +test('a TimeoutError opening a page is thrown, not reported', async () => { + const { browser, context } = browserFailingOn( + 'newPage', + new TimeoutError('waiting for target failed'), + ); + + await assert.rejects(render(browser), /waiting for target failed/); + assert.equal(context.closes, 1, 'the browser context must not be leaked'); +}); + +test('a failure creating the context is thrown, not reported', async () => { + const { browser } = browserFailingOn( + 'createBrowserContext', + new ProtocolError('Target.createBrowserContext timed out'), + ); + + await assert.rejects( + render(browser), + /Target.createBrowserContext timed out/, + ); +}); + +test('a failure after the page exists still propagates', async () => { + // `installSsrfGuard` is the first thing done with a live page, and a page + // that cannot be guarded is this sidecar failing too + const closes = { page: 0, context: 0 }; + const page = { + setRequestInterception: async () => { + throw new Error('interception unavailable'); + }, + close: async () => { + closes.page++; + }, + on: () => {}, + setViewport: () => {}, + }; + const browser = { + createBrowserContext: async () => ({ + newPage: async () => page, + close: async () => { + closes.context++; + }, + on: () => {}, + }), + } as unknown as pup.Browser; + + await assert.rejects(render(browser), /interception unavailable/); + assert.deepEqual( + closes, + { page: 1, context: 1 }, + 'the page and its context must still be torn down', + ); +}); + +// -- and a failing remote page still reports, it does not raise ------- + +/** + * A browser that opens a page normally and then fails the navigation with + * `error`, which is how a healthy sidecar meets a page it cannot load. + * Records page and context closes so a leak shows up as a failing assertion. + */ +function browserWhoseNavigationFails(error: Error): { + browser: pup.Browser; + closes: { page: number; context: number }; +} { + const closes = { page: 0, context: 0 }; + const page = { + setRequestInterception: async () => {}, + goto: async () => { + throw error; + }, + // the heap monitor polls this + evaluate: async () => 0, + close: async () => { + closes.page++; + }, + on: () => {}, + off: () => {}, + setViewport: () => {}, + }; + const browser = { + createBrowserContext: async () => ({ + newPage: async () => page, + close: async () => { + closes.context++; + }, + on: () => {}, + }), + } as unknown as pup.Browser; + + return { browser, closes }; +} + +/** A Chrome navigation failure, which arrives as a plain `Error`. */ +function netError(code: string): Error { + return new Error(`${code} at https://example.com/`); +} + +/** + * The other half of the pair, so the tests above are not satisfied by "throw + * everything". A site that refuses the connection is something we *did* + * observe, so it comes back as a status rather than an exception -- and it is + * a 503, the very code the sidecar-failure path must not borrow. + */ +test('an unreachable remote site is reported, not thrown', async () => { + const { browser, closes } = browserWhoseNavigationFails( + netError('net::ERR_CONNECTION_REFUSED'), + ); + + const result = await render(browser); + + assert.equal(result.status, 503); + assert.match(String(result.body), /Connection refused/); + assert.equal( + statusIsGood(result.status), + false, + 'the module must see this as a failed load', + ); + assert.deepEqual(closes, { page: 1, context: 1 }); +}); + +/** + * A certificate the site presents badly is likewise the site's doing, and the + * contract is entitled to see it: a contract may legitimately branch on "that + * host is not serving valid TLS". + */ +test('a certificate error is reported, not thrown', async () => { + const { browser, closes } = browserWhoseNavigationFails( + netError('net::ERR_CERT_AUTHORITY_INVALID'), + ); + + const result = await render(browser); + + assert.equal(result.status, 495); + assert.match(String(result.body), /SSL certificate error/); + assert.deepEqual(closes, { page: 1, context: 1 }); +}); + +// -- the two ambiguous cases stay on the observation channel ---------- + +/** + * Regression guard, not a description of a certainty. + * + * A name that does not resolve is either a domain that does not exist -- the + * site -- or a resolver of *ours* that is broken. The two are indistinguishable + * from inside one validator, and resolving that ambiguity is exactly what + * several validators and an equivalence principle are for: pre-judging it here + * would suppress the disagreement consensus exists to reconcile. So it stays + * returned and contract-visible, and this test fails if someone later + * "helpfully" reclassifies it as ours the way `ERR_INTERNET_DISCONNECTED` is. + */ +test('a name that does not resolve is still reported, not thrown', async () => { + const { browser, closes } = browserWhoseNavigationFails( + netError('net::ERR_NAME_NOT_RESOLVED'), + ); + + const result = await render(browser); + + assert.equal(result.status, 502); + assert.match(String(result.body), /DNS resolution failed/); + assert.equal( + statusIsGood(result.status), + false, + 'the module must see this as a failed load', + ); + assert.deepEqual(closes, { page: 1, context: 1 }); +}); + +/** + * The same guard for the other ambiguous case: a navigation timeout is either + * a slow site or a browser of ours that wedged, and it is not this process's + * place to decide which. Returned, not thrown. + */ +test('a navigation timeout is still reported, not thrown', async () => { + const { browser, closes } = browserWhoseNavigationFails( + new TimeoutError('Navigation timeout of 30000 ms exceeded'), + ); + + const result = await render(browser); + + assert.equal(result.status, 408); + assert.match(String(result.body), /Navigation timeout/); + assert.equal( + statusIsGood(result.status), + false, + 'the module must see this as a failed load', + ); + assert.deepEqual(closes, { page: 1, context: 1 }); +}); + +// -- and the one navigation error that is ours ------------------------ + +/** + * `net::ERR_INTERNET_DISCONNECTED` is not ambiguous: the request never left + * this host, so there is no observation to reconcile with anyone. It must + * leave by the thrown channel, which the caller turns into a `500` and the + * module into a fatal error, so the validator abstains. + * + * The message must say so in its own words. It is all that reaches the `500` + * body, and from there the module error's `ctx` and the operator-facing abort + * string; "webdriver unavailable" would send an operator to look at a browser + * that is working fine. + */ +test('a local network outage is thrown, not reported', async () => { + const { browser, closes } = browserWhoseNavigationFails( + netError('net::ERR_INTERNET_DISCONNECTED'), + ); + + await assert.rejects(render(browser), (e: Error) => { + assert.match( + e.message, + /local network/i, + 'the message must name the fault an operator has to go and fix', + ); + assert.match(e.message, /net::ERR_INTERNET_DISCONNECTED/); + assert.doesNotMatch( + e.message, + /webdriver unavailable/i, + 'the browser answered, so it must not be reported as the broken part', + ); + return true; + }); + assert.deepEqual( + closes, + { page: 1, context: 1 }, + 'the page and its context must still be torn down', + ); +}); diff --git a/webdriver/src/prj/src/render.ts b/webdriver/src/prj/src/render.ts new file mode 100644 index 00000000..be2fd77b --- /dev/null +++ b/webdriver/src/prj/src/render.ts @@ -0,0 +1,446 @@ +/** + * Rendering a page with an already-running browser. + * + * Split out of `index.ts` so it can be exercised without launching Chromium or + * binding a port: importing `index.ts` starts the HTTP server, and importing + * `browser/chrome.js` launches a browser at module scope. + * + * The contract this file upholds: everything it *returns* is an observation + * about the page, reported as a status the caller puts in `Resulting-Status` + * alongside a `200`, which the module surfaces to the contract as a catchable + * `WEBPAGE_LOAD_FAILED`. Everything it *throws* is this sidecar failing. The + * caller turns a throw into a `500`, which the module classifies as fatal, so + * the validator abstains rather than voting on a page it never observed. "This + * sidecar failing" includes the host it runs on: a machine with no network is + * ours too, and `LocalNetworkUnavailable` below is that case. + * + * The two channels must not be mixed. Folding a sidecar failure into a + * `Resulting-Status` -- 503 is the tempting one, since a page whose host + * refused the connection already reports 503 -- would make a broken sidecar + * indistinguishable from a real observation, and let the validator vote on it. + */ + +import type * as pup from 'puppeteer-core'; + +import * as logger from './logging.js'; +import * as ssrf from './ssrf.js'; +import { + envDurationMs, + envInt, + envSize, + formatDurationMs, +} from './duration.js'; + +interface NavigationOptions { + waitUntil?: pup.PuppeteerLifeCycleEvent; + timeout?: number; +} + +export interface RenderOptions { + loadTimeout?: number; + waitAfterLoaded?: number; + waitUntil?: pup.PuppeteerLifeCycleEvent; + maxPageHeapMB?: number; + maxResponseChars?: number; +} + +const STATUS_I_AM_A_TEAPOT = 418; +const STATUS_INSUFFICIENT_STORAGE = 507; + +const DEFAULT_MAX_PAGE_HEAP_MB = envInt('GVM_WEBDRIVER_MAX_PAGE_HEAP_MB', 1024); + +// Counted in characters, which is what bounds the string materialized in this +// process. One character encodes to at most 4 UTF-8 bytes on the wire, and +// occupies 2 bytes in V8 for the common (BMP) case. +const MAX_RESPONSE_CHARS = envSize('GVM_WEBDRIVER_MAX_RESPONSE', '5MiB'); + +const MAX_WAIT_AFTER_LOADED_MS = envDurationMs( + 'GVM_WEBDRIVER_MAX_WAIT_AFTER_LOADED', + '60s', +); + +function normalizeWhitespace(contents: string): string { + return contents + .split('\n') + .map((line) => line.trim().replace(/\s+/g, ' ')) + .join('\n') + .replace(/\n{2,}/g, '\n\n'); +} + +/** + * This machine has no network at all, which is not something we observed about + * the page: `net::ERR_INTERNET_DISCONNECTED` is Chrome saying the request never + * left the host. + * + * It is the one navigation error that leaves through the thrown channel. The + * others are ambiguous between the site and us, and stay returned: a name that + * does not resolve is either a domain that does not exist or a resolver of ours + * that is broken, and a timeout is either a slow site or a browser of ours that + * is wedged. Settling that is what several validators and an equivalence + * principle are for, so deciding it here would suppress the very disagreement + * consensus exists to reconcile. This case is different only because it is + * unambiguous -- there is no observation to reconcile. + * + * The wording says *local network* rather than *webdriver*: the browser and + * this sidecar are answering normally, and the `500` the caller sends carries + * only this text. + */ +export class LocalNetworkUnavailable extends Error { + constructor() { + super( + 'Local network fault: this host has no network route ' + + '(net::ERR_INTERNET_DISCONNECTED). The browser answered, so it is ' + + 'the local network that failed rather than the webdriver, and ' + + 'nothing about the page was observed.', + ); + this.name = 'LocalNetworkUnavailable'; + } +} + +/** + * What we observed happening to the page, as a status for `Resulting-Status`. + * + * `net::ERR_INTERNET_DISCONNECTED` is deliberately absent: it is ours rather + * than the page's, so `navigateToPage` throws it before reaching here. See + * [`LocalNetworkUnavailable`] for why the neighbouring cases are NOT treated + * the same way. + */ +function getNavigationErrorStatus(error: any): number { + if (error.name === 'TimeoutError') { + // Ambiguous on purpose: a slow site and a browser of ours that wedged + // are indistinguishable here, so the validators settle it, not us. + return 408; // Request Timeout + } else if (error.message?.includes('net::ERR_NAME_NOT_RESOLVED')) { + // Ambiguous in the same way: a domain that does not exist and a broken + // resolver of ours look identical from this side. + return 502; // Bad Gateway + } else if (error.message?.includes('net::ERR_CONNECTION_REFUSED')) { + return 503; // Service Unavailable + } else if (error.message?.includes('net::ERR_CERT_')) { + return 495; // SSL Certificate Error + } else if (error.message?.includes('net::ERR_BLOCKED_BY_CLIENT')) { + return 403; // Forbidden (SSRF guard) + } + return STATUS_I_AM_A_TEAPOT; // Unknown error +} + +function getNavigationErrorMessage(error: any): string { + if (error.name === 'TimeoutError') { + return 'Navigation timeout'; + } else if (error.message?.includes('net::ERR_NAME_NOT_RESOLVED')) { + return 'DNS resolution failed'; + } else if (error.message?.includes('net::ERR_CONNECTION_REFUSED')) { + return 'Connection refused'; + } else if (error.message?.includes('net::ERR_CERT_')) { + return 'SSL certificate error'; + } else if (error.message?.includes('net::ERR_BLOCKED_BY_CLIENT')) { + return 'Blocked by SSRF guard: address not allowed'; + } + return `Navigation error: ${error.message || 'Unknown error'}`; +} + +async function navigateToPage( + page: pup.Page, + targetUrl: string, + options: NavigationOptions = {}, +): Promise<{ status: number; error?: string; response?: pup.HTTPResponse }> { + const { waitUntil = 'domcontentloaded', timeout = 30000 } = options; + + try { + const response = await page.goto(targetUrl, { + waitUntil, + timeout, + }); + + if (!response) { + return { + status: STATUS_I_AM_A_TEAPOT, + error: 'Navigation did not result in a valid HTTP response', + }; + } + + return { status: response.status(), response }; + } catch (navigationError: any) { + logger.log('error', 'navigation Error', navigationError); + if (navigationError.message?.includes('net::ERR_INTERNET_DISCONNECTED')) { + // Thrown, not returned: this one is our fault, and the caller's `500` + // is what makes the module abort, so the validator abstains rather + // than voting on a page the request never reached. + throw new LocalNetworkUnavailable(); + } + const statusCode = getNavigationErrorStatus(navigationError); + const errorMessage = getNavigationErrorMessage(navigationError); + return { status: statusCode, error: errorMessage }; + } +} + +/** + * Read `innerText` or `innerHTML`, rejecting oversized documents *in the page*. + * + * The size test has to happen on the browser side. Pulling the string across + * first and measuring it here would already have paid the memory cost we are + * trying to avoid -- this process would hold the whole document no matter what + * any downstream limit says. + */ +async function extractBounded( + page: pup.Page, + kind: 'innerText' | 'innerHTML', + maxChars: number, +): Promise<string> { + const extracted = await page.evaluate( + (k, limit) => { + const raw = + k === 'innerText' ? document.body.innerText : document.body.innerHTML; + return raw.length > limit + ? { tooLarge: true as const, length: raw.length } + : { tooLarge: false as const, content: raw }; + }, + kind, + maxChars, + ); + + if (extracted.tooLarge) { + throw new ResponseLimitExceeded(extracted.length, maxChars); + } + return extracted.content; +} + +async function asText(page: pup.Page, maxChars: number) { + return normalizeWhitespace(await extractBounded(page, 'innerText', maxChars)); +} + +async function asHTML(page: pup.Page, maxChars: number) { + return await extractBounded(page, 'innerHTML', maxChars); +} + +async function asScreenshot(page: pup.Page, maxChars: number) { + // Screenshots are viewport-sized rather than full-page, so this is a + // backstop rather than a limit anyone should reach. + const image = await page.screenshot(); + if (image.length > maxChars) { + throw new ResponseLimitExceeded(image.length, maxChars); + } + return image; +} + +export class HeapLimitExceeded extends Error { + constructor(heapMB: number, maxHeapMB: number) { + super(`Page JS heap ${heapMB.toFixed(1)}MB exceeds limit ${maxHeapMB}MB`); + } +} + +export class ResponseLimitExceeded extends Error { + constructor(size: number, maxSize: number) { + super(`Rendered response ${size} exceeds limit ${maxSize}`); + } +} + +const HEAP_CHECK_INTERVAL_MS = envInt( + 'GVM_WEBDRIVER_HEAP_CHECK_INTERVAL_MS', + 200, +); + +async function withHeapMonitor<T>( + page: pup.Page, + maxHeapMB: number, + fn: () => Promise<T>, +): Promise<T> { + let stopped = false; + let stopResolve: () => void; + const stopPromise = new Promise<void>((r) => { + stopResolve = r; + }); + const monitor = (async () => { + while (!stopped) { + await new Promise((r) => setTimeout(r, HEAP_CHECK_INTERVAL_MS)); + if (stopped) break; + let totalMB: number; + try { + totalMB = await page.evaluate( + () => (performance as any).memory?.totalJSHeapSize / 1024 / 1024, + ); + } catch { + await stopPromise; + return; + } + logger.log('debug', 'heap monitor check', { + heapMB: totalMB.toFixed(1), + }); + if (totalMB > maxHeapMB) { + throw new HeapLimitExceeded(totalMB, maxHeapMB); + } + } + })(); + + try { + const result = await Promise.race([ + fn(), + monitor.then(() => undefined as never), + ]); + let totalMB: number; + try { + totalMB = await page.evaluate( + () => (performance as any).memory?.totalJSHeapSize / 1024 / 1024, + ); + } catch { + return result; + } + if (totalMB > maxHeapMB) { + throw new HeapLimitExceeded(totalMB, maxHeapMB); + } + return result; + } finally { + stopped = true; + stopResolve!(); + await monitor.catch(() => {}); + } +} + +export function statusIsGood(status: number): boolean { + return (status >= 200 && status < 300) || status === 304; +} + +/** + * Acquire the per-render browsing context and its page. + * + * Both calls are CDP round-trips (`Target.createBrowserContext`, + * `Target.createTarget`) that can hang or fail on their own, so they are kept + * in one place with the context closed if the page never materializes. The + * failure itself is re-raised, not translated -- see the file header. + */ +async function newRenderTarget( + browserInstance: pup.Browser, +): Promise<{ context: pup.BrowserContext; page: pup.Page }> { + // Each render runs in its own browser context so cookies, localStorage, + // IndexedDB, service workers and HSTS state never leak between tenants + // sharing this long-lived browser. + const context = await browserInstance.createBrowserContext(); + try { + return { context, page: await context.newPage() }; + } catch (error) { + await context.close().catch(() => {}); + // Logged here as well as re-raised: this is the last point that still + // knows the failure was ours rather than the page's, and the `500` the + // caller sends carries only a message. + logger.log('error', 'could not open a page for rendering', { + name: (error as Error).name, + error: (error as Error).message, + }); + throw error; + } +} + +export async function renderPageWithBrowser( + browserInstance: pup.Browser, + targetUrl: string, + mode: 'text' | 'html' | 'screenshot', + options: RenderOptions = {}, +): Promise<{ status: number; body: any }> { + const { + loadTimeout = 30000, + waitAfterLoaded = 0, + waitUntil = 'domcontentloaded', + maxPageHeapMB = DEFAULT_MAX_PAGE_HEAP_MB, + maxResponseChars = MAX_RESPONSE_CHARS, + } = options; + + // No contract input reaches `newRenderTarget` -- the target URL is not + // passed to it -- so anything it throws is this sidecar failing, never an + // observation about the page. It is therefore allowed to propagate: the + // caller turns it into a `500`, which the module classifies as fatal, and + // the validator abstains instead of reporting a page outcome it never + // observed. It must NOT be folded into a `Resulting-Status`, which is the + // channel reserved for what actually happened to the page. + const { context, page } = await newRenderTarget(browserInstance); + + try { + await ssrf.installSsrfGuard(page); + // A render needs exactly one page, and a popup cannot be guarded after + // the fact: nothing holds its first navigation while an async handler + // installs an interceptor, so it could already have reached an internal + // address. Chrome is launched with `--block-new-web-contents`, and any + // target that still appears is closed rather than trusted. + context.on('targetcreated', (target) => { + void (async () => { + const p = await target.page(); + if (p && p !== page) { + logger.log('warn', 'closing unexpected popup target', { + url: target.url(), + }); + await p.close().catch(() => {}); + } + })(); + }); + page.setViewport({ width: 1920 / 2, height: 1080 / 2 }); + + return await withHeapMonitor(page, maxPageHeapMB, async () => { + const navigationResult = await navigateToPage(page, targetUrl, { + waitUntil, + timeout: loadTimeout, + }); + + if (navigationResult.error) { + return { + status: navigationResult.status, + body: navigationResult.error, + }; + } + + const statusCode = navigationResult.status; + + if (statusIsGood(statusCode) && waitAfterLoaded > 0) { + let waitMs = Math.floor(waitAfterLoaded * 1000); + if (waitMs > MAX_WAIT_AFTER_LOADED_MS) { + logger.log('warn', 'waitAfterLoaded clamped to maximum', { + url: targetUrl, + requested: formatDurationMs(waitMs), + max: formatDurationMs(MAX_WAIT_AFTER_LOADED_MS), + }); + waitMs = MAX_WAIT_AFTER_LOADED_MS; + } + await new Promise((resolve) => setTimeout(resolve, waitMs)); + } + + let data; + switch (mode) { + case 'text': + data = await asText(page, maxResponseChars); + break; + case 'html': + data = await asHTML(page, maxResponseChars); + break; + case 'screenshot': + data = await asScreenshot(page, maxResponseChars); + break; + default: + data = 'Invalid mode'; + } + + return { status: statusCode, body: data }; + }); + } catch (e) { + // Both limits are a property of the page rather than of this sidecar -- + // the same document would exceed them anywhere -- so they belong on the + // returned channel, like any other page the contract could not load. + if (e instanceof HeapLimitExceeded) { + logger.log('warn', 'page heap limit exceeded', { + url: targetUrl, + error: e.message, + }); + } else if (e instanceof ResponseLimitExceeded) { + logger.log('warn', 'rendered response limit exceeded', { + url: targetUrl, + mode, + error: e.message, + }); + } else { + throw e; + } + + return { status: STATUS_INSUFFICIENT_STORAGE, body: e.message }; + } finally { + // Close the page and its context together; the context teardown is what + // actually discards the per-request browsing state. + await Promise.allSettled([page.close(), context.close()]); + } +} From 217b4dfaf184ae723f4dfcd725bf0cd790bcdee9 Mon Sep 17 00:00:00 2001 From: Kira <kp2pml30@gmail.com> Date: Wed, 9 Sep 2026 15:48:48 +0900 Subject: [PATCH 16/45] chore: sort out pending fixes and build/CI plumbing (#34) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(ci): materialize the full executor history for the branch sync πŸ’š A shallow gitlink checkout holds the pinned commit alone, so the push cannot be shown to build on the target branch and the server rejects even a genuine fast-forward. * fix(storage): make the final octet of a slot addressable πŸ› * chore(build): take git-third-party from its flake input πŸ—οΈ The vendored copy is replaced by a pinned flake input, which also carries the tool through `env.sh` and `get-all-git.py` -- both run before the dev shell, so they realize the package themselves. * chore(deps): move the executors to wasmtime 48 ⬆️ Wasmtime 48 declares rust-version 1.95, above the 1.94 the pinned channel provided, so the toolchain moves with the gitlinks. * chore(ci): watch vendored upstream sources for published advisories πŸ”’οΈ * chore: remove legacy compatibility layer * fix(ci): test emoji reaction πŸ› * chore(build): warm the webdriver image from a ninja target πŸ”¨ * chore(build): build the webdriver docker image as `webdriver/image` πŸ”¨ * chore(ci): build that target in the mock PR cell so the lazy spawn hits the cache πŸ’š * fix(tests): coerce jsonnet allocation recipients to `Address` recursively πŸ› * feat(expr): add internalError and keep the failure class on replay ✨ * fix(ci): set nix up before the source checkout πŸ’š `get-all-git.py` realizes `git-third-party` from the flake, so every job that updates third-party trees needs nix during the checkout step, not after it. The nixless commit-messages job wants git history rather than vendored sources, so it skips the update instead. --- .github/actions/get-src/action.yaml | 24 +- .../workflows/branch_run_tests_command.yaml | 3 +- .github/workflows/branch_wasmtime_watch.yaml | 61 ++ .github/workflows/incl_initial.yaml | 3 + .../incl_release_build_test_cell_test.yaml | 5 +- .../howto/committing/git-third-party.md | 63 +- docs/contributing/howto/setup.md | 4 +- .../03-wasi_genlayer_sdk/01-functions.rst | 4 +- docs/website/src/spec/changelog.rst | 4 + env.sh | 9 +- executors/v0.2.x | 2 +- executors/v0.3.x | 2 +- flake.lock | 63 ++ flake.nix | 26 +- implementation/src/manager/handlers.rs | 1 - implementation/src/manager/run.rs | 27 - implementation/src/manager/socket.rs | 1 - libs/deps/dependency-urls.json | 10 +- support/ci/__main__.py | 2 + support/ci/gh_common.py | 39 + support/ci/pipelines/tests.py | 6 +- support/ci/tools/sync_executor_branches.py | 20 +- support/ci/tools/wasmtime_watch.py | 256 +++++++ .../unit_tests/test_sync_executor_branches.py | 42 +- support/ci/unit_tests/test_wasmtime_watch.py | 311 ++++++++ support/ci/wasmtime_pins.py | 243 +++++++ support/nix/git-third-party.nix | 12 +- support/rust.nix | 2 +- support/scripts/get-all-git.py | 38 +- .../genvm-tool/genvm_tool/cmd_configure.py | 18 + support/tools/git-third-party/.editorconfig | 17 - support/tools/git-third-party/.gitignore | 3 - .../git-third-party/.pre-commit-config.yaml | 32 - support/tools/git-third-party/LICENSE | 674 ------------------ support/tools/git-third-party/README.md | 35 - support/tools/git-third-party/git-third-party | 447 ------------ support/tools/git-third-party/ruff.toml | 9 - support/zig.nix | 5 + .../runner/genvm_tool_plugins/integration.py | 27 +- tests/runner/gvm_extra/mock_host.py | 73 +- tests/system/cross-major/test.py | 8 +- 41 files changed, 1299 insertions(+), 1332 deletions(-) create mode 100644 .github/workflows/branch_wasmtime_watch.yaml create mode 100644 support/ci/tools/wasmtime_watch.py create mode 100644 support/ci/unit_tests/test_wasmtime_watch.py create mode 100644 support/ci/wasmtime_pins.py delete mode 100644 support/tools/git-third-party/.editorconfig delete mode 100644 support/tools/git-third-party/.gitignore delete mode 100644 support/tools/git-third-party/.pre-commit-config.yaml delete mode 100644 support/tools/git-third-party/LICENSE delete mode 100644 support/tools/git-third-party/README.md delete mode 100755 support/tools/git-third-party/git-third-party delete mode 100644 support/tools/git-third-party/ruff.toml diff --git a/.github/actions/get-src/action.yaml b/.github/actions/get-src/action.yaml index bbecea3a..5defdf19 100644 --- a/.github/actions/get-src/action.yaml +++ b/.github/actions/get-src/action.yaml @@ -2,7 +2,10 @@ name: GenVM get source description: Checkout sources, optionally update submodules/third-party, and (optionally) set up Nix. inputs: with_nix: - description: if should setup nix + description: >- + if the *job* needs nix after this action. Nix is also installed whenever + `third_party` is not `none`, so it can be present without this being set; + an installation cannot be undone. required: false default: "false" third_party: @@ -38,6 +41,17 @@ outputs: runs: using: composite steps: + # Before the checkout: `get-all-git.py` realizes `git-third-party` from the + # flake, so a third-party update needs nix here whether or not the job asked + # for one. The action itself only touches the runner, never the tree. + - name: setup nix + if: ${{ inputs.with_nix == 'true' || inputs.third_party != 'none' }} + # Pinned at every call site of these actions, `# <branch>` naming where the + # sha came from: a job re-run later has to run the action it first ran. + uses: genlayerlabs/github-actions/nix-setup@39b0a0d5e9bb27a1612d2e98b0f8509d31745157 # main + with: + github_token: ${{ inputs.github_token }} + cache_pull_token: ${{ inputs.nix_cache_pull_token }} - name: checkout submodules run: | cd "$GITHUB_WORKSPACE" @@ -54,11 +68,3 @@ runs: run: | cd "$GITHUB_WORKSPACE" python3 support/scripts/ci-changes.py --github-output - - name: setup nix - if: ${{ inputs.with_nix == 'true' }} - # Pinned at every call site of these actions, `# <branch>` naming where the - # sha came from: a job re-run later has to run the action it first ran. - uses: genlayerlabs/github-actions/nix-setup@39b0a0d5e9bb27a1612d2e98b0f8509d31745157 # main - with: - github_token: ${{ inputs.github_token }} - cache_pull_token: ${{ inputs.nix_cache_pull_token }} diff --git a/.github/workflows/branch_run_tests_command.yaml b/.github/workflows/branch_run_tests_command.yaml index 6069c73f..708defc3 100644 --- a/.github/workflows/branch_run_tests_command.yaml +++ b/.github/workflows/branch_run_tests_command.yaml @@ -31,7 +31,7 @@ jobs: actions: write contents: read issues: write - pull-requests: read + pull-requests: write steps: # issue_comment workflows run from the default branch. Keep that trusted # checkout: this handler dispatches PR code with credentials in scope @@ -57,6 +57,7 @@ jobs: permissions: contents: read issues: write + pull-requests: write steps: # workflow_run also resolves this workflow from the default branch. Never # check out or execute the completed run's potentially PR-authored tree diff --git a/.github/workflows/branch_wasmtime_watch.yaml b/.github/workflows/branch_wasmtime_watch.yaml new file mode 100644 index 00000000..cd51e4de --- /dev/null +++ b/.github/workflows/branch_wasmtime_watch.yaml @@ -0,0 +1,61 @@ +name: branch / wasmtime watch + +# Daily sweep for published advisories against the upstream sources the manager +# vendors: each repo in an executor line's `manifest.json` by its pinned commit, +# and each crate that comes out of one by its locked version. +# +# It gates nothing and writes no branch: it keeps one `wasmtime-maintenance` +# issue in sync with what OSV currently says. Nothing PR-triggered could do this +# job β€” a pin becomes vulnerable while standing still, with no PR involved. +# +# Closing the issue is a human ruling on the findings it names; the tool never +# closes or reopens one. See support/ci/tools/wasmtime_watch.py. + +on: + schedule: + # Daily, off the hour: a scheduled run on a busy minute is queued or dropped. + - cron: "17 5 * * *" + workflow_dispatch: + inputs: + dry_run: + description: Report findings without creating or editing the issue + type: boolean + required: false + default: false + +permissions: + contents: read + issues: write + +# Never cancel: a cancelled sweep can leave the issue describing a state that no +# longer holds, and the next run is a day away. +concurrency: + group: wasmtime-watch + cancel-in-progress: false + +defaults: + run: + shell: bash -exo pipefail {0} + +jobs: + watch: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + # Both files it reads are tracked inside the executor submodules, so this + # needs a checkout but not the materialized third-party trees β€” which is + # what lets it skip nix entirely. + - name: Get source + uses: ./.github/actions/get-src + with: + with_nix: "false" + third_party: none + github_token: ${{ secrets.GITHUB_TOKEN }} + - name: Sweep the vendored upstream sources + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + WASMTIME_REBASE_OWNER: ${{ vars.WASMTIME_REBASE_OWNER }} + run: >- + ./support/ci/run.sh tool wasmtime-watch + ${{ inputs.dry_run && '--dry-run' || '' }} diff --git a/.github/workflows/incl_initial.yaml b/.github/workflows/incl_initial.yaml index 863eb4af..45af4fee 100644 --- a/.github/workflows/incl_initial.yaml +++ b/.github/workflows/incl_initial.yaml @@ -71,6 +71,9 @@ jobs: uses: ./.github/actions/get-src with: with_nix: "false" + # Commit messages need history, not vendored sources, and asking for + # them would install nix here for nothing. + third_party: none pr: ${{ inputs.pr }} # Needed even with with_nix=false: on a dispatched run there is no # GITHUB_BASE_REF, so ci-changes.py resolves the base branch through diff --git a/.github/workflows/incl_release_build_test_cell_test.yaml b/.github/workflows/incl_release_build_test_cell_test.yaml index f500c572..760b08ca 100644 --- a/.github/workflows/incl_release_build_test_cell_test.yaml +++ b/.github/workflows/incl_release_build_test_cell_test.yaml @@ -57,7 +57,10 @@ jobs: - name: Get source uses: ./.github/actions/get-src with: - third_party: --all + # A nixless cell has no dev shell to build anything in: it replays + # prebuilt tarballs and reads no vendored source, so asking for them + # would install a nix nothing else here uses. + third_party: ${{ inputs.with_nix && '--all' || 'none' }} with_nix: ${{ inputs.with_nix && 'true' || 'false' }} github_token: ${{ secrets.GITHUB_TOKEN }} nix_cache_pull_token: ${{ secrets.NIX_CACHE_PULL_TOKEN }} diff --git a/docs/contributing/howto/committing/git-third-party.md b/docs/contributing/howto/committing/git-third-party.md index e38b2954..b24e42d6 100644 --- a/docs/contributing/howto/committing/git-third-party.md +++ b/docs/contributing/howto/committing/git-third-party.md @@ -5,22 +5,31 @@ git-ignored and materialized on demand. Why patches rather than a fork: [vendored-trees.md](../../explanation/vendored-trees.md). What is tracked, per line, is `.git-third-party/`: -1. `config.json` β€” per repo: upstream `url`, pinned base `commit`, `patches` - count, optional `submodules` list -2. `patches/<repo-path>/<n>` β€” numbered `git format-patch` files, 1-based, mbox - format, applied on top of the base commit +1. `manifest.json` β€” per repo: upstream `url`, pinned base `commit`, the + ordered `patches` list, optional `submodules` list +2. `patches/<repo-path>/<name>` β€” `git format-patch` files in mbox format, + applied on top of the base commit in the order the manifest lists. `<name>` + is a digest of the patch bytes, so inserting a patch renames nothing else +3. `.gitattributes`, `.gitignore` β€” written by the tool. The first keeps the + enclosing repo from normalizing bytes the names are derived from -The tool is `support/tools/git-third-party/git-third-party`, invoked as -`git third-party`, on `PATH` in the dev shell and through `env.sh`. It has no -`--help`; running it bare prints usage +The tool comes from the `git-third-party` flake input, on `PATH` in the dev +shell and through `env.sh`, and is invoked as `git third-party`. It has no +`--help`; running it bare prints usage. Bump it with +`nix flake update git-third-party` | Command | Effect | |---|---| | `add <PATH> <REPO_URL> <COMMIT>` | register a vendored repo (the path must already be git-ignored) and materialize it | | `update {--all \| <path>...}` | materialize: nested `git init` + fetch of the pinned commit, submodule update, `git am` of the patches | -| `save {--all \| <path>...}` | regenerate the patches from the commits above the base commit, updating the count | +| `save {--all \| <path>...}` | regenerate the patches from the commits above the base commit, rewriting the manifest's list | -`update` refuses a dirty tree. The result is a real nested git repo: base commit +`update` refuses a dirty tree β€” untracked files and dirty submodules count β€” +and refuses to discard commits in a managed checkout that `save` has not +captured. `git am` runs with `--keep-cr --whitespace=nowarn --no-3way`, so +local git config cannot rewrite what a patch applies, and a patch that only +landed by 3-way fallback now fails instead. Materialized checkouts get their +push URL disabled. The result is a real nested git repo: base commit plus one commit per patch. It always runs `git submodule update --init --recursive --depth 1` first, so `submodules: []` only skips the second pass and a list only adds a targeted one @@ -35,8 +44,8 @@ only skips the second pass and a list only adds a targeted one 4. Bump the gitlink in the manager ([submodules.md](submodules.md)) The nested repo is never pushed, only the patches persist. To bump upstream, -edit `commit` in `config.json` and run `update`; on an `am` conflict fix up the -nested repo's commits and `save` +edit `commit` in `manifest.json` and run `update`; on an `am` conflict fix up +the nested repo's commits and `save` ## Build Integration @@ -44,6 +53,36 @@ nested repo's commits and `save` executor submodule** β€” the config resolves from the current git toplevel, so running it at the manager root materializes nothing 2. Nix never calls the tool: `support/nix/git-third-party.nix` re-reads - `config.json` and uses `builtins.fetchGit` plus `pkgs.applyPatches`, ignoring + `manifest.json` and uses `builtins.fetchGit` plus `pkgs.applyPatches`, ignoring `submodules`. So nix sees the committed base plus patches β€” `save` and commit before building any flake package + +## Advisory Monitoring + +`branch / wasmtime watch` checks every vendored source of every active line +against [OSV](https://osv.dev) daily and keeps one `wasmtime-maintenance` issue +in sync with the result. It gates nothing and writes no branch. Run it by hand +with `./support/ci/run.sh tool wasmtime-watch --dry-run` + +Two things are queried per line, because neither covers the other: + +1. each repo in `manifest.json`, by its pinned upstream commit +2. each crate that comes out of one, by its locked version β€” a vendored tree is + dozens of crates (`cranelift-codegen`, `wasmparser`, `wiggle`), and that is + where most advisories against this stack are actually published + +A crate counts as vendored when `executor/Cargo.lock` gives it no `source` and +no tracked `Cargo.toml` in the executor repo declares it. Registry crates are +excluded because they are covered upstream; first-party crates, because they are +ours. A first-party crate with no committed manifest falls through and is +queried too β€” a dismissible false positive, chosen over a silent gap + +OSV knows nothing about the features GenVM disables or what the patch series +changes, so a hit is a finding to rule on, not a proven exposure + +The sweep edits its issue in place and never closes or reopens one: closing it +is the ruling that its findings are handled. The advisory ids that issue covered +are recorded in its body marker, so later sweeps stay quiet until an id outside +that set appears, at which point a fresh issue supersedes it. +`WASMTIME_REBASE_OWNER` (an Actions variable) is the login the issue is assigned +to at creation; sweeps never reassign it diff --git a/docs/contributing/howto/setup.md b/docs/contributing/howto/setup.md index 8be85bdb..1a860ccf 100644 --- a/docs/contributing/howto/setup.md +++ b/docs/contributing/howto/setup.md @@ -21,8 +21,8 @@ `?submodules=1` is mandatory on every manager flake ref; without it nix fails with `Path 'executors/v0.3.x' … is not tracked by Git` 3. Only outside the dev shell, and only for manual `git third-party` calls: - `source env.sh` puts `support/tools/git-third-party` on `PATH` and sources - `.env` if present + `source env.sh` realizes the pinned `git-third-party` flake package onto + `PATH` and sources `.env` if present Next: [build.md](building/build.md). Vendoring mechanics: [git-third-party.md](committing/git-third-party.md) diff --git a/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/01-functions.rst b/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/01-functions.rst index d8cf25f3..f624b49b 100644 --- a/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/01-functions.rst +++ b/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/01-functions.rst @@ -10,7 +10,7 @@ Requirements ~~~~~~~~~~~~ #. :term:`Sub-VM` must be in deterministic mode -#. index + buf_len must not overflow +#. index + buf_len must not exceed the :term:`Storage Slot` length ``storage_write`` ----------------- @@ -22,7 +22,7 @@ Requirements #. :term:`Sub-VM` must be in deterministic mode #. :term:`Sub-VM` must have write storage permission -#. index + buf_len must not overflow +#. index + buf_len must not exceed the :term:`Storage Slot` length #. :term:`Sub-VM` Storage slot must not be locked, unless the sender is in ``upgraders`` ``get_balance`` diff --git a/docs/website/src/spec/changelog.rst b/docs/website/src/spec/changelog.rst index dae6e3b3..22e85c9f 100644 --- a/docs/website/src/spec/changelog.rst +++ b/docs/website/src/spec/changelog.rst @@ -72,3 +72,7 @@ Changed code β€” e.g. exhausting the memory budget there reports :ref:`gvm-def-str-trie-value-vm-error-out-of-memory-wasm-memory` β€” instead of being reported as a bare ``invalid_contract`` +#. ``storage_read`` and ``storage_write`` bound an access by the + :term:`Storage Slot` length instead of by ``u32`` overflow, so a slot's final + octet is addressable; every access naming it was previously refused. See + :doc:`02-execution-environment/03-wasi_genlayer_sdk/01-functions` diff --git a/env.sh b/env.sh index 9396fb6f..2b474ab5 100755 --- a/env.sh +++ b/env.sh @@ -2,7 +2,14 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -export PATH="$SCRIPT_DIR/support/tools/git-third-party:$PATH" +# The tool comes from the pinned flake input, so outside the dev shell it has to +# be realized first. Already on PATH inside any dev shell +if ! command -v git-third-party > /dev/null +then + git_third_party_out=$(nix build --no-link --print-out-paths "$SCRIPT_DIR?submodules=1#git-third-party") + export PATH="$git_third_party_out/bin:$PATH" + unset git_third_party_out +fi if [ -f "$SCRIPT_DIR/.env" ] then diff --git a/executors/v0.2.x b/executors/v0.2.x index 750ca85e..bac7bb2a 160000 --- a/executors/v0.2.x +++ b/executors/v0.2.x @@ -1 +1 @@ -Subproject commit 750ca85e3f1d9d8cfe632a29b12b2b261d0b8df3 +Subproject commit bac7bb2af8928b5af403d195e75b6e1d4f882a7c diff --git a/executors/v0.3.x b/executors/v0.3.x index ee7b5513..a0041928 160000 --- a/executors/v0.3.x +++ b/executors/v0.3.x @@ -1 +1 @@ -Subproject commit ee7b55133b637acd22118ed06ff52f233b21366a +Subproject commit a00419281450ae8997704170fcb50d4a6ca9d8fc diff --git a/flake.lock b/flake.lock index 0156faf8..3831ad06 100644 --- a/flake.lock +++ b/flake.lock @@ -16,6 +16,22 @@ "type": "github" } }, + "flake-compat_2": { + "flake": false, + "locked": { + "lastModified": 1767039857, + "narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=", + "owner": "NixOS", + "repo": "flake-compat", + "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "flake-compat", + "type": "github" + } + }, "flake-utils": { "inputs": { "systems": [ @@ -57,6 +73,52 @@ "type": "github" } }, + "git-hooks_2": { + "inputs": { + "flake-compat": "flake-compat_2", + "nixpkgs": [ + "git-third-party", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1783008725, + "narHash": "sha256-jGiy6+sxjNWXSjp25uoJuNfyH9zBK1PEDY0lVoL4ibQ=", + "owner": "cachix", + "repo": "git-hooks.nix", + "rev": "bca82caa46d5ec0f5d422c61fb1e30bc51313cbe", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "git-hooks.nix", + "type": "github" + } + }, + "git-third-party": { + "inputs": { + "git-hooks": "git-hooks_2", + "nixpkgs": [ + "nixpkgs" + ], + "systems": [ + "systems" + ] + }, + "locked": { + "lastModified": 1785223192, + "narHash": "sha256-B0G3SjlmGM0m5hJVufWnw3dxi4NZ80rTRY59KmWtasY=", + "owner": "kp2pml30", + "repo": "git-third-party", + "rev": "a6e28ba6ca31215d994e8778b53abf142bd7f3dd", + "type": "github" + }, + "original": { + "owner": "kp2pml30", + "repo": "git-third-party", + "type": "github" + } + }, "nixpkgs": { "locked": { "lastModified": 1778443072, @@ -77,6 +139,7 @@ "inputs": { "flake-utils": "flake-utils", "git-hooks": "git-hooks", + "git-third-party": "git-third-party", "nixpkgs": "nixpkgs", "systems": "systems" } diff --git a/flake.nix b/flake.nix index f04ccffd..62e42f8a 100644 --- a/flake.nix +++ b/flake.nix @@ -12,6 +12,11 @@ url = "github:cachix/git-hooks.nix"; inputs.nixpkgs.follows = "nixpkgs"; }; + git-third-party = { + url = "github:kp2pml30/git-third-party"; + inputs.nixpkgs.follows = "nixpkgs"; + inputs.systems.follows = "systems"; + }; }; outputs = @@ -21,6 +26,7 @@ flake-utils, systems, git-hooks, + git-third-party, }: let for-systems = flake-utils.lib.eachDefaultSystem ( @@ -112,6 +118,7 @@ glibc nix genvm-tool + git-third-party.packages.${system}.default ]; packages-rust = [ custom-rust ]; packages-debug-test = with pkgs; [ @@ -159,7 +166,6 @@ python312Packages.pytest-xdist ]; shell-hook-base = '' - export PATH="$(pwd)/support/tools/git-third-party:$PATH" export CARGO_LD_LIBRARY_PATH="${toString pkgs.xz.out}/lib:${toString pkgs.zlib.out}/lib:${pkgs.stdenv.cc.cc.lib}/lib:${toString pkgs.glibc}/lib" export LLVM_PROFILE_FILE=/dev/null export LSQLITE3_SRC="${deps."lsqlite3-0.9.6"}" @@ -288,17 +294,8 @@ ]; }; # --- manager-owned languages: python, lua, ts, nix -------- - # git-third-party is a vendored helper tool (late imports by - # design); the old engine skipped it (keyed on the `.py` ext), - # so keep ruff off it. - ruff = { - enable = true; - excludes = [ "^support/tools/git-third-party/" ]; - }; - ruff-format = { - enable = true; - excludes = [ "^support/tools/git-third-party/" ]; - }; + ruff.enable = true; + ruff-format.enable = true; stylua = { enable = true; files = "\\.lua$"; @@ -715,9 +712,12 @@ # Inputs for producing the three platform artifacts and the # platform-independent universal artifact. // artifact-prepack-packages - # Utility packages (grouped separately). + # Utility packages (grouped separately). `git-third-party` is + # re-exported so `env.sh` can put the pinned tool on PATH without + # entering a dev shell. // { inherit genvm-tool; + git-third-party = git-third-party.packages.${system}.default; }; devShells.minimal = pkgs.mkShell { diff --git a/implementation/src/manager/handlers.rs b/implementation/src/manager/handlers.rs index 902f53db..3945704e 100644 --- a/implementation/src/manager/handlers.rs +++ b/implementation/src/manager/handlers.rs @@ -77,7 +77,6 @@ pub async fn handle_genvm_run( data: &[u8], ) -> Result<Json<serde_json::Value>> { let mut res: super::run::Request = calldata::decode_obj(data)?; - res.patch_legacy_method_key(); let modules_lock = if res.needs_modules() { let lock = Ctx::get_module_locks(ctx.gep(|x| &x.mod_ctx)).await; diff --git a/implementation/src/manager/run.rs b/implementation/src/manager/run.rs index 4a64176f..4fc3ecdc 100644 --- a/implementation/src/manager/run.rs +++ b/implementation/src/manager/run.rs @@ -1075,33 +1075,6 @@ impl Request { pub fn needs_modules(&self) -> bool { !self.no_modules && !self.is_sync && self.permissions.contains('n') } - - /// The legacy v0.2 ABI stored the contract method name under the calldata - /// key `"method"`; the current ABI uses the empty key `""`. A client sending - /// old-format calldata would otherwise reach the executor with a `"method"` - /// key its runner no longer understands. Detect that here, log an error so - /// the stale caller is visible, and rewrite the key in place. No-op for - /// well-formed (new-format) calldata or non-map calldata. - pub fn patch_legacy_method_key(&mut self) { - const LEGACY_METHOD_KEY: &str = "method"; - const NEW_METHOD_KEY: &str = ""; - - let mut value = match calldata::decode(&self.calldata) { - Ok(v @ calldata::Value::Map(_)) => v, - _ => return, - }; - let calldata::Value::Map(map) = &mut value else { - return; - }; - - if let Some(method) = map.remove(LEGACY_METHOD_KEY) { - log_error!( - "received legacy 'method' key in run calldata; patching it to the new empty key" - ); - map.insert(NEW_METHOD_KEY.to_owned(), method); - self.calldata = bytes::Bytes::from(calldata::encode(&value)); - } - } } fn default_max_execution_minutes() -> u64 { diff --git a/implementation/src/manager/socket.rs b/implementation/src/manager/socket.rs index 67b66612..bfd8c69c 100644 --- a/implementation/src/manager/socket.rs +++ b/implementation/src/manager/socket.rs @@ -491,7 +491,6 @@ where return Ok(()); } }; - req.patch_legacy_method_key(); if req .host_hello_data .get(1) diff --git a/libs/deps/dependency-urls.json b/libs/deps/dependency-urls.json index 52b56d10..d55ffaf9 100644 --- a/libs/deps/dependency-urls.json +++ b/libs/deps/dependency-urls.json @@ -46,12 +46,12 @@ "fetcher": "fetchzip" }, { - "name": "rust-channel-stable-2026-03-05", - "original_url": "https://static.rust-lang.org/dist/2026-03-05/channel-rust-stable.toml", + "name": "rust-channel-stable-2026-09-03", + "original_url": "https://static.rust-lang.org/dist/2026-09-03/channel-rust-stable.toml", "alternative_urls": [ - "https://storage.googleapis.com/genvm-artifacts/channel-rust-stable-2026-03-05.toml" + "https://storage.googleapis.com/genvm-artifacts/channel-rust-stable-2026-09-03.toml" ], - "hash": "sha256-qqF33vNuAdU5vua96VKVIwuc43j4EFeEXbjQ6+l4mO4=", + "hash": "sha256-p8h3Sl/YRByZfZTAKXdsvF6xEenXKrXSVvpphmZENH4=", "must_preload": false, "needed-for": [ "x86_64-linux", @@ -65,7 +65,7 @@ "executor-arm64-macos" ], "fetcher": "fetchurl", - "upload_as": "channel-rust-stable-2026-03-05.toml" + "upload_as": "channel-rust-stable-2026-09-03.toml" }, { "name": "binaryen-raw-128", diff --git a/support/ci/__main__.py b/support/ci/__main__.py index 994c1456..1ecd9726 100644 --- a/support/ci/__main__.py +++ b/support/ci/__main__.py @@ -16,6 +16,7 @@ rebase_watch, sync_executor_branches, versions, + wasmtime_watch, ) PIPELINES: list[ci_lib.Pipeline] = [ @@ -36,6 +37,7 @@ *open_executor_prs.COMMANDS, *rebase_watch.COMMANDS, *sync_executor_branches.COMMANDS, + *wasmtime_watch.COMMANDS, ] diff --git a/support/ci/gh_common.py b/support/ci/gh_common.py index ae7cafc3..7209c752 100644 --- a/support/ci/gh_common.py +++ b/support/ci/gh_common.py @@ -37,6 +37,7 @@ import shlex import subprocess import sys +import tempfile import time import urllib.parse @@ -355,3 +356,41 @@ def has_write_access(login: str) -> bool: if r.returncode != 0: return False return json.loads(r.stdout).get('permission') in WRITE_ROLES + + +def api_json(method: str, path: str, payload: dict, *, retry: bool = False) -> str: + """ + `gh api` with a nested JSON body, as the manager. + + Through a temporary file rather than `-f` fields: a payload that nests (a + check-run's `output.title`, an issue's `assignees` list) cannot be expressed + as `-f` pairs, and `--input -` would need stdin that `gh` above does not + plumb. Defaults to `retry=False` because every caller so far is a write, and + `gh`'s transient-failure markers can misread a request that in fact + succeeded. + """ + with tempfile.NamedTemporaryFile('w', suffix='.json') as body: + json.dump(payload, body) + body.flush() + return gh_manager( + 'api', '--method', method, path, '--input', body.name, retry=retry + ) + + +def ensure_label(name: str, color: str, description: str) -> None: + """ + Create a repo label if it is missing; already-exists (422) is the normal case. + """ + gh_manager( + 'api', + '--method', + 'POST', + f'repos/{repo()}/labels', + '-f', + f'name={name}', + '-f', + f'color={color}', + '-f', + f'description={description}', + check=False, + ) diff --git a/support/ci/pipelines/tests.py b/support/ci/pipelines/tests.py index 3193f4a0..7af4d1ab 100644 --- a/support/ci/pipelines/tests.py +++ b/support/ci/pipelines/tests.py @@ -248,6 +248,10 @@ def group(gh_step: str, name: str, stage: Stage, body: GithubStep) -> None: TEST_MOCK_BUILD_ALL = TEST_MOCK_BUILD_BINS.merge(TEST_MOCK_BUILD_RUNNERS) TEST_MOCK_PR = DefaultStepInfo( + # The webdriver service is spawned lazily, deep into the suite, under a spawn + # budget that a cold image build does not fit in; build it here so that call + # is a docker cache hit. + build_step=Stage(data=frozenset(['webdriver/image'])), tool_step=Stage( commands_pre=[ {'type': 'group-start', 'name': 'precompile'}, @@ -258,7 +262,7 @@ def group(gh_step: str, name: str, stage: Stage, body: GithubStep) -> None: tests_name_filter_for_or=None, tests_tags_filter_for_or='integration', ), - ) + ), ) TEST_MOCK_RELEASE = DefaultStepInfo( diff --git a/support/ci/tools/sync_executor_branches.py b/support/ci/tools/sync_executor_branches.py index 6e223c8f..f30c8219 100644 --- a/support/ci/tools/sync_executor_branches.py +++ b/support/ci/tools/sync_executor_branches.py @@ -70,9 +70,10 @@ def sync_line(line: str, manager_branch: str) -> tuple[str, str]: path = f'executors/{line}.x' target = target_branch(manager_branch, declared_branch(path)) - initialized = git( - 'submodule', 'update', '--init', '--depth', '1', '--', path, check=False - ) + # Full history: a shallow gitlink checkout holds the pinned commit alone, so + # git cannot show the push builds on the target branch and the server + # rejects even a genuine fast-forward + initialized = git('submodule', 'update', '--init', '--', path, check=False) if initialized.returncode != 0: detail = (initialized.stderr or initialized.stdout).strip() raise SyncError( @@ -88,6 +89,19 @@ def sync_line(line: str, manager_branch: str) -> tuple[str, str]: f'could not resolve the gitlink checkout for `{path}`' + (f': {detail}' if detail else '') ) + # A missing target is created by the push below; an existing one is checked + # here so a divergence is named instead of surfacing as a push rejection + fetched = git('-C', path, 'fetch', 'origin', f'refs/heads/{target}', check=False) + if fetched.returncode == 0: + contained = git( + '-C', path, 'merge-base', '--is-ancestor', 'FETCH_HEAD', sha, check=False + ) + if contained.returncode != 0: + raise SyncError( + f'`{target}` has diverged: its tip is not an ancestor of `{sha}`; ' + 'reconcile the executor branch with the manager gitlink by hand' + ) + result = git( '-C', path, diff --git a/support/ci/tools/wasmtime_watch.py b/support/ci/tools/wasmtime_watch.py new file mode 100644 index 00000000..ed77ac42 --- /dev/null +++ b/support/ci/tools/wasmtime_watch.py @@ -0,0 +1,256 @@ +#!/usr/bin/env python3 +""" +Report published advisories against the upstream sources the manager vendors. + +A pin does not have to move to become vulnerable, so no PR-triggered check can +catch this β€” the day an advisory is published, every pin that stood still +becomes a finding and no PR is involved. This sweeps on a schedule instead and +keeps a single tracking issue in sync with what OSV currently says. It gates +nothing and writes no branch; the measurement is `wasmtime_pins`. + +The automation never closes and never reopens an issue. Closing is the reader's +ruling on the findings that issue names β€” GenVM disables features and carries a +local patch stack, so whether an upstream advisory is reachable here is a +judgement OSV cannot make. So the advisory ids a closed issue covered are +recorded in its own marker, and a later sweep stays quiet while it finds nothing +outside that set. A new id files a fresh issue that supersedes it. + +The issue is found by label (the query GitHub can filter server-side) and +identified by a marker in its body: a label is not a namespace, so anything else +that adopts this label β€” a person filing by hand, another tool β€” must not be +mistaken for this tool's issue and rewritten. The marker is also where the +issue's own state lives, since GitHub gives an issue no field this tool owns. +""" + +import argparse +import dataclasses +import json +import os +import re + +import ci_lib +import gh_common +import wasmtime_pins +from gh_common import gh_manager as gh +from gh_common import repo +from tools.versions import active_lines +from wasmtime_pins import Advisory, Subject + +LABEL = 'wasmtime-maintenance' +LABEL_COLOR = 'b60205' +LABEL_DESCRIPTION = 'published advisories against vendored upstream sources' + +MARKER_NAME = 'genvm-ci/wasmtime-watch' +MARKER_RE = re.compile( + rf'<!-- {re.escape(MARKER_NAME)} advisories=(\S*)(?: supersedes=(\d+))? -->' +) + +OWNER_ENV = 'WASMTIME_REBASE_OWNER' + +# A backslash cannot appear inside an f-string expression before python 3.12, +# and the scheduled workflow runs whatever `python3` the runner ships +TABLE_PIPE = r'\|' + + +@dataclasses.dataclass(frozen=True) +class Marker: + """ + The state this tool keeps in an issue body. + """ + + ids: frozenset[str] + supersedes: int | None = None + + def render(self) -> str: + trailer = f' supersedes={self.supersedes}' if self.supersedes else '' + return f'<!-- {MARKER_NAME} advisories={",".join(sorted(self.ids))}{trailer} -->' + + +def parse_marker(body: str | None) -> Marker | None: + """ + The marker an issue body carries, or None if it carries none. + """ + match = MARKER_RE.search(body or '') + if match is None: + return None + return Marker( + ids=frozenset(id for id in match.group(1).split(',') if id), + supersedes=int(match.group(2)) if match.group(2) else None, + ) + + +def body_for( + findings: dict[Advisory, list[Subject]], owner: str | None, marker: Marker +) -> str: + if findings: + rows = '\n'.join( + f'| [{advisory.id}]({advisory.url}) ' + f'| {advisory.summary.replace("|", TABLE_PIPE)} ' + f'| {", ".join(str(subject) for subject in subjects)} |' + for advisory, subjects in findings.items() + ) + table = f'| Advisory | Summary | Affected |\n|---|---|---|\n{rows}' + else: + table = 'No vendored source currently matches a published advisory.' + + owner_line = f'@{owner}' if owner else f'**unset** (`{OWNER_ENV}`)' + supersedes = f'\nSupersedes #{marker.supersedes}.\n' if marker.supersedes else '' + + return f"""{marker.render()} +## Published advisories against vendored upstream sources + +{table} +{supersedes} +**Owner:** {owner_line} + +Each vendored repo is matched by its pinned upstream commit, and every crate +that comes out of one by its locked version. OSV knows nothing about the +features GenVM disables or the patches it carries, so a row is a finding to rule +on rather than a proven exposure. + +Fixing one goes through the usual flow β€” a manager feature branch, an executor +mirror at `pr/<line>/<feature>` targeting `<line>-dev`, then the standing +release gate. Nothing here writes to a branch. + +This issue is edited in place while it is open. **Close it once you have ruled +on the findings above**; it is never closed or reopened automatically, and a new +issue is filed only when an advisory outside this list appears. +""" + + +def title_for(findings: dict[Advisory, list[Subject]]) -> str: + return f'[wasmtime] {len(findings)} published advisory(ies) affect vendored sources' + + +def marker_issues(state: str) -> list[dict]: + """ + This tool's own issues in `state`, newest first. + + `sort=created`, because the newest closed issue is the one whose ruling is + current. + """ + out = gh( + 'api', + gh_common.api_path( + f'repos/{repo()}/issues', + state=state, + labels=LABEL, + per_page=100, + sort='created', + direction='desc', + ), + '--jq', + # The issues endpoint returns PRs too; they carry a `pull_request` key. + '.[] | select(has("pull_request") | not) | {number, title, body}', + ) + issues = [json.loads(line) for line in out.splitlines() if line.strip()] + return [issue for issue in issues if parse_marker(issue['body']) is not None] + + +def create_issue(title: str, body: str, owner: str | None) -> None: + gh_common.ensure_label(LABEL, LABEL_COLOR, LABEL_DESCRIPTION) + payload: dict = {'title': title, 'body': body, 'labels': [LABEL]} + if owner: + payload['assignees'] = [owner] + gh_common.api_json('POST', f'repos/{repo()}/issues', payload) + + +def update_issue(issue: dict, title: str, body: str) -> None: + """ + Edit the open issue in place, and only when something actually differs. + + A no-op PATCH still notifies everyone subscribed to the issue, and this runs + daily. Bodies are compared with line endings normalized, so a round-trip + through GitHub cannot make an unchanged body look different every morning. + + Assignees are deliberately left out: they are set once at creation, and a + sweep that re-asserted them would undo every hand-off. + """ + + def normalized(text: str) -> str: + return text.replace('\r\n', '\n') + + payload = {} + if issue['title'] != title: + payload['title'] = title + if normalized(issue['body']) != normalized(body): + payload['body'] = body + if not payload: + print(f'#{issue["number"]} is already up to date') + return + gh_common.api_json('PATCH', f'repos/{repo()}/issues/{issue["number"]}', payload) + + +def sweep(dry_run: bool) -> int: + owner = os.environ.get(OWNER_ENV) or None + if owner is None: + print(f'::warning::{OWNER_ENV} is unset; the issue will be unassigned') + + findings = wasmtime_pins.scan(active_lines()) + current = frozenset(advisory.id for advisory in findings) + + open_issues = marker_issues('open') + if len(open_issues) > 1: + # A past race, or someone filing by hand. Report it rather than picking + # one silently: two issues means two places a ruling can be recorded. + numbers = ', '.join(f'#{issue["number"]}' for issue in open_issues) + ci_lib.github_error(f'several open {LABEL} issues ({numbers}); resolve by hand') + return 1 + + if open_issues: + issue = open_issues[0] + # The supersede link is carried forward from the issue's own marker: it is + # a fact about which issue this one replaced, not about today's findings. + previous = parse_marker(issue['body']) + assert previous is not None, 'marker_issues returns only issues with a marker' + marker = Marker(ids=current, supersedes=previous.supersedes) + print(f'updating #{issue["number"]} ({len(findings)} finding(s))') + if not dry_run: + update_issue(issue, title_for(findings), body_for(findings, owner, marker)) + return 0 + + if not current: + print('no advisories and no open issue: nothing to report') + return 0 + + # Nothing open: the newest closed issue carries the last ruling. Everything it + # already named has been ruled on, so only something outside that set is news. + closed = marker_issues('closed') + ruled_on = parse_marker(closed[0]['body']).ids if closed else frozenset() + new = current - ruled_on + if not new: + print( + f'{len(current)} advisory(ies), all already ruled on in ' + f'#{closed[0]["number"]}: staying quiet' + ) + return 0 + + marker = Marker(ids=current, supersedes=closed[0]['number'] if closed else None) + print(f'filing an issue for {len(new)} new advisory(ies): {", ".join(sorted(new))}') + if not dry_run: + create_issue(title_for(findings), body_for(findings, owner, marker), owner) + return 0 + + +class WasmtimeWatch(ci_lib.Tool): + """ + Track published advisories against the vendored upstream sources. + """ + + def name(self) -> str: + return 'wasmtime-watch' + + def add_to(self, parser: argparse.ArgumentParser) -> None: + gh_common.add_args(parser, executor_repo=False, pr=False, head_ref=False) + parser.add_argument( + '--dry-run', + action='store_true', + help='report what would be filed or edited without touching any issue', + ) + + def handler(self, args: argparse.Namespace) -> int: + gh_common.set_ctx(gh_common.Ctx.from_args(args)) + return sweep(args.dry_run) + + +COMMANDS = [WasmtimeWatch()] diff --git a/support/ci/unit_tests/test_sync_executor_branches.py b/support/ci/unit_tests/test_sync_executor_branches.py index ad92b7ec..b625e591 100644 --- a/support/ci/unit_tests/test_sync_executor_branches.py +++ b/support/ci/unit_tests/test_sync_executor_branches.py @@ -11,18 +11,25 @@ def test_target_branch_derives_dev_branch(): assert sync.target_branch('v0.6-dev', 'v0.3.x') == 'v0.3-dev' -def test_sync_line_pushes_without_force(monkeypatch): - commands = [] - +def fake_git(commands, **outcomes): def git(*args, check=True): commands.append((args, check)) if args[:4] == ('config', '-f', '.gitmodules', '--get'): return SimpleNamespace(returncode=0, stdout='v0.3.x\n', stderr='') if args[:3] == ('-C', 'executors/v0.3.x', 'rev-parse'): return SimpleNamespace(returncode=0, stdout=f'{"a" * 40}\n', stderr='') + for prefix, returncode in outcomes.items(): + if prefix in args: + return SimpleNamespace(returncode=returncode, stdout='', stderr='') return SimpleNamespace(returncode=0, stdout='', stderr='') - monkeypatch.setattr(sync, 'git', git) + return git + + +def test_sync_line_pushes_without_force(monkeypatch): + commands = [] + + monkeypatch.setattr(sync, 'git', fake_git(commands)) target, sha = sync.sync_line('v0.3', 'v0.6-dev') assert target == 'v0.3-dev' @@ -36,6 +43,33 @@ def git(*args, check=True): f'{"a" * 40}:refs/heads/v0.3-dev', ) assert '--force' not in push + assert ('submodule', 'update', '--init', '--', 'executors/v0.3.x') == commands[1][0] + assert ('-C', 'executors/v0.3.x', 'fetch', 'origin', 'refs/heads/v0.3-dev') in [ + command[0] for command in commands + ] + + +def test_sync_line_reports_a_diverged_target(monkeypatch): + commands = [] + + monkeypatch.setattr(sync, 'git', fake_git(commands, **{'merge-base': 1})) + try: + sync.sync_line('v0.3', 'v0.6-dev') + except sync.SyncError as error: + assert '`v0.3-dev` has diverged' in str(error) + else: + raise AssertionError('a diverged target must not be pushed') + assert 'push' not in [command[0][2] for command in commands if len(command[0]) > 2] + + +def test_sync_line_creates_a_missing_target(monkeypatch): + commands = [] + + monkeypatch.setattr(sync, 'git', fake_git(commands, **{'fetch': 128})) + target, sha = sync.sync_line('v0.3', 'v0.6-dev') + + assert (target, sha) == ('v0.3-dev', 'a' * 40) + assert commands[-1][0][2] == 'push' def test_all_lines_are_attempted_before_failures_are_reported(monkeypatch, capsys): diff --git a/support/ci/unit_tests/test_wasmtime_watch.py b/support/ci/unit_tests/test_wasmtime_watch.py new file mode 100644 index 00000000..e9aedfdd --- /dev/null +++ b/support/ci/unit_tests/test_wasmtime_watch.py @@ -0,0 +1,311 @@ +import json + +import pytest +import wasmtime_pins +from tools import wasmtime_watch as watch +from wasmtime_pins import Advisory, Subject + +MANIFEST = { + 'repos': { + 'executor/third-party/wasmtime': {'commit': 'a' * 40}, + 'executor/third-party/wasm-tools': {'commit': 'b' * 40}, + } +} + +LOCK = """ +[[package]] +name = "wasmtime" +version = "48.0.1" + +[[package]] +name = "cranelift-codegen" +version = "0.123.0" + +[[package]] +name = "genvm" +version = "0.1.0" + +[[package]] +name = "serde" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +""" + + +@pytest.fixture +def tree(tmp_path, monkeypatch): + line = tmp_path / 'executors' / 'v0.3.x' + (line / '.git-third-party').mkdir(parents=True) + (line / 'executor').mkdir() + (line / wasmtime_pins.MANIFEST).write_text(json.dumps(MANIFEST)) + (line / wasmtime_pins.CARGO_LOCK).write_text(LOCK) + # `git ls-files` needs a real repo; the mapping it feeds is tested separately. + monkeypatch.setattr(wasmtime_pins, 'workspace_crates', lambda tree: {'genvm'}) + return tmp_path + + +def test_every_vendored_crate_is_a_subject_not_just_the_repo_basename(tree): + found = wasmtime_pins.subjects('v0.3', root=tree) + + # The repos, by pinned commit... + assert [(s.name, s.commit) for s in found if s.commit] == [ + ('wasm-tools', 'b' * 40), + ('wasmtime', 'a' * 40), + ] + # ...and every crate that comes out of one, by locked version. `wasmtime` is + # both; `cranelift-codegen` shares its name with no repo and is where most + # advisories against this stack are published. + assert [(s.name, s.version) for s in found if s.version] == [ + ('cranelift-codegen', '0.123.0'), + ('wasmtime', '48.0.1'), + ] + + +def test_registry_and_first_party_crates_are_not_subjects(tree): + names = {s.name for s in wasmtime_pins.subjects('v0.3', root=tree) if s.version} + + # `serde` has a source, so Dependabot covers it; `genvm` is ours. + assert 'serde' not in names + assert 'genvm' not in names + + +def test_a_manifest_with_no_vendored_repo_is_an_error_not_a_clean_bill(tree): + path = tree / 'executors' / 'v0.3.x' / wasmtime_pins.MANIFEST + path.write_text(json.dumps({'repos': {}})) + + with pytest.raises(ValueError, match='lists no vendored repos'): + wasmtime_pins.subjects('v0.3', root=tree) + + +def test_a_pin_without_a_full_commit_is_an_error(tree): + path = tree / 'executors' / 'v0.3.x' / wasmtime_pins.MANIFEST + path.write_text(json.dumps({'repos': {'x/wasmtime': {'commit': 'abc'}}})) + + with pytest.raises(ValueError, match='no usable commit pin'): + wasmtime_pins.subjects('v0.3', root=tree) + + +def test_workspace_crates_reads_tracked_manifests_only(tmp_path, monkeypatch): + (tmp_path / 'crate').mkdir() + (tmp_path / 'crate' / 'Cargo.toml').write_text('[package]\nname = "genvm"\n') + (tmp_path / 'workspace.toml').write_text('[workspace]\nmembers = []\n') + monkeypatch.setattr( + wasmtime_pins.subprocess, + 'run', + lambda *a, **kw: type('R', (), {'stdout': 'crate/Cargo.toml\nworkspace.toml\n'}), + ) + + # A virtual manifest declares no package and must not contribute a name. + assert wasmtime_pins.workspace_crates(tmp_path) == {'genvm'} + + +def test_a_subject_queries_osv_by_commit_or_by_crate_version(): + assert Subject(line='v0.3', name='wasmtime', commit='a' * 40).query() == { + 'commit': 'a' * 40 + } + assert Subject(line='v0.3', name='wasmtime', version='48.0.1').query() == { + 'package': {'name': 'wasmtime', 'ecosystem': 'crates.io'}, + 'version': '48.0.1', + } + + +def test_a_query_shared_between_lines_is_only_sent_once(monkeypatch): + sent = [] + monkeypatch.setattr(wasmtime_pins, '_osv', lambda query: sent.append(query) or []) + subject = Subject(line='v0.2', name='wasmparser', version='0.1') + cache: dict = {} + + wasmtime_pins.advisories_for(subject, cache) + wasmtime_pins.advisories_for( + Subject(line='v0.3', name='wasmparser', version='0.1'), cache + ) + + assert len(sent) == 1 + + +def test_canonical_id_is_stable_across_the_alias_that_matched(): + entry = {'id': 'RUSTSEC-2024-0001', 'aliases': ['GHSA-zzzz', 'CVE-2024-1']} + other = {'id': 'GHSA-zzzz', 'aliases': ['RUSTSEC-2024-0001', 'CVE-2024-1']} + + assert wasmtime_pins.canonical_id(entry) == wasmtime_pins.canonical_id(other) + assert wasmtime_pins.canonical_id({'id': 'OSV-1'}) == 'OSV-1' + + +def test_osv_pagination_is_bounded(monkeypatch): + monkeypatch.setattr( + wasmtime_pins, '_post', lambda page: {'vulns': [], 'next_page_token': 'more'} + ) + + with pytest.raises(RuntimeError, match='did not stop paginating'): + wasmtime_pins._osv({'commit': 'a' * 40}) + + +def findings(*ids: str) -> dict: + subject = Subject(line='v0.3', name='wasmtime', version='48.0.1') + return {Advisory(id=id, summary='boom | bang'): [subject] for id in ids} + + +def marker(*ids: str, supersedes: int | None = None) -> watch.Marker: + return watch.Marker(ids=frozenset(ids), supersedes=supersedes) + + +def test_the_marker_round_trips_the_state_the_issue_carries(): + body = watch.body_for( + findings('GHSA-a', 'GHSA-b'), 'someone', marker('GHSA-a', 'GHSA-b', supersedes=7) + ) + + assert watch.parse_marker(body) == marker('GHSA-a', 'GHSA-b', supersedes=7) + assert watch.parse_marker('no marker here') is None + assert watch.parse_marker(watch.body_for({}, None, marker())) == marker() + + +def test_the_body_names_the_owner_and_the_superseded_issue(): + assert '@someone' in watch.body_for(findings('GHSA-a'), 'someone', marker('GHSA-a')) + assert 'WASMTIME_REBASE_OWNER' in watch.body_for( + findings('GHSA-a'), None, marker('GHSA-a') + ) + assert 'Supersedes #7.' in watch.body_for( + findings('GHSA-a'), None, marker('GHSA-a', supersedes=7) + ) + + +def test_a_pipe_in_a_summary_does_not_break_the_table(): + body = watch.body_for(findings('GHSA-a'), None, marker('GHSA-a')) + row = next(line for line in body.splitlines() if line.startswith('| [GHSA-a]')) + + assert r'boom \| bang' in row + assert row.count('|') - row.count(r'\|') == 4 + + +def issue(number: int, ids: list[str], supersedes: int | None = None) -> dict: + return { + 'number': number, + 'title': watch.title_for(dict.fromkeys(ids, [])), + 'body': marker(*ids, supersedes=supersedes).render(), + } + + +def test_marker_issues_drops_prs_and_issues_that_are_not_ours(monkeypatch): + rows = [ + {'number': 1, 'title': 't', 'body': marker('GHSA-a').render()}, + {'number': 2, 'title': 't', 'body': 'filed by hand, same label'}, + ] + monkeypatch.setattr( + watch, 'gh', lambda *a: '\n'.join(json.dumps(row) for row in rows) + '\n' + ) + + assert [issue['number'] for issue in watch.marker_issues('open')] == [1] + + +@pytest.fixture +def sweep(monkeypatch): + """ + Drive `sweep` over canned findings and issues, capturing what it would write. + """ + calls = [] + monkeypatch.setattr(watch, 'active_lines', lambda: ['v0.3']) + monkeypatch.setattr( + watch, 'create_issue', lambda title, body, owner: calls.append(('create', body)) + ) + monkeypatch.setattr( + watch, + 'update_issue', + lambda issue, title, body: calls.append(('update', issue['number'], body)), + ) + monkeypatch.delenv(watch.OWNER_ENV, raising=False) + + def run(found: dict, *, open_issues: list, closed: list, dry_run: bool = False): + monkeypatch.setattr(watch.wasmtime_pins, 'scan', lambda lines: found) + monkeypatch.setattr( + watch, + 'marker_issues', + lambda state: open_issues if state == 'open' else closed, + ) + return watch.sweep(dry_run), calls + + return run + + +def test_an_open_issue_is_edited_rather_than_duplicated(sweep): + rc, calls = sweep( + findings('GHSA-a', 'GHSA-b'), open_issues=[issue(1, ['GHSA-a'])], closed=[] + ) + + assert rc == 0 + assert [call[:2] for call in calls] == [('update', 1)] + assert watch.parse_marker(calls[0][2]).ids == {'GHSA-a', 'GHSA-b'} + + +def test_an_update_keeps_the_supersede_link_of_the_issue_it_edits(sweep): + rc, calls = sweep( + findings('GHSA-a'), open_issues=[issue(1, ['GHSA-a'], supersedes=9)], closed=[] + ) + + assert rc == 0 + assert watch.parse_marker(calls[0][2]).supersedes == 9 + assert 'Supersedes #9.' in calls[0][2] + + +def test_an_open_issue_is_edited_even_once_the_findings_clear(sweep): + rc, calls = sweep({}, open_issues=[issue(1, ['GHSA-a'])], closed=[]) + + assert rc == 0 + assert watch.parse_marker(calls[0][2]).ids == set() + + +def test_several_open_issues_are_reported_instead_of_picked_between(sweep): + rc, calls = sweep( + findings('GHSA-a'), + open_issues=[issue(1, ['GHSA-a']), issue(2, ['GHSA-a'])], + closed=[], + ) + + assert (rc, calls) == (1, []) + + +def test_findings_already_ruled_on_do_not_reopen_the_conversation(sweep): + rc, calls = sweep( + findings('GHSA-a'), open_issues=[], closed=[issue(9, ['GHSA-a', 'GHSA-b'])] + ) + + assert (rc, calls) == (0, []) + + +def test_a_new_advisory_files_a_fresh_issue_superseding_the_closed_one(sweep): + rc, calls = sweep( + findings('GHSA-a', 'GHSA-new'), open_issues=[], closed=[issue(9, ['GHSA-a'])] + ) + + assert rc == 0 + assert [call[0] for call in calls] == ['create'] + assert watch.parse_marker(calls[0][1]) == marker('GHSA-a', 'GHSA-new', supersedes=9) + + +def test_nothing_is_filed_when_there_is_nothing_to_report(sweep): + assert sweep({}, open_issues=[], closed=[]) == (0, []) + + +def test_a_dry_run_writes_nothing(sweep): + rc, calls = sweep(findings('GHSA-a'), open_issues=[], closed=[], dry_run=True) + assert (rc, calls) == (0, []) + + rc, calls = sweep( + findings('GHSA-a'), open_issues=[issue(1, [])], closed=[], dry_run=True + ) + assert (rc, calls) == (0, []) + + +def test_an_unchanged_issue_is_not_patched(monkeypatch): + patched = [] + monkeypatch.setattr(watch.gh_common, 'api_json', lambda *a, **kw: patched.append(a)) + body = watch.body_for(findings('GHSA-a'), None, marker('GHSA-a')) + title = watch.title_for(findings('GHSA-a')) + + # The same content, and the same content after a CRLF round-trip through the + # API: a daily no-op PATCH notifies every subscriber. + watch.update_issue({'number': 1, 'title': title, 'body': body}, title, body) + watch.update_issue( + {'number': 1, 'title': title, 'body': body.replace('\n', '\r\n')}, title, body + ) + + assert patched == [] diff --git a/support/ci/wasmtime_pins.py b/support/ci/wasmtime_pins.py new file mode 100644 index 00000000..60e76e54 --- /dev/null +++ b/support/ci/wasmtime_pins.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python3 +""" +One definition of "which upstream sources does an executor line vendor, and are +any of them subject to a published advisory". + +The subjects live in the manager's own checkout: the gitlink at +`executors/<line>.x` is the snapshot that actually ships, so this reads them out +of the submodule tree rather than resolving the executor's `<line>-dev` / +`<line>.x` refs through the API β€” those move independently of what the manager +pins. Two files, because upstream is vendored twice over: + +1. `.git-third-party/manifest.json` names each vendored repo and the upstream + commit it is pinned at; +2. `executor/Cargo.lock` names the crates that come out of those repos. A + vendored tree is dozens of crates, not one β€” `cranelift-codegen`, + `wasmparser` and `wiggle` are where most advisories against this stack are + actually published, and none of them shares a name with the repo it lives in. + +Both are queried against OSV, because neither covers the other: a commit query +misses a crate whose advisory names no commit range, and a version query misses +a vendored tree pinned between releases. + +A crate is vendored if the lock gives it no `source` (so it is a path +dependency) and no tracked `Cargo.toml` in the executor repo declares it. The +residue of that rule is a first-party crate with no committed manifest, which +gets one pointless crates.io query β€” a false positive a reader can dismiss, +where the opposite mistake is a silent gap. + +Nothing here knows which features GenVM disables or what its patch series +changes, so a hit is a finding to rule on, not a proven exposure. +""" + +import dataclasses +import json +import subprocess +import time +import tomllib +import urllib.error +import urllib.request +from pathlib import Path + +import ci_lib + +MANIFEST = Path('.git-third-party') / 'manifest.json' +CARGO_LOCK = Path('executor') / 'Cargo.lock' +OSV_QUERY_URL = 'https://api.osv.dev/v1/query' + +# OSV is a third party on a daily job's critical path, so a blip must not cost a +# day of monitoring. Mirrors gh_common.gh's policy for the same reason. +_RETRY_ATTEMPTS = 4 +_RETRY_BACKOFF_S = 2.0 +# A server that keeps handing back a page token would otherwise spin until the +# job's timeout. No real query is anywhere near this deep. +_MAX_PAGES = 50 + + +@dataclasses.dataclass(frozen=True, order=True) +class Subject: + """ + One thing an advisory can be published against, in one executor line. + + Exactly one of `commit` (a vendored repo at its pinned upstream commit) and + `version` (a crate that comes out of one) is set. + """ + + line: str + name: str + version: str = '' + commit: str = '' + + def query(self) -> dict: + if self.commit: + return {'commit': self.commit} + return { + 'package': {'name': self.name, 'ecosystem': 'crates.io'}, + 'version': self.version, + } + + def __str__(self) -> str: + what = f'@{self.commit[:12]}' if self.commit else f' {self.version}' + return f'`{self.line}` {self.name}{what}' + + +@dataclasses.dataclass(frozen=True, order=True) +class Advisory: + id: str + summary: str + + @property + def url(self) -> str: + return f'https://osv.dev/vulnerability/{self.id}' + + +def canonical_id(entry: dict) -> str: + """ + The single id an OSV entry is tracked under here. + + OSV returns the same advisory under several aliases and reports whichever the + matching source used, so the raw `id` is not stable across queries β€” and this + id is what `wasmtime_watch` compares to decide whether a finding is new. GHSA + first because every advisory reaching a Rust crate has one; RUSTSEC next; the + OSV id only when neither exists. + """ + ids = sorted({entry['id'], *entry.get('aliases', [])}) + for prefix in ('GHSA-', 'RUSTSEC-'): + for candidate in ids: + if candidate.startswith(prefix): + return candidate + return entry['id'] + + +def workspace_crates(tree: Path) -> set[str]: + """ + Crate names declared by a tracked `Cargo.toml` in the executor repo. + + Tracked, not on disk: the vendored trees are git-ignored, so they are absent + from this list whether or not `git third-party update` has materialized them. + Globbing the filesystem instead would classify every vendored crate as + first-party the moment someone runs the tool on a materialized checkout, + which is exactly the failure that stays silent. + """ + paths = subprocess.run( + ['git', '-C', str(tree), 'ls-files', '*Cargo.toml'], + check=True, + text=True, + capture_output=True, + ).stdout.split() + + names = set() + for path in paths: + name = tomllib.loads((tree / path).read_text()).get('package', {}).get('name') + if isinstance(name, str): + names.add(name) + return names + + +def subjects(line: str, *, root: Path | None = None) -> list[Subject]: + """ + Everything of one executor line that an advisory could be published against. + + Needs the submodule checked out; `get-all-git.py --third-party=none` is + enough, since both files it reads are tracked. + """ + tree = (root or ci_lib.ROOT_DIR) / 'executors' / f'{line}.x' + repos = json.loads((tree / MANIFEST).read_bytes()).get('repos') + if not repos: + # A monitor that finds nothing to monitor is indistinguishable from a + # clean bill of health, so refuse to report one. + raise ValueError(f'{line}: {MANIFEST} lists no vendored repos') + + result = [] + for key, repo in sorted(repos.items()): + commit = repo.get('commit') + if not isinstance(commit, str) or len(commit) != 40: + raise ValueError(f'{line}: {key} has no usable commit pin: {commit!r}') + result.append(Subject(line=line, name=key.rsplit('/', 1)[-1], commit=commit)) + + first_party = workspace_crates(tree) + lock = tomllib.loads((tree / CARGO_LOCK).read_text()) + for package in sorted(lock.get('package', []), key=lambda p: str(p.get('name'))): + name, version = package.get('name'), package.get('version') + if 'source' in package or name in first_party: + continue + if not isinstance(name, str) or not isinstance(version, str): + raise ValueError(f'{line}: {CARGO_LOCK} has a package with no name/version') + result.append(Subject(line=line, name=name, version=version)) + return result + + +def _post(query: dict) -> dict: + request = urllib.request.Request( + OSV_QUERY_URL, + data=json.dumps(query).encode(), + headers={'Content-Type': 'application/json', 'User-Agent': 'genvm-ci'}, + method='POST', + ) + for attempt in range(_RETRY_ATTEMPTS): + try: + with urllib.request.urlopen(request, timeout=30) as response: + return json.load(response) + except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as error: + # A 4xx is the caller's fault and will fail identically on a replay. + terminal = isinstance(error, urllib.error.HTTPError) and error.code < 500 + if terminal or attempt + 1 == _RETRY_ATTEMPTS: + raise RuntimeError(f'OSV query {query} failed: {error}') from error + time.sleep(_RETRY_BACKOFF_S * (attempt + 1)) + raise AssertionError('unreachable') + + +def _osv(query: dict) -> list[dict]: + entries: list[dict] = [] + page = dict(query) + for _ in range(_MAX_PAGES): + response = _post(page) + entries.extend(response.get('vulns', [])) + token = response.get('next_page_token') + if not token: + return entries + page = {**query, 'page_token': token} + raise RuntimeError(f'OSV query {query} did not stop paginating') + + +def advisories_for(subject: Subject, cache: dict | None = None) -> list[Advisory]: + """ + Published advisories matching one subject. + + `cache` is keyed by the query rather than by the subject, so the two lines' + shared crates cost one request between them. + """ + query = subject.query() + key = json.dumps(query, sort_keys=True) + if cache is not None and key in cache: + return cache[key] + + found: dict[str, Advisory] = {} + for entry in _osv(query): + canonical = canonical_id(entry) + found[canonical] = Advisory( + id=canonical, + summary=(entry.get('summary') or 'no summary').replace('\n', ' '), + ) + result = sorted(found.values()) + if cache is not None: + cache[key] = result + return result + + +def scan( + lines: list[str], *, root: Path | None = None +) -> dict[Advisory, list[Subject]]: + """ + Every advisory affecting `lines`, with the subjects it was matched against. + """ + cache: dict[str, list[Advisory]] = {} + result: dict[Advisory, list[Subject]] = {} + for line in lines: + checked = subjects(line, root=root) + print(f'{line}: {len(checked)} subject(s)') + for subject in checked: + for advisory in advisories_for(subject, cache): + print(f' {subject}: {advisory.id}') + result.setdefault(advisory, []).append(subject) + return dict(sorted(result.items())) diff --git a/support/nix/git-third-party.nix b/support/nix/git-third-party.nix index 9b66dfd9..6159c1c1 100644 --- a/support/nix/git-third-party.nix +++ b/support/nix/git-third-party.nix @@ -1,8 +1,8 @@ # Library for genvm's `.git-third-party` convention. # -# A `.git-third-party` directory holds a `config.json`: -# { "repos": { "<repo-path>": { "url": ..., "commit": ..., "patches": <count> } } } -# and patch files at `patches/<repo-path>/<n>` (1-based). +# A `.git-third-party` directory holds a `manifest.json`: +# { "repos": { "<repo-path>": { "url": ..., "commit": ..., "patches": [<name>...] } } } +# and patch files at `patches/<repo-path>/<name>`, applied in the listed order. # # `load <dir>` fetches each repo at its pinned commit, applies its patches, and # returns an attrset { <repo-path> = <patched source derivation>; }. Callers @@ -12,7 +12,7 @@ let load = dir: let - repos = (builtins.fromJSON (builtins.readFile "${dir}/config.json")).repos; + repos = (builtins.fromJSON (builtins.readFile "${dir}/manifest.json")).repos; in builtins.mapAttrs ( path: cfg: @@ -28,9 +28,7 @@ let pkgs.applyPatches { name = "gtt-" + builtins.hashString "sha256" path + "-patched"; src = unpatched; - patches = builtins.genList ( - patch-no: "${dir}/patches/${path}/${toString (patch-no + 1)}" - ) cfg.patches; + patches = builtins.map (name: "${dir}/patches/${path}/${name}") cfg.patches; } ) repos; in diff --git a/support/rust.nix b/support/rust.nix index 986d5d9d..d3bd212f 100644 --- a/support/rust.nix +++ b/support/rust.nix @@ -28,7 +28,7 @@ let is-macos = systemAsGenVM == "arm64-macos"; - manifest-src = deps."rust-channel-stable-2026-03-05"; + manifest-src = deps."rust-channel-stable-2026-09-03"; manifest = builtins.fromTOML (builtins.readFile manifest-src); diff --git a/support/scripts/get-all-git.py b/support/scripts/get-all-git.py index e90e272e..42153de6 100755 --- a/support/scripts/get-all-git.py +++ b/support/scripts/get-all-git.py @@ -53,6 +53,33 @@ def git_output_bytes(cmd: list[str], cwd: Path) -> bytes: return subprocess.run(cmd, check=True, cwd=cwd, stdout=subprocess.PIPE).stdout +def git_third_party_bin(root: Path) -> Path: + """ + Realizes the pinned `git-third-party` and returns the directory holding it. + + This script runs before the dev shell, where the tool is not on PATH yet; + the flake input is what pins its version. + """ + found = shutil.which('git-third-party') + if found is not None: + return Path(found).parent + + out = subprocess.run( + [ + 'nix', + 'build', + '--no-link', + '--print-out-paths', + f'{root}?submodules=1#git-third-party', + ], + check=True, + cwd=root, + text=True, + capture_output=True, + ).stdout.strip() + return Path(out) / 'bin' + + def repo_root(value: str | None) -> Path: if value is not None: return Path(value).resolve() @@ -593,18 +620,19 @@ def main() -> None: root = repo_root(args.repo_root) env = os.environ.copy() - git_third_party = root / 'support' / 'tools' / 'git-third-party' - env['PATH'] = f'{git_third_party}{os.pathsep}{env.get("PATH", "")}' update_submodules_with_cache(root, env) if args.third_party == 'none': return + env['PATH'] = f'{git_third_party_bin(root)}{os.pathsep}{env.get("PATH", "")}' + third_party_args = shlex.split(args.third_party) - for cfg in sorted(root.glob('executors/*/.git-third-party/config.json')): - executor = cfg.parent.parent - run(['git', 'third-party', 'update', *third_party_args], executor, env) + for line in sorted(root.glob('executors/*/.git-third-party')): + if not (line / 'manifest.json').exists(): + continue + run(['git', 'third-party', 'update', *third_party_args], line.parent, env) if __name__ == '__main__': diff --git a/support/tools/genvm-tool/genvm_tool/cmd_configure.py b/support/tools/genvm-tool/genvm_tool/cmd_configure.py index ad1285ec..6436f4c3 100644 --- a/support/tools/genvm-tool/genvm_tool/cmd_configure.py +++ b/support/tools/genvm-tool/genvm_tool/cmd_configure.py @@ -547,6 +547,24 @@ def _runner_file(f: Path) -> bool: runners_build.var('pool', 'console') runners_build.finish() + # Warms the docker layer cache the integration suite's webdriver service + # relies on: a cold build of that image outlives the per-service spawn budget. + webdriver_dir = source_dir / 'webdriver' + webdriver_inputs = [ + f + for f in ninja.glob(webdriver_dir, '**/*') + # what .dockerignore drops is not part of the context + if f.is_file() and not ({'node_modules', 'dist'} & set(f.parts)) + ] + + n.build('phony', 'webdriver/image').add_dependency('target/webdriver-image').finish() + + webdriver_build = n.build('CUSTOM_COMMAND', 'target/webdriver-image') + webdriver_build.var('command', ['docker', 'build', '--progress=plain', webdriver_dir]) + webdriver_build.add_implicit_dependency(webdriver_inputs) + webdriver_build.var('pool', 'console') + webdriver_build.finish() + n.build('phony', 'cargo/fmt').add_dependency(n.all_format).finish() n.build('phony', 'cargo/clippy').add_dependency(n.all_clippy).finish() n.build('phony', 'cargo/clippy/fix').add_dependency(n.all_clippy_fix).finish() diff --git a/support/tools/git-third-party/.editorconfig b/support/tools/git-third-party/.editorconfig deleted file mode 100644 index de2e132b..00000000 --- a/support/tools/git-third-party/.editorconfig +++ /dev/null @@ -1,17 +0,0 @@ -root = true - -[*] -end_of_line = lf -insert_final_newline = true -charset = utf-8 -trim_trailing_whitespace = true -indent_style = tab -tab_width = 2 - -[*.toml] -indent_style = space -indent_size = 4 - -[*.{yml,yaml}] -indent_style = space -indent_size = 2 diff --git a/support/tools/git-third-party/.gitignore b/support/tools/git-third-party/.gitignore deleted file mode 100644 index 0416a3b7..00000000 --- a/support/tools/git-third-party/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -/.ruff_cache -__pycache__ -/third-party diff --git a/support/tools/git-third-party/.pre-commit-config.yaml b/support/tools/git-third-party/.pre-commit-config.yaml deleted file mode 100644 index 7ac2b46e..00000000 --- a/support/tools/git-third-party/.pre-commit-config.yaml +++ /dev/null @@ -1,32 +0,0 @@ -default_install_hook_types: - - pre-commit - - commit-msg - -repos: - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 - hooks: - - id: trailing-whitespace - - id: end-of-file-fixer - - id: check-added-large-files - - id: check-json - - id: check-yaml - - id: check-toml - - id: check-merge-conflict - - - repo: https://github.com/compilerla/conventional-pre-commit - rev: v4.0.0 - hooks: - - id: conventional-pre-commit - stages: [commit-msg] - args: [] - - - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.6.9 - hooks: - - id: ruff-format - - - repo: https://github.com/editorconfig-checker/editorconfig-checker.python - rev: 3.0.3 - hooks: - - id: editorconfig-checker diff --git a/support/tools/git-third-party/LICENSE b/support/tools/git-third-party/LICENSE deleted file mode 100644 index f288702d..00000000 --- a/support/tools/git-third-party/LICENSE +++ /dev/null @@ -1,674 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 3, 29 June 2007 - - Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/> - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The GNU General Public License is a free, copyleft license for -software and other kinds of works. - - The licenses for most software and other practical works are designed -to take away your freedom to share and change the works. By contrast, -the GNU General Public License is intended to guarantee your freedom to -share and change all versions of a program--to make sure it remains free -software for all its users. We, the Free Software Foundation, use the -GNU General Public License for most of our software; it applies also to -any other work released this way by its authors. You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -them if you wish), that you receive source code or can get it if you -want it, that you can change the software or use pieces of it in new -free programs, and that you know you can do these things. - - To protect your rights, we need to prevent others from denying you -these rights or asking you to surrender the rights. Therefore, you have -certain responsibilities if you distribute copies of the software, or if -you modify it: responsibilities to respect the freedom of others. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must pass on to the recipients the same -freedoms that you received. You must make sure that they, too, receive -or can get the source code. And you must show them these terms so they -know their rights. - - Developers that use the GNU GPL protect your rights with two steps: -(1) assert copyright on the software, and (2) offer you this License -giving you legal permission to copy, distribute and/or modify it. - - For the developers' and authors' protection, the GPL clearly explains -that there is no warranty for this free software. For both users' and -authors' sake, the GPL requires that modified versions be marked as -changed, so that their problems will not be attributed erroneously to -authors of previous versions. - - Some devices are designed to deny users access to install or run -modified versions of the software inside them, although the manufacturer -can do so. This is fundamentally incompatible with the aim of -protecting users' freedom to change the software. The systematic -pattern of such abuse occurs in the area of products for individuals to -use, which is precisely where it is most unacceptable. Therefore, we -have designed this version of the GPL to prohibit the practice for those -products. If such problems arise substantially in other domains, we -stand ready to extend this provision to those domains in future versions -of the GPL, as needed to protect the freedom of users. - - Finally, every program is threatened constantly by software patents. -States should not allow patents to restrict development and use of -software on general-purpose computers, but in those that do, we wish to -avoid the special danger that patents applied to a free program could -make it effectively proprietary. To prevent this, the GPL assures that -patents cannot be used to render the program non-free. - - The precise terms and conditions for copying, distribution and -modification follow. - - TERMS AND CONDITIONS - - 0. Definitions. - - "This License" refers to version 3 of the GNU General Public License. - - "Copyright" also means copyright-like laws that apply to other kinds of -works, such as semiconductor masks. - - "The Program" refers to any copyrightable work licensed under this -License. Each licensee is addressed as "you". "Licensees" and -"recipients" may be individuals or organizations. - - To "modify" a work means to copy from or adapt all or part of the work -in a fashion requiring copyright permission, other than the making of an -exact copy. The resulting work is called a "modified version" of the -earlier work or a work "based on" the earlier work. - - A "covered work" means either the unmodified Program or a work based -on the Program. - - To "propagate" a work means to do anything with it that, without -permission, would make you directly or secondarily liable for -infringement under applicable copyright law, except executing it on a -computer or modifying a private copy. Propagation includes copying, -distribution (with or without modification), making available to the -public, and in some countries other activities as well. - - To "convey" a work means any kind of propagation that enables other -parties to make or receive copies. Mere interaction with a user through -a computer network, with no transfer of a copy, is not conveying. - - An interactive user interface displays "Appropriate Legal Notices" -to the extent that it includes a convenient and prominently visible -feature that (1) displays an appropriate copyright notice, and (2) -tells the user that there is no warranty for the work (except to the -extent that warranties are provided), that licensees may convey the -work under this License, and how to view a copy of this License. If -the interface presents a list of user commands or options, such as a -menu, a prominent item in the list meets this criterion. - - 1. Source Code. - - The "source code" for a work means the preferred form of the work -for making modifications to it. "Object code" means any non-source -form of a work. - - A "Standard Interface" means an interface that either is an official -standard defined by a recognized standards body, or, in the case of -interfaces specified for a particular programming language, one that -is widely used among developers working in that language. - - The "System Libraries" of an executable work include anything, other -than the work as a whole, that (a) is included in the normal form of -packaging a Major Component, but which is not part of that Major -Component, and (b) serves only to enable use of the work with that -Major Component, or to implement a Standard Interface for which an -implementation is available to the public in source code form. A -"Major Component", in this context, means a major essential component -(kernel, window system, and so on) of the specific operating system -(if any) on which the executable work runs, or a compiler used to -produce the work, or an object code interpreter used to run it. - - The "Corresponding Source" for a work in object code form means all -the source code needed to generate, install, and (for an executable -work) run the object code and to modify the work, including scripts to -control those activities. However, it does not include the work's -System Libraries, or general-purpose tools or generally available free -programs which are used unmodified in performing those activities but -which are not part of the work. For example, Corresponding Source -includes interface definition files associated with source files for -the work, and the source code for shared libraries and dynamically -linked subprograms that the work is specifically designed to require, -such as by intimate data communication or control flow between those -subprograms and other parts of the work. - - The Corresponding Source need not include anything that users -can regenerate automatically from other parts of the Corresponding -Source. - - The Corresponding Source for a work in source code form is that -same work. - - 2. Basic Permissions. - - All rights granted under this License are granted for the term of -copyright on the Program, and are irrevocable provided the stated -conditions are met. This License explicitly affirms your unlimited -permission to run the unmodified Program. The output from running a -covered work is covered by this License only if the output, given its -content, constitutes a covered work. This License acknowledges your -rights of fair use or other equivalent, as provided by copyright law. - - You may make, run and propagate covered works that you do not -convey, without conditions so long as your license otherwise remains -in force. You may convey covered works to others for the sole purpose -of having them make modifications exclusively for you, or provide you -with facilities for running those works, provided that you comply with -the terms of this License in conveying all material for which you do -not control copyright. Those thus making or running the covered works -for you must do so exclusively on your behalf, under your direction -and control, on terms that prohibit them from making any copies of -your copyrighted material outside their relationship with you. - - Conveying under any other circumstances is permitted solely under -the conditions stated below. Sublicensing is not allowed; section 10 -makes it unnecessary. - - 3. Protecting Users' Legal Rights From Anti-Circumvention Law. - - No covered work shall be deemed part of an effective technological -measure under any applicable law fulfilling obligations under article -11 of the WIPO copyright treaty adopted on 20 December 1996, or -similar laws prohibiting or restricting circumvention of such -measures. - - When you convey a covered work, you waive any legal power to forbid -circumvention of technological measures to the extent such circumvention -is effected by exercising rights under this License with respect to -the covered work, and you disclaim any intention to limit operation or -modification of the work as a means of enforcing, against the work's -users, your or third parties' legal rights to forbid circumvention of -technological measures. - - 4. Conveying Verbatim Copies. - - You may convey verbatim copies of the Program's source code as you -receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice; -keep intact all notices stating that this License and any -non-permissive terms added in accord with section 7 apply to the code; -keep intact all notices of the absence of any warranty; and give all -recipients a copy of this License along with the Program. - - You may charge any price or no price for each copy that you convey, -and you may offer support or warranty protection for a fee. - - 5. Conveying Modified Source Versions. - - You may convey a work based on the Program, or the modifications to -produce it from the Program, in the form of source code under the -terms of section 4, provided that you also meet all of these conditions: - - a) The work must carry prominent notices stating that you modified - it, and giving a relevant date. - - b) The work must carry prominent notices stating that it is - released under this License and any conditions added under section - 7. This requirement modifies the requirement in section 4 to - "keep intact all notices". - - c) You must license the entire work, as a whole, under this - License to anyone who comes into possession of a copy. This - License will therefore apply, along with any applicable section 7 - additional terms, to the whole of the work, and all its parts, - regardless of how they are packaged. This License gives no - permission to license the work in any other way, but it does not - invalidate such permission if you have separately received it. - - d) If the work has interactive user interfaces, each must display - Appropriate Legal Notices; however, if the Program has interactive - interfaces that do not display Appropriate Legal Notices, your - work need not make them do so. - - A compilation of a covered work with other separate and independent -works, which are not by their nature extensions of the covered work, -and which are not combined with it such as to form a larger program, -in or on a volume of a storage or distribution medium, is called an -"aggregate" if the compilation and its resulting copyright are not -used to limit the access or legal rights of the compilation's users -beyond what the individual works permit. Inclusion of a covered work -in an aggregate does not cause this License to apply to the other -parts of the aggregate. - - 6. Conveying Non-Source Forms. - - You may convey a covered work in object code form under the terms -of sections 4 and 5, provided that you also convey the -machine-readable Corresponding Source under the terms of this License, -in one of these ways: - - a) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by the - Corresponding Source fixed on a durable physical medium - customarily used for software interchange. - - b) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by a - written offer, valid for at least three years and valid for as - long as you offer spare parts or customer support for that product - model, to give anyone who possesses the object code either (1) a - copy of the Corresponding Source for all the software in the - product that is covered by this License, on a durable physical - medium customarily used for software interchange, for a price no - more than your reasonable cost of physically performing this - conveying of source, or (2) access to copy the - Corresponding Source from a network server at no charge. - - c) Convey individual copies of the object code with a copy of the - written offer to provide the Corresponding Source. This - alternative is allowed only occasionally and noncommercially, and - only if you received the object code with such an offer, in accord - with subsection 6b. - - d) Convey the object code by offering access from a designated - place (gratis or for a charge), and offer equivalent access to the - Corresponding Source in the same way through the same place at no - further charge. You need not require recipients to copy the - Corresponding Source along with the object code. If the place to - copy the object code is a network server, the Corresponding Source - may be on a different server (operated by you or a third party) - that supports equivalent copying facilities, provided you maintain - clear directions next to the object code saying where to find the - Corresponding Source. Regardless of what server hosts the - Corresponding Source, you remain obligated to ensure that it is - available for as long as needed to satisfy these requirements. - - e) Convey the object code using peer-to-peer transmission, provided - you inform other peers where the object code and Corresponding - Source of the work are being offered to the general public at no - charge under subsection 6d. - - A separable portion of the object code, whose source code is excluded -from the Corresponding Source as a System Library, need not be -included in conveying the object code work. - - A "User Product" is either (1) a "consumer product", which means any -tangible personal property which is normally used for personal, family, -or household purposes, or (2) anything designed or sold for incorporation -into a dwelling. In determining whether a product is a consumer product, -doubtful cases shall be resolved in favor of coverage. For a particular -product received by a particular user, "normally used" refers to a -typical or common use of that class of product, regardless of the status -of the particular user or of the way in which the particular user -actually uses, or expects or is expected to use, the product. A product -is a consumer product regardless of whether the product has substantial -commercial, industrial or non-consumer uses, unless such uses represent -the only significant mode of use of the product. - - "Installation Information" for a User Product means any methods, -procedures, authorization keys, or other information required to install -and execute modified versions of a covered work in that User Product from -a modified version of its Corresponding Source. The information must -suffice to ensure that the continued functioning of the modified object -code is in no case prevented or interfered with solely because -modification has been made. - - If you convey an object code work under this section in, or with, or -specifically for use in, a User Product, and the conveying occurs as -part of a transaction in which the right of possession and use of the -User Product is transferred to the recipient in perpetuity or for a -fixed term (regardless of how the transaction is characterized), the -Corresponding Source conveyed under this section must be accompanied -by the Installation Information. But this requirement does not apply -if neither you nor any third party retains the ability to install -modified object code on the User Product (for example, the work has -been installed in ROM). - - The requirement to provide Installation Information does not include a -requirement to continue to provide support service, warranty, or updates -for a work that has been modified or installed by the recipient, or for -the User Product in which it has been modified or installed. Access to a -network may be denied when the modification itself materially and -adversely affects the operation of the network or violates the rules and -protocols for communication across the network. - - Corresponding Source conveyed, and Installation Information provided, -in accord with this section must be in a format that is publicly -documented (and with an implementation available to the public in -source code form), and must require no special password or key for -unpacking, reading or copying. - - 7. Additional Terms. - - "Additional permissions" are terms that supplement the terms of this -License by making exceptions from one or more of its conditions. -Additional permissions that are applicable to the entire Program shall -be treated as though they were included in this License, to the extent -that they are valid under applicable law. If additional permissions -apply only to part of the Program, that part may be used separately -under those permissions, but the entire Program remains governed by -this License without regard to the additional permissions. - - When you convey a copy of a covered work, you may at your option -remove any additional permissions from that copy, or from any part of -it. (Additional permissions may be written to require their own -removal in certain cases when you modify the work.) You may place -additional permissions on material, added by you to a covered work, -for which you have or can give appropriate copyright permission. - - Notwithstanding any other provision of this License, for material you -add to a covered work, you may (if authorized by the copyright holders of -that material) supplement the terms of this License with terms: - - a) Disclaiming warranty or limiting liability differently from the - terms of sections 15 and 16 of this License; or - - b) Requiring preservation of specified reasonable legal notices or - author attributions in that material or in the Appropriate Legal - Notices displayed by works containing it; or - - c) Prohibiting misrepresentation of the origin of that material, or - requiring that modified versions of such material be marked in - reasonable ways as different from the original version; or - - d) Limiting the use for publicity purposes of names of licensors or - authors of the material; or - - e) Declining to grant rights under trademark law for use of some - trade names, trademarks, or service marks; or - - f) Requiring indemnification of licensors and authors of that - material by anyone who conveys the material (or modified versions of - it) with contractual assumptions of liability to the recipient, for - any liability that these contractual assumptions directly impose on - those licensors and authors. - - All other non-permissive additional terms are considered "further -restrictions" within the meaning of section 10. If the Program as you -received it, or any part of it, contains a notice stating that it is -governed by this License along with a term that is a further -restriction, you may remove that term. If a license document contains -a further restriction but permits relicensing or conveying under this -License, you may add to a covered work material governed by the terms -of that license document, provided that the further restriction does -not survive such relicensing or conveying. - - If you add terms to a covered work in accord with this section, you -must place, in the relevant source files, a statement of the -additional terms that apply to those files, or a notice indicating -where to find the applicable terms. - - Additional terms, permissive or non-permissive, may be stated in the -form of a separately written license, or stated as exceptions; -the above requirements apply either way. - - 8. Termination. - - You may not propagate or modify a covered work except as expressly -provided under this License. Any attempt otherwise to propagate or -modify it is void, and will automatically terminate your rights under -this License (including any patent licenses granted under the third -paragraph of section 11). - - However, if you cease all violation of this License, then your -license from a particular copyright holder is reinstated (a) -provisionally, unless and until the copyright holder explicitly and -finally terminates your license, and (b) permanently, if the copyright -holder fails to notify you of the violation by some reasonable means -prior to 60 days after the cessation. - - Moreover, your license from a particular copyright holder is -reinstated permanently if the copyright holder notifies you of the -violation by some reasonable means, this is the first time you have -received notice of violation of this License (for any work) from that -copyright holder, and you cure the violation prior to 30 days after -your receipt of the notice. - - Termination of your rights under this section does not terminate the -licenses of parties who have received copies or rights from you under -this License. If your rights have been terminated and not permanently -reinstated, you do not qualify to receive new licenses for the same -material under section 10. - - 9. Acceptance Not Required for Having Copies. - - You are not required to accept this License in order to receive or -run a copy of the Program. Ancillary propagation of a covered work -occurring solely as a consequence of using peer-to-peer transmission -to receive a copy likewise does not require acceptance. However, -nothing other than this License grants you permission to propagate or -modify any covered work. These actions infringe copyright if you do -not accept this License. Therefore, by modifying or propagating a -covered work, you indicate your acceptance of this License to do so. - - 10. Automatic Licensing of Downstream Recipients. - - Each time you convey a covered work, the recipient automatically -receives a license from the original licensors, to run, modify and -propagate that work, subject to this License. You are not responsible -for enforcing compliance by third parties with this License. - - An "entity transaction" is a transaction transferring control of an -organization, or substantially all assets of one, or subdividing an -organization, or merging organizations. If propagation of a covered -work results from an entity transaction, each party to that -transaction who receives a copy of the work also receives whatever -licenses to the work the party's predecessor in interest had or could -give under the previous paragraph, plus a right to possession of the -Corresponding Source of the work from the predecessor in interest, if -the predecessor has it or can get it with reasonable efforts. - - You may not impose any further restrictions on the exercise of the -rights granted or affirmed under this License. For example, you may -not impose a license fee, royalty, or other charge for exercise of -rights granted under this License, and you may not initiate litigation -(including a cross-claim or counterclaim in a lawsuit) alleging that -any patent claim is infringed by making, using, selling, offering for -sale, or importing the Program or any portion of it. - - 11. Patents. - - A "contributor" is a copyright holder who authorizes use under this -License of the Program or a work on which the Program is based. The -work thus licensed is called the contributor's "contributor version". - - A contributor's "essential patent claims" are all patent claims -owned or controlled by the contributor, whether already acquired or -hereafter acquired, that would be infringed by some manner, permitted -by this License, of making, using, or selling its contributor version, -but do not include claims that would be infringed only as a -consequence of further modification of the contributor version. For -purposes of this definition, "control" includes the right to grant -patent sublicenses in a manner consistent with the requirements of -this License. - - Each contributor grants you a non-exclusive, worldwide, royalty-free -patent license under the contributor's essential patent claims, to -make, use, sell, offer for sale, import and otherwise run, modify and -propagate the contents of its contributor version. - - In the following three paragraphs, a "patent license" is any express -agreement or commitment, however denominated, not to enforce a patent -(such as an express permission to practice a patent or covenant not to -sue for patent infringement). To "grant" such a patent license to a -party means to make such an agreement or commitment not to enforce a -patent against the party. - - If you convey a covered work, knowingly relying on a patent license, -and the Corresponding Source of the work is not available for anyone -to copy, free of charge and under the terms of this License, through a -publicly available network server or other readily accessible means, -then you must either (1) cause the Corresponding Source to be so -available, or (2) arrange to deprive yourself of the benefit of the -patent license for this particular work, or (3) arrange, in a manner -consistent with the requirements of this License, to extend the patent -license to downstream recipients. "Knowingly relying" means you have -actual knowledge that, but for the patent license, your conveying the -covered work in a country, or your recipient's use of the covered work -in a country, would infringe one or more identifiable patents in that -country that you have reason to believe are valid. - - If, pursuant to or in connection with a single transaction or -arrangement, you convey, or propagate by procuring conveyance of, a -covered work, and grant a patent license to some of the parties -receiving the covered work authorizing them to use, propagate, modify -or convey a specific copy of the covered work, then the patent license -you grant is automatically extended to all recipients of the covered -work and works based on it. - - A patent license is "discriminatory" if it does not include within -the scope of its coverage, prohibits the exercise of, or is -conditioned on the non-exercise of one or more of the rights that are -specifically granted under this License. You may not convey a covered -work if you are a party to an arrangement with a third party that is -in the business of distributing software, under which you make payment -to the third party based on the extent of your activity of conveying -the work, and under which the third party grants, to any of the -parties who would receive the covered work from you, a discriminatory -patent license (a) in connection with copies of the covered work -conveyed by you (or copies made from those copies), or (b) primarily -for and in connection with specific products or compilations that -contain the covered work, unless you entered into that arrangement, -or that patent license was granted, prior to 28 March 2007. - - Nothing in this License shall be construed as excluding or limiting -any implied license or other defenses to infringement that may -otherwise be available to you under applicable patent law. - - 12. No Surrender of Others' Freedom. - - If conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot convey a -covered work so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you may -not convey it at all. For example, if you agree to terms that obligate you -to collect a royalty for further conveying from those to whom you convey -the Program, the only way you could satisfy both those terms and this -License would be to refrain entirely from conveying the Program. - - 13. Use with the GNU Affero General Public License. - - Notwithstanding any other provision of this License, you have -permission to link or combine any covered work with a work licensed -under version 3 of the GNU Affero General Public License into a single -combined work, and to convey the resulting work. The terms of this -License will continue to apply to the part which is the covered work, -but the special requirements of the GNU Affero General Public License, -section 13, concerning interaction through a network will apply to the -combination as such. - - 14. Revised Versions of this License. - - The Free Software Foundation may publish revised and/or new versions of -the GNU General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - - Each version is given a distinguishing version number. If the -Program specifies that a certain numbered version of the GNU General -Public License "or any later version" applies to it, you have the -option of following the terms and conditions either of that numbered -version or of any later version published by the Free Software -Foundation. If the Program does not specify a version number of the -GNU General Public License, you may choose any version ever published -by the Free Software Foundation. - - If the Program specifies that a proxy can decide which future -versions of the GNU General Public License can be used, that proxy's -public statement of acceptance of a version permanently authorizes you -to choose that version for the Program. - - Later license versions may give you additional or different -permissions. However, no additional obligations are imposed on any -author or copyright holder as a result of your choosing to follow a -later version. - - 15. Disclaimer of Warranty. - - THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY -APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT -HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY -OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM -IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF -ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. Limitation of Liability. - - IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS -THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE -USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF -DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD -PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), -EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF -SUCH DAMAGES. - - 17. Interpretation of Sections 15 and 16. - - If the disclaimer of warranty and limitation of liability provided -above cannot be given local legal effect according to their terms, -reviewing courts shall apply local law that most closely approximates -an absolute waiver of all civil liability in connection with the -Program, unless a warranty or assumption of liability accompanies a -copy of the Program in return for a fee. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -state the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - <one line to give the program's name and a brief idea of what it does.> - Copyright (C) <year> <name of author> - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see <https://www.gnu.org/licenses/>. - -Also add information on how to contact you by electronic and paper mail. - - If the program does terminal interaction, make it output a short -notice like this when it starts in an interactive mode: - - <program> Copyright (C) <year> <name of author> - This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, your program's commands -might be different; for a GUI interface, you would use an "about box". - - You should also get your employer (if you work as a programmer) or school, -if any, to sign a "copyright disclaimer" for the program, if necessary. -For more information on this, and how to apply and follow the GNU GPL, see -<https://www.gnu.org/licenses/>. - - The GNU General Public License does not permit incorporating your program -into proprietary programs. If your program is a subroutine library, you -may consider it more useful to permit linking proprietary applications with -the library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. But first, please read -<https://www.gnu.org/licenses/why-not-lgpl.html>. diff --git a/support/tools/git-third-party/README.md b/support/tools/git-third-party/README.md deleted file mode 100644 index af88763b..00000000 --- a/support/tools/git-third-party/README.md +++ /dev/null @@ -1,35 +0,0 @@ -# Git third party - -Tool for patching third-party libraries without need to fork them - -This is a small zero-dependency utility that is an alternative to [git-submodules](https://git-scm.com/book/en/v2/Git-Tools-Submodules) and [git-subtree](https://manpages.debian.org/testing/git-man/git-subtree.1.en.html). It stores only delta (changes) inside your tree - -This tool is not oriented for highly concurrent modification of third-party tools by multiple users - -## Usage -```bash -# adding third-party repository -git-third-party add third-party/RustPython https://github.com/RustPython/RustPython.git a13b99642b0bc13ca89d01768a7ddbec18fe8219 -# ^ git-third-party add <relative path> <repository url> <commit hash> - -# now you can modify it as a regular repository, commit your changes - -# save patches to push to your origin -git-third-party save third-party/RustPython -# ^ git-third-party save <relative path> - -# update -git-third-party update third-party/RustPython -# ^ git-third-party update <relative path> -``` - -## How it works? - -It stores (at `/.git-third-party/`) a configuration that describes all third party repositories and patches. `save` command updates the patches, `update` command reapplies them - -Directory `.git-third-party` must be controlled by git, while third-party repos should not - -Best effort is done to keep patches deterministic and strip all metadata from them, which includes: -- git version -- commit hash (which depends on committer) -- ... diff --git a/support/tools/git-third-party/git-third-party b/support/tools/git-third-party/git-third-party deleted file mode 100755 index e0b436a5..00000000 --- a/support/tools/git-third-party/git-third-party +++ /dev/null @@ -1,447 +0,0 @@ -#!/usr/bin/env python3 - -# -# This file is part of the git-third-party distribution (https://github.com/kp2pml30/git-third-party). -# Copyright (c) 2024-2025 Kira Prokopenko kp2pml30@gmail.com -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, version 3. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -# - -import abc -import io -import json -import logging -import os -import re -import subprocess -import sys -import textwrap -import typing -from pathlib import Path - -EXE_NAME = 'git third-party' - -logging.basicConfig(stream=sys.stderr, level=logging.INFO) - -# Runtime context, populated by `_init_context()` at the start of each `main()` -# call rather than at import time. Keeping import side-effect free lets this file -# be imported directly (e.g. by the test-suite) and driven in-process. -GIT = 'git' -top_dir: Path -cur_dir: Path -SAVED_FILES_PATH: Path -PATCHES_PATH: Path -CONFIG_PATH: Path -config: dict - - -def _init_context() -> None: - global GIT, top_dir, cur_dir, SAVED_FILES_PATH, PATCHES_PATH, CONFIG_PATH, config - - GIT = os.getenv('GIT', 'git') - - try: - top_dir_str = subprocess.check_output( - [GIT, 'rev-parse', '--show-toplevel'], text=True - ) - except Exception: - print( - f'could not execute `{GIT} rev-parse --show-toplevel`. You may wish to set `GIT` env variable', - file=sys.stderr, - ) - sys.exit(1) - - top_dir = Path(top_dir_str.strip()) - cur_dir = Path(os.getcwd()) - - SAVED_FILES_PATH = top_dir.joinpath('.git-third-party') - PATCHES_PATH = SAVED_FILES_PATH.joinpath('patches') - CONFIG_PATH = SAVED_FILES_PATH.joinpath('config.json') - - if not PATCHES_PATH.exists(): - logging.info(f'patches directory does not exist, creating path=`{PATCHES_PATH}`') - PATCHES_PATH.mkdir(parents=True, exist_ok=True) - - if not CONFIG_PATH.exists(): - config = {'repos': {}} - else: - config = json.loads(CONFIG_PATH.read_text()) - - -class Command(metaclass=abc.ABCMeta): - name: typing.ClassVar[str] - help: str - - @abc.abstractmethod - def handle(self, args: list[str]) -> None: ... - - def show_help(self, file: io.StringIO): - file.write(f'subcommand `{self.name}`:\n') - file.write(textwrap.indent(textwrap.dedent(self.help), prefix='\t').rstrip()) - file.write('\n') - - -def _get_patches_dir(name: str) -> Path: - return PATCHES_PATH.joinpath(*name.split('/')) - - -def _dirty_check(name: str) -> None: - target_dir = top_dir.joinpath(*name.split('/')) - # `--untracked-files=all` so a brand-new local file counts as dirt, and - # `--ignore-submodules=none` so a hostile `diff.ignoreSubmodules` config can - # not hide a modified submodule. A non-empty porcelain listing is dirt; it - # also names the offending paths, which we surface to the user. - rs = subprocess.run( - [GIT, 'status', '--porcelain', '--untracked-files=all', '--ignore-submodules=none'], - check=True, - cwd=target_dir, - text=True, - capture_output=True, - ) - if rs.stdout.strip(): - raise GitThirdPartyException( - f'repo {name} is dirty ( {target_dir} ):\n{rs.stdout.rstrip()}' - ) - - -def _patch_id(diff: str, cwd: Path) -> str | None: - """Stable patch-id of a diff/patch, or None when there is nothing to hash. - - `--stable` hashes only the diff hunks, ignoring the commit hash, author, - date and message β€” so a patch re-applied by `git am` compares equal to the - saved patch file it came from, which is exactly the metadata-independence the - rest of the tool relies on. - """ - rs = subprocess.run( - [GIT, 'patch-id', '--stable'], - input=diff, - text=True, - capture_output=True, - check=True, - cwd=cwd, - ) - out = rs.stdout.strip() - # Empty for a patch with no textual diff (e.g. an empty or merge commit). - return out.split()[0] if out else None - - -def _unsaved_commits_check(name: str) -> None: - """Refuse to overwrite local commits that were never `save`d. - - `update` detaches to the base commit and replays the *saved* patch series, so - any commit on top that no patch file records would be silently orphaned. We - compare the current history above the base against the saved patches by - patch-id, keep the longest matching prefix (K), and error when the checkout - is *ahead* β€” carrying `commits(base..HEAD) - K` unrecorded commits. - """ - conf = config['repos'][name] - target_dir = top_dir.joinpath(*name.split('/')) - base = conf['commit'] - - # Nothing checked out yet (fresh `add` / freshly `init`-ed dir): nothing to - # protect. - if ( - subprocess.run( - [GIT, 'rev-parse', '--verify', '--quiet', 'HEAD'], - check=False, - cwd=target_dir, - capture_output=True, - ).returncode - != 0 - ): - return - - # Base not in this checkout: we cannot relate HEAD to the saved series (most - # likely the pin was bumped). Let update fetch + reapply; the worktree dirty - # check still protects uncommitted work. - if ( - subprocess.run( - [GIT, 'cat-file', '-e', f'{base}^{{commit}}'], - check=False, - cwd=target_dir, - capture_output=True, - ).returncode - != 0 - ): - logging.warning( - f'base commit {base} not present in {name}, skipping unsaved-commits check' - ) - return - - revs = subprocess.run( - [GIT, 'rev-list', '--reverse', f'{base}..HEAD'], - check=True, - cwd=target_dir, - text=True, - capture_output=True, - ).stdout.split() - current_ids = [ - _patch_id( - subprocess.run( - [GIT, 'diff-tree', '--patch', '--no-commit-id', rev], - check=True, - cwd=target_dir, - text=True, - capture_output=True, - ).stdout, - target_dir, - ) - for rev in revs - ] - - patches_dir = _get_patches_dir(name) - saved_ids = [ - _patch_id(patches_dir.joinpath(str(i)).read_text(), target_dir) - for i in range(1, conf['patches'] + 1) - ] - - # Longest common prefix of patch-ids; a None (unhashable commit) stops it. - matched = 0 - for cur, saved in zip(current_ids, saved_ids): - if cur is None or cur != saved: - break - matched += 1 - - ahead = len(current_ids) - matched - if ahead > 0: - raise GitThirdPartyException( - f'repo {name} is {ahead} commit(s) ahead of its saved patches ' - f'( {target_dir} ); run `save` first or reset those commits' - ) - - -def _update_at(name: str) -> None: - conf = config['repos'][name] - target_dir = top_dir.joinpath(*name.split('/')) - logging.info(f'updating name={name} at path={target_dir}') - if not target_dir.exists(): - target_dir.mkdir(parents=True, exist_ok=True) - subprocess.run([GIT, 'init'], check=True, cwd=target_dir) - subprocess.run( - [GIT, 'remote', 'add', 'origin', conf['url']], check=True, cwd=target_dir - ) - - _dirty_check(name) - _unsaved_commits_check(name) - rs = subprocess.run( - [GIT, '-c', 'advice.detachedHead=false', 'checkout', conf['commit']], - check=False, - cwd=target_dir, - ) - if rs.returncode != 0: - logging.info(f'checking out {conf["commit"]} failed, fetching') - subprocess.run( - [GIT, 'fetch', 'origin', '--depth=1', conf['commit']], - check=True, - cwd=target_dir, - ) - subprocess.run( - [GIT, '-c', 'advice.detachedHead=false', 'checkout', conf['commit']], - check=True, - cwd=target_dir, - ) - subprocess.run( - [GIT, 'submodule', 'update', '--init', '--recursive', '--depth', '1'], - check=True, - cwd=target_dir, - ) - submodules = conf.get('submodules', None) - if submodules is None: - logging.info('updating all submodules') - subprocess.run( - [GIT, 'submodule', 'update', '--recursive', '--depth', '1'], - check=True, - cwd=target_dir, - ) - elif len(submodules) != 0: - logging.info(f'updating submodules {submodules}') - subprocess.run( - [GIT, 'submodule', 'update', '--recursive', '--depth', '1', '--'] + submodules, - check=True, - cwd=target_dir, - ) - # apply patches - patches_dir = _get_patches_dir(name) - patch_files = [patches_dir.joinpath(str(i)) for i in range(1, conf['patches'] + 1)] - if len(patch_files) != 0: - subprocess.run([GIT, 'am', *patch_files], cwd=target_dir, check=True) - else: - logging.warning('no patches detected') - - -class GitThirdPartyException(Exception): - pass - - -def _get_path_rel_to_dir(path: str) -> str: - target_dir = cur_dir.joinpath(Path(path)) - if top_dir not in target_dir.parents: - raise GitThirdPartyException( - f'target directory ({target_dir}) is not in git subtree ({top_dir})' - ) - return str(target_dir.relative_to(top_dir)).replace(os.sep, '/') - - -class AddCommand(Command): - name = 'add' - help = f"""\ - {EXE_NAME} {name} <PATH> <REPO URL> <COMMIT> - - Adds a new repository to track - Example: `{EXE_NAME} {name} third-party/my-cool-tool https://github.com/kp2pml30/git-third-party 4e6cc662a7e2b409e36d2412440c5670d6034431` - """ - - def handle(self, args: list[str]) -> None: - if len(args) != 3: - print(f'Unexpected arguments {args}', file=sys.stderr) - self.show_help(sys.stderr) - sys.exit(1) - path, repo, commit = args - - if re.search(r'[\\:;]', path): - print(f'bad path `{path}`', file=sys.stderr) - sys.exit(1) - name = _get_path_rel_to_dir(path) - if name in config['repos']: - print(f'repo ({name}) is already in config', file=sys.stderr) - sys.exit(1) - target_dir = top_dir.joinpath(*name.split('/')) - res = subprocess.run([GIT, 'check-ignore', target_dir], check=False) - if res.returncode != 0: - raise GitThirdPartyException( - f'target directory ({target_dir}) is not ignored by git' - ) - config['repos'][name] = { - 'url': repo, - 'commit': commit, - 'patches': 0, - } - _update_at(name) - - -def _get_all_names() -> list[str]: - return list(config['repos'].keys()) - - -def _get_names_from_paths(paths: list[str]) -> list[str]: - if len(paths) == 0: - raise GitThirdPartyException('excepted non-empty list of arguments') - if paths == ['--all']: - return _get_all_names() - return list(set(map(_get_path_rel_to_dir, paths))) - - -class UpdateCommand(Command): - name = 'update' - help = f"""\ - {EXE_NAME} {name} --all - {EXE_NAME} {name} relative/path/to/repo/1 relative/path/to/repo/2 - - restore state of specified repositories - """ - - def handle(self, args): - names = _get_names_from_paths(args) - for name in names: - _update_at(name) - - -class SaveCommand(Command): - name = 'save' - help = f"""\ - {EXE_NAME} {name} --all - {EXE_NAME} {name} relative/path/to/repo/1 relative/path/to/repo/2 - - record state of specified repositories - """ - - def handle(self, args): - names = _get_names_from_paths(args) - for name in names: - _save_at(name) - - -def _save_at(name: str) -> None: - _dirty_check(name) - target_dir = top_dir.joinpath(*name.split('/')) - logging.info(f'saving name={name} at path={target_dir}') - conf = config['repos'][name] - patches_dir = _get_patches_dir(name) - for i in range(1, conf['patches'] + 1): - patches_dir.joinpath(str(i)).unlink() - patches_dir.mkdir(parents=True, exist_ok=True) - - subprocess.run( - [ - GIT, - 'format-patch', - f'{conf["commit"]}..HEAD', - '--no-numbered', - '--zero-commit', - '--no-signature', - '--numbered-files', - '-o', - patches_dir, - ], - check=True, - cwd=target_dir, - ) - new_patches = subprocess.run( - [GIT, 'rev-list', '--count', f'{conf["commit"]}..HEAD'], - check=True, - cwd=target_dir, - text=True, - capture_output=True, - ) - patches_cnt = int(new_patches.stdout.strip()) - logging.info(f'number of patches is {patches_cnt}') - conf['patches'] = patches_cnt - - -def main(argv: list[str] | None = None) -> int: - argv = sys.argv[1:] if argv is None else list(argv) - - _init_context() - - mode = argv[0] if len(argv) > 0 else 'help' - - handlers: dict[str, Command] = {} - - for c in [UpdateCommand, SaveCommand, AddCommand]: - handlers[c.name] = c() - - mode_handler = handlers.get(mode, None) - if mode_handler is None: - print( - f'{EXE_NAME} expected one of following subcommands: ' + '|'.join(handlers.keys()), - file=sys.stderr, - ) - for k in handlers.values(): - k.show_help(sys.stderr) - return 1 - - try: - mode_handler.handle(argv[1:]) - except GitThirdPartyException as e: - print(*e.args, file=sys.stderr) - return 1 - - with open(CONFIG_PATH, 'wt') as f: - json.dump(config, f, indent='\t') - f.write('\n') - - return 0 - - -if __name__ == '__main__': # pragma: no cover - sys.exit(main()) diff --git a/support/tools/git-third-party/ruff.toml b/support/tools/git-third-party/ruff.toml deleted file mode 100644 index 34b109c4..00000000 --- a/support/tools/git-third-party/ruff.toml +++ /dev/null @@ -1,9 +0,0 @@ -indent-width = 2 - -[format] -indent-style = "tab" -line-ending = "lf" -quote-style = "single" - -docstring-code-format = true -skip-magic-trailing-comma = false diff --git a/support/zig.nix b/support/zig.nix index 96b431dd..4cce5318 100644 --- a/support/zig.nix +++ b/support/zig.nix @@ -36,6 +36,11 @@ let [[ "$arg" == */self-contained/crtendS.o ]]; then continue fi + # Zig's cc driver rejects the aarch64 erratum flag rustc passes; the + # workaround only matters on early Cortex-A53 revisions, not our targets. + if [[ "$arg" == -Wl,--fix-cortex-a53-843419 ]]; then + continue + fi if [[ "$arg" != --target=* ]] && \ [[ "$arg" != -framework ]] && \ [[ "$arg" != CoreFoundation ]] && \ diff --git a/tests/runner/genvm_tool_plugins/integration.py b/tests/runner/genvm_tool_plugins/integration.py index f2c7fb12..8ce58c2a 100644 --- a/tests/runner/genvm_tool_plugins/integration.py +++ b/tests/runner/genvm_tool_plugins/integration.py @@ -83,6 +83,19 @@ def _is_ignore_hash_enabled() -> bool: if genvm_tool.tests.util.environ.DEFAULT_FILTER(k, v) } + +def _normalize_message_fee_allocation( + allocation: fees.MessageAllocationNode, +) -> fees.MessageAllocationNode: + allocation = allocation.copy() + if isinstance(allocation['recipient'], str): + allocation['recipient'] = Address(allocation['recipient']) + allocation['children'] = [ + _normalize_message_fee_allocation(child) for child in allocation['children'] + ] + return allocation + + # `prepare` scripts run `cargo build`; the host `rustc` used for build # scripts/proc-macros needs libz etc. on LD_LIBRARY_PATH. Fold in # CARGO_LD_LIBRARY_PATH the same way cargo.py does for its own commands. @@ -429,12 +442,12 @@ async def run( base_mock_storage = MockStorage() if storage_json := top_level_conf.get('storage_json'): storage_b64 = json.loads(await gvm_io.read_file_text(Path(storage_json))) - base_mock_storage._storages = { - Address(a): { - base64.b64decode(k): bytearray(base64.b64decode(v)) for k, v in kv.items() + base_mock_storage.load( + { + Address(a): {base64.b64decode(k): base64.b64decode(v) for k, v in kv.items()} + for a, kv in storage_b64.items() } - for a, kv in storage_b64.items() - } + ) empty_storage = tmp_dir.joinpath('empty-storage.pickle') await gvm_io.write_file_bytes(empty_storage, pickle.dumps(base_mock_storage)) @@ -764,6 +777,10 @@ def resolve_executor(address, state, advisory_major): message_fee_allocation: list[fees.MessageAllocationNode] = single_conf.get( 'message_fee_allocation', default_message_fee_allocation ) + message_fee_allocation = [ + _normalize_message_fee_allocation(allocation) + for allocation in message_fee_allocation + ] if not single_conf['message'].get('is_init', False): code = None diff --git a/tests/runner/gvm_extra/mock_host.py b/tests/runner/gvm_extra/mock_host.py index 26dc21d4..2a86b114 100644 --- a/tests/runner/gvm_extra/mock_host.py +++ b/tests/runner/gvm_extra/mock_host.py @@ -21,16 +21,55 @@ ] +SLOT_SIZE = 2**32 +""" +Size of a single storage slot, as the guest ABI sees it. +""" + +_PAGE_SIZE = 4096 + + class MockStorage: - _storages: dict[Address, dict[bytes, bytearray]] + """ + Slot storage keeping only the pages that were actually touched. + + A slot spans `SLOT_SIZE` bytes, so a write near its end must not + materialize everything before it. + """ + + _storages: dict[Address, dict[bytes, dict[int, bytearray]]] def __init__(self): self._storages = {} + def load( + self, + contents: collections.abc.Mapping[ + Address, collections.abc.Mapping[bytes, collections.abc.Buffer] + ], + ) -> None: + """ + Replaces the contents with slots given from their first byte on. + """ + self._storages = {} + for account, slots in contents.items(): + for slot, data in slots.items(): + self.write(account, slot, 0, data) + + def slots(self, account: Address) -> collections.abc.Iterable[bytes]: + return self._storages.get(account, {}).keys() + def read(self, account: Address, slot: bytes, index: int, le: int) -> bytes: - res = self._storages.setdefault(account, {}) - res = res.setdefault(slot, bytearray()) - return res[index : index + le] + b'\x00' * (le - max(0, len(res) - index)) + assert index + le <= SLOT_SIZE, 'read past the end of the slot' + pages = self._pages(account, slot) + res = bytearray(le) + at = 0 + for page_no, offset, take in _spans(index, le): + page = pages.get(page_no) + if page is not None: + res[at : at + take] = page[offset : offset + take] + at += take + return bytes(res) def write( self, @@ -39,11 +78,29 @@ def write( index: int, what: collections.abc.Buffer, ) -> None: - res = self._storages.setdefault(account, {}) - res = res.setdefault(slot, bytearray()) what = memoryview(what) - res.extend(b'\x00' * (index + len(what) - len(res))) - memoryview(res)[index : index + len(what)] = what + assert index + len(what) <= SLOT_SIZE, 'write past the end of the slot' + pages = self._pages(account, slot) + at = 0 + for page_no, offset, take in _spans(index, len(what)): + page = pages.setdefault(page_no, bytearray(_PAGE_SIZE)) + page[offset : offset + take] = what[at : at + take] + at += take + + def _pages(self, account: Address, slot: bytes) -> dict[int, bytearray]: + return self._storages.setdefault(account, {}).setdefault(slot, {}) + + +def _spans(index: int, le: int) -> collections.abc.Iterator[tuple[int, int, int]]: + """ + Splits ``[index, index + le)`` into ``(page, offset in it, length)`` triples. + """ + while le > 0: + page_no, offset = divmod(index, _PAGE_SIZE) + take = min(_PAGE_SIZE - offset, le) + yield page_no, offset, take + index += take + le -= take _STOP_CONNECTIONS_TIMEOUT_S = 10.0 diff --git a/tests/system/cross-major/test.py b/tests/system/cross-major/test.py index c6a6e34c..6373d8f1 100644 --- a/tests/system/cross-major/test.py +++ b/tests/system/cross-major/test.py @@ -452,12 +452,12 @@ def _replace_code(self, address: Address, old_code: bytes, new_code: bytes) -> N new_code = typing.cast(bytes, resolve_runners(new_code, root_dir)) with open(self.storage_path, 'rb') as storage_file: storage: MockStorage = pickle.load(storage_file) - slots = storage._storages[address] code_slots = [ slot - for slot, data in slots.items() - if int.from_bytes(data[:4], byteorder='little') == len(old_code) - and bytes(data[4 : 4 + len(old_code)]) == old_code + for slot in storage.slots(address) + if int.from_bytes(storage.read(address, slot, 0, 4), byteorder='little') + == len(old_code) + and storage.read(address, slot, 4, len(old_code)) == old_code ] assert len(code_slots) == 1, code_slots storage.write( From 8271d7145bb02a40bf3dcc55919f701eb85cdbe6 Mon Sep 17 00:00:00 2001 From: Kira <kp2pml30@gmail.com> Date: Mon, 14 Sep 2026 16:18:03 +0900 Subject: [PATCH 17/45] chore: port pending executor and release hardening (#36) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(executor): bound retained RAM a contract can drive πŸ”’οΈ - intern directory descriptor path components instead of keeping owned strings - bound the host balance cache, which grew per distinct address for a whole run - spec: a sub-VM copies its caller's RAM budget rather than sharing it, so what a child spends never reduces what its caller may still spend * fix(executor): adopt deferred arguments and balance assertions πŸ”’οΈ * chore(ci): pin actions to digests and attest release assets πŸ”’οΈ * chore(ci): pin every third-party action to a commit digest, Renovate keeps them fresh πŸ”’οΈ * feat(ci): publish SHA256SUMS and a sigstore provenance attestation with each release ✨ * feat(release): pin executor tarballs by sha256 and state what a platform asset ships πŸ”’οΈ * feat(release): build-manifest copies a line's executor-sha256 into the manager manifest, validated per platform ✨ * feat(install): post-install refuses an executor download without a pinned hash or with a mismatching one πŸ”’οΈ * test(release): cover build-manifest pins, post-install download outcomes and the manager tolerating the sha256 key βœ… * docs(release): a platform asset bundles every executor line, the download is a fallback; how to verify an asset πŸ“ * chore(build): align v0.2 with Wasmtime 48 ⬆️ --- .github/workflows/branch_executor_prs.yaml | 2 +- .github/workflows/branch_forward.yaml | 2 +- .github/workflows/branch_pr_actions.yaml | 2 +- .github/workflows/branch_provision.yaml | 2 +- .../branch_provision_executor_prs.yaml | 2 +- .github/workflows/branch_rebase_watch.yaml | 2 +- .github/workflows/branch_release_gate_pr.yaml | 2 +- .github/workflows/branch_retarget.yaml | 2 +- .../workflows/branch_run_tests_command.yaml | 4 +- .github/workflows/branch_sync_executors.yaml | 2 +- .github/workflows/branch_wasmtime_watch.yaml | 2 +- .github/workflows/docs_nightly.yaml | 4 +- .github/workflows/incl_initial.yaml | 14 +- .../workflows/incl_release_build_test.yaml | 6 +- .../incl_release_build_test_cell_build.yaml | 6 +- .../incl_release_build_test_cell_test.yaml | 8 +- .github/workflows/queue.yaml | 6 +- .github/workflows/queue_test_cell.yaml | 11 +- .github/workflows/release.yaml | 37 +++- .github/workflows/webdriver-image.yaml | 12 +- .../howto/committing/submodules.md | 2 +- .../howto/releasing/release-build.md | 21 +- .../howto/releasing/versioning.md | 11 + docs/contributing/howto/testing/python.md | 16 +- .../src/spec/03-vm/03-ram-limiting.rst | 10 +- executors/v0.2.x | 2 +- executors/v0.3.x | 2 +- flake.nix | 10 +- implementation/src/manager/versioning_test.rs | 3 +- install/lib/python/post-install/__main__.py | 36 +++- renovate.json | 1 + support/manifest-base.yaml | 7 +- .../tools/genvm-tool/genvm_tool/manifest.py | 31 ++- .../genvm-tool/unit_tests/test_manifest.py | 95 +++++++++ .../test_post_install_executor_download.py | 195 ++++++++++++++++++ 35 files changed, 492 insertions(+), 78 deletions(-) create mode 100644 support/tools/genvm-tool/unit_tests/test_manifest.py create mode 100644 support/tools/genvm-tool/unit_tests/test_post_install_executor_download.py diff --git a/.github/workflows/branch_executor_prs.yaml b/.github/workflows/branch_executor_prs.yaml index f6f7d971..be5b979c 100644 --- a/.github/workflows/branch_executor_prs.yaml +++ b/.github/workflows/branch_executor_prs.yaml @@ -51,7 +51,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Open and link executor PR(s) env: MANAGER_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/branch_forward.yaml b/.github/workflows/branch_forward.yaml index 63085761..9603796a 100644 --- a/.github/workflows/branch_forward.yaml +++ b/.github/workflows/branch_forward.yaml @@ -36,7 +36,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: ssh-key: ${{ secrets.GENVM_CI_PRIVATE_KEY }} # Full history: a shallow checkout only has the version branch's diff --git a/.github/workflows/branch_pr_actions.yaml b/.github/workflows/branch_pr_actions.yaml index 41110e04..fda2a850 100644 --- a/.github/workflows/branch_pr_actions.yaml +++ b/.github/workflows/branch_pr_actions.yaml @@ -52,7 +52,7 @@ jobs: # Check out the PR head so the handler runs the PR's own CI scripts. # This runs PR-authored code β€” the `ci-safe` gate on the job above is # what makes that safe. - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: ref: refs/pull/${{ github.event.issue.number }}/head # The handler talks to GitHub through `gh` (GH_TOKEN below) and never diff --git a/.github/workflows/branch_provision.yaml b/.github/workflows/branch_provision.yaml index be727df7..24e3de93 100644 --- a/.github/workflows/branch_provision.yaml +++ b/.github/workflows/branch_provision.yaml @@ -26,7 +26,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: # Deploy key so branch-creating pushes bypass protection on the # (protected) version / dev branches. Full history so the diff --git a/.github/workflows/branch_provision_executor_prs.yaml b/.github/workflows/branch_provision_executor_prs.yaml index 3fae94d4..86fffd08 100644 --- a/.github/workflows/branch_provision_executor_prs.yaml +++ b/.github/workflows/branch_provision_executor_prs.yaml @@ -55,7 +55,7 @@ jobs: exit 1 fi - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: ref: refs/pull/${{ inputs.pr }}/head # Provisioning goes through the executor token via `gh`, never `git diff --git a/.github/workflows/branch_rebase_watch.yaml b/.github/workflows/branch_rebase_watch.yaml index 39d9a3fc..2422490c 100644 --- a/.github/workflows/branch_rebase_watch.yaml +++ b/.github/workflows/branch_rebase_watch.yaml @@ -53,7 +53,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Flag PRs that are behind their base env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/branch_release_gate_pr.yaml b/.github/workflows/branch_release_gate_pr.yaml index b84e2e9a..7384806c 100644 --- a/.github/workflows/branch_release_gate_pr.yaml +++ b/.github/workflows/branch_release_gate_pr.yaml @@ -27,7 +27,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: # Full history for comparing the remote branches fetch-depth: 0 diff --git a/.github/workflows/branch_retarget.yaml b/.github/workflows/branch_retarget.yaml index e782843d..34ad582c 100644 --- a/.github/workflows/branch_retarget.yaml +++ b/.github/workflows/branch_retarget.yaml @@ -30,7 +30,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Retarget PR to latest dev branch env: diff --git a/.github/workflows/branch_run_tests_command.yaml b/.github/workflows/branch_run_tests_command.yaml index 708defc3..ff74b988 100644 --- a/.github/workflows/branch_run_tests_command.yaml +++ b/.github/workflows/branch_run_tests_command.yaml @@ -35,7 +35,7 @@ jobs: steps: # issue_comment workflows run from the default branch. Keep that trusted # checkout: this handler dispatches PR code with credentials in scope - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: persist-credentials: false - name: Dispatch full tests @@ -61,7 +61,7 @@ jobs: steps: # workflow_run also resolves this workflow from the default branch. Never # check out or execute the completed run's potentially PR-authored tree - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: persist-credentials: false - name: Report full-test result diff --git a/.github/workflows/branch_sync_executors.yaml b/.github/workflows/branch_sync_executors.yaml index cd15d8b8..aa26775c 100644 --- a/.github/workflows/branch_sync_executors.yaml +++ b/.github/workflows/branch_sync_executors.yaml @@ -33,7 +33,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: # Reconcile the live branch tip. A rerun therefore repairs only refs # that still need an update diff --git a/.github/workflows/branch_wasmtime_watch.yaml b/.github/workflows/branch_wasmtime_watch.yaml index cd51e4de..4af27ce9 100644 --- a/.github/workflows/branch_wasmtime_watch.yaml +++ b/.github/workflows/branch_wasmtime_watch.yaml @@ -42,7 +42,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 # Both files it reads are tracked inside the executor submodules, so this # needs a checkout but not the materialized third-party trees β€” which is # what lets it skip nix entirely. diff --git a/.github/workflows/docs_nightly.yaml b/.github/workflows/docs_nightly.yaml index e2a39282..12fc51f6 100644 --- a/.github/workflows/docs_nightly.yaml +++ b/.github/workflows/docs_nightly.yaml @@ -53,7 +53,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 120 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 # The docs build spans the manager and every active executor line, each # in its own `gen-docs` nix shell, so it needs submodules and nix. @@ -68,7 +68,7 @@ jobs: run: ./support/ci/run.sh pipeline docs - name: Checkout the site repo - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: repository: ${{ env.SITE_REPO }} path: build/site diff --git a/.github/workflows/incl_initial.yaml b/.github/workflows/incl_initial.yaml index 45af4fee..57845190 100644 --- a/.github/workflows/incl_initial.yaml +++ b/.github/workflows/incl_initial.yaml @@ -29,7 +29,9 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false - name: Get source uses: ./.github/actions/get-src with: @@ -63,9 +65,10 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: fetch-depth: 0 + persist-credentials: false - name: Get source id: src uses: ./.github/actions/get-src @@ -95,9 +98,10 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: fetch-depth: 0 + persist-credentials: false - name: Ensure 0 commits behind base env: # BASE is resolved from the PR by the pipeline itself, so this works @@ -113,7 +117,9 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false - name: Check PR title is a valid commit subject env: # PR_TITLE likewise comes from the API, not the event payload. diff --git a/.github/workflows/incl_release_build_test.yaml b/.github/workflows/incl_release_build_test.yaml index dd4a1885..46c85933 100644 --- a/.github/workflows/incl_release_build_test.yaml +++ b/.github/workflows/incl_release_build_test.yaml @@ -61,7 +61,7 @@ jobs: artifact_notes: ${{ steps.plan.outputs.artifact_notes }} notes_file: ${{ steps.plan.outputs.notes_file }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: # Each line's manifest.json (its executor-version) lives in the # submodule, so the gitlinks must be materialized. @@ -80,7 +80,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: # make-release-notes diffs against the previous tag via `git describe`. fetch-depth: 0 @@ -99,7 +99,7 @@ jobs: fi # Echoed so a debug run needs no artifact download. cat "$NOTES_FILE" - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: ${{ needs.plan.outputs.artifact_notes }} path: ${{ needs.plan.outputs.notes_file }} diff --git a/.github/workflows/incl_release_build_test_cell_build.yaml b/.github/workflows/incl_release_build_test_cell_build.yaml index d79e09bb..0b25bc5c 100644 --- a/.github/workflows/incl_release_build_test_cell_build.yaml +++ b/.github/workflows/incl_release_build_test_cell_build.yaml @@ -39,7 +39,7 @@ jobs: runs-on: ${{ inputs.runs_on }} timeout-minutes: 150 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 # Guards the "Job:" step below: every key must have a step, every step a key. - name: Check job keys @@ -47,7 +47,7 @@ jobs: JOB_JSON: ${{ inputs.job_json }} run: ./support/ci/run.sh pipeline check-cell-keys --job-json "$JOB_JSON" build - - uses: insightsengineering/disk-space-reclaimer@v1 + - uses: insightsengineering/disk-space-reclaimer@dae9fabcb8febe09f6585471948acf9dc9a57489 # v1.1.2 with: tools-cache: false swap-storage: false @@ -80,7 +80,7 @@ jobs: exclude: genvm-cpython-objs - name: Upload artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: ${{ inputs.artifact_name }} # upload-artifact strips the common prefix, so entries land flat as diff --git a/.github/workflows/incl_release_build_test_cell_test.yaml b/.github/workflows/incl_release_build_test_cell_test.yaml index 760b08ca..1ac09490 100644 --- a/.github/workflows/incl_release_build_test_cell_test.yaml +++ b/.github/workflows/incl_release_build_test_cell_test.yaml @@ -47,7 +47,7 @@ jobs: runs-on: ${{ inputs.runs_on }} timeout-minutes: 90 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Check job keys env: @@ -67,12 +67,12 @@ jobs: - name: Setup Python if: ${{ inputs.setup_python }} - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: '3.12' - name: Download platform artifacts - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: ${{ inputs.platform_artifact }} path: ${{ runner.temp }}/release-artifacts @@ -81,7 +81,7 @@ jobs: # to run the runners with. - name: Download runners artifact if: ${{ inputs.runners_artifact != '' }} - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: ${{ inputs.runners_artifact }} path: ${{ runner.temp }}/release-artifacts diff --git a/.github/workflows/queue.yaml b/.github/workflows/queue.yaml index 598ccfb7..1e650d90 100644 --- a/.github/workflows/queue.yaml +++ b/.github/workflows/queue.yaml @@ -186,7 +186,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: ref: refs/pull/${{ github.event.pull_request.number || inputs.pr }}/head # Read-only gh API calls; nothing here pushes, so the checkout @@ -226,7 +226,9 @@ jobs: exit 1 ;; esac - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false - name: Plan test matrix id: plan env: diff --git a/.github/workflows/queue_test_cell.yaml b/.github/workflows/queue_test_cell.yaml index 9b67ad83..419711c7 100644 --- a/.github/workflows/queue_test_cell.yaml +++ b/.github/workflows/queue_test_cell.yaml @@ -49,9 +49,10 @@ jobs: GEMINIKEY: ${{ secrets.GEMINIKEY }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: lfs: true + persist-credentials: false # Guards the "Job:" steps below: every key must have a step, every step a key. - name: Check job keys @@ -59,7 +60,7 @@ jobs: JOB_JSON: ${{ inputs.job_json }} run: ./support/ci/run.sh pipeline check-cell-keys --job-json "$JOB_JSON" configure build tool - - uses: insightsengineering/disk-space-reclaimer@v1 + - uses: insightsengineering/disk-space-reclaimer@dae9fabcb8febe09f6585471948acf9dc9a57489 # v1.1.2 if: ${{ inputs.disk_reclaim }} with: tools-cache: false @@ -83,17 +84,17 @@ jobs: - name: Authenticate to Google Cloud if: ${{ inputs.gcp && env.GCP_SA_KEY != '' }} - uses: google-github-actions/auth@v2 + uses: google-github-actions/auth@c200f3691d83b41bf9bbd8638997a462592937ed # v2.1.13 with: credentials_json: ${{ secrets.GCP_SA_KEY }} - name: Set up Cloud SDK if: ${{ inputs.gcp && env.GCP_SA_KEY != '' }} - uses: google-github-actions/setup-gcloud@v2 + uses: google-github-actions/setup-gcloud@e427ad8a34f8676edf47cf7d7925499adf3eb74f # v2.2.1 - name: Set up Docker Buildx if: ${{ inputs.buildx }} - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - name: "Job: configure" uses: ./.github/actions/matrix-cell-step diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index dabab419..57b296d0 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -43,7 +43,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Ensure tag is free env: VERSION_OVERRIDE: ${{ inputs.version_override }} @@ -71,37 +71,39 @@ jobs: timeout-minutes: 30 permissions: contents: write # Needed for creating releases + id-token: write # Sigstore identity for the provenance attestation + attestations: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: # Full history + tags: the release-notes step diffs against the # previous tag via `git describe`. fetch-depth: 0 - name: Download release notes - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: ${{ needs.build-and-test.outputs.artifact_notes }} path: artifacts/notes # Each artifact holds exactly one tarball, already named as the asset it # is published as, so they all land in one dir and need no renaming. - name: Download universal artifact - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: ${{ needs.build-and-test.outputs.artifact_universal }} path: release-assets - name: Download amd64-linux artifact - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: ${{ needs.build-and-test.outputs.artifact_amd64_linux }} path: release-assets - name: Download arm64-linux artifact - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: ${{ needs.build-and-test.outputs.artifact_arm64_linux }} path: release-assets - name: Download arm64-macos artifact - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: ${{ needs.build-and-test.outputs.artifact_arm64_macos }} path: release-assets @@ -117,6 +119,21 @@ jobs: } done + # Plain checksums for a verifier without `gh`; the attestation below is + # what binds them to this workflow on this tag. + - name: Write checksums + run: | + set -euo pipefail + cd release-assets + sha256sum *.tar.xz | tee SHA256SUMS + + # Keyless sigstore provenance, verifiable with + # `gh attestation verify <asset> --owner genlayerlabs`. + - name: Attest provenance + uses: actions/attest-build-provenance@96278af6caaf10aea03fd8d33a09a777ca52d62f # v3.2.0 + with: + subject-path: release-assets/* + - name: Tag the release env: TAG: ${{ needs.build-and-test.outputs.tag }} @@ -126,9 +143,11 @@ jobs: git push origin "$TAG" - name: Create Release - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2.6.2 with: - files: release-assets/*.tar.xz + files: | + release-assets/*.tar.xz + release-assets/SHA256SUMS name: Release ${{ needs.build-and-test.outputs.tag }} tag_name: ${{ needs.build-and-test.outputs.tag }} body_path: artifacts/notes/${{ needs.build-and-test.outputs.notes_file }} diff --git a/.github/workflows/webdriver-image.yaml b/.github/workflows/webdriver-image.yaml index 09397889..ae991753 100644 --- a/.github/workflows/webdriver-image.yaml +++ b/.github/workflows/webdriver-image.yaml @@ -23,17 +23,17 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Log in to Docker Hub - uses: docker/login-action@v3 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: username: yeagerai password: ${{ secrets.DOCKER_PASSWORD }} - name: Docker Metadata id: meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0 with: images: | yeagerai/genlayer-genvm-webdriver @@ -42,13 +42,13 @@ jobs: type=raw,value=latest - name: Set up QEMU for docker - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - name: Build and push image - uses: docker/build-push-action@v6 + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 with: context: webdriver push: true diff --git a/docs/contributing/howto/committing/submodules.md b/docs/contributing/howto/committing/submodules.md index 8a89f141..97dd0e46 100644 --- a/docs/contributing/howto/committing/submodules.md +++ b/docs/contributing/howto/committing/submodules.md @@ -6,7 +6,7 @@ `runners/`, `support/`, `install/` 2. **Executor lines** β€” `executors/v0.2.x`, `executors/v0.3.x`: checkouts of the **same** repo, `genlayerlabs/genvm-executor`, on different branches. - Each line's `manifest.json` owns its `executor-version` + Each line's `manifest.json` owns its `executor-version` (and `executor-sha256`, see [versioning.md](../releasing/versioning.md)) 3. **`libs/unhardcoded-engine`** β€” the LLM policy engine used by `implementation/`; the build fails loudly when it is missing diff --git a/docs/contributing/howto/releasing/release-build.md b/docs/contributing/howto/releasing/release-build.md index 09e546b3..dd9dc20f 100644 --- a/docs/contributing/howto/releasing/release-build.md +++ b/docs/contributing/howto/releasing/release-build.md @@ -36,7 +36,26 @@ An asset is the prepacked tree itself, already named β€” nothing renames it: All of them go onto a single **genvm-manager** release, tagged with `version` from `.genvm-monorepo-root` ([versioning.md](versioning.md)). They overlay onto one install root, so a full install is one platform asset plus -`genvm-universal` +`genvm-universal`. The platform asset already carries every active executor +line; post-install's executor download is only a fallback for a root missing +one ([versioning.md](versioning.md)) + +## Verifying an Asset + +The publisher uploads `SHA256SUMS` next to the tarballs and records a keyless +sigstore provenance attestation for each of them, bound to `release.yaml` in +this repository at the commit the run was dispatched from (the tag is created +after the attestation, so it is not part of the identity). Verify a download +with either: + +```bash +sha256sum -c --ignore-missing SHA256SUMS +gh attestation verify genvm-amd64-linux.tar.xz --owner genlayerlabs +``` + +The second one is the stronger check: it proves the file came out of this +repository's release workflow, not merely that it matches a list published +alongside it To exercise the whole pipeline on a PR without releasing anything, add the `test-release-pipeline` label: `queue.yaml` then runs the same diff --git a/docs/contributing/howto/releasing/versioning.md b/docs/contributing/howto/releasing/versioning.md index 31676682..14e3b50f 100644 --- a/docs/contributing/howto/releasing/versioning.md +++ b/docs/contributing/howto/releasing/versioning.md @@ -12,6 +12,17 @@ Executor crates are versioned **independently**: a line's `manifest.json` holds `executor-version` and no manager script rewrites it +A platform release asset bundles every active line, so post-install downloads +a line only when the install root lacks it, from `executor_download_urls` in +`support/manifest-base.yaml`: `genvm-<platform>-executor.tar.xz` on a release +of the executor repo tagged with the line's `executor-version`. The download is +accepted only against `executor-sha256` in the line's `manifest.json`, a +`{platform: sha256}` map copied from that release's published checksums; an unpinned +platform is refused. So the executor release must exist before the manifest +that pins it is committed, and a re-release under the same tag needs a manifest +bump. No line pins anything today: the executor repo publishes no releases, and +the fallback fails closed until it does + Manager branches are named after the *manager* train, not an executor line: `v<X>` is the release branch, `v<X>-dev` the integration branch. Here `<X>` is major.minor, for example `v0.6` diff --git a/docs/contributing/howto/testing/python.md b/docs/contributing/howto/testing/python.md index e2a6f9ab..cf34808d 100644 --- a/docs/contributing/howto/testing/python.md +++ b/docs/contributing/howto/testing/python.md @@ -4,9 +4,10 @@ genvm-tool test run --filter-tag python ``` -Two suites: `genlayer-py-std` (in the primary executor line) and `support/ci` -(`unit_tests/`, the stdlib-only CI tools). The interpreter comes from a pinned -standalone flake, so pytest can be run directly, without the dev shell: +Three suites: `genlayer-py-std` (in the primary executor line), +`support/tools/genvm-tool` and `support/ci` (each under `unit_tests/`; the CI +tools are stdlib-only). The interpreter comes from a pinned standalone flake, so +pytest can be run directly, without the dev shell: ```bash cd executors/v0.3.x/runners/genlayer-py-std @@ -19,3 +20,12 @@ that nix environment For `support/ci` the same env works with no PYTHONPATH: `cd support/ci && "$env_dir/bin/pytest"` (its `conftest.py` puts the tools on the path) + +`support/tools/genvm-tool` has its own flake, and the package is imported from +the working tree: + +```bash +cd support/tools/genvm-tool +env_dir="$(nix build --no-link --print-out-paths path:.)" +PYTHONPATH="$PWD" "$env_dir/bin/pytest" unit_tests/ +``` diff --git a/docs/website/src/spec/03-vm/03-ram-limiting.rst b/docs/website/src/spec/03-vm/03-ram-limiting.rst index 6de65c44..d73cc9e1 100644 --- a/docs/website/src/spec/03-vm/03-ram-limiting.rst +++ b/docs/website/src/spec/03-vm/03-ram-limiting.rst @@ -7,7 +7,12 @@ The deterministic budget starts at 4294967295 octets (4 GiB). Every :ref:`gvm-def-gl-call-run-nondet` gets its own non-deterministic budget, starting at what its caller had remaining at the moment of the call, so a nondet block never gets more RAM than its caller had left. -All :term:`sub-VM` instances within one budget share it. +A :term:`sub-VM` does not share its caller's budget. It starts from a copy of +what the caller had remaining and spends from that copy, so what a +:term:`sub-VM` spends never reduces what its caller may still spend, and a +caller that spawns many children in sequence is not drained by them. Only the +charges for data the caller keeps after a child returns move to the caller; see +`RAM Release`_. .. _gvm-def-ram-consumption: @@ -104,7 +109,8 @@ RAM Release ----------- File content memory is released when the corresponding file descriptor is closed via ``fd_close``. -When a :term:`sub-VM` finishes execution, all remaining RAM consumed by it is released back to the shared budget. +When a :term:`sub-VM` finishes execution, its copy of the budget is discarded, +so every charge it still held is released at once. This applies to runner charges as well: memory consumed by loading or registering a runner is released when the registering :term:`sub-VM` finishes, like any other charge. diff --git a/executors/v0.2.x b/executors/v0.2.x index bac7bb2a..a27db421 160000 --- a/executors/v0.2.x +++ b/executors/v0.2.x @@ -1 +1 @@ -Subproject commit bac7bb2af8928b5af403d195e75b6e1d4f882a7c +Subproject commit a27db421e17c5d534f6955302df3f5357d572848 diff --git a/executors/v0.3.x b/executors/v0.3.x index a0041928..977f9456 160000 --- a/executors/v0.3.x +++ b/executors/v0.3.x @@ -1 +1 @@ -Subproject commit a00419281450ae8997704170fcb50d4a6ca9d8fc +Subproject commit 977f945679e6c8823bc859d270c5b680bd6d1d84 diff --git a/flake.nix b/flake.nix index 62e42f8a..6f4f854b 100644 --- a/flake.nix +++ b/flake.nix @@ -619,11 +619,11 @@ # ---- Release distribution bundles ------------------------- # `executor[-<platform>]` (no version) merges every active line's # executor tree (executor/<version>/...) into one bundle. The release - # publishes each line on its own (`executor-<version>[-<platform>]`, - # one release per line in the executor repo), so this merged bundle has - # no CI consumer; it stays as the convenient "give me every line" - # target for local use. Runners are NOT included; they ship - # platform-independently via artifact-prepack-genvm-universal. + # packs artifact-prepack-genvm-<platform> (manager + every line) + # instead, so this merged bundle has no CI consumer; it stays as the + # convenient "give me every line" target for local use. Runners are + # NOT included; they ship platform-independently via + # artifact-prepack-genvm-universal. combine-executors = suffix: pkgs.runCommand "genvm-executor${suffix}" diff --git a/implementation/src/manager/versioning_test.rs b/implementation/src/manager/versioning_test.rs index 0d76af3a..63b111dd 100644 --- a/implementation/src/manager/versioning_test.rs +++ b/implementation/src/manager/versioning_test.rs @@ -1,9 +1,10 @@ use super::*; /// The manifest the manager ships: two live lines, both semver major zero. +/// `sha256` is post-install's pin map; the manager must tolerate it. fn live_manifest() -> Manifest { serde_yaml::from_str( - "executor_versions:\n \"v0.2.17\":\n available_after: 2024-09-01T00:00:00Z\n \"v0.3.0-rc7\":\n available_after: 2024-09-01T00:00:00Z\n", + "executor_versions:\n \"v0.2.17\":\n available_after: 2024-09-01T00:00:00Z\n \"v0.3.0-rc7\":\n available_after: 2024-09-01T00:00:00Z\n sha256:\n amd64-linux: \"0000000000000000000000000000000000000000000000000000000000000000\"\n", ) .unwrap() } diff --git a/install/lib/python/post-install/__main__.py b/install/lib/python/post-install/__main__.py index 5caf5f32..615ba795 100644 --- a/install/lib/python/post-install/__main__.py +++ b/install/lib/python/post-install/__main__.py @@ -490,28 +490,48 @@ def download_runners_from_json( def download_executor(executor_version: str): """ - Fetch an executor line from its own release and unpack it at the install root. + Fetch an executor line's tarball and unpack it at the install root. - Since the repo split the manager release carries no executors: each line is - released separately, under its own executor-version. The tarball is laid out - as `executor/<version>/...`, i.e. relative to the install root, so it is + A platform release asset already bundles every active line, so this only + runs for an install root that lacks one (a `manager-<platform>` tree + installed on its own). The tarball is laid out as + `executor/<version>/...`, i.e. relative to the install root, so it is unpacked there. + + The tarball must match the sha256 the manager pinned for this platform in + its manifest (`executor_versions.<version>.sha256.<platform>`): the download + is over plain HTTPS from a release page and is native code, so no hash means + no download. """ templates = manifest.get('executor_download_urls', []) if not templates: raise RuntimeError('manifest has no executor_download_urls') + platform = f'{args.arch}-{args.os}' + pinned = manifest['executor_versions'][executor_version].get('sha256') or {} + expected = pinned.get(platform) + if not isinstance(expected, str): + raise RuntimeError( + f'manifest pins no sha256 for executor {executor_version} on {platform}; refusing to download' + ) + data = _download_template( f'executor {executor_version}', templates, { 'version': executor_version, - 'platform': f'{args.arch}-{args.os}', + 'platform': platform, 'arch': args.arch, 'os': args.os, }, ) + actual = hashlib.sha256(data).hexdigest() + if actual != expected.lower(): + raise RuntimeError( + f'sha256 mismatch for executor {executor_version} on {platform}: expected {expected}, got {actual}' + ) + with tarfile.open(fileobj=io.BytesIO(data), mode='r:xz') as tar: tar.extractall(genvm_root_dir, filter='data') @@ -547,9 +567,9 @@ def process_executor_version(executor_version: str): executor_root_dir = genvm_root_dir.joinpath('executor', executor_version) executor_executable = executor_root_dir.joinpath('bin', 'genvm') - # The manager bundle no longer ships the executors, so fetch what is missing - # before anything below expects it on disk. A failure here is only fatal if a - # missing executor is (--error-on-missing-executor). + # A platform asset bundles every line, so this only fetches what a partial + # install lacks. A failure here is only fatal if a missing executor is + # (--error-on-missing-executor). if args.executor_download and not executor_executable.exists(): logger.info(f'Executor {executor_version} is not installed, downloading it') try: diff --git a/renovate.json b/renovate.json index e0299bdc..2f338ea2 100644 --- a/renovate.json +++ b/renovate.json @@ -3,6 +3,7 @@ "extends": [ "config:recommended", "group:recommended", + "helpers:pinGitHubActionDigests", "group:allNonMajor", "group:githubArtifactActions", "group:rust-analyzerMonorepo", diff --git a/support/manifest-base.yaml b/support/manifest-base.yaml index 654bad6f..0eb8dbdc 100644 --- a/support/manifest-base.yaml +++ b/support/manifest-base.yaml @@ -4,8 +4,9 @@ # build-independent and lives under version control. runners_download_urls: - "https://storage.googleapis.com/gh-af/genvm_runners/{name}/{hash}.{ext}" -# Each executor line is released on its own, in its own repo, tagged with its -# executor-version β€” the manager bundle does not carry them. `{version}` is that -# tag and `{platform}` is the nix-style token (amd64-linux, arm64-macos, ...). +# Fallback source for a line missing from the install root (a platform release +# asset already bundles every active line). `{version}` is the line's +# executor-version and `{platform}` the nix-style token (amd64-linux, +# arm64-macos, ...). Accepted only against the line's pinned `executor-sha256`. executor_download_urls: - "https://github.com/genlayerlabs/genvm-executor/releases/download/{version}/genvm-{platform}-executor.tar.xz" diff --git a/support/tools/genvm-tool/genvm_tool/manifest.py b/support/tools/genvm-tool/genvm_tool/manifest.py index e6696d82..c9c1c8ca 100644 --- a/support/tools/genvm-tool/genvm_tool/manifest.py +++ b/support/tools/genvm-tool/genvm_tool/manifest.py @@ -9,7 +9,7 @@ Inputs, all under the monorepo root: * `.genvm-monorepo-root` -> `active-versions` (the executor trains) - * `executors/v<ver>.x/manifest.json` -> `executor-version` + `available-after` + * `executors/v<ver>.x/manifest.json` -> `executor-version`, `available-after`, `executor-sha256` * `support/manifest-base.yaml` -> static fields (runner download URLs) Used in-process by `genvm-tool configure` (dev) and exposed as the @@ -17,6 +17,7 @@ """ import json +import re from pathlib import Path from ruamel.yaml import YAML @@ -27,12 +28,30 @@ # Manager-root-relative; holds the build-independent fields (runner URLs). BASE_REL = Path('support') / 'manifest-base.yaml' +# The platforms a release ships (support/ci/pipelines/release.py PLATFORMS). +PLATFORMS = frozenset({'amd64-linux', 'arm64-linux', 'arm64-macos'}) +_SHA256 = re.compile(r'[0-9a-fA-F]{64}') + def _executor_manifest(root: Path, bare_version: str) -> dict: path = root / f'executors/v{bare_version}.x' / 'manifest.json' return json.loads(path.read_text()) +def _pins(bare_version: str, pins: dict) -> dict[str, str]: + """`executor-sha256` checked here, where a typo is a build error, not a refused install.""" + for platform, digest in pins.items(): + if platform not in PLATFORMS: + raise common.ToolError( + f'executors/v{bare_version}.x/manifest.json: executor-sha256 names unknown platform {platform!r}' + ) + if not isinstance(digest, str) or not _SHA256.fullmatch(digest): + raise common.ToolError( + f'executors/v{bare_version}.x/manifest.json: executor-sha256[{platform!r}] is not a hex sha256' + ) + return {platform: digest.lower() for platform, digest in sorted(pins.items())} + + def build(root: Path): """ Return the manifest document: `executor_versions` followed by the static @@ -43,9 +62,17 @@ def build(root: Path): em = _executor_manifest(root, version) # Quote the timestamp so YAML keeps it a string rather than parsing it # into a native timestamp (the manager expects an RFC 3339 string). - executor_versions[em['executor-version']] = { + entry = { 'available_after': DoubleQuotedScalarString(em['available-after']), } + # {platform: sha256} of the line's own release tarballs; post-install + # refuses to download a line whose platform has no pinned hash. + if 'executor-sha256' in em: + entry['sha256'] = { + k: DoubleQuotedScalarString(v) + for k, v in _pins(version, em['executor-sha256']).items() + } + executor_versions[em['executor-version']] = entry doc = YAML(typ='rt').load((root / BASE_REL).read_text()) doc.insert(0, 'executor_versions', executor_versions) diff --git a/support/tools/genvm-tool/unit_tests/test_manifest.py b/support/tools/genvm-tool/unit_tests/test_manifest.py new file mode 100644 index 00000000..4bdf1128 --- /dev/null +++ b/support/tools/genvm-tool/unit_tests/test_manifest.py @@ -0,0 +1,95 @@ +"""What build-manifest hands the manager, and that post-install reads it back.""" + +import importlib.util +import json +from pathlib import Path + +import pytest +from genvm_tool import common, manifest + +# post-install's own YAML reader is not a package; the generated file is +# checked against the parser that consumes it +MICRO_YAML = ( + common.find_root(Path(__file__).parent) + / 'install/lib/python/post-install/micro_yaml.py' +) + +BASE = 'runners_download_urls:\n - "https://example.invalid/{name}/{hash}.{ext}"\n' +PINS = {'arm64-macos': 'b' * 64, 'amd64-linux': 'A' * 64} +# what build-manifest emits for PINS: sorted, lowercased +WRITTEN = {'amd64-linux': 'a' * 64, 'arm64-macos': 'b' * 64} + + +def _micro_yaml_loads(): + spec = importlib.util.spec_from_file_location('micro_yaml', MICRO_YAML) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.loads + + +@pytest.fixture +def root(tmp_path): + """Two active lines; v0.3 pins its tarballs, v0.2 does not.""" + tmp_path.joinpath('.genvm-monorepo-root').write_text( + json.dumps({'active-versions': ['v0.3', 'v0.2']}) + ) + base = tmp_path / manifest.BASE_REL + base.parent.mkdir(parents=True) + base.write_text(BASE) + _line(tmp_path, '0.3', {'executor-sha256': PINS}) + _line(tmp_path, '0.2', {}) + return tmp_path + + +def _line(root: Path, line: str, extra: dict) -> None: + path = root / f'executors/v{line}.x/manifest.json' + path.parent.mkdir(parents=True) + path.write_text( + json.dumps( + { + 'executor-version': f'v{line}.0', + 'available-after': '2024-09-01T00:00:00Z', + **extra, + } + ) + ) + + +def test_pins_are_copied_per_platform_sorted_and_lowercased(root): + entry = manifest.build(root)['executor_versions']['v0.3.0'] + + assert list(entry['sha256']) == ['amd64-linux', 'arm64-macos'] + assert dict(entry['sha256']) == WRITTEN + + +@pytest.mark.parametrize( + 'pins', + [{'x86_64-linux': 'a' * 64}, {'amd64-linux': 'a' * 63}, {'amd64-linux': 'g' * 64}], + ids=['unknown-platform', 'short', 'not-hex'], +) +def test_a_malformed_pin_fails_the_build(root, pins): + path = root / 'executors/v0.3.x/manifest.json' + line = json.loads(path.read_text()) + line['executor-sha256'] = pins + path.write_text(json.dumps(line)) + + with pytest.raises(common.ToolError, match='executors/v0.3.x/manifest.json'): + manifest.build(root) + + +def test_a_line_without_pins_gets_no_sha256_key(root): + entry = manifest.build(root)['executor_versions']['v0.2.0'] + + assert dict(entry) == {'available_after': '2024-09-01T00:00:00Z'} + + +def test_the_written_file_reads_back_through_post_install(root, tmp_path): + out = tmp_path / 'data/manifest.yaml' + manifest.write(root, out) + + doc = _micro_yaml_loads()(out.read_text()) + + assert doc['executor_versions']['v0.3.0']['sha256'] == WRITTEN + assert doc['executor_versions']['v0.3.0']['available_after'] == '2024-09-01T00:00:00Z' + assert 'sha256' not in doc['executor_versions']['v0.2.0'] + assert doc['runners_download_urls'] == ['https://example.invalid/{name}/{hash}.{ext}'] diff --git a/support/tools/genvm-tool/unit_tests/test_post_install_executor_download.py b/support/tools/genvm-tool/unit_tests/test_post_install_executor_download.py new file mode 100644 index 00000000..35ba4b97 --- /dev/null +++ b/support/tools/genvm-tool/unit_tests/test_post_install_executor_download.py @@ -0,0 +1,195 @@ +"""Post-install fetches a missing executor line only against the hash its manifest pins.""" + +import functools +import hashlib +import http.server +import io +import json +import shutil +import subprocess +import sys +import tarfile +import threading +import typing +from pathlib import Path + +import pytest +from genvm_tool import common + +POST_INSTALL = ( + common.find_root(Path(__file__).parent) / 'install/lib/python/post-install' +) +VERSION = 'v0.3.0' +PLATFORM = 'amd64-linux' +ASSET = f'{VERSION}/genvm-{PLATFORM}-executor.tar.xz' + + +def _tarball() -> bytes: + """A line's release tarball: `executor/<version>/...`, relative to the install root.""" + payload = b'#!/bin/sh\n' + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode='w:xz') as tar: + info = tarfile.TarInfo(f'executor/{VERSION}/bin/genvm') + info.size = len(payload) + tar.addfile(info, io.BytesIO(payload)) + return buf.getvalue() + + +class _Handler(http.server.SimpleHTTPRequestHandler): + requested: typing.ClassVar[list[str]] = [] + + def do_GET(self): + self.requested.append(self.path) + super().do_GET() + + def log_message(self, *args): + pass + + +@pytest.fixture +def release(tmp_path): + """The executor release page, serving `tmp_path/www`; `requested` records every GET.""" + www = tmp_path / 'www' + www.mkdir() + _Handler.requested = [] + httpd = http.server.ThreadingHTTPServer( + ('127.0.0.1', 0), functools.partial(_Handler, directory=str(www)) + ) + threading.Thread(target=httpd.serve_forever, daemon=True).start() + yield www, f'http://127.0.0.1:{httpd.server_address[1]}' + httpd.shutdown() + httpd.server_close() + + +def _install_root(tmp_path: Path, url: str, pins: dict[str, object] | None) -> Path: + """A manager-only install: no executor on disk, so post-install must fetch it.""" + root = tmp_path / 'root' + shutil.copytree(POST_INSTALL, root / 'lib/python/post-install') + root.joinpath('bin').mkdir() + # not ELF, so the platform check leaves it alone; bin-check is off + root.joinpath('bin/genvm-modules').write_text('#!/bin/sh\n') + lines = [ + 'executor_versions:', + f' {VERSION}:', + ' available_after: "2024-09-01T00:00:00Z"', + ] + if pins is not None: + lines.append(' sha256:') + lines.extend( + f' {platform}: {json.dumps(digest)}' for platform, digest in pins.items() + ) + lines += [ + 'executor_download_urls:', + f' - "{url}/{{version}}/genvm-{{platform}}-executor.tar.xz"', + ] + root.joinpath('data').mkdir() + root.joinpath('data/manifest.yaml').write_text('\n'.join(lines) + '\n') + return root + + +def _run(root: Path, *extra: str) -> subprocess.CompletedProcess: + return subprocess.run( + [ + sys.executable, + '-B', + str(root / 'lib/python/post-install/__main__.py'), + '--default-steps=false', + '--executor-download=true', + '--runners-download=false', + '--os=linux', + '--arch=amd64', + *extra, + ], + capture_output=True, + text=True, + check=False, + ) + + +def test_a_tarball_matching_its_pin_is_unpacked(tmp_path, release): + www, url = release + data = _tarball() + www.joinpath(ASSET).parent.mkdir() + www.joinpath(ASSET).write_bytes(data) + root = _install_root( + tmp_path, url, {PLATFORM: hashlib.sha256(data).hexdigest().upper()} + ) + + result = _run(root) + + assert result.returncode == 0, result.stderr + assert root.joinpath('executor', VERSION, 'bin/genvm').read_bytes() == b'#!/bin/sh\n' + + +def test_a_tarball_that_does_not_match_its_pin_is_refused(tmp_path, release): + www, url = release + www.joinpath(ASSET).parent.mkdir() + www.joinpath(ASSET).write_bytes(_tarball()) + root = _install_root(tmp_path, url, {PLATFORM: '0' * 64}) + + result = _run(root) + + assert result.returncode != 0 + assert f'sha256 mismatch for executor {VERSION} on {PLATFORM}' in result.stderr + assert not root.joinpath('executor').exists() + + +@pytest.mark.parametrize( + 'pins', + [None, {'arm64-macos': '0' * 64}, {PLATFORM: 123}], + ids=['none', 'other-platform', 'not-a-string'], +) +def test_an_unpinned_platform_is_not_even_downloaded(tmp_path, release, pins): + www, url = release + www.joinpath(ASSET).parent.mkdir() + www.joinpath(ASSET).write_bytes(_tarball()) + root = _install_root(tmp_path, url, pins) + + result = _run(root) + + assert result.returncode != 0 + assert ( + f'manifest pins no sha256 for executor {VERSION} on {PLATFORM}' in result.stderr + ) + assert _Handler.requested == [] + + +def test_a_release_without_the_asset_unpacks_nothing(tmp_path, release): + _, url = release + root = _install_root(tmp_path, url, {PLATFORM: '0' * 64}) + + result = _run(root) + + assert result.returncode != 0 + assert f'failed to download executor {VERSION}' in result.stderr + assert not root.joinpath('executor').exists() + + +@pytest.mark.parametrize( + 'pins', [None, {PLATFORM: '0' * 64}], ids=['unpinned', 'mismatch'] +) +def test_a_missing_executor_is_a_warning_when_asked(tmp_path, release, pins): + www, url = release + www.joinpath(ASSET).parent.mkdir() + www.joinpath(ASSET).write_bytes(_tarball()) + root = _install_root(tmp_path, url, pins) + + result = _run(root, '--error-on-missing-executor=false') + + assert result.returncode == 0, result.stderr + assert f'Could not download executor {VERSION}' in result.stderr + assert not root.joinpath('executor').exists() + + +def test_an_installed_line_is_left_alone(tmp_path, release): + """The bundled line wins: the download is a fallback, not a refresh.""" + _, url = release + root = _install_root(tmp_path, url, None) + installed = root / 'executor' / VERSION / 'bin/genvm' + installed.parent.mkdir(parents=True) + installed.write_text('#!/bin/sh\n') + + result = _run(root) + + assert result.returncode == 0, result.stderr + assert _Handler.requested == [] From 775036dc7804bd93b9998fd22395aa1b2e79f365 Mon Sep 17 00:00:00 2001 From: kp2pml30 <kp2pml30@gmail.com> Date: Mon, 14 Sep 2026 20:19:48 +0900 Subject: [PATCH 18/45] fix(ci): repair docs workflow --- .github/workflows/docs_nightly.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/docs_nightly.yaml b/.github/workflows/docs_nightly.yaml index 12fc51f6..881fc577 100644 --- a/.github/workflows/docs_nightly.yaml +++ b/.github/workflows/docs_nightly.yaml @@ -71,6 +71,10 @@ jobs: uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: repository: ${{ env.SITE_REPO }} + # The site repo is private, and `ref` is what keeps checkout from + # asking the REST API for the default branch with this repo's token, + # which cannot see it. The deploy key covers the clone itself. + ref: main path: build/site ssh-key: ${{ secrets.DOCS_DEPLOY_KEY }} From 054ea0667c71b06865674448c0709969cbbe2334 Mon Sep 17 00:00:00 2001 From: Kira <kp2pml30@gmail.com> Date: Tue, 15 Sep 2026 21:05:41 +0900 Subject: [PATCH 19/45] feat(executor): use named fee buckets and align reserves (#31) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(executor): adopt named fee buckets and opaque outputs ✨πŸ’₯ Expose named resource caps through the host protocol while keeping leader-proposed output opaque until each executor validates it. * fix(executor): guard the overlay split, correct fee docs πŸ›πŸ“ * test(executor): bump v0.2 for re-recorded send_message goldens βœ… --- .genvm-tool.py | 1 + crates/modules-interfaces/src/domain.rs | 4 +- .../modules-interfaces/src/domain/fees/abi.rs | 32 ++-- .../modules-interfaces/src/domain/fees/mod.rs | 17 +- docs/schemas/default-config.json | 14 +- docs/website/src/impl-spec/04-fees.rst | 167 +++++++++++------- .../src/impl-spec/appendix/manager-api.yaml | 11 +- .../03-wasi_genlayer_sdk/02-gl_call.rst | 32 +++- docs/website/src/spec/03-vm/05-result.rst | 10 +- docs/website/src/spec/changelog.rst | 46 +++++ executors/v0.2.x | 2 +- executors/v0.3.x | 2 +- implementation/src/manager/run.rs | 4 +- implementation/src/manager/run_test.rs | 2 +- .../runner/genvm_tool_plugins/integration.py | 35 +++- tests/runner/origin/base_host.py | 34 +++- tests/runner/origin/leader_public_data.py | 88 ++------- .../runner/origin/test_leader_public_data.py | 22 ++- .../system/cross-major-observability/test.py | 4 +- tests/system/cross-major/test.py | 4 +- tests/system/manager-socket/test.py | 4 +- tests/system/permits/test.py | 6 +- 22 files changed, 316 insertions(+), 225 deletions(-) diff --git a/.genvm-tool.py b/.genvm-tool.py index 008dab25..ffdb2cae 100644 --- a/.genvm-tool.py +++ b/.genvm-tool.py @@ -256,6 +256,7 @@ def collect_integration(ctx: genvm_tool.tests.stage.collection.Context): integration.integration_test_directory( ctx, cases_dir=ctx.shared.root_dir / 'tests' / 'system' / 'cross-major' / 'cases', + executor_major=3, reroute_to=build_info['executor_versions']['v0.3'], save_hashes=False, manager_service=manager_service, diff --git a/crates/modules-interfaces/src/domain.rs b/crates/modules-interfaces/src/domain.rs index ab89ff98..9819ae1d 100644 --- a/crates/modules-interfaces/src/domain.rs +++ b/crates/modules-interfaces/src/domain.rs @@ -35,7 +35,7 @@ pub struct ExecutionData { #[serde(default)] #[calldata(default = default_host_hello_data)] pub host_hello_data: Vec<Bytes>, - pub bucket_totals: Vec<num_bigint::BigInt>, + pub bucket_totals: std::collections::BTreeMap<String, num_bigint::BigInt>, /// Host-provided `node` fee constants (moved off `host_data`). pub gas_data: std::collections::BTreeMap<String, String>, /// Message-fee allocation tree passed alongside the execution. @@ -410,7 +410,7 @@ pub struct ReportedResult { pub nondet_disagreement: Option<u32>, pub leader_public_data: bytes::Bytes, - pub data_fees_remaining: Vec<primitive_types::U256>, + pub data_fees_remaining: std::collections::BTreeMap<String, primitive_types::U256>, pub data_fees_consumed: BucketsConsumed, pub llm_consumed_gen_wei: primitive_types::U256, diff --git a/crates/modules-interfaces/src/domain/fees/abi.rs b/crates/modules-interfaces/src/domain/fees/abi.rs index 89d938c8..778c9cf0 100644 --- a/crates/modules-interfaces/src/domain/fees/abi.rs +++ b/crates/modules-interfaces/src/domain/fees/abi.rs @@ -50,46 +50,34 @@ fn push_address(buf: &mut Vec<u8>, addr: Option<&genlayer_calldata::Address>) { } } -/// `callKey` is already a 32-byte word; `None` is the `CALL_KEY_WILDCARD` = `bytes32(0)`. +/// `callKey` is already a 32-byte word; `None` is `CALL_KEY_WILDCARD`. fn push_call_key(buf: &mut Vec<u8>, call_key: Option<&crate::abi_stub::CallKey>) { match call_key { Some(ck) => buf.extend_from_slice(&ck.0), - None => buf.extend_from_slice(&[0u8; 32]), + None => buf.extend_from_slice(&super::CALL_KEY_WILDCARD.0), } } -/// ABI-encodes the nested allocation tree as the chain's flat -/// `MessageAllocationNode[]` representation (matching `abi.encode(nodes)`). -pub(super) fn encode(roots: &[MessageAllocationNode]) -> Vec<u8> { - // Pre-order flatten: (node, parentIndex). Parents always precede children, - // so the parent's array index is already assigned when a child is visited. +/// Encodes the transport payload consumed by `decodeAllocationSubtree`. +pub(super) fn encode(root: &MessageAllocationNode) -> Vec<u8> { let mut flat: Vec<(&MessageAllocationNode, U256)> = Vec::new(); - fn flatten<'a>( - nodes: &'a [MessageAllocationNode], - parent: U256, - out: &mut Vec<(&'a MessageAllocationNode, U256)>, - ) { - for node in nodes { - let my_index = U256::from(out.len() as u64); - out.push((node, parent)); - flatten(&node.children, my_index, out); + let mut queue = std::collections::VecDeque::from([(root, NODE_ROOT_SENTINEL)]); + while let Some((node, parent_index)) = queue.pop_front() { + let node_index = U256::from(flat.len()); + flat.push((node, parent_index)); + for child in &node.children { + queue.push_back((child, node_index)); } } - flatten(roots, NODE_ROOT_SENTINEL, &mut flat); - // Encode each element as its own self-contained dynamic tuple. let elements: Vec<Vec<u8>> = flat .iter() .map(|(node, parent_index)| encode_node(node, *parent_index)) .collect(); - // `abi.encode(MessageAllocationNode[])`: leading offset (0x20) to the array. let mut buf = Vec::new(); push_word(&mut buf, 0x20); - // Dynamic array of dynamic elements: length, then per-element head offsets - // (relative to the start of the head region, i.e. right after the length), - // then the element tails. push_word(&mut buf, elements.len() as u64); let mut offset = elements.len() * 32; for element in &elements { diff --git a/crates/modules-interfaces/src/domain/fees/mod.rs b/crates/modules-interfaces/src/domain/fees/mod.rs index c0efedde..bea5819b 100644 --- a/crates/modules-interfaces/src/domain/fees/mod.rs +++ b/crates/modules-interfaces/src/domain/fees/mod.rs @@ -4,6 +4,11 @@ use primitive_types::U256; use crate::On; +pub const CALL_KEY_WILDCARD: crate::abi_stub::CallKey = crate::abi_stub::CallKey([ + 0xc5, 0xd2, 0x46, 0x01, 0x86, 0xf7, 0x23, 0x3c, 0x92, 0x7e, 0x7d, 0xb2, 0xdc, 0xc7, 0x03, 0xc0, + 0xe5, 0x00, 0xb6, 0x53, 0xca, 0x82, 0x27, 0x3b, 0x7b, 0xfa, 0xd8, 0x04, 0x5d, 0x85, 0xa4, 0x70, +]); + #[derive( Debug, Clone, @@ -83,7 +88,7 @@ pub struct MessageAllocationNode { /// Target contract address; `None` means wildcard (any recipient). pub recipient: Option<genlayer_calldata::Address>, /// `None` = wildcard: all call keys for this recipient - /// (chain sentinel: `CALL_KEY_WILDCARD` = `bytes32(0)`). + /// (chain sentinel: `CALL_KEY_WILDCARD` = `keccak256("")`). pub call_key: Option<crate::abi_stub::CallKey>, /// Max budget for matching messages. pub budget: U256, @@ -94,12 +99,10 @@ pub struct MessageAllocationNode { } impl MessageAllocationNode { - /// ABI-encodes the nested allocation tree as the chain's flat - /// `MessageAllocationNode[]` representation (matching `abi.encode(nodes)`), - /// flattening children to parent-pointer form via pre-order traversal so that - /// every parent precedes its children and their `parentIndex` is well-defined. - pub fn abi_encode(roots: &[MessageAllocationNode]) -> Vec<u8> { - abi::encode(roots) + /// ABI-encodes this matched node and its descendants for transport to the chain. + /// The matched node is element 0 and descendants are in BFS order. + pub fn abi_encode(&self) -> Vec<u8> { + abi::encode(self) } #[allow(clippy::if_same_then_else)] diff --git a/docs/schemas/default-config.json b/docs/schemas/default-config.json index ccce9e0c..fd3c2d5b 100644 --- a/docs/schemas/default-config.json +++ b/docs/schemas/default-config.json @@ -351,19 +351,17 @@ "genvm-fees-bucket-conf": { "type": "object", "properties": { - "bucket_no": { + "buckets": { "oneOf": [ { - "type": "integer", - "minimum": 0, - "maximum": 255 + "type": "string", + "minLength": 1 }, { "type": "array", "items": { - "type": "integer", - "minimum": 0, - "maximum": 255 + "type": "string", + "minLength": 1 }, "minItems": 1 } @@ -379,7 +377,7 @@ "description": "cost charged per change, evaluated with the `units` variable; see `expr_prelude`" } }, - "required": ["bucket_no", "delta_expr"], + "required": ["buckets", "delta_expr"], "additionalProperties": false }, "genvm-fees-conf": { diff --git a/docs/website/src/impl-spec/04-fees.rst b/docs/website/src/impl-spec/04-fees.rst index 0aa0d5a8..9b2212af 100644 --- a/docs/website/src/impl-spec/04-fees.rst +++ b/docs/website/src/impl-spec/04-fees.rst @@ -7,11 +7,11 @@ expression evaluator lives in ``executor/crates/common/src/expr/``. Three independent moving parts are involved: -#. **Buckets** β€” opaque integer reservoirs (``U256``) that GenVM debits during - execution. Their initial totals come from the host (typically the consensus - layer) on a per-call basis. +#. **Buckets** β€” named ``U256`` reservoirs that GenVM debits during execution. + Their initial totals come from the host (typically the consensus layer) on a + per-call basis. #. **Bucket configs** β€” operator-provided rules that bind a *named* charge (storage - pages, message receipts, nondet output bytes, message fees) to a *bucket number* + pages, message receipts, nondet output bytes, message fees) to a *bucket name* and an expression that computes the per-event cost. #. **Expressions** β€” a small typed lambda calculus used to write the cost rules. It is evaluated once at startup (for ``subtract_on_start_expr``) and once per @@ -25,10 +25,11 @@ The ``fees`` block of the GenVM config (see ``doc/schemas/default-config.json``, "fees": { "expr_prelude": "<expression>", - "storage": { "bucket_no": N, "subtract_on_start_expr": "...", "delta_expr": "..." }, - "message_receipt": { "bucket_no": N, "subtract_on_start_expr": "...", "delta_expr": "..." }, - "nondet_output": { "bucket_no": N, "subtract_on_start_expr": "...", "delta_expr": "..." }, - "message_fee": { "bucket_no": N, "subtract_on_start_expr": "...", "delta_expr": "..." } + "storage": { "buckets": "execution_data_gas", "subtract_on_start_expr": "...", "delta_expr": "..." }, + "message_receipt": { "buckets": ["execution_data_gas", "submitted_messages", "submitted_messages_count"], "subtract_on_start_expr": "...", "delta_expr": "..." }, + "nondet_output": { "buckets": ["execution_data_gas", "nondet_outputs"], "subtract_on_start_expr": "...", "delta_expr": "..." }, + "message_fee": { "buckets": "message_fee", "subtract_on_start_expr": "...", "delta_expr": "..." }, + "event": { "buckets": "execution_data_gas", "subtract_on_start_expr": "...", "delta_expr": "..." } } ``expr_prelude`` @@ -36,43 +37,47 @@ The ``fees`` block of the GenVM config (see ``doc/schemas/default-config.json``, intended use is shared ``let`` bindings (or Y-combinator definitions) that the per-bucket expressions can reference. -``bucket_no`` - Either a single index, or an array of indices, into the ``Vec<U256>`` of bucket - totals the host passes to ``DataLimit::new``. With an array, a scalar ``delta_expr`` +``buckets`` + Either a single name, or an array of names, in the bucket-total map the host + passes to ``DataLimit::new``. With an array, a scalar ``delta_expr`` result is charged against every listed bucket, while an array result is charged element-wise (lengths must match); all debits in one charge are atomic - (all-or-nothing). Two bucket configs MAY share the same index; in that case both + (all-or-nothing). Two bucket configs MAY share the same name; in that case both charge against the same reservoir. The ``message_fee`` / ``message_receipt`` pair - has special atomic-debit behaviour when they share a bucket (``consume_message_fee`` - in ``fees.rs:332``). + has special atomic-debit behaviour when they share a bucket ``subtract_on_start_expr`` - A bare numeric expression. Evaluated once at startup with ``node`` bound to the - host-provided gas constants (see below). The resulting ``U256`` is debited from - the bucket immediately. Defaults to ``"0"`` if omitted. + A numeric expression or array of numeric expressions. It follows the same scalar + versus element-wise array rule as ``delta_expr``, including exact array-length + matching. Evaluated once at startup with ``node`` bound to the host-provided gas + constants (see below). The resulting ``U256`` values are debited immediately. + Defaults to ``"0"`` if omitted. ``delta_expr`` A function ``\attrs = body`` (one or more ``\`` lambdas) whose ``attrs`` parameter is an object containing the variables that the charge depends on. Evaluated once per charge; the resulting integer is debited from the bucket. -The four named buckets and their ``attrs`` shape are: +The five fee rules and their ``attrs`` shape are: ==================== ====================================================================== Name ``attrs`` keys ==================== ====================================================================== ``storage`` ``pages``: number of pages allocated -``message_receipt`` ``isInternal``, ``isDeploy``, ``calldataLength``, ``codeLength`` +``message_receipt`` ``isFirstMessage``, ``isInternal``, ``isDeploy``, + ``rotationsCount``, ``calldataLength``, ``codeLength``, + ``subtreeLength`` ``nondet_output`` ``outputLength`` -``message_fee`` ``isInternal``, ``onAcceptance`` (when internal), - ``matchedFeeParams``: an object with +``message_fee`` ``isInternal``, ``matchedFeeParams``: an object with ``leaderTimeunitsAllocation``, ``validatorTimeunitsAllocation``, ``executionBudgetPerRound``, ``rotations`` (array) +``event`` ``blobSize``, ``topicsCount`` ==================== ====================================================================== The ground truth for these names is ``fees.rs`` (search for ``calculate_bucket`` -call sites). +call sites). ``isFirstMessage`` is true until the first message emission is +successfully accumulated; event emissions do not clear it. The Expression Language ----------------------- @@ -159,6 +164,17 @@ Bucket expressions reference these as e.g. ``node.gasPerChangedSlot`` and the node-provided values let consensus parameters drift independently of the executor binary. +``node.overlaySplitBps`` carries the combined developer and DAO share of the +time-unit fee pool. Internal-message primary reserves use:: + + timeUnitPool + + floor(timeUnitPool * overlaySplitBps / (10000 - overlaySplitBps)) + + executionTerm + +This minimum primary fee is the same for messages emitted on acceptance and +finalization. The execution term is not grossed up. Omitting the field makes +internal-message fee evaluation fail rather than silently undercharge + The closing of ``node`` happens at startup: ``\node = <prelude> <code>`` is parsed and applied to the resolved ``node`` object. The returned value (a number for ``subtract_on_start_expr``, a function for @@ -185,21 +201,23 @@ Per process, per non-batched call: #. The evaluator parses ``expr_prelude + gas_data[i].value`` for every ``gas_data`` entry and builds the ``node`` object. Parse or evaluation errors here surface as ``parsing gas_data constant 'X'`` / ``evaluating gas_data constant 'X'``. -#. For each of the four buckets the evaluator builds ``\node = <prelude> <code>`` +#. For each of the five fee rules the evaluator builds ``\node = <prelude> <code>`` for ``subtract_on_start_expr`` and ``delta_expr``, applies the ``node`` object, and stores the resulting ``Value`` (a rational and a function respectively). -#. The startup cost is debited immediately. Underflow surfaces as the bucket's - configured OOM ``VmError`` (see the ``VmError::oom().*`` enum chain at the call - site). +#. For top-level runs, the startup cost is debited immediately. Underflow surfaces + as the bucket's configured OOM ``VmError`` (see the ``VmError::oom().*`` enum + chain at the call site). Nested runs receive zero-valued bucket placeholders and + skip startup debits because their caller already paid them. #. During the run, every ``consume_*`` function in ``fees.rs`` packs the per-event ``attrs`` into an object, applies the bucket's ``delta`` function, converts the result to ``U256`` (rejecting non-integers and negatives), and debits the reservoir. -Bucket totals are mutated under a single ``tokio::sync::Mutex<Vec<U256>>`` so the -debits are sequentially consistent across concurrent sub-VMs. ``message_fee`` and +Bucket totals are mutated under a single ``tokio::sync::Mutex<HashMap<String, +U256>>`` so the debits are sequentially consistent across concurrent sub-VMs. +``message_fee`` and ``message_receipt`` debits are atomic with respect to each other: when they share a -``bucket_no``, the costs are summed and debited once; otherwise both reservoirs +bucket name, the costs are summed and debited once; otherwise both reservoirs are checked before either is decremented. This avoids partial charges when a combined send-and-deliver flow runs out of fee budget mid-call. @@ -230,43 +248,49 @@ A minimal, illustrative ``fees`` block:: "expr_prelude": "let Y = \\f = (\\x = f (x x)) (\\x = f (x x)) in", "storage": { - "bucket_no": 0, + "buckets": "execution_data_gas", "subtract_on_start_expr": "node.gasPerStorageBootstrap", "delta_expr": "\\a = a.pages * node.gasPerStoragePage" }, "message_receipt": { - "bucket_no": 1, + "buckets": "message_fee", "delta_expr": "\\a = if a.isDeploy then node.gasPerDeployByte * (a.calldataLength + a.codeLength)\n else node.gasPerCallByte * a.calldataLength" }, "nondet_output": { - "bucket_no": 2, + "buckets": "nondet_outputs", "delta_expr": "\\a = a.outputLength * node.gasPerNondetByte" }, "message_fee": { - "bucket_no": 1, + "buckets": "message_fee", "delta_expr": - "\\a = if a.isInternal\n then (if a.onAcceptance then node.gasInternalAccept else node.gasInternalFinal)\n else floor (a.matchedFeeParams.executionBudgetPerRound * arrayLen a.matchedFeeParams.rotations)" + "\\a = if a.isInternal\n then node.gasInternal * arrayLen a.matchedFeeParams.rotations\n else a.matchedFeeParams.gasLimit * a.matchedFeeParams.maxGasPrice" + }, + "event": { + "buckets": "execution_data_gas", + "delta_expr": "\\a = (a.topicsCount + a.blobSize) * node.gasPerStoragePage" } } -``message_fee`` and ``message_receipt`` share ``bucket_no = 1`` here, so an outbound -message debits both costs atomically against bucket 1. +``message_fee`` and ``message_receipt`` share the ``message_fee`` bucket here, so an +outbound message debits both costs atomically against it. Message-Fee Allocation Matching ------------------------------- The ``a.matchedFeeParams`` above is selected per outbound message from the call's -allocation list (``accumulator.message_fee_allocation``). A node -(``domain/fees.rs``, ``matches_internal`` / ``matches_external``) matches on kind -(``External`` for ``EmitExternalMessage``, else ``Internal``), ``on`` (internal -only), and -``recipient`` / ``call_key`` (each a wildcard ``None`` or an exact match). Selection -is **first-match-wins** in list order (``find_map``); no match yields the -``fee no_matching_allocation`` VM error. - -Wildcards are not reordered, so a wildcard node shadows every more-specific node -after it: producers are advised to sort more-specific nodes ahead of wildcard ones. +allocation list (``accumulator.message_fee_allocation``). Kind is matched first +(``External`` for ``EmitExternalMessage``, else ``Internal``), followed by exact +``recipient`` and ``call_key``. A per-recipient ``call_key`` wildcard is tried only +after the exact key; executor-only open-bucket recipient wildcards are less specific +than either. List order therefore cannot make a wildcard shadow an exact allocation. + +For internal messages, ``on`` is checked after the allocation key is resolved, so a +phase mismatch on an exact allocation does not fall through to a wildcard. For +external messages, an exhausted exact allocation spills to the per-recipient +``call_key`` wildcard. If neither key has an allocation, the external message uses +the legacy unallocated path and consumes only its receipt cost. Existing but exhausted +candidates yield an allocation-budget error. Funding modes ~~~~~~~~~~~~~~ @@ -274,8 +298,12 @@ Funding modes An outgoing internal message is funded one of two ways: - **Allocation-matched (default).** The fee is matched against the allocation - tree as above, capped by the matched node's ``budget``, and consumes both the - ``message_fee`` and ``message_receipt`` buckets. + tree as above. After the expression computes the primary reserve, the executor + adds the budgets of the matched node's direct children; deeper budgets are + already contained by their direct parent. A declared budget exceeding the + matched node's remaining ``budget`` is rejected with an allocation-budget + error; otherwise it consumes ``message_fee`` atomically with the + ``message_receipt`` charge. - **Balance-funded (``use_balance``).** When an ``EmitInternalMessage`` / ``EmitInternalDeployMessage`` sets ``use_balance`` (the chain's ``useBalance``, gated on @@ -287,6 +315,27 @@ An outgoing internal message is funded one of two ways: excluded from the sender pool on-chain); only ``message_receipt`` is. The emitted allocation subtree is empty, so nested child messages must each fund themselves. +For either phase, an internal message therefore declares:: + + minPrimaryFees(feeParams) + sum(directChildAllocation.budget) + +For balance funding the sum is zero. The primary fee already covers the child's +configured lifecycle, including appeals; the remainder becomes the child's +message-fee bucket. External messages declare zero + +The parent allocation's aggregate capacity is a separate invariant. If +``L = appealRounds + 1`` novel executions may each emit a child carrying budget +``C``, preserving that descendant capacity for every execution requires:: + + allocationBudget >= L * (minPrimaryFees + C) + +The lifecycle multiplier therefore belongs to this parent allocation-capacity +check, not to one emitted child's declared budget + +The current minimum ``L * minPrimaryFees + C`` reserves ``C`` only once. Whether +an allocation promises descendant capacity once or once per accepted execution +remains unspecified; this does not change the per-message formula + Because ``fee_params`` is guest-supplied, it is validated before it reaches the fee evaluator (``validate_balance_fee``): empty ``rotations`` and any zero price cap (``max_price_gen_per_time_unit`` / ``storage_fee_max_gas_price`` / @@ -294,17 +343,17 @@ An outgoing internal message is funded one of two ways: reveal-time ``FeeValueMustBeNonZero`` checks. Magnitudes are bounded too (prices/budgets below 2\ :sup:`96`, counts β€” time units and rotations entries β€” below 2\ :sup:`32`) so the worst-case metered floor provably fits - in ``U256`` and cannot trip the evaluator's overflow abort. - - The floor replicates the chain's ``minMessagePrimaryFees`` rather than the - allocation-matched floor: the consensus term is charged at the guest's - ``max_price_gen_per_time_unit`` (the funding cap), not the node's - ``genPerTimeUnit`` (the allocation path keeps the ``genPerTimeUnit`` multiplier - β€” this branch is selected by the ``balanceFunded`` flag threaded into the - ``message_fee`` expression). Two further node-configured floors fire as + in ``U256`` without saturating. + + The floor replicates the chain's ``minMessagePrimaryFees``: the consensus term + is charged at the guest's ``max_price_gen_per_time_unit`` funding cap, then the + developer and DAO overlay is grossed up on that time-unit pool. Two further + node-configured floors fire as ``VMError``\ s: ``fee below_minimum`` when a non-zero ``execution_budget_per_round`` is below ``node.messageBudgetFloor`` (the chain's ``BudgetTooLow``), and ``fee too_many_rounds`` when ``rotations`` implies more rounds than the validator - table (``node.validatorsPerRound``) covers. The per-phase time-unit floor - (``fee below_minimum`` against ``node.minTimeUnitsPerPhase``) applies on both - funding paths. + table (``node.validatorsPerRound``) covers. Unless both allocations are zero, + the leader and validator time units must also fit the host-provided + ``minProposeTimeout`` / ``maxProposeTimeout`` and ``minCommitTimeout`` / + ``maxCommitTimeout`` ranges; violations produce + ``fee phase_timeout_out_of_bounds`` on both funding paths. diff --git a/docs/website/src/impl-spec/appendix/manager-api.yaml b/docs/website/src/impl-spec/appendix/manager-api.yaml index 2d150638..c4e6e339 100644 --- a/docs/website/src/impl-spec/appendix/manager-api.yaml +++ b/docs/website/src/impl-spec/appendix/manager-api.yaml @@ -809,13 +809,14 @@ components: default: 20 description: Maximum execution time in minutes bucket_totals: - type: array - items: + type: object + minProperties: 1 + additionalProperties: type: string description: | - Per-bucket data-fee balances, as big integer strings, indexed by the - bucket numbers the executor's fee config references. No default: a - request that omits it is rejected. + Per-bucket data-fee balances, as big integer strings keyed by the + non-empty names the executor's fee config references. No default: + a request that omits it is rejected. host_data: type: string description: Host-specific data as JSON string diff --git a/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/02-gl_call.rst b/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/02-gl_call.rst index 2710685e..98445c47 100644 --- a/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/02-gl_call.rst +++ b/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/02-gl_call.rst @@ -135,6 +135,19 @@ Requirements #. When ``use_balance`` is set: :ref:`gvm-perm-use-balance-for-message-fees` and ``fee_params`` #. ``calldata`` satisfies :ref:`gvm-def-contract-call-conv` +Allocation-funded fees +~~~~~~~~~~~~~~~~~~~~~~ + +For an allocation-funded internal message, ``declaredBudget`` is the metered +primary reserve plus the budgets of the direct children in its emitted allocation +subtree. The same amount is deducted from the matching allocation and the +transaction's message-fee pool. Grandchild budgets are already contained within +their direct parent's budget and are not added again. The primary reserve includes +the consensus developer and DAO gross-up on its time-unit portion; the execution +budget is not grossed up. This formula is identical for messages emitted on +acceptance and finalization; the primary reserve already covers the child's +configured lifecycle, including appeals + .. _gvm-gl-call-balance-fees: Balance-funded fees @@ -168,7 +181,8 @@ Semantics: transaction's ``declaredBudget`` β€” the contract balance is the only bound. The consensus term is charged at the guest's ``max_price_gen_per_time_unit`` cap (matching the chain's ``minMessagePrimaryFees``), not the node's live - ``genPerTimeUnit``, so the fee scales with the cap. + ``genPerTimeUnit``, so the fee scales with the cap. The consensus developer and + DAO gross-up applies to this time-unit portion but not the execution budget - The message is excluded from allocation matching, so no matching node is required (and none is consulted). - The contract must be able to cover ``value + metered_fee`` from its balance; @@ -191,16 +205,22 @@ Semantics: ``validator_time_units_allocation``, each ``rotations`` entry) below 2\ :sup:`32`. These bounds keep the metered floor within ``U256``. -Metering additionally enforces node-configured floors, surfaced as ``VMError``\ s: +Metering additionally enforces node-configured bounds, surfaced as ``VMError``\ s: -- ``fee below_minimum`` β€” a per-phase time-unit allocation below - ``node.minTimeUnitsPerPhase``, or a non-zero ``execution_budget_per_round`` below +- :ref:`gvm-def-str-trie-value-vm-error-fee-phase-timeout-out-of-bounds` β€” + unless both time-unit allocations are + zero, the leader allocation is outside ``node.minProposeTimeout`` through + ``node.maxProposeTimeout``, or the validator allocation is outside + ``node.minCommitTimeout`` through ``node.maxCommitTimeout``. +- :ref:`gvm-def-str-trie-value-vm-error-fee-below-minimum` β€” a non-zero + ``execution_budget_per_round`` below ``node.messageBudgetFloor`` (the chain's ``BudgetTooLow``). -- ``fee too_many_rounds`` β€” ``rotations`` implies more consensus rounds than the +- :ref:`gvm-def-str-trie-value-vm-error-fee-too-many-rounds` β€” ``rotations`` + implies more consensus rounds than the node's validator table supports (on-chain ``MAX_ROUNDS``). ``EmitInternalDeployMessage`` Message ------------------------------------- +------------------------------------- Deploys new intelligent contract to blockchain. diff --git a/docs/website/src/spec/03-vm/05-result.rst b/docs/website/src/spec/03-vm/05-result.rst index 94ddc9b2..39aa88eb 100644 --- a/docs/website/src/spec/03-vm/05-result.rst +++ b/docs/website/src/spec/03-vm/05-result.rst @@ -121,9 +121,10 @@ caller and MUST NOT be included in ``LeaderPublicData`` Leader Output Format -------------------- -``LeaderPublicData`` is encoded as an RLP list containing every non-deterministic -block result, in execution order, followed by the byte string ``padded``. Each -result uses :ref:`gvm-def-subvm-result-encoding` +``LeaderPublicData`` uses :ref:`gvm-def-calldata-encoding` with the shape +``{"nd_outs": bytes[]}``. The array contains every non-deterministic block result +in execution order; each element uses :ref:`gvm-def-subvm-result-encoding`. Empty +bytes and all other representations are malformed Contract Result Encoding ------------------------ @@ -184,7 +185,8 @@ with the following keys (in this order): #. ``backtrace`` #. ``data`` β€” the contract result value #. ``data_fees_consumed`` -#. ``data_fees_remaining`` +#. ``data_fees_remaining`` β€” map from each fee-bucket name to its remaining + ``U256`` balance #. ``emissions`` β€” emitted messages and events, in emission order #. ``kind`` β€” the :ref:`gvm-def-vm-result` result code #. ``storage_deltas`` diff --git a/docs/website/src/spec/changelog.rst b/docs/website/src/spec/changelog.rst index 22e85c9f..1e0bbbd7 100644 --- a/docs/website/src/spec/changelog.rst +++ b/docs/website/src/spec/changelog.rst @@ -5,6 +5,24 @@ User-observable changes to the specification, grouped by executor line (see :doc:`01-core-architecture/03-versioning`). Internal refactors are omitted; see the repository history for those +.. _gvm-changelog-v0-2: + +v0.2 +---- + +Breaking +~~~~~~~~ + +#. ``LeaderPublicData`` uses calldata ``{"nd_outs": bytes[]}`` instead of an + RLP list with a trailing ``padded`` sentinel. Its gas accounting uses a + conservative 64-byte frame plus a 64-byte header and 32-byte-padded data for + each output +#. Fee-bucket references are human-readable names. The execution request's + ``bucket_totals`` and result's ``data_fees_remaining`` fields are maps keyed + by bucket name, and executor fee rules use ``buckets`` with 1 name or an + array of names. Numeric bucket arrays and the former ``bucket_no`` config + field are rejected + .. _gvm-changelog-v0-3: v0.3 @@ -13,6 +31,23 @@ v0.3 Breaking ~~~~~~~~ +#. ``LeaderPublicData`` uses calldata ``{"nd_outs": bytes[]}`` instead of an + RLP list with a trailing ``padded`` sentinel. Its fee and output-cap + accounting uses a conservative 64-byte frame plus a 64-byte header and + 32-byte-padded data for each output +#. Each outbound message consumes 1 unit from the + ``submitted_messages_count`` bucket in addition to its byte and gas charges. + Submitted-message byte accounting includes the canonical ABI array frame, + struct head, fee parameters, payload, and allocation subtree. Receipt-gas + accounting includes the host-provided ``receiptWrapperBytes`` governance value +#. Message-reveal gas and ABI-array overhead are charged on the first successfully + emitted message rather than at execution startup. Message-free executions do + not consume that reveal allowance +#. Fee-bucket references are human-readable names. The execution request's + ``bucket_totals`` and result's ``data_fees_remaining`` fields are maps keyed + by bucket name, and executor fee rules use ``buckets`` with 1 name or an + array of names. Numeric bucket arrays and the former ``bucket_no`` config + field are rejected #. The pre-finalization state is spelled *decided* everywhere it is named. :ref:`gvm-def-enum-storage-view` reads ``latest_finalized`` (1) and ``latest_decided`` (2) instead of ``latest_final`` and ``latest_non_final``, @@ -48,10 +83,21 @@ Breaking ``/``, no backslash, no empty, ``.`` or ``..`` path component, no trailing ``/`` on a file #. Entries sharing a name resolve to the last of them +#. Hosts must provide separate ``minProposeTimeout``, ``maxProposeTimeout``, + ``minCommitTimeout`` and ``maxCommitTimeout`` gas-data values. The former + ``minTimeUnitsPerPhase`` value is no longer read Changed ~~~~~~~ +#. Every internal message declares its minimum primary fee plus its direct child + allocation budgets, independent of whether it is emitted on acceptance or + finalization. Balance-funded messages have no allocation subtree and declare + only the primary fee; external messages declare zero +#. Unless both time-unit allocations are zero, internal message emission checks + leader units against propose bounds and validator units against commit bounds; + violations report + :ref:`gvm-def-str-trie-value-vm-error-fee-phase-timeout-out-of-bounds` #. A missing or malformed runner archive/comment header is now reported as :ref:`gvm-def-str-trie-value-vm-error-invalid-contract-runner-absent` or :ref:`gvm-def-str-trie-value-vm-error-invalid-contract-runner-malformed` diff --git a/executors/v0.2.x b/executors/v0.2.x index a27db421..e4ea0dac 160000 --- a/executors/v0.2.x +++ b/executors/v0.2.x @@ -1 +1 @@ -Subproject commit a27db421e17c5d534f6955302df3f5357d572848 +Subproject commit e4ea0dac1bbe5151bbdef8332f451397deef35f0 diff --git a/executors/v0.3.x b/executors/v0.3.x index 977f9456..c6c6f3ce 160000 --- a/executors/v0.3.x +++ b/executors/v0.3.x @@ -1 +1 @@ -Subproject commit 977f945679e6c8823bc859d270c5b680bd6d1d84 +Subproject commit c6c6f3ceb289ffa4d8e517649349ec6cc0a94f4b diff --git a/implementation/src/manager/run.rs b/implementation/src/manager/run.rs index 4fc3ecdc..4e0ff97e 100644 --- a/implementation/src/manager/run.rs +++ b/implementation/src/manager/run.rs @@ -982,7 +982,7 @@ pub struct Request { #[serde(default = "default_max_execution_minutes")] #[calldata(default = default_max_execution_minutes)] pub max_execution_minutes: u64, - pub bucket_totals: Vec<num_bigint::BigInt>, + pub bucket_totals: std::collections::BTreeMap<String, num_bigint::BigInt>, pub host_data: String, #[calldata(serialize_with = encode_datetime_rfc3339, deserialize_with = decode_datetime_rfc3339)] pub timestamp: chrono::DateTime<chrono::Utc>, @@ -2128,7 +2128,7 @@ impl Ctx { is_sync: true, debug_mode: genvm_common::DebugMode::Disabled, max_execution_minutes: parent_req.max_execution_minutes, - bucket_totals: Vec::new(), + bucket_totals: std::collections::BTreeMap::new(), host_data: parent_req.host_data.clone(), timestamp: envelope.message.transaction_timestamp, host: parent_req.host.clone(), diff --git a/implementation/src/manager/run_test.rs b/implementation/src/manager/run_test.rs index 9f1427bc..1c20883b 100644 --- a/implementation/src/manager/run_test.rs +++ b/implementation/src/manager/run_test.rs @@ -554,7 +554,7 @@ fn clean_reported() -> genvm_modules_interfaces::ReportedResult { emissions: Vec::new(), nondet_disagreement: None, leader_public_data: bytes::Bytes::new(), - data_fees_remaining: Vec::new(), + data_fees_remaining: std::collections::BTreeMap::new(), data_fees_consumed: Default::default(), llm_consumed_gen_wei: primitive_types::U256::zero(), } diff --git a/tests/runner/genvm_tool_plugins/integration.py b/tests/runner/genvm_tool_plugins/integration.py index 8ce58c2a..45ee7226 100644 --- a/tests/runner/genvm_tool_plugins/integration.py +++ b/tests/runner/genvm_tool_plugins/integration.py @@ -308,6 +308,7 @@ class IntegrationSingleCase(genvm_tool.tests.test.Case): description: genvm_tool.tests.test.Description jsonnet_path: Path cases_dir: Path + executor_major: int reroute_to: str manager_service: genvm_tool.tests.stage.collection.Service tree_path: str @@ -666,7 +667,12 @@ async def _run_single_step( leader_public_data = None else: leader_nondet_fixture = single_conf.get('leader_nondet', None) - if leader_nondet_fixture is None: + # `leader_nondet` fills a well-formed envelope; this one replaces the + # envelope itself, which is the only way to reach the decode failure + raw_envelope = single_conf.get('leader_public_data_raw', None) + if raw_envelope is not None: + leader_public_data = bytes(raw_envelope) + elif leader_nondet_fixture is None: leader_public_data = await gvm_io.read_file_bytes( path_to_which_leader_puts_result ) @@ -765,14 +771,24 @@ def resolve_executor(address, state, advisory_major): request_extra['permissions'] = case_permissions if hook_calls: request_extra['hook_cross_contract_calls'] = True - dflt_bucket = 2**200 - bucket_totals: list[int] = single_conf.get('bucket_totals', [dflt_bucket] * 10) + bucket_totals = base_host.default_bucket_totals(self._test_case.executor_major) + bucket_totals.update(single_conf.get('bucket_totals', {})) default_message_fee_allocation = [ fees.DEFAULT_EXTERNAL_MESSAGE_ALLOC, fees.DEFAULT_INTERNAL_FIN_MESSAGE_ALLOC, fees.DEFAULT_INTERNAL_DEC_MESSAGE_ALLOC, ] + if self._test_case.executor_major >= 3: + for index, allocation in enumerate(default_message_fee_allocation[1:], 1): + allocation = allocation.copy() + allocation['fee_params'] = { + 'Internal': { + **allocation['fee_params']['Internal'], + 'max_price_gen_per_time_unit': 1, + } + } + default_message_fee_allocation[index] = allocation message_fee_allocation: list[fees.MessageAllocationNode] = single_conf.get( 'message_fee_allocation', default_message_fee_allocation @@ -910,10 +926,10 @@ def resolve_executor(address, state, advisory_major): messages_content.append('\n') # The executor's own execution hash, i.e. the value consensus actually - # compares. It folds kind, data, backtrace, storage changes, sub-VM - # hashes, wasm store hashes and both fee buckets. Emissions are not in - # it; those are covered by the `messages` semantics component, which - # only runs in the main mode. + # compares. It folds kind (with fatality coalesced away), data, backtrace, + # storage deltas, sub-VM hashes, wasm store hashes, both fee counters and + # emissions -- so a mode that emits different messages fails here, not + # only in the main-mode `messages` semantics component. hash_data = res.execution_hash semantics_parts = { @@ -1329,6 +1345,7 @@ def integration_test_single_executor( integration_test_directory( ctx, cases_dir=CASES_DIR, + executor_major=int(executor_version.removeprefix('v').split('.')[1]), reroute_to=reroute_to, save_hashes=save_hashes, manager_service=manager_service, @@ -1342,6 +1359,7 @@ def integration_test_directory( ctx: genvm_tool.tests.stage.collection.Context, *, cases_dir: Path, + executor_major: int, reroute_to: str, save_hashes: bool, manager_service: genvm_tool.tests.stage.collection.Service, @@ -1375,6 +1393,7 @@ def integration_test_directory( jsonnet_file, jsonnet_cache, cases_dir=cases_dir, + executor_major=executor_major, reroute_to=reroute_to, save_hashes=save_hashes, manager_service=manager_service, @@ -1399,6 +1418,7 @@ def _single_integration_test( jsonnet_cache: JsonnetCache, *, cases_dir: Path, + executor_major: int, reroute_to: str, save_hashes: bool, manager_service: genvm_tool.tests.stage.collection.Service, @@ -1562,6 +1582,7 @@ def _single_integration_test( description=step_desc, jsonnet_path=jsonnet_file, cases_dir=cases_dir, + executor_major=executor_major, reroute_to=reroute_to, manager_service=manager_service, tree_path=tree_path, diff --git a/tests/runner/origin/base_host.py b/tests/runner/origin/base_host.py index bd4c9f04..1a185608 100644 --- a/tests/runner/origin/base_host.py +++ b/tests/runner/origin/base_host.py @@ -68,13 +68,34 @@ 'bootloaderOverhead': '0', 'fixedProposeReceiptGas': '0', 'fixedMessageRevealGas': '0', + 'lockedReceiptGasPrice': '1', + # 0 disables the overlay unless a fee test opts into a specific chain split + 'overlaySplitBps': '0', + 'receiptWrapperBytes': '1024', + 'minProposeTimeout': '1', + 'maxProposeTimeout': '340282366920938463463374607431768211455', + 'minCommitTimeout': '1', + 'maxCommitTimeout': '340282366920938463463374607431768211455', 'genPerTimeUnit': '0', - # 0 = no per-phase timeunit floor, so default-allocation tests are unaffected. - 'minTimeUnitsPerPhase': '0', # 0 = no per-round execution-budget floor for balance-funded messages. 'messageBudgetFloor': '0', } +_DEFAULT_BUCKET_TOTALS: dict[str, int] = { + 'execution_data_gas': 2**200, + 'message_fee': 2**200, + 'nondet_outputs': 2**200, + 'submitted_messages': 2**200, + 'submitted_messages_count': 2**200, +} + + +def default_bucket_totals(executor_major: int) -> dict[str, int]: + totals = _DEFAULT_BUCKET_TOTALS.copy() + if executor_major >= 3: + totals['submitted_messages_count'] = 2**200 + return totals + class Context(typing.Protocol): logger: Logger @@ -503,7 +524,7 @@ class ConsumedResult: result_storage_deltas: list[tuple[bytes, bytes]] = field(default_factory=list) result_emissions: list[ResultEmission] = field(default_factory=list) result_leader_public_data: bytes = b'' - data_fees_remaining: list[int] = field(default_factory=list) + data_fees_remaining: dict[str, int] = field(default_factory=dict) @classmethod def internal_error(cls, message: str) -> 'ConsumedResult': @@ -552,7 +573,7 @@ def decode(cls, raw: typing.Any) -> 'ConsumedResult': result_storage_deltas=decoded.get('storage_deltas', []), result_emissions=decoded.get('emissions', []), result_leader_public_data=decoded.get('leader_public_data', b''), - data_fees_remaining=decoded.get('data_fees_remaining', []), + data_fees_remaining=decoded.get('data_fees_remaining', {}), ) @@ -572,7 +593,7 @@ class RunHostAndProgramRes: result_storage_deltas: list[tuple[bytes, bytes]] result_emissions: list[ResultEmission] result_leader_public_data: bytes - data_fees_remaining: list[int] + data_fees_remaining: dict[str, int] metrics: dict[str, typing.Any] | None = None vm_error_description: str | None = None @@ -1161,8 +1182,7 @@ async def run_genvm( gas_data: dict[str, str] | None = None, host: str, extra_args: collections.abc.Sequence[str] = (), - # default config fee buckets use bucket_no 0 and 1 - bucket_totals: list[int], + bucket_totals: dict[str, int], code: bytes | None = None, calldata: bytes, leader_public_data: bytes | None = None, diff --git a/tests/runner/origin/leader_public_data.py b/tests/runner/origin/leader_public_data.py index b2fb25bb..f11a9ae4 100644 --- a/tests/runner/origin/leader_public_data.py +++ b/tests/runner/origin/leader_public_data.py @@ -2,8 +2,7 @@ from dataclasses import dataclass from typing import Self - -_PADDING = b'padded' +from . import calldata as gvm_calldata @dataclass @@ -11,35 +10,19 @@ class LeaderPublicData: nondet_block_outputs: list[bytes] def encode(self) -> bytes: - payload = b''.join( - _encode_bytes(value) for value in (*self.nondet_block_outputs, _PADDING) - ) - return _encode_length(len(payload), 0xC0, 0xF7) + payload + return gvm_calldata.encode({'nd_outs': self.nondet_block_outputs}) @classmethod def decode(cls, encoded: collections.abc.Buffer) -> Self: - data = bytes(encoded) - if not data: - return cls([]) - - payload_start, payload_len = _decode_length(data, 0, list_=True) - payload_end = payload_start + payload_len - if payload_end != len(data): - raise ValueError('trailing leader public data') - - outputs: list[bytes] = [] - cursor = payload_start - while cursor < payload_end: - value_start, value_len = _decode_length(data, cursor, list_=False) - value_end = value_start + value_len - if value_end > payload_end: - raise ValueError('leader public data item exceeds list') - outputs.append(data[value_start:value_end]) - cursor = value_end - - if not outputs or outputs[-1] != _PADDING: - raise ValueError('leader public data padding is missing') - return cls(outputs[:-1]) + decoded = gvm_calldata.decode(bytes(encoded)) + if ( + not isinstance(decoded, dict) + or decoded.keys() != {'nd_outs'} + or not isinstance(decoded['nd_outs'], list) + or any(not isinstance(output, bytes) for output in decoded['nd_outs']) + ): + raise ValueError('invalid leader public data') + return cls(decoded['nd_outs']) def encode(value: LeaderPublicData) -> bytes: @@ -48,52 +31,3 @@ def encode(value: LeaderPublicData) -> bytes: def decode(encoded: collections.abc.Buffer) -> LeaderPublicData: return LeaderPublicData.decode(encoded) - - -def _encode_bytes(value: bytes) -> bytes: - if len(value) == 1 and value[0] < 0x80: - return value - return _encode_length(len(value), 0x80, 0xB7) + value - - -def _encode_length(length: int, short_base: int, long_base: int) -> bytes: - if length <= 55: - return bytes([short_base + length]) - length_bytes = length.to_bytes((length.bit_length() + 7) // 8, 'big') - return bytes([long_base + len(length_bytes)]) + length_bytes - - -def _decode_length( - encoded: bytes, - offset: int, - *, - list_: bool, -) -> tuple[int, int]: - if offset >= len(encoded): - raise ValueError('truncated leader public data') - - prefix = encoded[offset] - short_base = 0xC0 if list_ else 0x80 - long_base = 0xF7 if list_ else 0xB7 - if not list_ and prefix < 0x80: - return offset, 1 - if prefix < short_base or prefix > long_base + 8: - raise ValueError('invalid RLP prefix') - if prefix <= long_base: - if not list_ and prefix == 0x81 and offset + 1 < len(encoded): - if encoded[offset + 1] < 0x80: - raise ValueError('non-canonical RLP string') - return offset + 1, prefix - short_base - - length_len = prefix - long_base - length_start = offset + 1 - length_end = length_start + length_len - if length_end > len(encoded): - raise ValueError('truncated RLP length') - length_bytes = encoded[length_start:length_end] - if length_bytes[0] == 0: - raise ValueError('non-canonical RLP length') - length = int.from_bytes(length_bytes, 'big') - if length <= 55: - raise ValueError('non-canonical long RLP value') - return length_end, length diff --git a/tests/runner/origin/test_leader_public_data.py b/tests/runner/origin/test_leader_public_data.py index d2cf81f6..258e2ead 100644 --- a/tests/runner/origin/test_leader_public_data.py +++ b/tests/runner/origin/test_leader_public_data.py @@ -1,5 +1,6 @@ import pytest +from . import calldata as gvm_calldata from .leader_public_data import LeaderPublicData, decode, encode @@ -8,15 +9,20 @@ def test_round_trip_from_memoryview() -> None: assert decode(memoryview(encode(data))) == data -def test_preserves_legacy_encoding() -> None: - assert LeaderPublicData([b'test']).encode() == b'\xcc\x84test\x86padded' +def test_has_stable_encoding() -> None: + assert LeaderPublicData([b'a', b'bc']).encode() == b'\x0e\x07nd_outs\x15\x0ba\x13bc' -def test_empty_timeout_decodes_as_no_outputs() -> None: - assert LeaderPublicData.decode(b'') == LeaderPublicData([]) - - -@pytest.mark.parametrize('encoded', [b'\xc0', b'\xc7\x86padded\x00', b'\xc2\x81\x01']) +@pytest.mark.parametrize( + 'encoded', + [ + b'', + b'\xcc\x84test\x86padded', + gvm_calldata.encode({}), + gvm_calldata.encode({'nd_outs': [1]}), + gvm_calldata.encode({'nd_outs': []}) + b'\x00', + ], +) def test_rejects_invalid_encoding(encoded: bytes) -> None: - with pytest.raises(ValueError): + with pytest.raises((ValueError, gvm_calldata.DecodingError)): LeaderPublicData.decode(encoded) diff --git a/tests/system/cross-major-observability/test.py b/tests/system/cross-major-observability/test.py index ca4b92cc..ab279f5b 100644 --- a/tests/system/cross-major-observability/test.py +++ b/tests/system/cross-major-observability/test.py @@ -165,7 +165,9 @@ async def _execute( 'hook_cross_contract_calls', True ), }, - bucket_totals=[2**200] * 20, + bucket_totals=base.base_host.default_bucket_totals( + typing.cast(int, kwargs['line']) + ), ) if ( kwargs.get('apply_changes', True) diff --git a/tests/system/cross-major/test.py b/tests/system/cross-major/test.py index 6373d8f1..c58da887 100644 --- a/tests/system/cross-major/test.py +++ b/tests/system/cross-major/test.py @@ -403,7 +403,7 @@ async def _execute( 'no_modules': True, 'hook_cross_contract_calls': hook_cross_contract_calls, }, - bucket_totals=[2**200] * 20, + bucket_totals=base_host.default_bucket_totals(line), ) if apply_changes and result.result_kind == host_fns.ResultCode.RETURN: assert mock_host.storage is not None @@ -500,7 +500,7 @@ def _raw_request( 'code': None, 'calldata': _calldata('call', ADDR_BUSY_V02), 'leader_public_data': None, - 'bucket_totals': [2**200] * 20, + 'bucket_totals': base_host.default_bucket_totals(line), 'gas_data': base_host.DEFAULT_GAS_DATA, 'message_fee_allocation': [], 'initial_time_units_allocation': 60, diff --git a/tests/system/manager-socket/test.py b/tests/system/manager-socket/test.py index 96c44b31..d6a73ee6 100644 --- a/tests/system/manager-socket/test.py +++ b/tests/system/manager-socket/test.py @@ -180,7 +180,7 @@ def _run_request( 'code': (root / HELLO_WORLD).read_bytes(), 'calldata': gvm_calldata.encode({}), 'leader_public_data': None, - 'bucket_totals': [2**200] * 20, + 'bucket_totals': base_host.default_bucket_totals(3), 'gas_data': base_host.DEFAULT_GAS_DATA, 'message_fee_allocation': [], 'initial_time_units_allocation': 60, @@ -884,7 +884,7 @@ async def _run_fixture_over_socket(self, manager: _ManagerFixture, name: str): timeout=30, debug_mode='unsafe', request_extra={'no_modules': True}, - bucket_totals=[2**200] * 20, + bucket_totals=base_host.default_bucket_totals(3), ) async def _unix_parent_and_stale_socket(self): diff --git a/tests/system/permits/test.py b/tests/system/permits/test.py index f6b110e4..6f4d6610 100644 --- a/tests/system/permits/test.py +++ b/tests/system/permits/test.py @@ -226,7 +226,7 @@ async def _run_background_genvm( ), request_extra={'no_modules': True}, shutdown_early=shut_downer, - bucket_totals=[2**200] * 20, + bucket_totals=base_host.default_bucket_totals(3), ) except Exception: pass # expected when cancelled or contract crashes @@ -245,7 +245,7 @@ async def _assert_blocks(self, manager_uri: str, code: bytes) -> bool: 'extra_args': [], 'code': code, 'calldata': gvm_calldata.encode({}), - 'bucket_totals': [2**200] * 20, + 'bucket_totals': base_host.default_bucket_totals(3), 'no_modules': True, 'leader_public_data': b'', 'initial_time_units_allocation': 60, @@ -295,7 +295,7 @@ async def _assert_socket_cancel_queued( 'extra_args': [], 'code': code, 'calldata': gvm_calldata.encode({}), - 'bucket_totals': [2**200] * 20, + 'bucket_totals': base_host.default_bucket_totals(3), 'no_modules': True, 'leader_public_data': b'', 'initial_time_units_allocation': 60, From be558e0742ef3853c8f7fae397d9e4fe852ea351 Mon Sep 17 00:00:00 2001 From: Kira <kp2pml30@gmail.com> Date: Thu, 17 Sep 2026 02:03:50 +0900 Subject: [PATCH 20/45] feat: add a clippy CI cell and tag log records with an audience (#41) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(ci): add a clippy cell to the full matrix πŸ’š * feat(manager): carry log audience through the sink and evict introspector first ✨ * feat(manager): tag log call sites with audience ✨ * fix(manager): no provider could handle prompt is an operator log πŸ› * docs(impl-spec): describe the log record format and audience πŸ“ * docs(spec): fix references that name nothing πŸ“ Every one of these pointed at a label the build never generates, so the page either dropped the link or the term was simply wrong: two emission constants that do not exist, a fee error spelled after a code that is not in the ABI, two mistyped enum and error labels, an under-long heading underline, and a generated Lua reference that is not part of the site. * docs(spec): specify the runner count and path limits πŸ“ Documents what the v0.3 executor now enforces: a per-sub-VM runner count cap refused as an out-of-memory error, a length and separator bound on the path argument of path_open and path_filestat_get, and the raised retained-data charges. Bumps the executor gitlink to the commits that implement them. * fix(executor): account for ZIP runner metadata πŸ”’οΈ --- docs/contributing/howto/building/build.md | 7 + .../01-core-architecture/04-executor.rst | 9 +- .../src/impl-spec/03-greyboxing/index.rst | 7 - docs/website/src/impl-spec/appendix/index.rst | 1 + .../src/impl-spec/appendix/log-record.rst | 93 ++++++++ .../02-execution-environment/02-wasip1.rst | 7 + .../03-wasi_genlayer_sdk/02-gl_call.rst | 26 ++- .../02-execution-environment/04-runners.rst | 23 +- docs/website/src/spec/03-vm/01-startup.rst | 6 +- .../src/spec/03-vm/03-ram-limiting.rst | 21 +- .../src/spec/appendix/internal-constants.rst | 29 ++- docs/website/src/spec/changelog.rst | 4 +- executors/v0.2.x | 2 +- executors/v0.3.x | 2 +- implementation/src/common/mod.rs | 219 ++++++++++++++---- implementation/src/llm/mod.rs | 8 +- implementation/src/llm/providers.rs | 28 +-- implementation/src/main.rs | 1 + implementation/src/manager/check_install.rs | 6 +- implementation/src/manager/handlers.rs | 2 +- implementation/src/manager/mod.rs | 2 +- implementation/src/manager/run.rs | 136 ++++++----- implementation/src/manager/socket.rs | 4 +- implementation/src/scripting/ctx/dflt.rs | 5 +- implementation/src/scripting/mod.rs | 2 +- implementation/src/scripting/pool.rs | 2 +- implementation/src/web/handler.rs | 1 + implementation/tests/log_sink.rs | 203 ++++++++++++++++ install/config/genvm-llm-default.lua | 18 +- install/lib/genvm-lua/lib-genvm.lua | 4 +- support/ci/ci_lib.py | 9 + support/ci/pipelines/checks.py | 84 ++++++- support/ci/pipelines/tests.py | 18 ++ support/ci/unit_tests/test_cargo_clippy.py | 102 ++++++++ .../runner/genvm_tool_plugins/integration.py | 4 - tests/runner/genvm_tool_plugins/ninja.py | 12 +- 36 files changed, 903 insertions(+), 204 deletions(-) create mode 100644 docs/website/src/impl-spec/appendix/log-record.rst create mode 100644 implementation/tests/log_sink.rs create mode 100644 support/ci/unit_tests/test_cargo_clippy.py diff --git a/docs/contributing/howto/building/build.md b/docs/contributing/howto/building/build.md index 38c91739..17def2df 100644 --- a/docs/contributing/howto/building/build.md +++ b/docs/contributing/howto/building/build.md @@ -20,12 +20,19 @@ shows up as ninja's `missing and no known rule to make it` | `all/data` | runner data via nix | | `all/runners` | the runners themselves via nix, x86_64 Linux only | | `codegen` | generated sources ([genvm-tool.md](../genvm-tool.md)) | +| `cargo/clippy` | clippy over every registered crate, warnings fatal | +| `cargo/clippy/fix` | the same lints, applying every machine-applicable fix | +| `cargo/fmt` | `cargo fmt` over every crate | Outputs land in `build/out`: `bin/genvm-modules`, `bin/genvm-manager`, `bin/genvm-post-install`, and `executor/<version>/bin/genvm` per built line, where the version comes from `executors/<line>.x/manifest.json` and is recorded in `build/info.json` +The full CI matrix has a `clippy` cell running `cargo/clippy`; when it fails it +reruns `cargo/clippy/fix` and prints the resulting patch in the job log and +summary, so run the target locally before pushing + Release packages: [release-build.md](../releasing/release-build.md). Runners: [runners.md](runners.md) diff --git a/docs/website/src/impl-spec/01-core-architecture/04-executor.rst b/docs/website/src/impl-spec/01-core-architecture/04-executor.rst index 0fa4cf6a..bde9b9c4 100644 --- a/docs/website/src/impl-spec/01-core-architecture/04-executor.rst +++ b/docs/website/src/impl-spec/01-core-architecture/04-executor.rst @@ -20,7 +20,7 @@ executor installs no signal handlers and has no graceful-shutdown path, so it **can be killed at any moment**, between any two operations, without notice. Implications ------------ +------------ - The executor keeps no durable state of its own. All persistent state lives in the host and is written only as part of delivering a result. A killed executor @@ -41,9 +41,10 @@ Output **capture** is a derived property with three states: - ``disabled`` β€” nothing is captured into the result: the executor's stdout/stderr go to ``/dev/null`` and its logs are *forwarded to the manager's own log* (so they are not lost, just not returned in the response). -- ``bounded`` β€” captured into the result, but bounded: at most the 128 most - recent log entries are kept (oldest dropped), and stdout/stderr are truncated - to a 4 MiB tail each. +- ``bounded`` β€” captured into the result, but bounded: at most 128 log entries + are kept, evicted by audience first and by age second (see + :doc:`../appendix/log-record`), and stdout/stderr are truncated to a 4 MiB + tail each. - ``unbounded`` β€” captured into the result in full. When capture is ``disabled`` the result's log/stdout/stderr fields are empty diff --git a/docs/website/src/impl-spec/03-greyboxing/index.rst b/docs/website/src/impl-spec/03-greyboxing/index.rst index c20928fd..d6440ad0 100644 --- a/docs/website/src/impl-spec/03-greyboxing/index.rst +++ b/docs/website/src/impl-spec/03-greyboxing/index.rst @@ -160,10 +160,3 @@ of the LLM module config and MUST contain the documented ``#{...}`` placeholders Missing placeholders are a config error and surface as :ref:`gvm-def-internal-error` at module startup. - -Generated Reference -------------------- - -The auto-generated signature reference for the Lua tables exposed to scripts -(``lib.rs.*``, ``llm.rs.*``, ``web.rs.*``, the ``Prompt`` shape, etc.) lives in -:doc:`01-lua-api`. diff --git a/docs/website/src/impl-spec/appendix/index.rst b/docs/website/src/impl-spec/appendix/index.rst index 5569c364..941d3571 100644 --- a/docs/website/src/impl-spec/appendix/index.rst +++ b/docs/website/src/impl-spec/appendix/index.rst @@ -7,3 +7,4 @@ Appendix host-loop manager-api manager-socket + log-record diff --git a/docs/website/src/impl-spec/appendix/log-record.rst b/docs/website/src/impl-spec/appendix/log-record.rst new file mode 100644 index 00000000..0a7fac96 --- /dev/null +++ b/docs/website/src/impl-spec/appendix/log-record.rst @@ -0,0 +1,93 @@ +Log Record Format +================= + +Every log line the executor and the manager emit is one JSON object. The same +shape is used on stderr, in the manager's own log and in the ``genvm_log`` +artifact served over the manager socket (see :doc:`manager-socket`). + +Fields +------ + +.. list-table:: + :header-rows: 1 + :widths: 16 14 70 + + * - Key + - Type + - Meaning + * - ``level`` + - string + - One of ``trace``, ``debug``, ``info``, ``warn``, ``error``. Always the first key + * - ``audience`` + - string + - Who the record is meant for, see below. Always the second key + * - ``target`` + - string + - Rust module path of the call site + * - ``message`` + - string + - Human-readable text + * - ``ts`` + - string + - Wall-clock timestamp + * - *anything else* + - any + - Structured captures of the call site. Long strings and byte buffers are truncated + to a per-record byte limit, currently 128 bytes + +``audience`` is a reserved key: a capture named that way fails to compile and a +hand-built record carrying one has it dropped. + +Audience +-------- + +The audience is a tag only, it never takes part in filtering. It answers *who +should act on this line*: + +.. list-table:: + :header-rows: 1 + :widths: 18 82 + + * - Value + - Reader + * - ``user`` + - The contract developer or caller: the record is a consequence of the contract's + own input, code or budget + * - ``operator`` + - The node runner: configuration, environment, providers, broken internal + invariants, panics + * - ``introspector`` + - Someone debugging GenVM itself: lifecycle tracing, state dumps, timings. The + default when a call site names none + +Rules that follow from the tag: + +1. A ``user`` record may only carry scalar captures (display, error, debug, + id); bulk dumps such as calldata, byte buffers and serialised values are + rejected at compile time, and the byte limit is clamped to 128 regardless of + configuration +2. Panics are ``operator`` +3. Records from a v0.2.x executor, which knows nothing about audiences, are + tagged ``introspector`` by the manager +4. A line the executor emitted that is not valid JSON is wrapped by the manager + into ``{"level": "error", "audience": "operator", "message": "genvm log", + "line": <base64>}`` + +Lua scripts log through ``lib.log { level = ..., audience = ..., message = ... }``; +an absent or unknown ``audience`` means ``introspector``. + +Manager Sink +------------ + +Under ``bounded`` capture (see :doc:`../01-core-architecture/04-executor`) the +manager keeps at most 128 records per execution and evicts by audience rather +than by age: + +1. While the cap has never been hit, everything is queued +2. On the first overflow every ``introspector`` record is dropped, a marker + ``{"level": "warn", "audience": "operator", "message": "introspector logs dropped"}`` + is appended, and from then on ``introspector`` records are discarded on arrival +3. On the next overflow the sink degrades to a plain queue that drops the oldest + ``user`` or ``operator`` record; ``introspector`` records stay discarded + +The marker counts toward the cap. ``unbounded`` capture keeps every record. diff --git a/docs/website/src/spec/02-execution-environment/02-wasip1.rst b/docs/website/src/spec/02-execution-environment/02-wasip1.rst index 8ef9f26a..e94a975b 100644 --- a/docs/website/src/spec/02-execution-environment/02-wasip1.rst +++ b/docs/website/src/spec/02-execution-environment/02-wasip1.rst @@ -164,6 +164,13 @@ Path Resolution ``path_*`` functions take a directory :term:`FD` and a path. Resolution is purely lexical and identical in both modes: +#. The supplied path must be at most + :ref:`gvm-def-consts-value-top-limits-vfs-path-len` octets long and contain + at most :ref:`gvm-def-consts-value-top-limits-vfs-path-components` ``/`` + separators, otherwise the call fails with ``Inval``. Both bound the argument + as written, before any normalization, so a separator that yields no component + still counts. Neither bound applies to the directory descriptor's own path, + which is bounded by how the :ref:`gvm-def-vfs` was populated. #. The directory descriptor's own path and the supplied path are concatenated and split on ``/``. Empty and ``.`` components are dropped, a ``..`` component pops the preceding one, and a ``..`` that would escape the root is diff --git a/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/02-gl_call.rst b/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/02-gl_call.rst index 98445c47..15dee7ae 100644 --- a/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/02-gl_call.rst +++ b/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/02-gl_call.rst @@ -207,14 +207,12 @@ Semantics: Metering additionally enforces node-configured bounds, surfaced as ``VMError``\ s: -- :ref:`gvm-def-str-trie-value-vm-error-fee-phase-timeout-out-of-bounds` β€” - unless both time-unit allocations are - zero, the leader allocation is outside ``node.minProposeTimeout`` through - ``node.maxProposeTimeout``, or the validator allocation is outside +- :ref:`gvm-def-str-trie-value-vm-error-fee-below-minimum` β€” either a non-zero + ``execution_budget_per_round`` below ``node.messageBudgetFloor`` (the chain's + ``BudgetTooLow``), or β€” unless both time-unit allocations are zero β€” a leader + allocation outside ``node.minProposeTimeout`` through + ``node.maxProposeTimeout`` or a validator allocation outside ``node.minCommitTimeout`` through ``node.maxCommitTimeout``. -- :ref:`gvm-def-str-trie-value-vm-error-fee-below-minimum` β€” a non-zero - ``execution_budget_per_round`` below - ``node.messageBudgetFloor`` (the chain's ``BudgetTooLow``). - :ref:`gvm-def-str-trie-value-vm-error-fee-too-many-rounds` β€” ``rotations`` implies more consensus rounds than the node's validator table supports (on-chain ``MAX_ROUNDS``). @@ -370,15 +368,19 @@ Semantics that returns the same runner id. Otherwise :ref:`gvm-def-consts-value-memory-limiter-consts-runner-load-cost` plus ``code`` length is charged against the caller's RAM budget before the archive is parsed; on success, the runner -enters the caller's loaded set. +also incurs its :ref:`metadata charge <gvm-def-runner-load-charge>` and enters +the caller's loaded set -The outcomes, in check order, are: +The outcomes are: #. Missing :ref:`gvm-def-det-mode`: the call fails with ``Forbidden``. Nothing is charged and no state changes. -#. Insufficient memory for the charge: the :term:`sub-VM` exits with an - out-of-memory :ref:`gvm-def-vm-error`. Nothing is charged and the runner is - not registered. +#. Insufficient memory for the base cost and ``code`` length: the :term:`sub-VM` + exits with :ref:`gvm-def-str-trie-value-vm-error-out-of-memory`. Nothing is + charged and the runner is not registered +#. Insufficient memory for metadata while loading: the :term:`sub-VM` exits + with :ref:`gvm-def-str-trie-value-vm-error-out-of-memory` and the runner is + not registered #. Malformed archive: the call fails with a deterministic invalid-contract :ref:`gvm-def-vm-error`. The charge is retained until the :term:`sub-VM` finishes, and the runner is not in the loaded set. diff --git a/docs/website/src/spec/02-execution-environment/04-runners.rst b/docs/website/src/spec/02-execution-environment/04-runners.rst index 725ea338..ea027225 100644 --- a/docs/website/src/spec/02-execution-environment/04-runners.rst +++ b/docs/website/src/spec/02-execution-environment/04-runners.rst @@ -364,15 +364,30 @@ Runner actions are executed left-recursively, until :ref:`gvm-def-start-wasm` is If it was not reached, it will result in a :ref:`gvm-def-vm-error` with ``invalid_contract runner malformed`` code. +.. _gvm-def-runner-load-charge: + Loading a :term:`runner` goes through a single **load action**, defined per :term:`sub-VM`. Each :term:`sub-VM` owns a **loaded-runner set**: the runner ids it has already loaded. The load action for an id is: - if the id is already in the :term:`sub-VM`'s loaded set, nothing is charged; -- otherwise :ref:`gvm-def-consts-value-memory-limiter-consts-runner-load-cost` plus the runner's size in octets is charged as +- otherwise, when the loaded set already holds + :ref:`gvm-def-consts-value-top-limits-max-runners` ids, the load fails with + :ref:`gvm-def-str-trie-value-vm-error-out-of-memory` and nothing is charged β€” + a count cap is refused exactly like an exhausted RAM budget; +- otherwise :ref:`gvm-def-consts-value-memory-limiter-consts-runner-load-cost` + plus the runner's raw size and metadata cost in octets is charged as :ref:`gvm-def-ram-consumption` against the :term:`sub-VM`'s RAM budget, and the id is then added to the loaded set. +The raw size is the length of the runner's code or archive bytes. For a ZIP +runner, the metadata cost is the sum of +:ref:`gvm-def-consts-value-memory-limiter-consts-zip-file-cost` plus the UTF-8 +filename length in octets for each distinct non-directory entry. Repeated +filenames count once, using the last entry. Non-ZIP runners have no additional +metadata charge. Insufficient RAM for either charge exits the :term:`sub-VM` +with :ref:`gvm-def-str-trie-value-vm-error-out-of-memory` + Whether the executor has the archive cached internally is not observable: the charge depends only on the :term:`sub-VM`'s own load history, never on cache state. The same runner loaded by different :term:`sub-VM` instances is charged @@ -388,9 +403,9 @@ A load action occurs when: - receiving a custom-runner grant at :term:`sub-VM` creation (see :ref:`gvm-meta-property-custom-runners`). -For a ``chain:`` runner the size is the length of the code blob read from -storage. A ``chain:`` load costs the same as any other load of that size β€” -there is no doubled charge and no separate fee component. +For a ``chain:`` runner the raw size is the length of the code blob read from +storage. The same content has the same load charge regardless of its source; +there is no doubled charge and no separate fee component .. _gvm-def-custom-runner-visibility: diff --git a/docs/website/src/spec/03-vm/01-startup.rst b/docs/website/src/spec/03-vm/01-startup.rst index 358bf604..0c5dae81 100644 --- a/docs/website/src/spec/03-vm/01-startup.rst +++ b/docs/website/src/spec/03-vm/01-startup.rst @@ -96,7 +96,7 @@ A new :term:`sub-VM` is created for: - :ref:`gvm-def-gl-call-run-nondet` Creation is rejected with -:ref:`gvm-def-str-trie-value-vm-error-out-of-vm-recursion` if the new +:ref:`gvm-def-str-trie-value-vm-error-out-of-subvm-recursion` if the new :ref:`gvm_vm_field_depth` is greater than or equal to :ref:`gvm-def-consts-value-top-limits-vm-recursion`. @@ -157,10 +157,10 @@ The child is read-only. - :ref:`gvm-perm-use-balance-for-message-fees` is false - :ref:`gvm_vm_field_state_mode` is the requested storage view (*param* - ``state``); a request of :ref:`gvm-def-enum-value-storage-type-default` + ``state``); a request of :ref:`gvm-def-enum-value-storage-view-default` keeps the parent's value (the plain copy rule), so by default the callee observes a view at least as recent as its caller's. Because the child - cannot write, its :ref:`gvm-def-enum-value-storage-type-default` view is + cannot write, its :ref:`gvm-def-enum-value-storage-view-default` view is the decided state: it never includes the calling transaction's uncommitted writes (see :ref:`contract-execution-flow`). - :ref:`gvm_vm_field_topmost_runner_id` is the callee's contract runner. diff --git a/docs/website/src/spec/03-vm/03-ram-limiting.rst b/docs/website/src/spec/03-vm/03-ram-limiting.rst index d73cc9e1..1a892a13 100644 --- a/docs/website/src/spec/03-vm/03-ram-limiting.rst +++ b/docs/website/src/spec/03-vm/03-ram-limiting.rst @@ -44,13 +44,16 @@ The following operations consume RAM: - **File mapping**: :ref:`gvm-def-consts-value-memory-limiter-consts-file-mapping` octets base cost plus the length of the filename in bytes - **File descriptor allocation**: :ref:`gvm-def-consts-value-memory-limiter-consts-fd-allocation` octets per descriptor - **Runner loading**: the first load of a :term:`runner` in a :term:`sub-VM` - costs :ref:`gvm-def-consts-value-memory-limiter-consts-runner-load-cost` plus the runner's size in octets. A runner already + consumes its :ref:`load charge <gvm-def-runner-load-charge>`, including ZIP + metadata. A runner already in that :term:`sub-VM`'s loaded set costs nothing, and the charge is released when the :term:`sub-VM` finishes, like any other charge. Loading covers spawning the entry-point runner, ``Depends``/``With`` actions, the ``MapFile`` and ``RegisterRunner`` ``gl_call``\ s, and receiving a custom-runner grant at sub-VM creation (see :doc:`../02-execution-environment/04-runners` and - :ref:`gvm-meta-property-custom-runners`) + :ref:`gvm-meta-property-custom-runners`). A :term:`sub-VM` holds at most + :ref:`gvm-def-consts-value-top-limits-max-runners` runners: a load past that + fails the same way an exhausted budget does, and charges nothing - **Storage writes**: writing to a 32-octet aligned region of a :term:`Storage Slot` costs :ref:`gvm-def-consts-value-memory-limiter-consts-new-storage-page` octets the @@ -58,17 +61,13 @@ The following operations consume RAM: already written from its caller, and repeated writes to a region, cost nothing - **Emissions**: each emitted message or event costs :ref:`gvm-def-consts-value-memory-limiter-consts-execution-emission-base-size` - octets, - plus its retained calldata, code, allocation subtree, topics, event data, and - :ref:`gvm-def-consts-value-memory-limiter-consts-calldata-arg-element-size` - octets per - retained positional argument, - :ref:`gvm-def-consts-value-memory-limiter-consts-calldata-kwarg-entry-size` - octets per - retained keyword argument, and + octets, plus the encoded length of each payload it retains β€” calldata, code, + allocation subtree, topics and event data β€” and :ref:`gvm-def-consts-value-memory-limiter-consts-message-fee-rotation-element-size` octets - per retained message-fee rotation + per retained message-fee rotation. Calldata is charged by its + :ref:`encoded <gvm-def-calldata-encoding>` length, so positional and keyword + arguments carry no charge of their own - **Nondeterministic outputs**: each output costs :ref:`gvm-def-consts-value-memory-limiter-consts-nondet-output-base-size` octets plus its encoded length on every role diff --git a/docs/website/src/spec/appendix/internal-constants.rst b/docs/website/src/spec/appendix/internal-constants.rst index f9258503..24fa3481 100644 --- a/docs/website/src/spec/appendix/internal-constants.rst +++ b/docs/website/src/spec/appendix/internal-constants.rst @@ -34,7 +34,7 @@ Value: ``96`` runner_load_cost ~~~~~~~~~~~~~~~~ -Value: ``4096`` +Value: ``1048576`` .. _gvm-def-consts-value-memory-limiter-consts-vm-spawn-cost: @@ -62,21 +62,28 @@ Value: ``128`` execution_emission_base_size ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Value: ``256`` +Value: ``1024`` .. _gvm-def-consts-value-memory-limiter-consts-message-fee-rotation-element-size: message_fee_rotation_element_size ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Value: ``32`` +Value: ``128`` .. _gvm-def-consts-value-memory-limiter-consts-nondet-output-base-size: nondet_output_base_size ~~~~~~~~~~~~~~~~~~~~~~~ -Value: ``32`` +Value: ``128`` + +.. _gvm-def-consts-value-memory-limiter-consts-zip-file-cost: + +zip_file_cost +~~~~~~~~~~~~~ + +Value: ``128`` .. _gvm-def-consts-top-limits: @@ -134,6 +141,13 @@ max_fds Value: ``1024`` +.. _gvm-def-consts-value-top-limits-max-runners: + +max_runners +~~~~~~~~~~~ + +Value: ``128`` + .. _gvm-def-consts-value-top-limits-wasm-call-depth: wasm_call_depth @@ -155,6 +169,13 @@ vfs_path_components Value: ``128`` +.. _gvm-def-consts-value-top-limits-vfs-path-len: + +vfs_path_len +~~~~~~~~~~~~ + +Value: ``16384`` + .. _gvm-def-consts-runner-limits: runner_limits diff --git a/docs/website/src/spec/changelog.rst b/docs/website/src/spec/changelog.rst index 1e0bbbd7..a7cd80ff 100644 --- a/docs/website/src/spec/changelog.rst +++ b/docs/website/src/spec/changelog.rst @@ -31,6 +31,8 @@ v0.3 Breaking ~~~~~~~~ +#. ZIP :term:`runner` loads charge per-entry metadata in addition to the raw + archive size and base load cost (see :ref:`runner load charges <gvm-def-runner-load-charge>`) #. ``LeaderPublicData`` uses calldata ``{"nd_outs": bytes[]}`` instead of an RLP list with a trailing ``padded`` sentinel. Its fee and output-cap accounting uses a conservative 64-byte frame plus a 64-byte header and @@ -97,7 +99,7 @@ Changed #. Unless both time-unit allocations are zero, internal message emission checks leader units against propose bounds and validator units against commit bounds; violations report - :ref:`gvm-def-str-trie-value-vm-error-fee-phase-timeout-out-of-bounds` + :ref:`gvm-def-str-trie-value-vm-error-fee-below-minimum` #. A missing or malformed runner archive/comment header is now reported as :ref:`gvm-def-str-trie-value-vm-error-invalid-contract-runner-absent` or :ref:`gvm-def-str-trie-value-vm-error-invalid-contract-runner-malformed` diff --git a/executors/v0.2.x b/executors/v0.2.x index e4ea0dac..5b799909 160000 --- a/executors/v0.2.x +++ b/executors/v0.2.x @@ -1 +1 @@ -Subproject commit e4ea0dac1bbe5151bbdef8332f451397deef35f0 +Subproject commit 5b799909780a3abda8fe9c055f23772b9220645b diff --git a/executors/v0.3.x b/executors/v0.3.x index c6c6f3ce..cc63964c 160000 --- a/executors/v0.3.x +++ b/executors/v0.3.x @@ -1 +1 @@ -Subproject commit c6c6f3ceb289ffa4d8e517649349ec6cc0a94f4b +Subproject commit cc63964cb7b6adcdaa41febb78364a6fc7ff4592 diff --git a/implementation/src/common/mod.rs b/implementation/src/common/mod.rs index 1a5cb697..005fc27b 100644 --- a/implementation/src/common/mod.rs +++ b/implementation/src/common/mod.rs @@ -291,7 +291,7 @@ pub(crate) fn module_error_to_wire<R>( } } Err(err) => { - log_error_into!(&LoggerWithId, error:ah = &err, genvm_id:id = genvm_id.0; "handler fatal error"); + log_error_into!(@operator, &LoggerWithId, error:ah = &err, genvm_id:id = genvm_id.0; "handler fatal error"); genvm_modules_interfaces::Result::FatalError(format!("{err:#}")) } } @@ -371,7 +371,7 @@ where .context("handling"); if let Err(close) = handler.cleanup().await { - log_error_into!(&LoggerWithId, error:ah = &close, genvm_id:id = genvm_id.0; "cleanup error"); + log_error_into!(@operator, &LoggerWithId, error:ah = &close, genvm_id:id = genvm_id.0; "cleanup error"); } res @@ -392,7 +392,7 @@ pub async fn handle_stream<T, R, S, P: MessageHandlerProvider<T, R>>( log_trace!("reading hello"); let hello = match read_hello(&mut stream).await { Err(e) => { - log_error!(error:ah = &e; "read hello failed"); + log_error!(@operator, error:ah = &e; "read hello failed"); return; } Ok(None) => return, @@ -407,7 +407,7 @@ pub async fn handle_stream<T, R, S, P: MessageHandlerProvider<T, R>>( match handler_provider.create_execution_context(hello) { Ok(ctx) => ctx, Err(e) => { - log_error!(error:ah = &e; "failed to create execution context"); + log_error!(@operator, error:ah = &e; "failed to create execution context"); return; } } @@ -418,7 +418,7 @@ pub async fn handle_stream<T, R, S, P: MessageHandlerProvider<T, R>>( .scope(genvm_id, async { log_debug_into!(&LoggerWithId, genvm_id:id = genvm_id.0; "peer accepted"); if let Err(e) = loop_one_impl(handler_provider, &mut stream, exec_ctx).await { - log_error_into!(&LoggerWithId, error:ah = &e, genvm_id:id = genvm_id.0; "internal loop error"); + log_error_into!(@operator, &LoggerWithId, error:ah = &e, genvm_id:id = genvm_id.0; "internal loop error"); } log_debug_into!(&LoggerWithId, genvm_id:id = genvm_id.0; "peer done"); }) @@ -537,7 +537,7 @@ where let cleanup_path = path.to_owned(); let _dropper = sync::DropGuard::new(move || { if let Err(e) = std::fs::remove_file(&cleanup_path) { - log_error!(error:err = &e, socket_path:? = cleanup_path; "cleaning up unix socket failed"); + log_error!(@operator, error:err = &e, socket_path:? = cleanup_path; "cleaning up unix socket failed"); } }); @@ -854,7 +854,7 @@ pub fn redirect_policy() -> reqwest::redirect::Policy { _ => false, }; if bad { - log_warn!(url = attempt.url().as_str(); "redirect to non-globally-routable IP-literal rejected"); + log_warn!(@user, url = attempt.url().as_str(); "redirect to non-globally-routable IP-literal rejected"); return attempt.error(NoRoutableAddress); } let downgrade = attempt @@ -862,7 +862,7 @@ pub fn redirect_policy() -> reqwest::redirect::Policy { .last() .is_some_and(|prev| prev.scheme() == "https" && attempt.url().scheme() == "http"); if downgrade { - log_warn!(url = attempt.url().as_str(); "HTTPS->HTTP downgrade redirect rejected"); + log_warn!(@user, url = attempt.url().as_str(); "HTTPS->HTTP downgrade redirect rejected"); return attempt.error(SchemeDowngrade); } attempt.follow() @@ -925,7 +925,7 @@ pub fn setup_cancels( continue; } - log_warn!(old = parent_pid, new_parent_pid = new_parent_pid; "parent pid changed, closing"); + log_warn!(@operator, old = parent_pid, new_parent_pid = new_parent_pid; "parent pid changed, closing"); canceller(); }, _ = token.chan.closed() => { @@ -947,67 +947,196 @@ pub enum LogSinkElement { impl LogSinkElement { pub fn into_json(self) -> serde_json::Map<String, serde_json::Value> { - match self { - LogSinkElement::Map(v) => v, - LogSinkElement::Line(text) => serde_json::Map::from_iter([ - ("level".into(), serde_json::Value::String("info".into())), - ( - "message".into(), - serde_json::Value::String("genvm log".into()), - ), - ("line".into(), text.into()), - ]), - LogSinkElement::Raw(s) => { - if let Ok(v) = serde_json::from_slice(&s) { - v - } else { + // the fallback carries the audience of an entry that names none: a line the + // executor failed to format ours is an operator matter, while a well-formed + // one from an executor that knows nothing about audiences (v0.2.x) is not + let (mut map, fallback) = match self { + LogSinkElement::Map(v) => (v, logger::Audience::Introspector), + LogSinkElement::Line(text) => ( + serde_json::Map::from_iter([ + ("level".into(), serde_json::Value::String("info".into())), + ( + "message".into(), + serde_json::Value::String("genvm log".into()), + ), + ("line".into(), text.into()), + ]), + logger::Audience::Operator, + ), + LogSinkElement::Raw(s) => match serde_json::from_slice(&s) { + Ok(v) => (v, logger::Audience::Introspector), + Err(_) => { let mut as_encoded = String::new(); base64::prelude::BASE64_STANDARD.encode_string(s, &mut as_encoded); - serde_json::Map::from_iter([ - ("level".into(), serde_json::Value::String("error".into())), - ( - "message".into(), - serde_json::Value::String("genvm log".into()), - ), - ("line".into(), as_encoded.into()), - ]) + ( + serde_json::Map::from_iter([ + ("level".into(), serde_json::Value::String("error".into())), + ( + "message".into(), + serde_json::Value::String("genvm log".into()), + ), + ("line".into(), as_encoded.into()), + ]), + logger::Audience::Operator, + ) } - } - } + }, + }; + + let audience = parse_audience(&map).unwrap_or(fallback); + let level = map + .remove("level") + .unwrap_or_else(|| serde_json::Value::String("info".into())); + map.remove("audience"); + + // `level` and `audience` lead the record, as they do in the executor's own format + let mut out = serde_json::Map::with_capacity(map.len() + 2); + out.insert("level".into(), level); + out.insert( + "audience".into(), + serde_json::Value::String(audience.to_string()), + ); + out.extend(map); + + out } } -/// Maximum number of buffered log entries kept per execution when not in debug -/// mode; older entries are evicted to bound memory usage. +fn parse_audience(map: &serde_json::Map<String, serde_json::Value>) -> Option<logger::Audience> { + use std::str::FromStr; + + logger::Audience::from_str(map.get("audience")?.as_str()?).ok() +} + +/// Number of buffered log entries per execution that makes the sink start +/// evicting when not in debug mode. See [`LogSinkInner::push`] for what goes first. pub const LOG_SINK_LIMIT: usize = 128; +/// Message of the entry that reports the compaction to the node runner +pub const INTROSPECTOR_DROPPED: &str = "introspector logs dropped"; + +/// What the sink does when it is full. It only ever advances, and it is reached +/// by hitting the cap, so a run that stays under it keeps every line. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +enum Eviction { + #[default] + Fresh, + NoIntrospector, + Fifo, +} + +struct LogSinkEntry { + audience: logger::Audience, + data: serde_json::Map<String, serde_json::Value>, +} + +impl LogSinkEntry { + fn new(elem: LogSinkElement) -> Self { + let data = elem.into_json(); + + Self { + audience: parse_audience(&data).unwrap_or_default(), + data, + } + } + + fn introspector_dropped_marker() -> Self { + Self::new(LogSinkElement::Map(serde_json::Map::from_iter([ + ("level".into(), serde_json::Value::String("warn".into())), + ( + "audience".into(), + serde_json::Value::String(logger::Audience::Operator.to_string()), + ), + ( + "message".into(), + serde_json::Value::String(INTROSPECTOR_DROPPED.into()), + ), + ]))) + } +} + +fn make_room_for_one(queue: &mut std::collections::VecDeque<LogSinkEntry>) { + while queue.len() + 1 > LOG_SINK_LIMIT { + queue.pop_front(); + } +} + +#[derive(Default)] +struct LogSinkState { + queue: std::collections::VecDeque<LogSinkEntry>, + eviction: Eviction, +} + #[derive(Default)] pub struct LogSinkInner { - queue: crossbeam::queue::SegQueue<LogSinkElement>, + /// Per execution: every sink has its own eviction state. + state: std::sync::Mutex<LogSinkState>, debug: bool, } impl LogSinkInner { pub fn new(debug: bool) -> Self { Self { - queue: crossbeam::queue::SegQueue::new(), + state: Default::default(), debug, } } - /// Appends an entry. In debug mode the sink is unbounded; otherwise the - /// oldest entries are dropped to keep at most [`LOG_SINK_LIMIT`] buffered. + /// Appends an entry. In debug mode the sink is unbounded; otherwise it holds at + /// most [`LOG_SINK_LIMIT`] entries: the first overflow sacrifices every + /// introspector entry, and only a later one starts dropping the oldest of any + /// audience. pub fn push(&self, elem: LogSinkElement) { - if !self.debug { - while self.queue.len() >= LOG_SINK_LIMIT { - self.queue.pop(); + let entry = LogSinkEntry::new(elem); + + let mut state = self.state.lock().unwrap(); + + if self.debug { + state.queue.push_back(entry); + return; + } + + // introspector entries are sacrificed for good: once the cap was hit they are + // never wanted back, not even by the fifo phase + if state.eviction != Eviction::Fresh && entry.audience == logger::Audience::Introspector { + return; + } + + if state.queue.len() + 1 > LOG_SINK_LIMIT { + if state.eviction == Eviction::Fresh { + state + .queue + .retain(|e| e.audience != logger::Audience::Introspector); + state.eviction = Eviction::NoIntrospector; + + // the marker is an entry like any other and takes a slot of its own + make_room_for_one(&mut state.queue); + state + .queue + .push_back(LogSinkEntry::introspector_dropped_marker()); + + // the entry that triggered the compaction is subject to it too + if entry.audience == logger::Audience::Introspector { + return; + } + } else { + state.eviction = Eviction::Fifo; } + + make_room_for_one(&mut state.queue); } - self.queue.push(elem); + + state.queue.push_back(entry); } - pub fn pop(&self) -> Option<LogSinkElement> { - self.queue.pop() + pub fn buffered(&self) -> usize { + self.state.lock().unwrap().queue.len() + } + + pub fn drain(&self) -> Vec<serde_json::Map<String, serde_json::Value>> { + let mut state = self.state.lock().unwrap(); + + state.queue.drain(..).map(|e| e.data).collect() } } diff --git a/implementation/src/llm/mod.rs b/implementation/src/llm/mod.rs index a97801c9..46b28bc7 100644 --- a/implementation/src/llm/mod.rs +++ b/implementation/src/llm/mod.rs @@ -63,7 +63,7 @@ pub async fn create_vm(config: &sync::DArc<config::Config>) -> anyhow::Result<Us scripting::load_script(&vm, &config.mod_base.lua_script_path) .await .with_context(|| { - format!("loading script from {}", &config.mod_base.lua_script_path) + format!("loading script from {}", config.mod_base.lua_script_path) })?; // get functions populated by script @@ -115,10 +115,10 @@ pub async fn create_llm_module( v.script_config.models.retain(|_k, v| v.enabled); if v.script_config.models.is_empty() { - log_warn!(backend = k; "models are empty"); + log_warn!(@operator, backend = k; "models are empty"); v.enabled = false; } else if v.key.is_empty() { - log_warn!(backend = k; "could not detect key for backend"); + log_warn!(@operator, backend = k; "could not detect key for backend"); v.enabled = false; } } @@ -126,7 +126,7 @@ pub async fn create_llm_module( config.backends.retain(|_k, v| v.enabled); if config.backends.is_empty() { - log_error!("no valid backend detected") + log_error!(@operator; "no valid backend detected") } if !allow_empty_backends && config.backends.is_empty() { diff --git a/implementation/src/llm/providers.rs b/implementation/src/llm/providers.rs index 03f66a31..083ee8e9 100644 --- a/implementation/src/llm/providers.rs +++ b/implementation/src/llm/providers.rs @@ -90,7 +90,7 @@ impl TokenUsage { /// clamping and logging avoids the silent wrap that could otherwise under-bill. fn narrow_token_count(v: u64) -> u32 { u32::try_from(v).unwrap_or_else(|_| { - log_error!(value = v; "provider reported token count exceeding u32; saturating to u32::MAX"); + log_error!(@operator, value = v; "provider reported token count exceeding u32; saturating to u32::MAX"); u32::MAX }) } @@ -147,7 +147,7 @@ pub trait Provider { // Unparseable output still cost the operator; charge the usage with an // empty object rather than dropping the tokens by erroring. let parsed = serde_json::from_str(&json_str).unwrap_or_else(|e| { - log_warn!(json = json_str, error = e.to_string(); "unparseable json response, charging usage with empty object"); + log_warn!(@operator, json = json_str, error = e.to_string(); "unparseable json response, charging usage with empty object"); serde_json::Map::new() }); @@ -166,7 +166,7 @@ pub trait Provider { if let Some(val) = result_val { Ok(ProviderResponse::new(val, res.tokens)) } else { - log_error!(result:? = res.result; "no result in reason, returning false"); + log_error!(@operator, result:? = res.result; "no result in reason, returning false"); Ok(ProviderResponse::new(false, res.tokens)) } @@ -344,7 +344,7 @@ impl Provider for OpenAICompatible { .client .post(&url) .header("Content-Type", "application/json") - .header("Authorization", &format!("Bearer {}", &self.config.key)) + .header("Authorization", &format!("Bearer {}", self.config.key)) .body(request.clone()); let res = scripting::send_request_get_lua_compatible_response_json( &ctx.metrics, @@ -373,7 +373,7 @@ impl Provider for OpenAICompatible { } let response = response.unwrap_or_else(|| { - log_warn!(body:serde = res.body; "openai: missing content, charging usage with empty response"); + log_warn!(@operator, body:serde = res.body; "openai: missing content, charging usage with empty response"); "" }); @@ -427,7 +427,7 @@ impl Provider for OpenAICompatible { .client .post(&url) .header("Content-Type", "application/json") - .header("Authorization", &format!("Bearer {}", &self.config.key)) + .header("Authorization", &format!("Bearer {}", self.config.key)) .body(request.clone()); let res = scripting::send_request_get_lua_compatible_response_json( &ctx.metrics, @@ -445,7 +445,7 @@ impl Provider for OpenAICompatible { .pointer("/choices/0/message/content") .and_then(|v| if v.is_null() { Some("") } else { v.as_str() }) .unwrap_or_else(|| { - log_warn!(body:serde = res.body; "openai: missing json content, charging usage with empty response"); + log_warn!(@operator, body:serde = res.body; "openai: missing json content, charging usage with empty response"); "" }); @@ -556,7 +556,7 @@ impl Provider for OLlama { .and_then(|v| v.get("response")) .and_then(|v| v.as_str()) .unwrap_or_else(|| { - log_warn!(body:serde = res.body; "ollama: missing response, charging usage with empty response"); + log_warn!(@operator, body:serde = res.body; "ollama: missing response, charging usage with empty response"); "" }); Ok(ProviderResponse::new(response.to_owned(), tokens)) @@ -618,7 +618,7 @@ impl Provider for OLlama { .and_then(|v| v.get("response")) .and_then(|v| v.as_str()) .unwrap_or_else(|| { - log_warn!(body:serde = res.body; "ollama: missing json response, charging usage with empty response"); + log_warn!(@operator, body:serde = res.body; "ollama: missing json response, charging usage with empty response"); "" }); Ok(ProviderResponse::new(sanitize_json_str(response), tokens)) @@ -714,7 +714,7 @@ impl Provider for Gemini { } let res = res.unwrap_or_else(|| { - log_warn!(body:serde = res_json.body; "gemini: missing text, charging usage with empty response"); + log_warn!(@operator, body:serde = res_json.body; "gemini: missing text, charging usage with empty response"); "" }); Ok(ProviderResponse::new(res.into(), tokens)) @@ -778,7 +778,7 @@ impl Provider for Gemini { } let res = res.unwrap_or_else(|| { - log_warn!(body:serde = res_json.body; "gemini: missing json text, charging usage with empty response"); + log_warn!(@operator, body:serde = res_json.body; "gemini: missing json text, charging usage with empty response"); "" }); @@ -891,7 +891,7 @@ impl Provider for Anthropic { .pointer("/content/0/text") .and_then(|x| x.as_str()) .unwrap_or_else(|| { - log_warn!(body:serde = res.body; "anthropic: missing text, charging usage with empty response"); + log_warn!(@operator, body:serde = res.body; "anthropic: missing text, charging usage with empty response"); "" }); @@ -968,7 +968,7 @@ impl Provider for Anthropic { .and_then(|x| x.as_object()) .cloned() .unwrap_or_else(|| { - log_warn!(body:serde = res.body; "anthropic: missing json input, charging usage with empty object"); + log_warn!(@operator, body:serde = res.body; "anthropic: missing json input, charging usage with empty object"); serde_json::Map::new() }); @@ -1041,7 +1041,7 @@ impl Provider for Anthropic { .pointer("/content/0/input/result") .and_then(|x| x.as_bool()) .unwrap_or_else(|| { - log_error!(body:serde = res.body; "no result in reason, returning false"); + log_error!(@operator, body:serde = res.body; "no result in reason, returning false"); false }); diff --git a/implementation/src/main.rs b/implementation/src/main.rs index 36716bbc..47c4dc4d 100644 --- a/implementation/src/main.rs +++ b/implementation/src/main.rs @@ -29,6 +29,7 @@ fn main() -> Result<()> { } .inspect_err(|e| { log_error!( + @operator, error:ah = e; "error in main" ); diff --git a/implementation/src/manager/check_install.rs b/implementation/src/manager/check_install.rs index 95eff07e..d129b13f 100644 --- a/implementation/src/manager/check_install.rs +++ b/implementation/src/manager/check_install.rs @@ -53,7 +53,7 @@ pub fn run(cli: &CliArgs, args: &Args) -> Result<()> { patch: version.patch + 1, }; if manifest.executor_versions.contains_key(&next) { - log_info!(version:? = version; "skipping executor: a newer patch exists"); + log_info!(@operator, version:? = version; "skipping executor: a newer patch exists"); continue; } @@ -77,7 +77,7 @@ pub fn run(cli: &CliArgs, args: &Args) -> Result<()> { .with_context(|| format!("spawning {genvm:?}"))?; if !status.success() { - log_error!(version = orig_key.as_str(), status:? = status; "executor check failed"); + log_error!(@operator, version = orig_key.as_str(), status:? = status; "executor check failed"); failures.push(orig_key.clone()); } } @@ -89,7 +89,7 @@ pub fn run(cli: &CliArgs, args: &Args) -> Result<()> { ); } - log_info!("check-install succeeded for all executors"); + log_info!(@operator; "check-install succeeded for all executors"); Ok(()) } diff --git a/implementation/src/manager/handlers.rs b/implementation/src/manager/handlers.rs index 3945704e..50b20151 100644 --- a/implementation/src/manager/handlers.rs +++ b/implementation/src/manager/handlers.rs @@ -76,7 +76,7 @@ pub async fn handle_genvm_run( ctx: sync::DArc<AppContext>, data: &[u8], ) -> Result<Json<serde_json::Value>> { - let mut res: super::run::Request = calldata::decode_obj(data)?; + let res: super::run::Request = calldata::decode_obj(data)?; let modules_lock = if res.needs_modules() { let lock = Ctx::get_module_locks(ctx.gep(|x| &x.mod_ctx)).await; diff --git a/implementation/src/manager/mod.rs b/implementation/src/manager/mod.rs index bc0a2dec..c3b48d65 100644 --- a/implementation/src/manager/mod.rs +++ b/implementation/src/manager/mod.rs @@ -236,7 +236,7 @@ fn error_response(status: StatusCode, error: impl Into<String>) -> Response { } fn internal_error(error: anyhow::Error) -> Response { - log_error!(err:ah = &error; "internal server error"); + log_error!(@operator, err:ah = &error; "internal server error"); error_response(StatusCode::INTERNAL_SERVER_ERROR, format!("{:#}", error)) } diff --git a/implementation/src/manager/run.rs b/implementation/src/manager/run.rs index 4e0ff97e..8ca0d162 100644 --- a/implementation/src/manager/run.rs +++ b/implementation/src/manager/run.rs @@ -236,7 +236,7 @@ async fn spawn_module_relay( handler(Box::new(stream), Some(exec_ctx)).await; } Err(e) => { - log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, module = module_name, error:err = e; "failed to create async stream"); + log_error_into!(@operator, &LoggerWithId, genvm_id:id = genvm_id.0, module = module_name, error:err = e; "failed to create async stream"); } } @@ -386,7 +386,7 @@ impl SingleGenVMContext { out.push(b'\n'); } Err(e) => { - log_error!(error:err = e; "failed to serialize genvm log artifact line"); + log_error!(@operator, error:err = e; "failed to serialize genvm log artifact line"); } } } @@ -540,7 +540,7 @@ impl Ctx { let min = self.min_permits(); if permits < min { - log_warn!(permits = permits, min = min; "cannot set permits below the most expensive run"); + log_warn!(@operator, permits = permits, min = min; "cannot set permits below the most expensive run"); return permits_lock.max; } @@ -708,7 +708,7 @@ impl Ctx { #[cfg(target_os = "macos")] fn detect_free_gigabytes() -> usize { - log_warn!("automatic permits detection is not supported on macOS, using default value"); + log_warn!(@operator; "automatic permits detection is not supported on macOS, using default value"); 32 } @@ -751,6 +751,7 @@ impl Ctx { }; let permits = if permits < min_permits { log_warn!( + @operator, permits = permits, min = min_permits; "permits are below the most expensive run, raising" ); @@ -821,7 +822,7 @@ async fn gc_step(ctx: &sync::DArc<Ctx>) { }; if val.strict_deadline < now && val.terminal_event.get().is_none() { - log_warn_into!(&LoggerWithId, genvm_id:id = key.0; "genvm execution exceeded strict deadline, terminating"); + log_warn_into!(@operator, &LoggerWithId, genvm_id:id = key.0; "genvm execution exceeded strict deadline, terminating"); let _ = ctx.request_finish(key, FinishCause::Deadline).await; } } @@ -840,7 +841,7 @@ async fn gc_step(ctx: &sync::DArc<Ctx>) { }; let passed = now.signed_duration_since(terminal_at); if passed > retention { - log_warn_into!(&LoggerWithId, genvm_id:id = k.0; "removing zombie genvm execution context"); + log_warn_into!(@operator, &LoggerWithId, genvm_id:id = k.0; "removing zombie genvm execution context"); if let Some(host_genvm_id) = &v.host_genvm_id { expired_host_ids.push((host_genvm_id.clone(), *k)); } @@ -1085,6 +1086,7 @@ trait LogAppender { async fn append_structured( &mut self, level: logger::Level, + audience: logger::Audience, data: serde_json::Map<String, serde_json::Value>, ); async fn append_text(&mut self, serde_err: serde_json::Error, text: &str); @@ -1116,9 +1118,10 @@ impl LogAppender for LogAppenderToLog { async fn append_structured( &mut self, level: logger::Level, + audience: logger::Audience, data: serde_json::Map<String, serde_json::Value>, ) { - log_with_level_into!(level, &LoggerWithId, log:serde = data, genvm_id:id = self.0.0; "genvm log"); + log_with_level_into!(level, @(audience), &LoggerWithId, log:serde = data, genvm_id:id = self.0.0; "genvm log"); } #[inline(always)] async fn append_text(&mut self, serde_err: serde_json::Error, text: &str) { @@ -1133,11 +1136,13 @@ impl LogAppender for LogAppenderToValue { async fn append_structured( &mut self, level: logger::Level, + audience: logger::Audience, mut data: serde_json::Map<String, serde_json::Value>, ) { log_trace!(log:serde = data; "genvm log"); data.insert("level".to_owned(), level.to_string().into()); + data.insert("audience".to_owned(), audience.to_string().into()); self.0.push(LogSinkElement::Map(data)); } @@ -1148,6 +1153,13 @@ impl LogAppender for LogAppenderToValue { } } +fn take_string(map: &mut serde_json::Map<String, serde_json::Value>, key: &str) -> Option<String> { + match map.remove(key)? { + serde_json::Value::String(s) => Some(s), + _ => None, + } +} + async fn read_log_pipe<LA: LogAppender>( reader_fd: std::os::unix::io::OwnedFd, mut sink: impl DerefMut<Target = LA>, @@ -1165,20 +1177,15 @@ async fn read_log_pipe<LA: LogAppender>( } match serde_json::from_str(line) { Ok::<serde_json::Map<String, serde_json::Value>, _>(mut log_record) => { - let level = log_record - .remove("level") - .map(|x| { - if let serde_json::Value::String(s) = x { - Some(s) - } else { - None - } - }) - .unwrap_or(None) + let level = take_string(&mut log_record, "level") .and_then(|x| logger::Level::from_str(&x).ok()) .unwrap_or(logger::Level::Info); - sink.append_structured(level, log_record).await; + let audience = take_string(&mut log_record, "audience") + .and_then(|x| logger::Audience::from_str(&x).ok()) + .unwrap_or_default(); + + sink.append_structured(level, audience, log_record).await; } Err(e) => { sink.append_text(e, line).await; @@ -1247,7 +1254,7 @@ async fn pipe_read<P: tokio::io::AsyncReadExt + Unpin>( } if let Err(e) = to.set(result) { - log_error!(error:err = e; "failed to set stdout/stderr content"); + log_error!(@operator, error:err = e; "failed to set stdout/stderr content"); } std::mem::drop(permit); @@ -1277,6 +1284,16 @@ impl ManagerHostStreamState { } } +struct ManagerHostStream { + full_ctx: sync::DArc<crate::manager::AppContext>, + exec_ctx: sync::DArc<SingleGenVMContext>, + parent_req: Arc<Request>, + consumed_result: sync::DArc<tokio::sync::OnceCell<Vec<u8>>>, + genvm_id: GenVMId, + is_top_level: bool, + state: Arc<ManagerHostStreamState>, +} + struct NestedRunRegistration(sync::DArc<SingleGenVMContext>); impl NestedRunRegistration { @@ -1430,7 +1447,7 @@ fn guard_top_level_consumed_result(data: Vec<u8>, genvm_id: GenVMId) -> anyhow:: genvm_modules_interfaces::ResultCode::FatalVmError, "top-level executor returned a fatal VM error after its publication boundary" ); - log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0; "top-level executor returned a fatal VM error; downgrading it to vm_error"); + log_error_into!(@operator, &LoggerWithId, genvm_id:id = genvm_id.0; "top-level executor returned a fatal VM error; downgrading it to vm_error"); Ok(downgrade_fatal_reported_result(reported)) } @@ -1530,21 +1547,25 @@ fn nested_effect(reported: &genvm_modules_interfaces::ReportedResult) -> Option< fn read_manager_host_stream( parent_fd: FdWrapper, - full_ctx: sync::DArc<crate::manager::AppContext>, - exec_ctx: sync::DArc<SingleGenVMContext>, - parent_req: Arc<Request>, - consumed_result: sync::DArc<tokio::sync::OnceCell<Vec<u8>>>, - genvm_id: GenVMId, - is_top_level: bool, - stream_state: Arc<ManagerHostStreamState>, + stream: ManagerHostStream, ) -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send>> { Box::pin(async move { use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let ManagerHostStream { + full_ctx, + exec_ctx, + parent_req, + consumed_result, + genvm_id, + is_top_level, + state: stream_state, + } = stream; + let file = match parent_fd.into_async_fd() { Ok(f) => f, Err(e) => { - log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, error:err = e; "failed to create async fd for manager host stream"); + log_error_into!(@operator, &LoggerWithId, genvm_id:id = genvm_id.0, error:err = e; "failed to create async fd for manager host stream"); stream_state.close(); return; } @@ -1563,7 +1584,7 @@ fn read_manager_host_stream( return; } Err(e) => { - log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, error:err = e; "failed to read method from manager host stream"); + log_error_into!(@operator, &LoggerWithId, genvm_id:id = genvm_id.0, error:err = e; "failed to read method from manager host stream"); return; } } @@ -1573,7 +1594,7 @@ fn read_manager_host_stream( let data = match read_length_prefixed(&mut reader, u32::MAX as usize).await { Ok(data) => data, Err(e) => { - log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, error:ah = &e; "failed to read consumed result"); + log_error_into!(@operator, &LoggerWithId, genvm_id:id = genvm_id.0, error:ah = &e; "failed to read consumed result"); return; } }; @@ -1581,7 +1602,7 @@ fn read_manager_host_stream( match guard_top_level_consumed_result(data, genvm_id) { Ok(data) => data, Err(e) => { - log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, error:ah = &e; "refusing invalid top-level consume_result"); + log_error_into!(@operator, &LoggerWithId, genvm_id:id = genvm_id.0, error:ah = &e; "refusing invalid top-level consume_result"); return; } } @@ -1593,11 +1614,11 @@ fn read_manager_host_stream( let mut writer = writer.lock().await; if let Err(e) = writer.write_all(&[0]).await { - log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, error:err = e; "failed to send ACK for consume_result"); + log_error_into!(@operator, &LoggerWithId, genvm_id:id = genvm_id.0, error:err = e; "failed to send ACK for consume_result"); return; } if let Err(e) = writer.flush().await { - log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, error:err = e; "failed to flush ACK for consume_result"); + log_error_into!(@operator, &LoggerWithId, genvm_id:id = genvm_id.0, error:err = e; "failed to flush ACK for consume_result"); return; } } @@ -1608,7 +1629,7 @@ fn read_manager_host_stream( let data = match read_length_prefixed(&mut reader, u32::MAX as usize).await { Ok(data) => data, Err(e) => { - log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, error:ah = &e; "failed to read run_nested request"); + log_error_into!(@operator, &LoggerWithId, genvm_id:id = genvm_id.0, error:ah = &e; "failed to read run_nested request"); return; } }; @@ -1635,20 +1656,20 @@ fn read_manager_host_stream( { Ok(reply) => reply, Err(e) => { - log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, error:ah = &e; "nested run failed"); + log_error_into!(@operator, &LoggerWithId, genvm_id:id = genvm_id.0, error:ah = &e; "nested run failed"); nested_internal_error() } } } Err(e) => { - log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, error:err = e; "failed to decode run_nested envelope"); + log_error_into!(@operator, &LoggerWithId, genvm_id:id = genvm_id.0, error:err = e; "failed to decode run_nested envelope"); nested_internal_error() } }; let encoded = calldata::encode_obj(&reply); let mut writer = writer.lock().await; if let Err(e) = write_length_prefixed(&mut *writer, &encoded).await { - log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, error:ah = &e; "failed to send run_nested reply"); + log_error_into!(@operator, &LoggerWithId, genvm_id:id = genvm_id.0, error:ah = &e; "failed to send run_nested reply"); } } Ok(host_fns::Methods::ResolveCallContractExecutor) => { @@ -1657,26 +1678,26 @@ fn read_manager_host_stream( // node's host, and a null reply means "stay in-process". let mut request = [0u8; calldata::ADDRESS_SIZE + 2]; if let Err(e) = reader.read_exact(&mut request).await { - log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, error:err = e; "failed to read resolve_call_contract_executor request"); + log_error_into!(@operator, &LoggerWithId, genvm_id:id = genvm_id.0, error:err = e; "failed to read resolve_call_contract_executor request"); return; } let encoded = calldata::encode_obj(&calldata::Value::Null); let mut writer = writer.lock().await; if let Err(e) = writer.write_all(&[host_fns::Errors::Ok as u8]).await { - log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, error:err = e; "failed to send resolve_call_contract_executor status"); + log_error_into!(@operator, &LoggerWithId, genvm_id:id = genvm_id.0, error:err = e; "failed to send resolve_call_contract_executor status"); return; } if let Err(e) = write_length_prefixed(&mut *writer, &encoded).await { - log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, error:ah = &e; "failed to send resolve_call_contract_executor reply"); + log_error_into!(@operator, &LoggerWithId, genvm_id:id = genvm_id.0, error:ah = &e; "failed to send resolve_call_contract_executor reply"); return; } } Ok(method) => { - log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, method = method as u8; "unexpected method on manager host stream"); + log_error_into!(@operator, &LoggerWithId, genvm_id:id = genvm_id.0, method = method as u8; "unexpected method on manager host stream"); return; } Err(()) => { - log_error_into!(&LoggerWithId, genvm_id:id = genvm_id.0, method = method_buf[0]; "unknown method on manager host stream"); + log_error_into!(@operator, &LoggerWithId, genvm_id:id = genvm_id.0, method = method_buf[0]; "unknown method on manager host stream"); return; } } @@ -1700,14 +1721,6 @@ async fn wait_for_output(exec: &SingleGenVMContext) { log_debug!(id = exec.id; "stdout/stderr sem acquired"); } -fn drain_log_sink(log_sink: &LogSink) -> Vec<serde_json::Map<String, serde_json::Value>> { - let mut as_vec = Vec::new(); - while let Some(data) = log_sink.pop() { - as_vec.push(data.into_json()); - } - as_vec -} - async fn finish_execution( exec: &SingleGenVMContext, status: Option<std::process::ExitStatus>, @@ -1736,7 +1749,7 @@ async fn finish_execution( let stdout = exec.stdout.get().map(|x| x.as_str()).unwrap_or(""); let stderr = exec.stderr.get().map(|x| x.as_str()).unwrap_or(""); - let genvm_log = drain_log_sink(&exec.log_sink); + let genvm_log = exec.log_sink.drain(); let cause = exec.finish_cause().unwrap_or(default_cause); let exit_code = status.and_then(|status| status.code()).map(i64::from); @@ -1754,7 +1767,7 @@ async fn finish_execution( }; let event = exec.finished_event(&done); if let Err(e) = exec.result.set(done) { - log_warn!(error:err = e; "error setting genvm result; it can happen rarely due to concurrency"); + log_warn!(@operator, error:err = e; "error setting genvm result; it can happen rarely due to concurrency"); return false; } exec.publish_terminal(event) @@ -2095,7 +2108,7 @@ impl Ctx { "nested CallContract message value must be zero" ); if parent_req.host.trim_start().starts_with("fd://") { - log_error_into!(&LoggerWithId, genvm_id:id = exec_ctx.id.0, host = parent_req.host; "cannot start nested executor because host 0 is not addressable"); + log_error_into!(@operator, &LoggerWithId, genvm_id:id = exec_ctx.id.0, host = parent_req.host; "cannot start nested executor because host 0 is not addressable"); anyhow::bail!("host 0 uses fd:// and cannot be re-dialed by a nested executor"); } @@ -2303,6 +2316,7 @@ async fn resolve_selector( } log_warn!( + @operator, major = major, genvm_id:id = genvm_id.0; "no line provides the requested major, falling back to the newest one" @@ -2477,13 +2491,15 @@ async fn run_genvm_process( let _manager_stream_guard = sync::DropGuard::new(move || manager_stream_state_on_drop.close()); tokio::spawn(read_manager_host_stream( manager_parent, - full_ctx.clone(), - exec_ctx.clone(), - Arc::new(req.clone()), - consumed_result, - genvm_id, - is_top_level, - manager_stream_state.clone(), + ManagerHostStream { + full_ctx: full_ctx.clone(), + exec_ctx: exec_ctx.clone(), + parent_req: Arc::new(req.clone()), + consumed_result, + genvm_id, + is_top_level, + state: manager_stream_state.clone(), + }, )); if let Some(mut stdin_task) = stdin_task { diff --git a/implementation/src/manager/socket.rs b/implementation/src/manager/socket.rs index bfd8c69c..fe962544 100644 --- a/implementation/src/manager/socket.rs +++ b/implementation/src/manager/socket.rs @@ -482,7 +482,7 @@ where } async fn handle_run(&mut self, request_id: u64, payload: &[u8]) -> Result<()> { - let RunPayload::Run(mut req) = match decode_payload::<RunPayload>(payload) { + let RunPayload::Run(req) = match decode_payload::<RunPayload>(payload) { Ok(req) => req, Err(e) => { self.writer @@ -806,7 +806,7 @@ pub async fn handle_connection(socket: WebSocket, ctx: sync::DArc<AppContext>) { Ok(Ok(())) => {} Ok(Err(e)) => log_debug!(error:ah = e; "manager websocket writer stopped"), Err(e) if e.is_cancelled() => {} - Err(e) => log_warn!(error:err = e; "manager websocket writer task failed"), + Err(e) => log_warn!(@operator, error:err = e; "manager websocket writer task failed"), } } diff --git a/implementation/src/scripting/ctx/dflt.rs b/implementation/src/scripting/ctx/dflt.rs index bd0a90cd..e561bcde 100644 --- a/implementation/src/scripting/ctx/dflt.rs +++ b/implementation/src/scripting/ctx/dflt.rs @@ -77,9 +77,12 @@ pub fn create_global( let level = as_serde.remove("level"); let level = level.and_then(|x| x.as_str().map(|x| x.to_owned())).map(|x| logger::Level::from_str(&x).unwrap_or(logger::Level::Info)).unwrap_or(logger::Level::Info); + let audience = as_serde.remove("audience"); + let audience = audience.and_then(|x| x.as_str().and_then(|x| logger::Audience::from_str(x).ok())).unwrap_or_default(); + let script_message = as_serde.remove("message").and_then(|x| x.as_str().map(|x| x.to_owned())).unwrap_or_else(|| "<none>".to_owned()); - log_with_level_into!(level, &LoggerWithId, log:serde = as_serde, genvm_id:id = crate::common::get_genvm_id().0; "script_log: {script_message}"); + log_with_level_into!(level, @(audience), &LoggerWithId, log:serde = as_serde, genvm_id:id = crate::common::get_genvm_id().0; "script_log: {script_message}"); Ok(()) })?)?; diff --git a/implementation/src/scripting/mod.rs b/implementation/src/scripting/mod.rs index 4b61b5b7..ac317676 100644 --- a/implementation/src/scripting/mod.rs +++ b/implementation/src/scripting/mod.rs @@ -224,7 +224,7 @@ where P: AsRef<std::path::Path> + Into<String> + std::fmt::Debug, { let script_contents = std::fs::read_to_string(&path) - .with_context(|| format!("reading script from {:?}", &path))?; + .with_context(|| format!("reading script from {:?}", path))?; let chunk = vm.load(script_contents); let mut name = String::from("@"); diff --git a/implementation/src/scripting/pool.rs b/implementation/src/scripting/pool.rs index 19aaa2da..6d8d4f7c 100644 --- a/implementation/src/scripting/pool.rs +++ b/implementation/src/scripting/pool.rs @@ -66,7 +66,7 @@ impl<T, C, E: 'static> Pool<T, C, E> { let vm = match self.inner.free.pop() { Some(vm) => vm, None => { - log_warn!("vm pool empty, building a vm on demand"); + log_warn!(@operator; "vm pool empty, building a vm on demand"); Arc::new((self.inner.factory)().await?) } }; diff --git a/implementation/src/web/handler.rs b/implementation/src/web/handler.rs index 3b445ad1..7fc5692f 100644 --- a/implementation/src/web/handler.rs +++ b/implementation/src/web/handler.rs @@ -59,6 +59,7 @@ impl common::MessageHandler<web_iface::Message, RenderAnswer> for Handler { let mut wait_after_loaded = payload.wait_after_loaded.as_secs_f64(); if wait_after_loaded > max_wait { log_warn!( + @user, requested = wait_after_loaded, max = max_wait; "wait_after_loaded clamped to maximum" diff --git a/implementation/tests/log_sink.rs b/implementation/tests/log_sink.rs new file mode 100644 index 00000000..a29785b9 --- /dev/null +++ b/implementation/tests/log_sink.rs @@ -0,0 +1,203 @@ +use genvm_modules::common::{LogSinkElement, LogSinkInner, INTROSPECTOR_DROPPED, LOG_SINK_LIMIT}; + +type Record = serde_json::Map<String, serde_json::Value>; + +fn entry(audience: &str, seq: usize) -> LogSinkElement { + LogSinkElement::Map(serde_json::Map::from_iter([ + ( + "audience".into(), + serde_json::Value::String(audience.into()), + ), + ("seq".into(), seq.into()), + ])) +} + +fn fill(sink: &LogSinkInner, audience: &str, count: usize) { + for seq in 0..count { + sink.push(entry(audience, seq)); + } +} + +fn audiences(drained: &[Record]) -> Vec<String> { + drained + .iter() + .map(|x| x["audience"].as_str().unwrap().to_owned()) + .collect() +} + +fn markers(drained: &[Record]) -> usize { + drained + .iter() + .filter(|x| x.get("message").and_then(|m| m.as_str()) == Some(INTROSPECTOR_DROPPED)) + .count() +} + +#[test] +fn fresh_sink_keeps_every_entry_under_the_cap() { + let sink = LogSinkInner::new(false); + fill(&sink, "introspector", LOG_SINK_LIMIT - 1); + + let drained = sink.drain(); + assert_eq!(drained.len(), LOG_SINK_LIMIT - 1); + assert_eq!(markers(&drained), 0); +} + +#[test] +fn cap_hit_drops_introspector_entries_and_adds_a_marker() { + let sink = LogSinkInner::new(false); + for seq in 0..LOG_SINK_LIMIT { + let audience = if seq % 4 == 0 { "user" } else { "introspector" }; + sink.push(entry(audience, seq)); + } + sink.push(entry("operator", LOG_SINK_LIMIT)); + + let drained = sink.drain(); + let auds = audiences(&drained); + + assert!( + !auds.contains(&"introspector".to_owned()), + "introspector entries survived: {auds:?}" + ); + assert_eq!( + markers(&drained), + 1, + "expected exactly one marker: {auds:?}" + ); + assert_eq!( + drained.last().unwrap()["seq"].as_u64(), + Some(LOG_SINK_LIMIT as u64), + "the incoming entry must be kept" + ); + assert_eq!( + auds.iter().filter(|x| *x == "user").count(), + LOG_SINK_LIMIT / 4 + ); +} + +#[test] +fn compacted_sink_discards_introspector_entries_at_push() { + let sink = LogSinkInner::new(false); + // the last one triggers the compaction and is compacted away with the rest + fill(&sink, "introspector", LOG_SINK_LIMIT + 1); + + sink.push(entry("introspector", 1000)); + sink.push(entry("user", 1001)); + sink.push(entry("operator", 1002)); + + let drained = sink.drain(); + let auds = audiences(&drained); + + assert_eq!( + auds, + vec![ + "operator".to_owned(), // the marker + "user".to_owned(), + "operator".to_owned(), + ], + "only the marker and the later non-introspector entries stay" + ); +} + +#[test] +fn second_cap_hit_drops_the_oldest_entry_of_any_audience() { + let sink = LogSinkInner::new(false); + fill(&sink, "user", LOG_SINK_LIMIT); + + // overflows while fresh: nothing to compact, so the marker and this entry each + // cost an oldest entry + sink.push(entry("user", 1000)); + sink.push(entry("user", 1001)); + + let drained = sink.drain(); + assert_eq!(drained.len(), LOG_SINK_LIMIT); + // three slots were needed: the marker and the two entries + assert_eq!( + drained[0]["seq"].as_u64(), + Some(3), + "the oldest entries must be the ones dropped" + ); + assert_eq!(drained.last().unwrap()["seq"].as_u64(), Some(1001)); +} + +#[test] +fn a_chatty_run_never_grows_past_the_cap() { + let sink = LogSinkInner::new(false); + for seq in 0..LOG_SINK_LIMIT * 100 { + let audience = match seq % 3 { + 0 => "introspector", + 1 => "user", + _ => "operator", + }; + sink.push(entry(audience, seq)); + assert!( + sink.buffered() <= LOG_SINK_LIMIT, + "sink grew to {} at {seq}", + sink.buffered() + ); + } + fill(&sink, "introspector", LOG_SINK_LIMIT * 10); + + let drained = sink.drain(); + assert_eq!(drained.len(), LOG_SINK_LIMIT); + assert!( + !audiences(&drained).contains(&"introspector".to_owned()), + "introspector entries came back once the fifo phase started" + ); + assert_eq!( + drained.last().unwrap()["seq"].as_u64(), + Some((LOG_SINK_LIMIT * 100 - 1) as u64), + "the newest entry must be kept" + ); +} + +#[test] +fn debug_sink_never_evicts() { + let sink = LogSinkInner::new(true); + fill(&sink, "introspector", LOG_SINK_LIMIT * 3); + + let drained = sink.drain(); + assert_eq!(drained.len(), LOG_SINK_LIMIT * 3); + assert_eq!(markers(&drained), 0); +} + +#[test] +fn an_entry_without_an_audience_is_an_introspector_one() { + let sink = LogSinkInner::new(false); + sink.push(LogSinkElement::Map(serde_json::Map::from_iter([( + "level".into(), + serde_json::Value::String("info".into()), + )]))); + sink.push(LogSinkElement::Raw(br#"{"level":"info"}"#.to_vec())); + + let drained = sink.drain(); + assert_eq!(audiences(&drained), vec!["introspector", "introspector"]); +} + +/// Neither shape is our JSON: the node runner is the one who can act on it +#[test] +fn a_line_the_executor_did_not_format_is_an_operator_one() { + let sink = LogSinkInner::new(false); + sink.push(LogSinkElement::Line("raw text".into())); + sink.push(LogSinkElement::Raw(b"\xff not json".to_vec())); + + let drained = sink.drain(); + assert_eq!(audiences(&drained), vec!["operator", "operator"]); + assert_eq!(drained[0]["line"].as_str(), Some("raw text")); +} + +#[test] +fn a_garbage_audience_is_replaced_and_the_leading_keys_are_ordered() { + let sink = LogSinkInner::new(false); + sink.push(LogSinkElement::Map(serde_json::Map::from_iter([ + ("message".into(), serde_json::Value::String("m".into())), + ("audience".into(), serde_json::Value::from(7)), + ("level".into(), serde_json::Value::String("warn".into())), + ]))); + + let drained = sink.drain(); + assert_eq!( + drained[0].keys().collect::<Vec<_>>(), + vec!["level", "audience", "message"] + ); + assert_eq!(drained[0]["audience"].as_str(), Some("introspector")); +} diff --git a/install/config/genvm-llm-default.lua b/install/config/genvm-llm-default.lua index d49f31bd..b545a9b3 100644 --- a/install/config/genvm-llm-default.lua +++ b/install/config/genvm-llm-default.lua @@ -180,8 +180,8 @@ _G.host = { now_ms = function() return lib.rs.monotonic_ms() end, - log = function(level, event, fields) - lib.log { level = level, message = event, fields = fields } + log = function(level, event, fields, audience) + lib.log { level = level, message = event, fields = fields, audience = audience } end, } @@ -256,12 +256,17 @@ local function run_candidate(ctx, mapped_prompt, timeout, request) local as_user_error = lib.rs.as_user_error(result) if as_user_error == nil then - lib.log { level = "warning", message = "non-user-error", original = result } + lib.log { level = "warning", audience = "operator", message = "non-user-error", original = result } error(result) end local status = as_user_error.ctx and as_user_error.ctx.status - lib.log { level = "warning", message = "provider failed, looking for next", error = as_user_error } + lib.log { + level = "warning", + audience = "operator", + message = "provider failed, looking for next", + error = as_user_error, + } return nil, { @@ -280,7 +285,7 @@ local function wait_until(until_ms) if delay <= 0 then return end - lib.log { level = "debug", message = "policy backoff", seconds = delay } + lib.log { level = "debug", audience = "operator", message = "policy backoff", seconds = delay } lib.rs.sleep_seconds(math.min(delay, MAX_WAIT_SECONDS)) end @@ -291,6 +296,7 @@ local function dispatch_prompt(ctx, mapped_prompt, remaining_gen) if timeout and timeout < 1 then lib.log { level = "warning", + audience = "user", message = "computed timeout is very low, failing immediately", timeout = timeout, } @@ -328,6 +334,7 @@ local function dispatch_prompt(ctx, mapped_prompt, remaining_gen) lib.log { level = "error", + audience = "operator", message = "no provider could handle prompt", error = step.result.error, trace = step.result.trace, @@ -470,6 +477,7 @@ function Teardown(ctx) if not ok then lib.log { level = "warning", + audience = "operator", message = "failed to persist llm stats during teardown", error = tostring(err), } diff --git a/install/lib/genvm-lua/lib-genvm.lua b/install/lib/genvm-lua/lib-genvm.lua index 73f39a70..29f6436d 100644 --- a/install/lib/genvm-lua/lib-genvm.lua +++ b/install/lib/genvm-lua/lib-genvm.lua @@ -34,7 +34,9 @@ local value2json = require("value2json") ---@type RS M.rs = __dflt ---@diagnostic disable-line ---- Log a message or structured data. The argument table may contain `level` and `message` fields. +--- Log a message or structured data. The argument table may contain `level`, `message` and +--- `audience` fields. `audience` is `"user" | "operator" | "introspector"` and defaults to +--- `"introspector"`. ---@param arg any value to log; converted to JSON via `value2json` before sending M.log = function(arg) M.rs.log_json(value2json(arg)) diff --git a/support/ci/ci_lib.py b/support/ci/ci_lib.py index 394f76b0..1b50c02e 100644 --- a/support/ci/ci_lib.py +++ b/support/ci/ci_lib.py @@ -104,6 +104,15 @@ def github_error( print(f'::error{msg_prefix}::{message.replace(chr(10), " ")}') +def github_step_summary(markdown: str) -> None: + """Append markdown to the job summary; a no-op outside GitHub Actions.""" + path = os.environ.get('GITHUB_STEP_SUMMARY') + if not path: + return + with open(path, 'a', encoding='utf-8') as f: + f.write(markdown.rstrip('\n') + '\n\n') + + def run( command: list[str | Path], *, diff --git a/support/ci/pipelines/checks.py b/support/ci/pipelines/checks.py index 61416cf4..ad70bfd5 100644 --- a/support/ci/pipelines/checks.py +++ b/support/ci/pipelines/checks.py @@ -275,25 +275,87 @@ def handler(self, args: argparse.Namespace) -> int: return 0 +SUMMARY_DIFF_LIMIT = 512 * 1024 + + +def _worktree_diff() -> str: + # `--submodule=diff`: executor crates live in submodules, whose changes would + # otherwise collapse into an unusable "dirty gitlink" line. + return ci_lib.output(['git', 'diff', '--submodule=diff']) + + +def _summary_patch(diff: str) -> str: + # GitHub caps a job summary at 1 MiB and drops the whole file past it; cut on + # a line boundary so what is shown stays an appliable patch, and the log above + # always carries the full one. + encoded = diff.encode() + if len(encoded) <= SUMMARY_DIFF_LIMIT: + return f'```diff\n{diff}```\n' + # The cap is on bytes, and a cut can land inside a multi-byte character. + cut = encoded[:SUMMARY_DIFF_LIMIT].decode(errors='ignore') + shown = cut.rsplit('\n', 1)[0] + return f'```diff\n{shown}\n```\n\nPatch truncated, see the job log.\n' + + class CargoClippy(ci_lib.Pipeline): """ - Check cargo clippy availability and enumerate crates. + Lint every registered crate, reporting a failure as an appliable patch. + + Clippy names the lint but not the edit. On failure the `--fix` edges rerun + over the same tree and the resulting diff is printed and put in the job + summary, so the author can apply it verbatim; lints `--fix` cannot rewrite + stay visible in the diagnostics above it. + + Must run in a dev shell carrying the pinned toolchain β€” clippy is a component + of it, and the linted crates must be the ones the rest of CI builds. """ def name(self) -> str: return 'cargo-clippy' def handler(self, args: argparse.Namespace) -> int: - if ci_lib.run(['cargo', 'clippy', '--version'], check=False).returncode != 0: - print('ERROR: cargo clippy not installed') - return 1 - for path in ci_lib.output(['git', 'ls-files']).splitlines(): - if not path.endswith('Cargo.toml'): - continue - if path == 'runners/nix/trg/py/modules/genvm-cpython-ext/Cargo.toml': - continue - print(f'clippy in {path}') - return 0 + # Lint edges declare no inputs, so nothing regenerates sources for them; a + # drifted generated file would be linted as if it were current. + with ci_lib.github_group('codegen'): + ci_lib.run(['ninja', '-C', 'build', 'codegen']) + + # Anything dirty here is not clippy's doing and must not be presented as + # its patch below. + pre_existing = ci_lib.output(['git', 'diff', '--name-only']).split() + + # `-k 0`: report every crate's diagnostics in one run, not just the first + # failing one. + with ci_lib.github_group('clippy'): + ok = ( + ci_lib.run( + ['ninja', '-k', '0', '-C', 'build', 'cargo/clippy'], check=False + ).returncode + == 0 + ) + if ok: + return 0 + + with ci_lib.github_group('suggested patch'): + if pre_existing: + print( + 'the worktree was already dirty before the fix run, so the patch below ' + f'also carries unrelated changes to: {" ".join(pre_existing)}' + ) + ci_lib.run(['ninja', '-k', '0', '-C', 'build', 'cargo/clippy/fix'], check=False) + diff = _worktree_diff() + if diff.strip(): + print(diff) + ci_lib.github_step_summary( + '### `cargo clippy` failed\n\n' + 'Machine-applicable part of the fix:\n\n' + _summary_patch(diff) + ) + else: + print('`cargo clippy --fix` rewrote nothing; fix the diagnostics by hand') + + ci_lib.github_error( + 'cargo clippy failed; see the diagnostics and the suggested patch above' + ) + return 1 COMMANDS = [CommitHooks(), CommitMessages(), BehindCheck(), PrTitle(), CargoClippy()] diff --git a/support/ci/pipelines/tests.py b/support/ci/pipelines/tests.py index 7af4d1ab..2a301559 100644 --- a/support/ci/pipelines/tests.py +++ b/support/ci/pipelines/tests.py @@ -235,6 +235,23 @@ def group(gh_step: str, name: str, stage: Stage, body: GithubStep) -> None: ) ) +# Not `configure_step`: that configures in `minimal`, which carries no +# toolchain, and configure bakes the `cargo` it finds on PATH into build.ninja β€” +# on a hosted runner, the ambient rustup shim, whose lint set is not the pinned +# one. +TEST_CLIPPY = DefaultStepInfo( + build_step=Stage( + data=frozenset(), + commands_pre=[ + step_nix_develop('.?submodules=1#rust-test', ['genvm-tool', 'configure', '--ci']), + step_nix_develop( + '.?submodules=1#rust-test', + ['./support/ci/run.sh', 'pipeline', 'cargo-clippy'], + ), + ], + ), +) + TEST_MOCK_BUILD_BINS = DefaultStepInfo( configure_step=Stage(None), build_step=Stage(data=frozenset(['all/bin'])), @@ -304,6 +321,7 @@ def _cell( QUEUE_CELLS = [ _cell('other', TEST_OTHER, fuzz_host=True), + _cell('clippy', TEST_CLIPPY, disk_reclaim=True), _cell('rust-executors', TEST_RUST_EXECUTORS, fuzz_host=True, disk_reclaim=True), _cell('rust-manager', TEST_RUST_MANAGER, fuzz_host=True, disk_reclaim=True), _cell( diff --git a/support/ci/unit_tests/test_cargo_clippy.py b/support/ci/unit_tests/test_cargo_clippy.py new file mode 100644 index 00000000..a4fe46e7 --- /dev/null +++ b/support/ci/unit_tests/test_cargo_clippy.py @@ -0,0 +1,102 @@ +import argparse +import subprocess + +import ci_lib +import pytest +from pipelines import checks + + +class Recorder: + """Stand-in for ci_lib.run: records commands and replays canned exit codes.""" + + def __init__(self, codes: dict[str, int] | None = None): + self.commands: list[list[str]] = [] + self.codes = codes or {} + + def __call__(self, command, **kwargs) -> subprocess.CompletedProcess: + command = [str(arg) for arg in command] + self.commands.append(command) + code = next((c for k, c in self.codes.items() if k in command), 0) + if code != 0 and kwargs.get('check', True): + raise subprocess.CalledProcessError(code, command) + return subprocess.CompletedProcess(command, code) + + +@pytest.fixture +def clippy(monkeypatch): + recorder = Recorder() + monkeypatch.setattr(ci_lib, 'run', recorder) + monkeypatch.setattr(ci_lib, 'output', lambda _command: '') + return recorder + + +def run() -> int: + return checks.CargoClippy().handler(argparse.Namespace()) + + +def test_clean_tree_skips_the_fix_run(clippy): + assert run() == 0 + assert not any('cargo/clippy/fix' in c for c in clippy.commands) + + +def test_failure_reports_the_fix_patch(monkeypatch, clippy, capsys): + clippy.codes = {'cargo/clippy': 1} + monkeypatch.setattr( + ci_lib, 'output', lambda command: '' if '--name-only' in command else 'a patch' + ) + summary = [] + monkeypatch.setattr(ci_lib, 'github_step_summary', summary.append) + + assert run() == 1 + assert any('cargo/clippy/fix' in c for c in clippy.commands) + assert 'a patch' in capsys.readouterr().out + assert '```diff\na patch```' in summary[0] + + +def test_failure_without_a_patch_says_so(monkeypatch, clippy, capsys): + clippy.codes = {'cargo/clippy': 1} + monkeypatch.setattr( + ci_lib, 'github_step_summary', lambda _text: pytest.fail('no patch') + ) + + assert run() == 1 + assert 'rewrote nothing' in capsys.readouterr().out + + +def test_pre_existing_changes_are_disclaimed(monkeypatch, clippy, capsys): + clippy.codes = {'cargo/clippy': 1} + monkeypatch.setattr(ci_lib, 'output', lambda _command: 'src/lib.rs') + monkeypatch.setattr(ci_lib, 'github_step_summary', lambda _text: None) + + assert run() == 1 + assert 'already dirty' in capsys.readouterr().out + + +def test_summary_patch_is_cut_on_a_line_boundary(monkeypatch): + monkeypatch.setattr(checks, 'SUMMARY_DIFF_LIMIT', 10) + patch = checks._summary_patch('+one\n+two\n+three\n') + assert '+one\n+two' in patch + assert '+three' not in patch + assert 'truncated' in patch + + +def test_summary_patch_is_cut_on_bytes(monkeypatch): + monkeypatch.setattr(checks, 'SUMMARY_DIFF_LIMIT', 10) + # Five 2-byte characters already fill the limit, so nothing but the first + # line survives a cut that would land mid-character. + patch = checks._summary_patch('+ééééé\n+tail\n') + assert '+tail' not in patch + assert 'truncated' in patch + + +def test_step_summary_is_a_noop_outside_actions(monkeypatch): + monkeypatch.delenv('GITHUB_STEP_SUMMARY', raising=False) + ci_lib.github_step_summary('ignored') + + +def test_step_summary_appends(monkeypatch, tmp_path): + path = tmp_path / 'summary.md' + monkeypatch.setenv('GITHUB_STEP_SUMMARY', str(path)) + ci_lib.github_step_summary('first') + ci_lib.github_step_summary('second') + assert path.read_text() == 'first\n\nsecond\n\n' diff --git a/tests/runner/genvm_tool_plugins/integration.py b/tests/runner/genvm_tool_plugins/integration.py index 45ee7226..752d2625 100644 --- a/tests/runner/genvm_tool_plugins/integration.py +++ b/tests/runner/genvm_tool_plugins/integration.py @@ -865,10 +865,6 @@ def resolve_executor(address, state, advisory_major): assert len(k) == 36 index = int.from_bytes(k[32:], byteorder='big') index *= 32 - if index >= 4096: - logger.warning( - 'suspicious storage writing', index=index, key=k.hex(), value=v.hex() - ) mock_host.storage.write( mock_host.running_address, k[:32], diff --git a/tests/runner/genvm_tool_plugins/ninja.py b/tests/runner/genvm_tool_plugins/ninja.py index 6eefdbef..a6a47384 100644 --- a/tests/runner/genvm_tool_plugins/ninja.py +++ b/tests/runner/genvm_tool_plugins/ninja.py @@ -334,7 +334,12 @@ def register_cargo( self.build('phony_touch', files_trg).add_implicit_dependency(all_files).finish() - clippy_lints = ['--', '-A', 'clippy::upper_case_acronyms', '-Dwarnings'] + # Everything past `--` reaches clippy-driver as rustc flags. + clippy_allowed = ['clippy::upper_case_acronyms'] + clippy_rustc_flags = ['--'] + [ + flag for lint in clippy_allowed for flag in ('-A', lint) + ] + clippy_lints = clippy_rustc_flags + ['-Dwarnings'] # Lint/format edges declare no inputs and produce no output file: cargo does # its own staleness tracking, and a stale ninja stamp hid real diagnostics. @@ -350,12 +355,15 @@ def register_cargo( clippy.finish() self.all_clippy.append('target/' + rel_path + '/clippy') + # No `-Dwarnings` on the fix edge: `cargo fix` re-verifies after rewriting + # and discards a crate's edits when it no longer compiles, so denying the + # lints that `--fix` cannot rewrite would throw away the ones it can. fix = self.build('cargo', 'target/' + rel_path + '/clippy/fix') fix.var('subcommand', 'clippy') fix.var('wd', crate_dir) fix.var( 'extra_args', - extra_args + ['--fix', '--allow-dirty', '--allow-staged'] + clippy_lints, + extra_args + ['--fix', '--allow-dirty', '--allow-staged'] + clippy_rustc_flags, ) fix.var('env', CARGO_ENV) if target_dir is not None: From bedbdd800e3e2983e1837054d83b6b6bb522a529 Mon Sep 17 00:00:00 2001 From: Kira <kp2pml30@gmail.com> Date: Thu, 17 Sep 2026 22:28:36 +0900 Subject: [PATCH 21/45] feat(manager): configure executor worker concurrency, harden post-install (#43) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: drain executor logs and polish CI diagnostics πŸ› * feat(manager): configure executor worker concurrency ✨ * chore(release): advance manager and executor release candidates πŸš€ * fix(manager): pass worker control in execution input πŸ› * chore(executor): pin worker ordering regression test βœ… * chore(post-install): hash-pin venv deps and warn on unverified runners πŸ”’οΈ * chore(post-install): drop the never-published executor download πŸ”₯ * fix(post-install): keep accepting --arch from forwarding callers πŸ› --- .agents/skills/agentic-fuzzing/SKILL.md | 7 +- .genvm-monorepo-root | 2 +- crates/modules-interfaces/src/domain.rs | 7 + .../howto/committing/submodules.md | 2 +- .../howto/releasing/release-build.md | 3 +- .../howto/releasing/versioning.md | 13 +- .../contributing/howto/testing/integration.md | 3 +- docs/schemas/default-config.json | 5 + .../src/impl-spec/appendix/log-record.rst | 2 + .../src/impl-spec/appendix/manager-api.yaml | 8 + .../src/impl-spec/appendix/manager-socket.rst | 6 + docs/website/src/spec/changelog.rst | 10 + executors/v0.2.x | 2 +- executors/v0.3.x | 2 +- implementation/src/manager/mod.rs | 6 + implementation/src/manager/run.rs | 393 ++++++++++-------- implementation/src/manager/run_test.rs | 121 ++++++ install/bin/genvm-post-install | 21 +- install/config/genvm-manager.yaml | 1 + install/lib/python/post-install/__main__.py | 90 +--- .../lib/python/post-install/create_venv.py | 13 +- .../lib/python/post-install/requirements.txt | 158 ++++++- support/ci/pipelines/checks.py | 14 +- support/ci/unit_tests/test_cargo_clippy.py | 9 +- support/manifest-base.yaml | 6 - .../scripts/refresh-requirements-hashes.py | 96 +++++ .../tools/genvm-tool/genvm_tool/manifest.py | 28 +- .../genvm-tool/unit_tests/test_manifest.py | 44 +- .../test_post_install_executor_download.py | 195 --------- .../runner/genvm_tool_plugins/integration.py | 3 +- tests/runner/origin/base_host.py | 4 +- tests/system/cross-major/test.py | 1 + 32 files changed, 718 insertions(+), 557 deletions(-) create mode 100755 support/scripts/refresh-requirements-hashes.py delete mode 100644 support/tools/genvm-tool/unit_tests/test_post_install_executor_download.py diff --git a/.agents/skills/agentic-fuzzing/SKILL.md b/.agents/skills/agentic-fuzzing/SKILL.md index d7b00ca0..cca08a35 100644 --- a/.agents/skills/agentic-fuzzing/SKILL.md +++ b/.agents/skills/agentic-fuzzing/SKILL.md @@ -29,10 +29,9 @@ Sandbox escape and native UB are AFL's job, not this one ## Before you can find anything -Nothing. The v0.3 line returns `save-hashes: False` from -`executors/v0.3.x/.genvm-tool.py`, but that only stops the committed `.hash` -sidecars from being tracked β€” the leader-vs-validator/sync comparison, which is -what a severity-2 finding consists of, runs regardless. Leave the setting alone. +Nothing. The v0.3 line tracks committed `.hash` sidecars. Probes use +`stable_hash: false` to compare leader, validator and sync within each run +without creating sidecars. Leave the line's `save-hashes` setting alone. (It used to be `ignore-hash: True` and it did switch off every comparison, including that one. That was a footgun and it is gone.) diff --git a/.genvm-monorepo-root b/.genvm-monorepo-root index e11b5104..78cefacd 100644 --- a/.genvm-monorepo-root +++ b/.genvm-monorepo-root @@ -1,5 +1,5 @@ { - "version": "v0.6.0-rc1", + "version": "v0.6.0-rc2", "active-versions": [ "v0.3", "v0.2" diff --git a/crates/modules-interfaces/src/domain.rs b/crates/modules-interfaces/src/domain.rs index 9819ae1d..02e37de6 100644 --- a/crates/modules-interfaces/src/domain.rs +++ b/crates/modules-interfaces/src/domain.rs @@ -15,6 +15,10 @@ fn default_none<T>() -> Option<T> { None } +fn default_allow_two_workers() -> bool { + true +} + #[derive( Debug, Clone, @@ -24,6 +28,9 @@ fn default_none<T>() -> Option<T> { genlayer_calldata::Decode, )] pub struct ExecutionData { + #[serde(default = "default_allow_two_workers")] + #[calldata(default = default_allow_two_workers)] + pub allow_two_workers: bool, pub calldata: Bytes, pub message: super::abi_stub::MessageData, pub host_data: String, diff --git a/docs/contributing/howto/committing/submodules.md b/docs/contributing/howto/committing/submodules.md index 97dd0e46..8a89f141 100644 --- a/docs/contributing/howto/committing/submodules.md +++ b/docs/contributing/howto/committing/submodules.md @@ -6,7 +6,7 @@ `runners/`, `support/`, `install/` 2. **Executor lines** β€” `executors/v0.2.x`, `executors/v0.3.x`: checkouts of the **same** repo, `genlayerlabs/genvm-executor`, on different branches. - Each line's `manifest.json` owns its `executor-version` (and `executor-sha256`, see [versioning.md](../releasing/versioning.md)) + Each line's `manifest.json` owns its `executor-version` 3. **`libs/unhardcoded-engine`** β€” the LLM policy engine used by `implementation/`; the build fails loudly when it is missing diff --git a/docs/contributing/howto/releasing/release-build.md b/docs/contributing/howto/releasing/release-build.md index dd9dc20f..714f1739 100644 --- a/docs/contributing/howto/releasing/release-build.md +++ b/docs/contributing/howto/releasing/release-build.md @@ -37,8 +37,7 @@ All of them go onto a single **genvm-manager** release, tagged with `version` from `.genvm-monorepo-root` ([versioning.md](versioning.md)). They overlay onto one install root, so a full install is one platform asset plus `genvm-universal`. The platform asset already carries every active executor -line; post-install's executor download is only a fallback for a root missing -one ([versioning.md](versioning.md)) +line ([versioning.md](versioning.md)) ## Verifying an Asset diff --git a/docs/contributing/howto/releasing/versioning.md b/docs/contributing/howto/releasing/versioning.md index 14e3b50f..bf1e037b 100644 --- a/docs/contributing/howto/releasing/versioning.md +++ b/docs/contributing/howto/releasing/versioning.md @@ -12,16 +12,9 @@ Executor crates are versioned **independently**: a line's `manifest.json` holds `executor-version` and no manager script rewrites it -A platform release asset bundles every active line, so post-install downloads -a line only when the install root lacks it, from `executor_download_urls` in -`support/manifest-base.yaml`: `genvm-<platform>-executor.tar.xz` on a release -of the executor repo tagged with the line's `executor-version`. The download is -accepted only against `executor-sha256` in the line's `manifest.json`, a -`{platform: sha256}` map copied from that release's published checksums; an unpinned -platform is refused. So the executor release must exist before the manifest -that pins it is committed, and a re-release under the same tag needs a manifest -bump. No line pins anything today: the executor repo publishes no releases, and -the fallback fails closed until it does +A platform release asset bundles every active line, and nothing else ships an +executor on its own: the executor repo publishes no releases, so an install root +gets its lines from the manager asset alone Manager branches are named after the *manager* train, not an executor line: `v<X>` is the release branch, `v<X>-dev` the integration branch. Here `<X>` is diff --git a/docs/contributing/howto/testing/integration.md b/docs/contributing/howto/testing/integration.md index cdff315f..98a2a1b2 100644 --- a/docs/contributing/howto/testing/integration.md +++ b/docs/contributing/howto/testing/integration.md @@ -68,8 +68,7 @@ against the leader's runtime hash dev mode, where every execution hash moves ([modify-runner.md](../extending/modify-runner.md)), not for a new case. A line can stop tracking sidecars altogether by returning `{'save-hashes': False}` from -`integration()` in its `.genvm-tool.py`, as v0.3.x currently does while its -hashes still move +`integration()` in its `.genvm-tool.py`. Both active lines track sidecars Neither switch turns off the leader-vs-validator/sync comparison: with sidecars disabled a non-main mode is compared against the main mode's hash from the same diff --git a/docs/schemas/default-config.json b/docs/schemas/default-config.json index fd3c2d5b..b5dd9365 100644 --- a/docs/schemas/default-config.json +++ b/docs/schemas/default-config.json @@ -174,6 +174,11 @@ "type": "string", "description": "path to manifest file" }, + "allow_two_workers": { + "type": "boolean", + "default": true, + "description": "overlap deterministic execution with nondeterministic validation (v0.3+)" + }, "permits": { "type": ["object", "null"], "default": null, diff --git a/docs/website/src/impl-spec/appendix/log-record.rst b/docs/website/src/impl-spec/appendix/log-record.rst index 0a7fac96..e90d6333 100644 --- a/docs/website/src/impl-spec/appendix/log-record.rst +++ b/docs/website/src/impl-spec/appendix/log-record.rst @@ -91,3 +91,5 @@ than by age: ``user`` or ``operator`` record; ``introspector`` records stay discarded The marker counts toward the cap. ``unbounded`` capture keeps every record. +The manager reads each executor's log pipe to EOF before collecting the +execution result, including on early process exit diff --git a/docs/website/src/impl-spec/appendix/manager-api.yaml b/docs/website/src/impl-spec/appendix/manager-api.yaml index c4e6e339..b90ea207 100644 --- a/docs/website/src/impl-spec/appendix/manager-api.yaml +++ b/docs/website/src/impl-spec/appendix/manager-api.yaml @@ -943,6 +943,14 @@ components: description: | Seeds the chain's recursion budget, replacing the executor's own `VM_RECURSION`. Honored from `unsafe` up. + allow_two_workers: + type: boolean + nullable: true + default: null + description: | + Overrides the manager's `allow_two_workers` config. v0.2 warns + when false and keeps its existing scheduling. + Honored from `unsafe` up. Null keeps the configured value. responses: ErrorResponse: diff --git a/docs/website/src/impl-spec/appendix/manager-socket.rst b/docs/website/src/impl-spec/appendix/manager-socket.rst index c5e3561d..d2874ffe 100644 --- a/docs/website/src/impl-spec/appendix/manager-socket.rst +++ b/docs/website/src/impl-spec/appendix/manager-socket.rst @@ -126,6 +126,12 @@ Request payload: the same logical structure as the deprecated budget, replacing the executor's own ``VM_RECURSION``, so a boundary test need not spend one executor process per unit of budget. Honored from ``unsafe``. + - ``allow_two_workers`` (bool, optional) -- overrides the manager config of + the same name from ``unsafe``. The config defaults to true. In v0.3+, false + makes deterministic execution await each submitted nondeterministic + validation task before continuing; null keeps the configured value. + The manager passes this in the execution input. v0.2 warns when false + and keeps its existing scheduling. Nondeterministic calls remain allowed. Response:: diff --git a/docs/website/src/spec/changelog.rst b/docs/website/src/spec/changelog.rst index a7cd80ff..b33d1625 100644 --- a/docs/website/src/spec/changelog.rst +++ b/docs/website/src/spec/changelog.rst @@ -33,6 +33,16 @@ Breaking #. ZIP :term:`runner` loads charge per-entry metadata in addition to the raw archive size and base load cost (see :ref:`runner load charges <gvm-def-runner-load-charge>`) +#. RAM overheads increase for + :ref:`runner loads <gvm-def-consts-value-memory-limiter-consts-runner-load-cost>`, + :ref:`emissions <gvm-def-consts-value-memory-limiter-consts-execution-emission-base-size>`, + :ref:`message-fee rotations <gvm-def-consts-value-memory-limiter-consts-message-fee-rotation-element-size>` + and :ref:`nondeterministic outputs <gvm-def-consts-value-memory-limiter-consts-nondet-output-base-size>` +#. Each :term:`sub-VM` is limited to + :ref:`gvm-def-consts-value-top-limits-max-runners` loaded runners; exceeding + the cap exits with :ref:`gvm-def-str-trie-value-vm-error-out-of-memory` +#. VFS paths are limited to + :ref:`gvm-def-consts-value-top-limits-vfs-path-len` UTF-8 octets #. ``LeaderPublicData`` uses calldata ``{"nd_outs": bytes[]}`` instead of an RLP list with a trailing ``padded`` sentinel. Its fee and output-cap accounting uses a conservative 64-byte frame plus a 64-byte header and diff --git a/executors/v0.2.x b/executors/v0.2.x index 5b799909..75e4d1fd 160000 --- a/executors/v0.2.x +++ b/executors/v0.2.x @@ -1 +1 @@ -Subproject commit 5b799909780a3abda8fe9c055f23772b9220645b +Subproject commit 75e4d1fd5945aadea065f6928e60155ffca63797 diff --git a/executors/v0.3.x b/executors/v0.3.x index cc63964c..224be9bf 160000 --- a/executors/v0.3.x +++ b/executors/v0.3.x @@ -1 +1 @@ -Subproject commit cc63964cb7b6adcdaa41febb78364a6fc7ff4592 +Subproject commit 224be9bf543d6fd910d28e3285d26142db6a84ba diff --git a/implementation/src/manager/mod.rs b/implementation/src/manager/mod.rs index c3b48d65..a04466f5 100644 --- a/implementation/src/manager/mod.rs +++ b/implementation/src/manager/mod.rs @@ -35,6 +35,12 @@ pub struct Config { pub execution_retention: run::ManagerDuration, #[serde(default = "default_max_message_bytes")] pub max_message_bytes: usize, + #[serde(default = "default_allow_two_workers")] + pub allow_two_workers: bool, +} + +fn default_allow_two_workers() -> bool { + true } /// Concurrency budget, denominated in gigabytes of RAM. diff --git a/implementation/src/manager/run.rs b/implementation/src/manager/run.rs index 8ca0d162..3c2651a4 100644 --- a/implementation/src/manager/run.rs +++ b/implementation/src/manager/run.rs @@ -1,7 +1,7 @@ use std::{ collections::HashMap, ops::DerefMut, - os::fd::{AsRawFd, FromRawFd}, + os::fd::AsRawFd, str::FromStr, sync::{ atomic::{AtomicBool, AtomicU16, AtomicUsize, Ordering}, @@ -957,6 +957,10 @@ pub struct UnsafeOverrides { #[serde(default)] #[calldata(default = default_initial_recursion)] pub initial_recursion: Option<u32>, + /// Override worker concurrency from `debug_mode >= Unsafe`. + #[serde(default)] + #[calldata(default = Option::default)] + pub allow_two_workers: Option<bool>, } #[serde_as] @@ -1096,9 +1100,7 @@ trait LogAppender { /// manager's own log instead of buffering them into the result. struct LogAppenderToLog(GenVMId); -// `read_log_pipe` takes `impl DerefMut<Target = LA>`. The value-capturing -// appender is wrapped in `Arc<Mutex<..>>` (which derefs to it), but this -// forwarding appender holds no shared state, so it derefs to itself. +// Let `read_log_pipe` own this appender while borrowing the capturing one. impl std::ops::Deref for LogAppenderToLog { type Target = Self; @@ -1196,6 +1198,18 @@ async fn read_log_pipe<LA: LogAppender>( Ok(()) } +async fn collect_process_logs( + process: impl std::future::Future<Output = anyhow::Result<std::process::ExitStatus>>, + log_reader: impl std::future::Future<Output = std::io::Result<()>>, + genvm_id: GenVMId, +) -> anyhow::Result<std::process::ExitStatus> { + let (result, logs) = tokio::join!(process, log_reader); + if let Err(error) = logs { + log_warn_into!(@operator, &LoggerWithId, genvm_id:id = genvm_id.0, error:err = error; "failed to read genvm logs"); + } + result +} + /// Maximum bytes of stdout/stderr kept (as a tail) per execution when not in /// debug mode, to bound manager memory. const OUTPUT_TAIL_LIMIT: usize = 4 * 1024 * 1024; @@ -1786,18 +1800,27 @@ fn fail_to_start(exec: &SingleGenVMContext, error: anyhow::Error) { /// Creates a pipe for GenVM logging. Returns `(read_fd, write_fd)` with /// the read end set to nonblocking + cloexec. fn create_log_pipe() -> anyhow::Result<(std::os::unix::io::OwnedFd, std::os::unix::io::OwnedFd)> { - let mut read_write_fd = [0; 2]; - let result_code = unsafe { libc::pipe(std::ptr::from_mut(&mut read_write_fd).cast()) }; - if result_code != 0 { - anyhow::bail!("failed to create pipe for genvm logging: {result_code}"); - } - - let read_fd = unsafe { FdWrapper::from_raw_fd(read_write_fd[0]) }; - let write_fd = unsafe { std::os::unix::io::OwnedFd::from_raw_fd(read_write_fd[1]) }; - + let (read_fd, write_fd) = std::io::pipe()?; + let read_fd = FdWrapper::new(read_fd.into()); read_fd.set_nonblocking(true)?; - read_fd.set_cloexec(true)?; - Ok((read_fd.into_inner(), write_fd)) + Ok((read_fd.into_inner(), write_fd.into())) +} + +fn configure_log_pipe( + proc: &mut tokio::process::Command, + log_write_fd: &std::os::unix::io::OwnedFd, +) { + proc.arg(format!("--log-fd={}", log_write_fd.as_raw_fd())); + let log_fd = log_write_fd.as_raw_fd(); + // Only this executor may inherit the writer, or unrelated children delay EOF. + unsafe { + proc.pre_exec(move || { + if libc::fcntl(log_fd, libc::F_SETFD, 0) == -1 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }); + } } /// Builds the `tokio::process::Command` for GenVM with all standard arguments @@ -1811,7 +1834,7 @@ fn build_genvm_command( let mut proc = tokio::process::Command::new(command_path); proc.stdin(std::process::Stdio::piped()); - proc.arg(format!("--log-fd={}", log_write_fd.as_raw_fd())); + configure_log_pipe(&mut proc, log_write_fd); proc.arg("run"); proc.args(&req.extra_args); @@ -1917,7 +1940,10 @@ fn strict_deadline_from_request(req: &Request) -> chrono::DateTime<chrono::Utc> + chrono::Duration::from_std(duration).unwrap_or_else(|_| chrono::Duration::hours(24)) } -fn execution_data_from_request(req: &Request) -> genvm_modules_interfaces::ExecutionData { +fn execution_data_from_request( + req: &Request, + config: &super::Config, +) -> genvm_modules_interfaces::ExecutionData { let mut method_hosts = vec![0; host_fns::Methods::SIZE]; method_hosts[host_fns::Methods::ConsumeResult as usize] = 1; method_hosts[host_fns::Methods::RunNested as usize] = 1; @@ -1926,6 +1952,12 @@ fn execution_data_from_request(req: &Request) -> genvm_modules_interfaces::Execu } genvm_modules_interfaces::ExecutionData { + allow_two_workers: if req.debug_mode >= genvm_common::DebugMode::Unsafe { + req.unsafe_overrides.allow_two_workers + } else { + None + } + .unwrap_or(config.allow_two_workers), calldata: req.calldata.clone(), message: req.message.clone(), host_data: req.host_data.clone(), @@ -2163,7 +2195,7 @@ impl Ctx { // routing as its parent. hook_cross_contract_calls: parent_req.hook_cross_contract_calls, }; - let mut execution_data = execution_data_from_request(&req); + let mut execution_data = execution_data_from_request(&req, &full_ctx.config); execution_data.code = None; execution_data.leader_public_data = None; execution_data.record_actions.clear(); @@ -2248,7 +2280,7 @@ async fn supervise_genvm_inner( modules_lock: Box<dyn std::any::Any + Send + Sync>, permits: tokio::sync::OwnedSemaphorePermit, ) -> anyhow::Result<()> { - let execution_data = execution_data_from_request(&req); + let execution_data = execution_data_from_request(&req, &full_ctx.config); let result = run_genvm_process( full_ctx, exec_ctx.clone(), @@ -2450,187 +2482,190 @@ async fn run_genvm_process( let execution_data_bytes = bytes::Bytes::from(calldata::encode_obj(&execution_data)); - // Spawn log reader: capture into the sink, or (when capture is disabled) - // forward to the manager log instead of buffering into the result. - if capture == Capture::Disabled { - tokio::spawn(read_log_pipe(read_fd, LogAppenderToLog(genvm_id))); - } else { - let logger = Arc::new(tokio::sync::Mutex::new(LogAppenderToValue( - log_sink.clone(), - ))); - let l = logger.clone().lock_owned().await; - tokio::spawn(read_log_pipe(read_fd, l)); - } - - // Spawn child process, then drop child-side FDs - let mut child = proc.spawn()?; - log_debug_into!(&LoggerWithId, genvm_id:id = genvm_id.0, pid:? = child.id(); "genvm process started"); - std::mem::drop(module_child_fds); - std::mem::drop(manager_child); - - let stdin_task = child - .stdin - .take() - .map(|stdin| spawn_stdin_writer(stdin, execution_data_bytes)); - - let stdout = child.stdout.take(); - let stderr = child.stderr.take(); - let stdout_perm = if stdout.is_some() { - Some(exec_ctx.stdout_stderr_sem.clone().acquire_owned().await?) - } else { - None - }; - let stderr_perm = if stderr.is_some() { - Some(exec_ctx.stdout_stderr_sem.clone().acquire_owned().await?) - } else { - None + let log_reader = async { + if capture == Capture::Disabled { + read_log_pipe(read_fd, LogAppenderToLog(genvm_id)).await + } else { + let mut logger = LogAppenderToValue(log_sink.clone()); + read_log_pipe(read_fd, &mut logger).await + } }; - let manager_stream_state = Arc::new(ManagerHostStreamState::default()); - let manager_stream_state_on_drop = manager_stream_state.clone(); - let _manager_stream_guard = sync::DropGuard::new(move || manager_stream_state_on_drop.close()); - tokio::spawn(read_manager_host_stream( - manager_parent, - ManagerHostStream { - full_ctx: full_ctx.clone(), - exec_ctx: exec_ctx.clone(), - parent_req: Arc::new(req.clone()), - consumed_result, - genvm_id, - is_top_level, - state: manager_stream_state.clone(), - }, - )); + let process = async { + // Spawn child process, then drop child-side FDs + let mut child = proc.spawn()?; + drop(write_fd); + log_debug_into!(&LoggerWithId, genvm_id:id = genvm_id.0, pid:? = child.id(); "genvm process started"); + std::mem::drop(module_child_fds); + std::mem::drop(manager_child); - if let Some(mut stdin_task) = stdin_task { - let deadline_duration = exec_ctx - .strict_deadline - .signed_duration_since(chrono::Utc::now()) - .to_std() - .unwrap_or_default(); - let stop = wait_for_process_stop(&exec_ctx, caller_stream.as_deref(), deadline_duration); - tokio::pin!(stop); - tokio::select! { - result = &mut stdin_task => match result { - Ok(Ok(())) => { - log_debug_into!(&LoggerWithId, genvm_id:id = genvm_id.0; "execution data written"); - } - Ok(Err(e)) => { - let _ = child.start_kill(); - let _ = child.wait().await; - anyhow::bail!("failed to write execution data to child stdin: {e}"); - } - Err(e) => { - let _ = child.start_kill(); - let _ = child.wait().await; - anyhow::bail!("stdin write task panicked: {e}"); - } + let stdin_task = child + .stdin + .take() + .map(|stdin| spawn_stdin_writer(stdin, execution_data_bytes)); + + let stdout = child.stdout.take(); + let stderr = child.stderr.take(); + let stdout_perm = if stdout.is_some() { + Some(exec_ctx.stdout_stderr_sem.clone().acquire_owned().await?) + } else { + None + }; + let stderr_perm = if stderr.is_some() { + Some(exec_ctx.stdout_stderr_sem.clone().acquire_owned().await?) + } else { + None + }; + + let manager_stream_state = Arc::new(ManagerHostStreamState::default()); + let manager_stream_state_on_drop = manager_stream_state.clone(); + let _manager_stream_guard = + sync::DropGuard::new(move || manager_stream_state_on_drop.close()); + tokio::spawn(read_manager_host_stream( + manager_parent, + ManagerHostStream { + full_ctx: full_ctx.clone(), + exec_ctx: exec_ctx.clone(), + parent_req: Arc::new(req.clone()), + consumed_result, + genvm_id, + is_top_level, + state: manager_stream_state.clone(), }, - status = child.wait() => { - stdin_task.abort(); - return match status { - Ok(status) => Ok(status), + )); + + if let Some(mut stdin_task) = stdin_task { + let deadline_duration = exec_ctx + .strict_deadline + .signed_duration_since(chrono::Utc::now()) + .to_std() + .unwrap_or_default(); + let stop = + wait_for_process_stop(&exec_ctx, caller_stream.as_deref(), deadline_duration); + tokio::pin!(stop); + tokio::select! { + result = &mut stdin_task => match result { + Ok(Ok(())) => { + log_debug_into!(&LoggerWithId, genvm_id:id = genvm_id.0; "execution data written"); + } + Ok(Err(e)) => { + let _ = child.start_kill(); + let _ = child.wait().await; + anyhow::bail!("failed to write execution data to child stdin: {e}"); + } Err(e) => { let _ = child.start_kill(); let _ = child.wait().await; - Err(e.into()) + anyhow::bail!("stdin write task panicked: {e}"); } - }; - } - reason = &mut stop => { - if reason == ProcessStop::Deadline { - exec_ctx.request_finish(FinishCause::Deadline); + }, + status = child.wait() => { + stdin_task.abort(); + return match status { + Ok(status) => Ok(status), + Err(e) => { + let _ = child.start_kill(); + let _ = child.wait().await; + Err(e.into()) + } + }; + } + reason = &mut stop => { + if reason == ProcessStop::Deadline { + exec_ctx.request_finish(FinishCause::Deadline); + } + stdin_task.abort(); + let _ = child.start_kill(); + return child.wait().await.map_err(Into::into); } - stdin_task.abort(); - let _ = child.start_kill(); - return child.wait().await.map_err(Into::into); } } - } - let mut child = Some(child); - if is_top_level { - let mut stored_child = exec_ctx.process_handle.lock().await; - *stored_child = child.take(); - } + let mut child = Some(child); + if is_top_level { + let mut stored_child = exec_ctx.process_handle.lock().await; + *stored_child = child.take(); + } - if let RunResources::TopLevel { - permits, - modules_lock, - } = resources - { - exec_ctx - .all_permits - .store(Some(Box::new((permits, modules_lock)))); - } + if let RunResources::TopLevel { + permits, + modules_lock, + } = resources + { + exec_ctx + .all_permits + .store(Some(Box::new((permits, modules_lock)))); + } - let out_limit = (capture == Capture::Bounded).then_some(OUTPUT_TAIL_LIMIT); - if let Some(stdout) = stdout { - tokio::spawn(pipe_read( - stdout, - exec_ctx.gep(|x| &x.stdout), - stdout_perm.expect("stdout permit must exist when stdout is piped"), - out_limit, - )); - } - if let Some(stderr) = stderr { - tokio::spawn(pipe_read( - stderr, - exec_ctx.gep(|x| &x.stderr), - stderr_perm.expect("stderr permit must exist when stderr is piped"), - out_limit, - )); - } + let out_limit = (capture == Capture::Bounded).then_some(OUTPUT_TAIL_LIMIT); + if let Some(stdout) = stdout { + tokio::spawn(pipe_read( + stdout, + exec_ctx.gep(|x| &x.stdout), + stdout_perm.expect("stdout permit must exist when stdout is piped"), + out_limit, + )); + } + if let Some(stderr) = stderr { + tokio::spawn(pipe_read( + stderr, + exec_ctx.gep(|x| &x.stderr), + stderr_perm.expect("stderr permit must exist when stderr is piped"), + out_limit, + )); + } - log_sink_guard.forget(); - if is_top_level { - exec_ctx.publish_started(); - } + log_sink_guard.forget(); + if is_top_level { + exec_ctx.publish_started(); + } - let deadline_duration = exec_ctx - .strict_deadline - .signed_duration_since(chrono::Utc::now()) - .to_std() - .unwrap_or_default(); - let mut child = if is_top_level { - let mut stored_child = exec_ctx.process_handle.lock().await; - stored_child - .take() - .ok_or_else(|| anyhow::anyhow!("process handle missing"))? - } else { - child - .take() - .ok_or_else(|| anyhow::anyhow!("nested process handle missing"))? - }; + let deadline_duration = exec_ctx + .strict_deadline + .signed_duration_since(chrono::Utc::now()) + .to_std() + .unwrap_or_default(); + let mut child = if is_top_level { + let mut stored_child = exec_ctx.process_handle.lock().await; + stored_child + .take() + .ok_or_else(|| anyhow::anyhow!("process handle missing"))? + } else { + child + .take() + .ok_or_else(|| anyhow::anyhow!("nested process handle missing"))? + }; - let mut kill_sent = false; - let stop = wait_for_process_stop(&exec_ctx, caller_stream.as_deref(), deadline_duration); - tokio::pin!(stop); - let status = loop { - if kill_sent { - break child.wait().await?; - } - tokio::select! { - status = child.wait() => match status { - Ok(status) => break status, - Err(e) => { + let mut kill_sent = false; + let stop = wait_for_process_stop(&exec_ctx, caller_stream.as_deref(), deadline_duration); + tokio::pin!(stop); + let status = loop { + if kill_sent { + break child.wait().await?; + } + tokio::select! { + status = child.wait() => match status { + Ok(status) => break status, + Err(e) => { + let _ = child.start_kill(); + let _ = child.wait().await; + return Err(e.into()); + } + }, + reason = &mut stop => { + if reason == ProcessStop::Deadline { + exec_ctx.request_finish(FinishCause::Deadline); + } let _ = child.start_kill(); - let _ = child.wait().await; - return Err(e.into()); + kill_sent = true; } - }, - reason = &mut stop => { - if reason == ProcessStop::Deadline { - exec_ctx.request_finish(FinishCause::Deadline); - } - let _ = child.start_kill(); - kill_sent = true; } - } + }; + manager_stream_state.close(); + log_debug_into!(&LoggerWithId, genvm_id:id = genvm_id.0, status = status; "genvm exited"); + Ok(status) }; - manager_stream_state.close(); - log_debug_into!(&LoggerWithId, genvm_id:id = genvm_id.0, status = status; "genvm exited"); - Ok(status) + collect_process_logs(process, log_reader, genvm_id).await } #[cfg(test)] diff --git a/implementation/src/manager/run_test.rs b/implementation/src/manager/run_test.rs index 1c20883b..f4db869f 100644 --- a/implementation/src/manager/run_test.rs +++ b/implementation/src/manager/run_test.rs @@ -751,6 +751,127 @@ async fn terminal_event_is_published_once() { assert!(matches!(rx.has_changed(), Ok(false))); } +#[tokio::test] +async fn process_completion_waits_for_final_logs_on_success_and_error() { + use std::io::Write; + use std::os::unix::process::ExitStatusExt; + + for failed in [false, true] { + let exec = fake_execution(GenVMId(1), None); + let (read_fd, write_fd) = create_log_pipe().unwrap(); + let mut logger = LogAppenderToValue(exec.log_sink.clone()); + let (release, ready) = tokio::sync::oneshot::channel(); + let process = async move { + let mut writer = std::fs::File::from(write_fd); + writeln!( + writer, + r#"{{"level":"info","audience":"user","message":"final"}}"# + )?; + if failed { + anyhow::bail!("process failed"); + } + Ok(std::process::ExitStatus::from_raw(0)) + }; + let logs = async { + ready.await.unwrap(); + read_log_pipe(read_fd, &mut logger).await + }; + let result = collect_process_logs(process, logs, exec.id); + tokio::pin!(result); + + assert!( + futures_util::poll!(result.as_mut()).is_pending(), + "returned before the log reader completed (failed={failed})" + ); + release.send(()).unwrap(); + let result = tokio::time::timeout(std::time::Duration::from_secs(5), result) + .await + .unwrap(); + assert_eq!(result.is_err(), failed); + finish_execution(&exec, None, FinishCause::Exited).await; + let logs = &exec.result.get().unwrap().genvm_log; + assert_eq!(logs.len(), 1); + assert_eq!(logs[0]["message"], "final"); + } +} + +#[tokio::test] +async fn spawn_failure_closes_the_log_pipe() { + let (read_fd, write_fd) = create_log_pipe().unwrap(); + let process = async { + let mut child = tokio::process::Command::new("/dev/null/genvm").spawn()?; + drop(write_fd); + Ok(child.wait().await?) + }; + let mut logger = LogAppenderToValue(Arc::new(LogSinkInner::new(false))); + let result = tokio::time::timeout( + std::time::Duration::from_secs(5), + collect_process_logs(process, read_log_pipe(read_fd, &mut logger), GenVMId(1)), + ) + .await + .expect("log pipe remained open after spawn failure"); + assert!(result.is_err(), "spawn unexpectedly succeeded"); +} + +#[tokio::test] +async fn log_read_failure_preserves_process_status() { + use std::os::unix::process::ExitStatusExt; + + let status = std::process::ExitStatus::from_raw(7 << 8); + let result = collect_process_logs( + async { Ok(status) }, + async { Err(std::io::Error::other("log read failed")) }, + GenVMId(1), + ) + .await + .unwrap(); + assert_eq!(result, status); +} + +#[tokio::test] +async fn only_the_intended_child_inherits_the_log_writer() { + let (read_fd, write_fd) = create_log_pipe().unwrap(); + let mut unrelated = tokio::process::Command::new("bash") + .args(["-c", "read -r line"]) + .stdin(std::process::Stdio::piped()) + .kill_on_drop(true) + .spawn() + .unwrap(); + + let mut command = tokio::process::Command::new("bash"); + command.args([ + "-c", + r#"printf '%s\n' '{"level":"info","message":"child"}' >&"${1#--log-fd=}""#, + "executor", + ]); + command.kill_on_drop(true); + configure_log_pipe(&mut command, &write_fd); + let process = async { + let mut child = command.spawn()?; + drop(write_fd); + Ok(child.wait().await?) + }; + let sink = Arc::new(LogSinkInner::new(false)); + let mut logger = LogAppenderToValue(sink.clone()); + let result = tokio::time::timeout( + std::time::Duration::from_secs(5), + collect_process_logs(process, read_log_pipe(read_fd, &mut logger), GenVMId(1)), + ) + .await; + let unrelated_running = unrelated.try_wait().unwrap().is_none(); + unrelated.kill().await.unwrap(); + + let status = result.expect("unrelated child delayed log EOF").unwrap(); + assert!(status.success(), "child failed: {status}"); + assert!( + unrelated_running, + "unrelated child exited before the EOF check" + ); + let logs = sink.drain(); + assert_eq!(logs.len(), 1); + assert_eq!(logs[0]["message"], "child"); +} + #[test] fn under_or_at_limit_is_unchanged() { let mut s = String::from("hello"); diff --git a/install/bin/genvm-post-install b/install/bin/genvm-post-install index 948894b1..20889c08 100755 --- a/install/bin/genvm-post-install +++ b/install/bin/genvm-post-install @@ -46,6 +46,25 @@ if has_help_arg: genvm_root_dir = Path(__file__).parent.parent +def drop_requirement(lines: list[str], name: str) -> list[str]: + """ + Drop a pin together with the `--hash` lines it continues into; leaving those + behind would attach them to the next requirement.""" + kept = [] + dropping = False + for line in lines: + if dropping: + dropping = line.rstrip().endswith('\\') + continue + # Anchored to the pinned name, so a comment or another package that merely + # contains it is not dropped along with its hashes. + if line.split('==', 1)[0].strip() == name: + dropping = line.rstrip().endswith('\\') + continue + kept.append(line) + return kept + + def requirements_for(base: Path) -> Path: """ The requirements to install into the venv. @@ -58,7 +77,7 @@ def requirements_for(base: Path) -> Path: if not args.use_patchelf: return base - kept = [line for line in base.read_text().splitlines() if 'lief' not in line] + kept = drop_requirement(base.read_text().splitlines(), 'lief') filtered = genvm_root_dir.joinpath('data', 'venvs', 'requirements-patchelf.txt') filtered.parent.mkdir(parents=True, exist_ok=True) filtered.write_text('\n'.join(kept) + '\n') diff --git a/install/config/genvm-manager.yaml b/install/config/genvm-manager.yaml index 930a47c3..ff9048d0 100644 --- a/install/config/genvm-manager.yaml +++ b/install/config/genvm-manager.yaml @@ -7,6 +7,7 @@ log_disable: tracing*,polling*,tungstenite*,tokio_tungstenite* log_level: info manifest_path: ${exeDir}/../data/manifest.yaml +allow_two_workers: true # overlap deterministic execution with nondeterministic validation permits: # 1 permit is 1 GiB of RAM total: null # if null, free memory times `per_gib` per_gib: 1 # permits per gibibyte of free memory, an integer or a ratio (1/4 = one permit per 4 GiB) diff --git a/install/lib/python/post-install/__main__.py b/install/lib/python/post-install/__main__.py index 615ba795..4962ae63 100644 --- a/install/lib/python/post-install/__main__.py +++ b/install/lib/python/post-install/__main__.py @@ -1,6 +1,5 @@ import argparse import hashlib -import io import json import logging import os @@ -8,7 +7,6 @@ import shlex import shutil import subprocess -import tarfile import traceback import typing from pathlib import Path @@ -51,6 +49,9 @@ def str_to_bool_or_none(value): default=target_os, help='Target operating system (linux/macos)', ) +# Callers forwarding a cross-target install pass the whole platform, so the +# architecture is still accepted; no step acts on it β€” only the OS decides what +# is patched and what a binary is checked against. parser.add_argument( '--arch', type=str, @@ -83,14 +84,6 @@ def str_to_bool_or_none(value): ], ) -step_names = [ - 'executor-download', - 'runners-download', - 'bin-patch', - 'bin-check', - 'precompile', -] - parser.add_argument( '--default-steps', type=str_to_bool, @@ -103,12 +96,6 @@ def str_to_bool_or_none(value): default=None, help='Default value for download steps (default: use --default-steps)', ) -parser.add_argument( - '--executor-download', - type=str_to_bool_or_none, - default=None, - help='Enable/disable executor download step (default: use --default-download)', -) parser.add_argument( '--runners-download', type=str_to_bool_or_none, @@ -140,8 +127,6 @@ def str_to_bool_or_none(value): if args.default_download is None: args.default_download = args.default_steps -if args.executor_download is None: - args.executor_download = args.default_download if args.runners_download is None: args.runners_download = args.default_download if args.bin_patch is None: @@ -452,6 +437,13 @@ def download_runners_from_json( logger.warning(f'Executor path {file} does not exist, skipping') return logger.info(f'checking that all runners are present for {file}') + if not verify_hash: + logger.warning( + f'!!! downloading runners for {file} WITHOUT hash verification: ' + 'code fetched over the network is written to disk unchecked. It is ' + 'rejected only by the later check-install step, which --precompile=false ' + 'skips entirely' + ) all_runners = _load_registry(file) for name, hashes in all_runners.items(): @@ -488,56 +480,6 @@ def download_runners_from_json( cur_dst.write_bytes(data) -def download_executor(executor_version: str): - """ - Fetch an executor line's tarball and unpack it at the install root. - - A platform release asset already bundles every active line, so this only - runs for an install root that lacks one (a `manager-<platform>` tree - installed on its own). The tarball is laid out as - `executor/<version>/...`, i.e. relative to the install root, so it is - unpacked there. - - The tarball must match the sha256 the manager pinned for this platform in - its manifest (`executor_versions.<version>.sha256.<platform>`): the download - is over plain HTTPS from a release page and is native code, so no hash means - no download. - """ - templates = manifest.get('executor_download_urls', []) - if not templates: - raise RuntimeError('manifest has no executor_download_urls') - - platform = f'{args.arch}-{args.os}' - pinned = manifest['executor_versions'][executor_version].get('sha256') or {} - expected = pinned.get(platform) - if not isinstance(expected, str): - raise RuntimeError( - f'manifest pins no sha256 for executor {executor_version} on {platform}; refusing to download' - ) - - data = _download_template( - f'executor {executor_version}', - templates, - { - 'version': executor_version, - 'platform': platform, - 'arch': args.arch, - 'os': args.os, - }, - ) - - actual = hashlib.sha256(data).hexdigest() - if actual != expected.lower(): - raise RuntimeError( - f'sha256 mismatch for executor {executor_version} on {platform}: expected {expected}, got {actual}' - ) - - with tarfile.open(fileobj=io.BytesIO(data), mode='r:xz') as tar: - tar.extractall(genvm_root_dir, filter='data') - - logger.info(f'Unpacked executor {executor_version} into {genvm_root_dir}') - - all_executor_versions = list(manifest.get('executor_versions', {}).keys()) all_executor_versions.sort() @@ -567,18 +509,6 @@ def process_executor_version(executor_version: str): executor_root_dir = genvm_root_dir.joinpath('executor', executor_version) executor_executable = executor_root_dir.joinpath('bin', 'genvm') - # A platform asset bundles every line, so this only fetches what a partial - # install lacks. A failure here is only fatal if a missing executor is - # (--error-on-missing-executor). - if args.executor_download and not executor_executable.exists(): - logger.info(f'Executor {executor_version} is not installed, downloading it') - try: - download_executor(executor_version) - except Exception as e: - if args.error_on_missing_executor: - raise - logger.warning(f'Could not download executor {executor_version}: {e}') - # Refuse a wrong-OS executor before anything trusts it, regardless of which # steps run (a missing file is a no-op here; missing-file policy is handled # per-step below). Mirrors the unconditional modules_executable check. diff --git a/install/lib/python/post-install/create_venv.py b/install/lib/python/post-install/create_venv.py index 058dc809..ae6eb122 100644 --- a/install/lib/python/post-install/create_venv.py +++ b/install/lib/python/post-install/create_venv.py @@ -37,8 +37,19 @@ def create_venv(genvm_root_path: Path, requirements_path: Path) -> Path: logger.info('installing dependencies') + # `--require-hashes`: the venv is built by fetching wheels over the network, + # so a pin alone is not enough β€” every requirement must match a sha256 listed + # in the file. pip refuses the whole install if any line lacks one. subprocess.run( - [actual_python, '-m', 'pip', 'install', '-r', requirements_path], + [ + actual_python, + '-m', + 'pip', + 'install', + '--require-hashes', + '-r', + requirements_path, + ], check=True, text=True, ) diff --git a/install/lib/python/post-install/requirements.txt b/install/lib/python/post-install/requirements.txt index 4c36ff85..73d48fc6 100644 --- a/install/lib/python/post-install/requirements.txt +++ b/install/lib/python/post-install/requirements.txt @@ -1,3 +1,155 @@ -# only needed for ELF interpreter patching (see post-install __main__.py) -lief==0.17.0; platform_system == "Linux" -pyyaml==6.0.3 +# Installed with --require-hashes, so every pin carries the sha256 of every +# distribution PyPI publishes for it. Refresh after a version bump with +# `support/scripts/refresh-requirements-hashes.py`. +# +# lief is only needed for ELF interpreter patching (see post-install +# __main__.py); `--use-patchelf` drops it and gets its own venv. + +lief==0.17.0; platform_system == "Linux" \ + --hash=sha256:044e77a80e00bd242f358744ddb98f50ada5c0a50091016d66ae02f4a6baf694 \ + --hash=sha256:0496c899ddd6c4c5faf31432ad06a963f5ecef6cab70902bed02b53c41151935 \ + --hash=sha256:0867d6f06caaae5a9df38b493f66db409b9b8af0277d7b4373f27146d1dd9134 \ + --hash=sha256:0b25ef9cfcfeebab42e51e09da01fda26ea2d847a3b568e51669e900706a4926 \ + --hash=sha256:0c4a820d8ef1e7bd53c0613bc4e00e80663680441fb37218e6e7a273d91d92f7 \ + --hash=sha256:112773fb27ba0ccf1abfd62e5e7e9ca8332d46990304c8286613e22c6ab0f5e4 \ + --hash=sha256:1893c42e32c675ce6d304936605ab28bbd63642a87a89e196eaee9b092280e42 \ + --hash=sha256:1bba12f5c20486d9f463bd999705c537a12976b9e3912388b232e8d12ae41c6c \ + --hash=sha256:1c0737b38413d2e353bec2fb4928f2a3bd434fc69415d6a5d88a799fd2b193b7 \ + --hash=sha256:276d6b1551bbaea230c23f6e4a829d6f6c9fd8568de5f3c893fe381213f898ed \ + --hash=sha256:298dd958aa80f54073a2683d964b892e7156c171095a77eb8a16c8c9d05e3bde \ + --hash=sha256:2a7c8045993470e21300ec590a9e64f02215e789f29c04b9c65ebdff68a45f22 \ + --hash=sha256:2eba6a8d1b52ec98a4bb73b18b83c394f038ab86ebe59e02ae12b13c96b1bd7c \ + --hash=sha256:2f661e0ff4d6ea8964ae079d46510c040bb0bfe24b760ed7ed93154d4ffa4f36 \ + --hash=sha256:33820c66288d447772153c7c9cab356cbb693b5ea0f87bb49e682aa919e08830 \ + --hash=sha256:340155975e25ed68f39fb43b0023cb8571f3171da1b3e6fa7a40441c4e547208 \ + --hash=sha256:35612f424d21612ca21a5b5b97fe42d1fc6928aa250c8bbf39e8eb7f83c67175 \ + --hash=sha256:35b18177821142c9b5ad657e7fbd7f82a9540246fec6ef3876cff9fe8086e4dc \ + --hash=sha256:36d7fcdcfb3136eb8dbaaa1b200026c7c16b78102ed402de54065a777016feb5 \ + --hash=sha256:3882aefeaad59815e605c2e10c38908b96b27800005525c5d8bc0371d115b0d6 \ + --hash=sha256:38e9b4d340158f2399dc00700899606c825dc9437809e8a69133496b6ee39db8 \ + --hash=sha256:3f5a06405c2f837f921da6223e694efc1a3a8800946a38cc5cec6f7eadf4e4c8 \ + --hash=sha256:467b25c857239271077c5ac9d97d8c1fb0282ce84096f2518ef03c244660ef10 \ + --hash=sha256:4899be50a745de11dd35de6b1bfbaac2afa4c62bf14153ffc7b6acb39abb9605 \ + --hash=sha256:4c1e2774a68f8d78043a684675e700c978656efdfbceb5177e262e7b93aa2e41 \ + --hash=sha256:4ca2d4e1e1c24dd1806b50b181c596af98ac40486032f2b8ca82d15e5beb771e \ + --hash=sha256:536e458043529fced8415158296db5768b377b6f60900c9efc79a4ff827e08a9 \ + --hash=sha256:5634aaaf3381c409284229dd7b9987e847f96cda38d14a8861e979cfdf7c789f \ + --hash=sha256:56db6556f68d51202f9a64f89492de8bc559629ce5ee60f2d3b7e2af81c77a60 \ + --hash=sha256:580c4f5370f569591f39a10c442a0d2333b895e457247c133e81020dfafd7e26 \ + --hash=sha256:580f09dfa152d8a15f2cce6578f0a945b0e5ba6148de614857620b087c62fbdd \ + --hash=sha256:630df2bc20c531ca5610db4a2ca393fab4699a6834ff6d7b937eb795d72592c7 \ + --hash=sha256:7198558b3f1803e361a38fc26142d60f3d24b08d9b1811a046ced3ed0fa233b9 \ + --hash=sha256:77a541669af2a4373e518961c1ead16695c6bfe942f95e0454439e52d50a4c25 \ + --hash=sha256:7a4a0276861c86c191bfc9668f172de1fc0c7ffda13a212395a57432392c8638 \ + --hash=sha256:7b5dc554f0ac995fe5d4e99e1f0387799446a9615830723d9810b4fd8da3c8ab \ + --hash=sha256:838a1a837288088ad9cb548b5dac49b3290da72279f3f833bf6e712ccff36749 \ + --hash=sha256:86334c239da1fe30acad7eb341d525c611b040017b8480bc7fddad8508a0a064 \ + --hash=sha256:87ebee7d2fb130750c611a6d9eac614463eb58e82b54695f2c48a9aa3402cd56 \ + --hash=sha256:89506b0a4a262bf9cefaaaaa628c2ffbf0857b8d1056e6a9fca24748470a4485 \ + --hash=sha256:9346d4916301e43c1d4ebb20cd90656ef52e30bfb0fbc181c247979b533da333 \ + --hash=sha256:a86ca76e834887019eeb3163bf2953b90a6e7518d4c0793e7f2ff2e8e8fa9585 \ + --hash=sha256:aea5b7f108e0ebef75fc3b081d6a72e9a7bd4bae5ac5b4d97fe001292fefec9d \ + --hash=sha256:b306e2c34baa7a1d5a061b828f88a5fc0d008992b76d972305c52529f17623e6 \ + --hash=sha256:b3314afd2c8739b2891cae666cd4bb08e14aa0b4c5b08f012a09e82ff362bd36 \ + --hash=sha256:b6680261786d3c27236a6334c441be81230bf2c425f28155b9dcf51aa87720f8 \ + --hash=sha256:b7e71c8b41a9af4d4d973bcf29b8a8970e2d2df94065d3ff043e53f4910d5c08 \ + --hash=sha256:bb977aaf049fa3e67c9757037dd8bd70140759d43b9a0a2b9dd055a18ba5fd9f \ + --hash=sha256:bcdcc52ee83bc2a16229d0df6595199576725db4555e35ac880852b7576534ed \ + --hash=sha256:c05f5a130bbb36b943d10a03b66f86293d99a886f83eff6c5efaf6b8b0e48c31 \ + --hash=sha256:c0ca980c27339611ea496f3405dc33cb21db6c61ed590ca615f239a4ce8545dc \ + --hash=sha256:c3829edaa0fe8487e58d1af7580cbfabd58f19f67edb45d6ae907ae9487aab9e \ + --hash=sha256:c491b40224bdc357c92f76179889c12ac3121b94f665e39827694b6665d0b656 \ + --hash=sha256:c6611056d3aab185fdda95b20e9fc406d7a79e165ab3c40cbadcd0fb49b1e232 \ + --hash=sha256:c84267d47f776dbe2f2cac56f0b038dc43c13d2d9b9a077fce796adbe3dcc4de \ + --hash=sha256:cd2d0e1960ee314c7930911b527228926f4ff2dcef3ca22a4b411cb7054f0d66 \ + --hash=sha256:d28c65782b91a3b65dc4337a29490b03cc8a2693eff2a9d41049b20e63608c12 \ + --hash=sha256:d5f3b87aef3817ba2ed226ff263bc37d85acc262d18367bc36839922e2866ced \ + --hash=sha256:d6f399793c386a349ffdeb908563fc5b0e107504fcfb1acd84776dac33031239 \ + --hash=sha256:d97c08dbfbb8a380347ca45b1f3efcf56f0d54d149e12d1cf16ad40abe039afd \ + --hash=sha256:dc6567e75247bf3dd0db14b8b2b6d9423a3790244a9c6cbf38c7391344e56179 \ + --hash=sha256:de2d838238c4e578bf53627fe9290cdc2f537dcb19b37075e56fab538320b026 \ + --hash=sha256:de370a6c4c861f67ca19718b7f4645eae5000bfe70a69bfa9172fbb84f86370c \ + --hash=sha256:e5057a509eb514c8284b282af2228c13ba15973d925ccb2747f488ab6c7b4f97 \ + --hash=sha256:e543273dcd52dfe48f08a1a0f32eb2ee24601485cf82de7b7d54a1e56d8aa6c7 \ + --hash=sha256:ed0ffe35bb18c965476cc71a5329538d615ef2708dd0aec19f2c6d914f6d0db5 \ + --hash=sha256:f3e918f80fcdc2d9e7d6b20805720db4a64abb5b797c620ff6c6e76d3ec4cf0a \ + --hash=sha256:f4d0e264368b54e54796e56b578681c3bd5eeef7c2d280acaaa918d896883aaf \ + --hash=sha256:f8a33db4573bedae481431fa31d608178af99b6f94e409b6b27c3f5b8067a6b7 \ + --hash=sha256:fb36b4d992dfc8dc592aeda6178dbf0ee51814a4ca49f84b569909cdc27089b0 \ + --hash=sha256:fc0114040f3518fdc97a4fc8aa626131da39d63ca28a3d898fd93f475b2b584a \ + --hash=sha256:fcbf36f9e41e083f9405f89e5c5c7e31ab8145bd1a5d0830e16d99eb368ca826 \ + --hash=sha256:fde04c8fe340358f35c94735405522437e13a52e91619b3ea643027d2f321b22 +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 diff --git a/support/ci/pipelines/checks.py b/support/ci/pipelines/checks.py index ad70bfd5..c01e4726 100644 --- a/support/ci/pipelines/checks.py +++ b/support/ci/pipelines/checks.py @@ -345,10 +345,16 @@ def handler(self, args: argparse.Namespace) -> int: diff = _worktree_diff() if diff.strip(): print(diff) - ci_lib.github_step_summary( - '### `cargo clippy` failed\n\n' - 'Machine-applicable part of the fix:\n\n' + _summary_patch(diff) - ) + if pre_existing: + ci_lib.github_step_summary( + '### `cargo clippy` failed\n\n' + 'Patch omitted because the worktree was already dirty before the fix run.' + ) + else: + ci_lib.github_step_summary( + '### `cargo clippy` failed\n\n' + 'Machine-applicable part of the fix:\n\n' + _summary_patch(diff) + ) else: print('`cargo clippy --fix` rewrote nothing; fix the diagnostics by hand') diff --git a/support/ci/unit_tests/test_cargo_clippy.py b/support/ci/unit_tests/test_cargo_clippy.py index a4fe46e7..193f1f8f 100644 --- a/support/ci/unit_tests/test_cargo_clippy.py +++ b/support/ci/unit_tests/test_cargo_clippy.py @@ -63,13 +63,18 @@ def test_failure_without_a_patch_says_so(monkeypatch, clippy, capsys): assert 'rewrote nothing' in capsys.readouterr().out -def test_pre_existing_changes_are_disclaimed(monkeypatch, clippy, capsys): +def test_pre_existing_changes_are_excluded_from_summary(monkeypatch, clippy, capsys): clippy.codes = {'cargo/clippy': 1} monkeypatch.setattr(ci_lib, 'output', lambda _command: 'src/lib.rs') - monkeypatch.setattr(ci_lib, 'github_step_summary', lambda _text: None) + summary = [] + monkeypatch.setattr(ci_lib, 'github_step_summary', summary.append) assert run() == 1 assert 'already dirty' in capsys.readouterr().out + assert len(summary) == 1 + assert 'Patch omitted' in summary[0] + assert 'src/lib.rs' not in summary[0] + assert '```diff' not in summary[0] def test_summary_patch_is_cut_on_a_line_boundary(monkeypatch): diff --git a/support/manifest-base.yaml b/support/manifest-base.yaml index 0eb8dbdc..c9d5d07c 100644 --- a/support/manifest-base.yaml +++ b/support/manifest-base.yaml @@ -4,9 +4,3 @@ # build-independent and lives under version control. runners_download_urls: - "https://storage.googleapis.com/gh-af/genvm_runners/{name}/{hash}.{ext}" -# Fallback source for a line missing from the install root (a platform release -# asset already bundles every active line). `{version}` is the line's -# executor-version and `{platform}` the nix-style token (amd64-linux, -# arm64-macos, ...). Accepted only against the line's pinned `executor-sha256`. -executor_download_urls: - - "https://github.com/genlayerlabs/genvm-executor/releases/download/{version}/genvm-{platform}-executor.tar.xz" diff --git a/support/scripts/refresh-requirements-hashes.py b/support/scripts/refresh-requirements-hashes.py new file mode 100755 index 00000000..03dd33c6 --- /dev/null +++ b/support/scripts/refresh-requirements-hashes.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 + +import argparse +import json +import re +import sys +import urllib.request +from pathlib import Path + +DEFAULT_REQUIREMENTS = ( + Path(__file__).parent.parent.parent + / 'install' + / 'lib' + / 'python' + / 'post-install' + / 'requirements.txt' +) + +# `name==version` plus an optional `; marker`; anything else is not a pin we can +# hash (a range has no single set of files, and --require-hashes rejects it). +PIN = re.compile(r'^(?P<name>[A-Za-z0-9._-]+)==(?P<version>[^\s;\\]+)') + + +def fetch_hashes(name: str, version: str) -> list[str]: + url = f'https://pypi.org/pypi/{name}/{version}/json' + try: + with urllib.request.urlopen(url) as f: + doc = json.load(f) + except Exception as e: + raise RuntimeError(f'cannot read {url}: {e}') from e + files = doc['urls'] + if not files: + raise RuntimeError(f'PyPI publishes no files for {name}=={version}') + return sorted(f['digests']['sha256'] for f in files) + + +def rewrite(text: str) -> str: + out: list[str] = [] + for line in text.splitlines(): + stripped = line.strip() + if stripped.startswith('--hash=') or stripped == '\\': + continue + if not stripped or stripped.startswith('#'): + out.append(line) + continue + pin = PIN.match(stripped) + if pin is None: + raise RuntimeError( + f'{stripped!r} is not a `name==version` pin; --require-hashes needs one' + ) + hashes = fetch_hashes(pin['name'], pin['version']) + print(f'{pin["name"]}=={pin["version"]}: {len(hashes)} distributions') + requirement = stripped.rstrip('\\').strip() + out.append(requirement + ' \\') + out.extend(f' --hash=sha256:{h} \\' for h in hashes[:-1]) + out.append(f' --hash=sha256:{hashes[-1]}') + return '\n'.join(out) + '\n' + + +def main() -> int: + parser = argparse.ArgumentParser( + description='Refresh the --hash lines of a pinned requirements file from PyPI' + ) + parser.add_argument( + 'requirements', + type=Path, + nargs='?', + default=DEFAULT_REQUIREMENTS, + help='requirements file to rewrite in place', + ) + parser.add_argument( + '--check', + action='store_true', + help='do not write; exit non-zero if the file is out of date', + ) + args = parser.parse_args() + + old = args.requirements.read_text() + new = rewrite(old) + + if args.check: + if old != new: + print(f'{args.requirements}: hashes are out of date') + return 1 + return 0 + + if old != new: + args.requirements.write_text(new) + print(f'{args.requirements}: updated') + else: + print(f'{args.requirements}: already up to date') + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/support/tools/genvm-tool/genvm_tool/manifest.py b/support/tools/genvm-tool/genvm_tool/manifest.py index c9c1c8ca..a855e9d5 100644 --- a/support/tools/genvm-tool/genvm_tool/manifest.py +++ b/support/tools/genvm-tool/genvm_tool/manifest.py @@ -9,7 +9,7 @@ Inputs, all under the monorepo root: * `.genvm-monorepo-root` -> `active-versions` (the executor trains) - * `executors/v<ver>.x/manifest.json` -> `executor-version`, `available-after`, `executor-sha256` + * `executors/v<ver>.x/manifest.json` -> `executor-version`, `available-after` * `support/manifest-base.yaml` -> static fields (runner download URLs) Used in-process by `genvm-tool configure` (dev) and exposed as the @@ -17,7 +17,6 @@ """ import json -import re from pathlib import Path from ruamel.yaml import YAML @@ -28,30 +27,12 @@ # Manager-root-relative; holds the build-independent fields (runner URLs). BASE_REL = Path('support') / 'manifest-base.yaml' -# The platforms a release ships (support/ci/pipelines/release.py PLATFORMS). -PLATFORMS = frozenset({'amd64-linux', 'arm64-linux', 'arm64-macos'}) -_SHA256 = re.compile(r'[0-9a-fA-F]{64}') - def _executor_manifest(root: Path, bare_version: str) -> dict: path = root / f'executors/v{bare_version}.x' / 'manifest.json' return json.loads(path.read_text()) -def _pins(bare_version: str, pins: dict) -> dict[str, str]: - """`executor-sha256` checked here, where a typo is a build error, not a refused install.""" - for platform, digest in pins.items(): - if platform not in PLATFORMS: - raise common.ToolError( - f'executors/v{bare_version}.x/manifest.json: executor-sha256 names unknown platform {platform!r}' - ) - if not isinstance(digest, str) or not _SHA256.fullmatch(digest): - raise common.ToolError( - f'executors/v{bare_version}.x/manifest.json: executor-sha256[{platform!r}] is not a hex sha256' - ) - return {platform: digest.lower() for platform, digest in sorted(pins.items())} - - def build(root: Path): """ Return the manifest document: `executor_versions` followed by the static @@ -65,13 +46,6 @@ def build(root: Path): entry = { 'available_after': DoubleQuotedScalarString(em['available-after']), } - # {platform: sha256} of the line's own release tarballs; post-install - # refuses to download a line whose platform has no pinned hash. - if 'executor-sha256' in em: - entry['sha256'] = { - k: DoubleQuotedScalarString(v) - for k, v in _pins(version, em['executor-sha256']).items() - } executor_versions[em['executor-version']] = entry doc = YAML(typ='rt').load((root / BASE_REL).read_text()) diff --git a/support/tools/genvm-tool/unit_tests/test_manifest.py b/support/tools/genvm-tool/unit_tests/test_manifest.py index 4bdf1128..e1e8c78b 100644 --- a/support/tools/genvm-tool/unit_tests/test_manifest.py +++ b/support/tools/genvm-tool/unit_tests/test_manifest.py @@ -15,9 +15,6 @@ ) BASE = 'runners_download_urls:\n - "https://example.invalid/{name}/{hash}.{ext}"\n' -PINS = {'arm64-macos': 'b' * 64, 'amd64-linux': 'A' * 64} -# what build-manifest emits for PINS: sorted, lowercased -WRITTEN = {'amd64-linux': 'a' * 64, 'arm64-macos': 'b' * 64} def _micro_yaml_loads(): @@ -29,19 +26,19 @@ def _micro_yaml_loads(): @pytest.fixture def root(tmp_path): - """Two active lines; v0.3 pins its tarballs, v0.2 does not.""" + """Two active lines.""" tmp_path.joinpath('.genvm-monorepo-root').write_text( json.dumps({'active-versions': ['v0.3', 'v0.2']}) ) base = tmp_path / manifest.BASE_REL base.parent.mkdir(parents=True) base.write_text(BASE) - _line(tmp_path, '0.3', {'executor-sha256': PINS}) - _line(tmp_path, '0.2', {}) + _line(tmp_path, '0.3') + _line(tmp_path, '0.2') return tmp_path -def _line(root: Path, line: str, extra: dict) -> None: +def _line(root: Path, line: str) -> None: path = root / f'executors/v{line}.x/manifest.json' path.parent.mkdir(parents=True) path.write_text( @@ -49,38 +46,16 @@ def _line(root: Path, line: str, extra: dict) -> None: { 'executor-version': f'v{line}.0', 'available-after': '2024-09-01T00:00:00Z', - **extra, } ) ) -def test_pins_are_copied_per_platform_sorted_and_lowercased(root): - entry = manifest.build(root)['executor_versions']['v0.3.0'] +def test_every_active_line_is_listed_with_its_availability(root): + versions = manifest.build(root)['executor_versions'] - assert list(entry['sha256']) == ['amd64-linux', 'arm64-macos'] - assert dict(entry['sha256']) == WRITTEN - - -@pytest.mark.parametrize( - 'pins', - [{'x86_64-linux': 'a' * 64}, {'amd64-linux': 'a' * 63}, {'amd64-linux': 'g' * 64}], - ids=['unknown-platform', 'short', 'not-hex'], -) -def test_a_malformed_pin_fails_the_build(root, pins): - path = root / 'executors/v0.3.x/manifest.json' - line = json.loads(path.read_text()) - line['executor-sha256'] = pins - path.write_text(json.dumps(line)) - - with pytest.raises(common.ToolError, match='executors/v0.3.x/manifest.json'): - manifest.build(root) - - -def test_a_line_without_pins_gets_no_sha256_key(root): - entry = manifest.build(root)['executor_versions']['v0.2.0'] - - assert dict(entry) == {'available_after': '2024-09-01T00:00:00Z'} + assert sorted(versions) == ['v0.2.0', 'v0.3.0'] + assert dict(versions['v0.3.0']) == {'available_after': '2024-09-01T00:00:00Z'} def test_the_written_file_reads_back_through_post_install(root, tmp_path): @@ -89,7 +64,6 @@ def test_the_written_file_reads_back_through_post_install(root, tmp_path): doc = _micro_yaml_loads()(out.read_text()) - assert doc['executor_versions']['v0.3.0']['sha256'] == WRITTEN assert doc['executor_versions']['v0.3.0']['available_after'] == '2024-09-01T00:00:00Z' - assert 'sha256' not in doc['executor_versions']['v0.2.0'] + assert doc['executor_versions']['v0.2.0']['available_after'] == '2024-09-01T00:00:00Z' assert doc['runners_download_urls'] == ['https://example.invalid/{name}/{hash}.{ext}'] diff --git a/support/tools/genvm-tool/unit_tests/test_post_install_executor_download.py b/support/tools/genvm-tool/unit_tests/test_post_install_executor_download.py deleted file mode 100644 index 35ba4b97..00000000 --- a/support/tools/genvm-tool/unit_tests/test_post_install_executor_download.py +++ /dev/null @@ -1,195 +0,0 @@ -"""Post-install fetches a missing executor line only against the hash its manifest pins.""" - -import functools -import hashlib -import http.server -import io -import json -import shutil -import subprocess -import sys -import tarfile -import threading -import typing -from pathlib import Path - -import pytest -from genvm_tool import common - -POST_INSTALL = ( - common.find_root(Path(__file__).parent) / 'install/lib/python/post-install' -) -VERSION = 'v0.3.0' -PLATFORM = 'amd64-linux' -ASSET = f'{VERSION}/genvm-{PLATFORM}-executor.tar.xz' - - -def _tarball() -> bytes: - """A line's release tarball: `executor/<version>/...`, relative to the install root.""" - payload = b'#!/bin/sh\n' - buf = io.BytesIO() - with tarfile.open(fileobj=buf, mode='w:xz') as tar: - info = tarfile.TarInfo(f'executor/{VERSION}/bin/genvm') - info.size = len(payload) - tar.addfile(info, io.BytesIO(payload)) - return buf.getvalue() - - -class _Handler(http.server.SimpleHTTPRequestHandler): - requested: typing.ClassVar[list[str]] = [] - - def do_GET(self): - self.requested.append(self.path) - super().do_GET() - - def log_message(self, *args): - pass - - -@pytest.fixture -def release(tmp_path): - """The executor release page, serving `tmp_path/www`; `requested` records every GET.""" - www = tmp_path / 'www' - www.mkdir() - _Handler.requested = [] - httpd = http.server.ThreadingHTTPServer( - ('127.0.0.1', 0), functools.partial(_Handler, directory=str(www)) - ) - threading.Thread(target=httpd.serve_forever, daemon=True).start() - yield www, f'http://127.0.0.1:{httpd.server_address[1]}' - httpd.shutdown() - httpd.server_close() - - -def _install_root(tmp_path: Path, url: str, pins: dict[str, object] | None) -> Path: - """A manager-only install: no executor on disk, so post-install must fetch it.""" - root = tmp_path / 'root' - shutil.copytree(POST_INSTALL, root / 'lib/python/post-install') - root.joinpath('bin').mkdir() - # not ELF, so the platform check leaves it alone; bin-check is off - root.joinpath('bin/genvm-modules').write_text('#!/bin/sh\n') - lines = [ - 'executor_versions:', - f' {VERSION}:', - ' available_after: "2024-09-01T00:00:00Z"', - ] - if pins is not None: - lines.append(' sha256:') - lines.extend( - f' {platform}: {json.dumps(digest)}' for platform, digest in pins.items() - ) - lines += [ - 'executor_download_urls:', - f' - "{url}/{{version}}/genvm-{{platform}}-executor.tar.xz"', - ] - root.joinpath('data').mkdir() - root.joinpath('data/manifest.yaml').write_text('\n'.join(lines) + '\n') - return root - - -def _run(root: Path, *extra: str) -> subprocess.CompletedProcess: - return subprocess.run( - [ - sys.executable, - '-B', - str(root / 'lib/python/post-install/__main__.py'), - '--default-steps=false', - '--executor-download=true', - '--runners-download=false', - '--os=linux', - '--arch=amd64', - *extra, - ], - capture_output=True, - text=True, - check=False, - ) - - -def test_a_tarball_matching_its_pin_is_unpacked(tmp_path, release): - www, url = release - data = _tarball() - www.joinpath(ASSET).parent.mkdir() - www.joinpath(ASSET).write_bytes(data) - root = _install_root( - tmp_path, url, {PLATFORM: hashlib.sha256(data).hexdigest().upper()} - ) - - result = _run(root) - - assert result.returncode == 0, result.stderr - assert root.joinpath('executor', VERSION, 'bin/genvm').read_bytes() == b'#!/bin/sh\n' - - -def test_a_tarball_that_does_not_match_its_pin_is_refused(tmp_path, release): - www, url = release - www.joinpath(ASSET).parent.mkdir() - www.joinpath(ASSET).write_bytes(_tarball()) - root = _install_root(tmp_path, url, {PLATFORM: '0' * 64}) - - result = _run(root) - - assert result.returncode != 0 - assert f'sha256 mismatch for executor {VERSION} on {PLATFORM}' in result.stderr - assert not root.joinpath('executor').exists() - - -@pytest.mark.parametrize( - 'pins', - [None, {'arm64-macos': '0' * 64}, {PLATFORM: 123}], - ids=['none', 'other-platform', 'not-a-string'], -) -def test_an_unpinned_platform_is_not_even_downloaded(tmp_path, release, pins): - www, url = release - www.joinpath(ASSET).parent.mkdir() - www.joinpath(ASSET).write_bytes(_tarball()) - root = _install_root(tmp_path, url, pins) - - result = _run(root) - - assert result.returncode != 0 - assert ( - f'manifest pins no sha256 for executor {VERSION} on {PLATFORM}' in result.stderr - ) - assert _Handler.requested == [] - - -def test_a_release_without_the_asset_unpacks_nothing(tmp_path, release): - _, url = release - root = _install_root(tmp_path, url, {PLATFORM: '0' * 64}) - - result = _run(root) - - assert result.returncode != 0 - assert f'failed to download executor {VERSION}' in result.stderr - assert not root.joinpath('executor').exists() - - -@pytest.mark.parametrize( - 'pins', [None, {PLATFORM: '0' * 64}], ids=['unpinned', 'mismatch'] -) -def test_a_missing_executor_is_a_warning_when_asked(tmp_path, release, pins): - www, url = release - www.joinpath(ASSET).parent.mkdir() - www.joinpath(ASSET).write_bytes(_tarball()) - root = _install_root(tmp_path, url, pins) - - result = _run(root, '--error-on-missing-executor=false') - - assert result.returncode == 0, result.stderr - assert f'Could not download executor {VERSION}' in result.stderr - assert not root.joinpath('executor').exists() - - -def test_an_installed_line_is_left_alone(tmp_path, release): - """The bundled line wins: the download is a fallback, not a refresh.""" - _, url = release - root = _install_root(tmp_path, url, None) - installed = root / 'executor' / VERSION / 'bin/genvm' - installed.parent.mkdir(parents=True) - installed.write_text('#!/bin/sh\n') - - result = _run(root) - - assert result.returncode == 0, result.stderr - assert _Handler.requested == [] diff --git a/tests/runner/genvm_tool_plugins/integration.py b/tests/runner/genvm_tool_plugins/integration.py index 752d2625..d1173785 100644 --- a/tests/runner/genvm_tool_plugins/integration.py +++ b/tests/runner/genvm_tool_plugins/integration.py @@ -823,7 +823,8 @@ def resolve_executor(address, state, advisory_major): leader_public_data=leader_public_data, message_fee_allocation=message_fee_allocation, unsafe_overrides=base_host.UnsafeOverrides( - reroute_to=single_conf.get('reroute_to', reroute_to) + reroute_to=single_conf.get('reroute_to', reroute_to), + allow_two_workers=single_conf.get('allow_two_workers'), ), request_extra=request_extra, # A case may state the major the host claims, for the diff --git a/tests/runner/origin/base_host.py b/tests/runner/origin/base_host.py index 1a185608..41d24f4a 100644 --- a/tests/runner/origin/base_host.py +++ b/tests/runner/origin/base_host.py @@ -127,17 +127,19 @@ class UnsafeOverrides: Request overrides that reach boundaries production traffic cannot. Each member states the `debug_mode` the manager requires before it applies: - `reroute_to` from `safe`, `initial_recursion` from `unsafe`. With debugging + `reroute_to` from `safe`, other overrides from `unsafe`. With debugging disabled none of them take effect. """ reroute_to: str = '' initial_recursion: int | None = None + allow_two_workers: bool | None = None def as_request_field(self) -> dict[str, typing.Any]: return { 'reroute_to': self.reroute_to, 'initial_recursion': self.initial_recursion, + 'allow_two_workers': self.allow_two_workers, } diff --git a/tests/system/cross-major/test.py b/tests/system/cross-major/test.py index c58da887..9e629581 100644 --- a/tests/system/cross-major/test.py +++ b/tests/system/cross-major/test.py @@ -424,6 +424,7 @@ async def _deploy(self, line: int, address: Address, code: bytes): calldata=gvm_calldata.encode({}), code=code, is_init=True, + timeout=10 * 60, ) assert result.result_kind == host_fns.ResultCode.RETURN, result assert len(result.execution_hash) == 32, result.execution_hash From 89c20400f77b647b0e7649f58be0212a4bd6f004 Mon Sep 17 00:00:00 2001 From: Kira <kp2pml30@gmail.com> Date: Fri, 18 Sep 2026 17:29:05 +0900 Subject: [PATCH 22/45] =?UTF-8?q?feat(fees):=20charge=20LLM=20tokens=20and?= =?UTF-8?q?=20permit=20deploy=20balance=20fees=20=E2=9C=A8=20(#44)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/impl-spec/03-greyboxing/index.rst | 12 ++ .../src/spec/03-vm/02-meta-properties.rst | 2 + .../spec/04-contract-interface/03-storage.rst | 3 +- executors/v0.3.x | 2 +- implementation/tests/policy_dispatch.rs | 125 +++++++++++++++++- install/config/genvm-llm-default.lua | 20 ++- 6 files changed, 153 insertions(+), 11 deletions(-) diff --git a/docs/website/src/impl-spec/03-greyboxing/index.rst b/docs/website/src/impl-spec/03-greyboxing/index.rst index d6440ad0..fadc6bb0 100644 --- a/docs/website/src/impl-spec/03-greyboxing/index.rst +++ b/docs/website/src/impl-spec/03-greyboxing/index.rst @@ -21,6 +21,18 @@ Retrieving Data from :term:`Host` Host can provide additional data to the :term:`Module` to help it make decisions, only transaction id and node address are required, as they are required for signing requests. +Token Charges +------------- + +The default Lua policy charges 1/4 time unit per 1,000 provider-reported total +tokens, using the host's ``gas_data.genPerTimeUnit`` price. A model can override +the rate with ``meta.time_units_per_1k_tokens``, a non-negative rational string +such as ``"1/2"`` or ``"0"``. Invalid rates fail before calling that provider + +The charge is ``ceil(total_tokens * genPerTimeUnit * rate / 1000)`` in GEN-wei, +computed with exact rational arithmetic. Missing or zero ``genPerTimeUnit`` +keeps the charge at zero + Current Built-in Filters ------------------------ diff --git a/docs/website/src/spec/03-vm/02-meta-properties.rst b/docs/website/src/spec/03-vm/02-meta-properties.rst index 50d7a774..071b2d32 100644 --- a/docs/website/src/spec/03-vm/02-meta-properties.rst +++ b/docs/website/src/spec/03-vm/02-meta-properties.rst @@ -140,3 +140,5 @@ flag has no effect on ``EmitExternalMessage``. Unlike the meta-properties above, this one is stored as a bit in the root slot's inline ``permissions`` bitfield and read before execution begins. The bit offset is the corresponding member of :ref:`gvm-def-enum-permissions`. + +A deployment is the exception: permission is granted unconditionally. diff --git a/docs/website/src/spec/04-contract-interface/03-storage.rst b/docs/website/src/spec/04-contract-interface/03-storage.rst index 7685c50e..edfe1ab5 100644 --- a/docs/website/src/spec/04-contract-interface/03-storage.rst +++ b/docs/website/src/spec/04-contract-interface/03-storage.rst @@ -61,7 +61,8 @@ the following data: the slot it points to (same 4-byte-length-prefixed layout). This lets a contract serve its code from an arbitrary slot, including one shared via a ``chain:<address>:<d|f>:<slot>`` runner id. - ``permissions``: (offset 37) A 32-byte (``u256``) little-endian permission bitfield read by - the executor at the start of every load. Bit ``n`` corresponds to the permission whose value + the executor at the start of every load but a deployment, which grants every permission in it + instead (see :ref:`gvm-perm-use-balance-for-message-fees`). Bit ``n`` corresponds to the permission whose value is ``n`` (currently only bit ``0``, ``can_use_balance_for_message_fees``). It is not reserved: contracts may set it (see ``Root.get_permission`` / ``Root.set_permission`` in the Python SDK). diff --git a/executors/v0.3.x b/executors/v0.3.x index 224be9bf..e813da59 160000 --- a/executors/v0.3.x +++ b/executors/v0.3.x @@ -1 +1 @@ -Subproject commit 224be9bf543d6fd910d28e3285d26142db6a84ba +Subproject commit e813da599329bb848efd058d9e20ae660d740ebe diff --git a/implementation/tests/policy_dispatch.rs b/implementation/tests/policy_dispatch.rs index c2d7f061..4eb44cf6 100644 --- a/implementation/tests/policy_dispatch.rs +++ b/implementation/tests/policy_dispatch.rs @@ -98,17 +98,29 @@ fn build_config( ) -> ( sync::DArc<config::Config>, Arc<BTreeMap<String, Box<dyn providers::Provider + Send + Sync>>>, +) { + build_config_with_model_meta(backends, serde_json::Value::Null) +} + +fn build_config_with_model_meta( + backends: &[(&FakeBackend, i64, bool)], + model_meta: serde_json::Value, +) -> ( + sync::DArc<config::Config>, + Arc<BTreeMap<String, Box<dyn providers::Provider + Send + Sync>>>, ) { let mut cfg_backends = BTreeMap::new(); let mut provider_map = BTreeMap::new(); for (fake, priority, supports_json) in backends { + let mut model = model_cfg(*supports_json); + model.meta = model_meta.clone(); let backend = config::BackendConfig { enabled: true, provider: config::Provider::OpenaiCompatible, key: "<empty>".to_owned(), script_config: config::ScriptBackendConfig { - models: BTreeMap::from([("model".to_owned(), model_cfg(*supports_json))]), + models: BTreeMap::from([("model".to_owned(), model)]), meta: serde_json::json!({ "priority": priority }), timeout: None, }, @@ -163,10 +175,33 @@ async fn run_prompt( config: &sync::DArc<config::Config>, providers: Arc<BTreeMap<String, Box<dyn providers::Provider + Send + Sync>>>, format: llm_iface::OutputFormat, +) -> anyhow::Result<String> { + run_priced_prompt( + config, + providers, + format, + None, + primitive_types::U256::zero(), + ) + .await +} + +async fn run_priced_prompt( + config: &sync::DArc<config::Config>, + providers: Arc<BTreeMap<String, Box<dyn providers::Provider + Send + Sync>>>, + format: llm_iface::OutputFormat, + time_unit_price: Option<&str>, + expected_charge: primitive_types::U256, ) -> anyhow::Result<String> { let user_vm = create_vm(config).await.unwrap(); - let hello = common::tests::get_hello(); + let mut hello = common::tests::get_hello(); + if let Some(price) = time_unit_price { + Arc::get_mut(&mut hello) + .unwrap() + .gas_data + .insert("genPerTimeUnit".to_owned(), price.to_owned()); + } let metrics = sync::DArc::new(Metrics::default()); let scripting_ctx = scripting::create_ctx_part( &hello, @@ -210,6 +245,10 @@ async fn run_prompt( .call_fn(&user_vm.data.exec_prompt, (ctx_lua, payload, fuel)) .await?; let table = res.as_table().unwrap(); + assert_eq!( + scripting::rat::lua_rat_to_u256(table).unwrap(), + expected_charge + ); let data: llm_iface::PromptAnswerData = user_vm.vm.from_value(table.get("data").unwrap()).unwrap(); match data { @@ -218,6 +257,88 @@ async fn run_prompt( } } +#[tokio::test] +async fn token_charge_rounds_up_the_total_using_rational_prices() { + common::tests::setup(); + let hits = Arc::new(Mutex::new(Vec::new())); + let backend = spawn_fake("priced", vec![200], hits); + let (config, providers) = build_config(&[(&backend, 0, true)]); + + for (price, charge) in [ + ("0", 0u64), + ("1", 1), + ("4000", 2), + ("4001", 3), + ("8001/2", 3), + ("9007199254742001", 4503599627372), + ] { + let answer = run_priced_prompt( + &config, + providers.clone(), + llm_iface::OutputFormat::Text, + Some(price), + primitive_types::U256::from(charge), + ) + .await + .unwrap(); + assert_eq!(answer, "ok"); + } +} + +#[tokio::test] +async fn model_meta_overrides_the_token_rate() { + common::tests::setup(); + let hits = Arc::new(Mutex::new(Vec::new())); + let backend = spawn_fake("priced", vec![200], hits); + for (rate, charge) in [("0", 0u64), ("1/2", 4)] { + let (config, providers) = build_config_with_model_meta( + &[(&backend, 0, true)], + serde_json::json!({ "time_units_per_1k_tokens": rate }), + ); + let answer = run_priced_prompt( + &config, + providers, + llm_iface::OutputFormat::Text, + Some("4000"), + primitive_types::U256::from(charge), + ) + .await + .unwrap(); + assert_eq!(answer, "ok"); + } +} + +#[tokio::test] +async fn invalid_token_rates_fail_before_calling_a_provider() { + common::tests::setup(); + let hits = Arc::new(Mutex::new(Vec::new())); + let backend = spawn_fake("priced", vec![200], hits.clone()); + for rate in [ + serde_json::json!("-1/4"), + serde_json::json!("invalid"), + serde_json::json!(0.25), + serde_json::json!(false), + ] { + let (config, providers) = build_config_with_model_meta( + &[(&backend, 0, true)], + serde_json::json!({ "time_units_per_1k_tokens": rate }), + ); + let error = run_prompt(&config, providers, llm_iface::OutputFormat::Text) + .await + .unwrap_err(); + let message = format!("{error:#}"); + assert!( + message.contains("time_units_per_1k_tokens") + || message.contains("failed to parse rational"), + "unexpected error: {message}" + ); + } + assert!( + hits.lock().unwrap().is_empty(), + "invalid pricing must not call a provider" + ); +} + // Higher priority is tried first; an overloaded provider falls through to the // next in priority order; the survivor's answer is returned. #[tokio::test] diff --git a/install/config/genvm-llm-default.lua b/install/config/genvm-llm-default.lua index b545a9b3..e71ffc2b 100644 --- a/install/config/genvm-llm-default.lua +++ b/install/config/genvm-llm-default.lua @@ -8,16 +8,21 @@ local sqlite3 = require("lsqlite3") -- Instead, each genvm creates a session, which has a single `ctx` object, -- which is preserved across multiple calls ---- Gas (gen) charged per token for a given provider/model. ---- TODO: per-model pricing is not configured yet, so this always returns 0. ---- Once a price table exists, look up the gen-per-token rate here so that token ---- usage is converted into gen and charged to the host as fuel. +--- GEN-wei charged per token, before rounding the total charge. ---@param ctx any execution context ---@param provider string provider id ---@param model string model name ----@return number|Rat gen charged per token +---@return Rat GEN-wei charged per token local function gen_per_token(ctx, provider, model) - return rat.zero + local meta = llm.providers[provider].models[model].meta + local rate = meta and meta.time_units_per_1k_tokens + if rate == nil then + rate = "1/4" + end + assert(type(rate) == "string", "time_units_per_1k_tokens must be a rational string") + rate = rat.new(rate) + assert(rate >= rat.zero, "time_units_per_1k_tokens must be non-negative") + return (ctx.gen_per_time_unit or rat.zero) * rate / rat.new(1000) end local function get_or_create_stats(ctx, provider, model) @@ -241,11 +246,12 @@ local function run_candidate(ctx, mapped_prompt, timeout, request) } lib.log { level = "trace", message = "calling exec_prompt_in_provider", request = call } + local token_price = gen_per_token(ctx, call.provider, call.model) local success, result = pcall(exec_update_policy_data, ctx, call, function(res) -- convert token usage into gen using the per-model rate, and report it -- back as the gen consumed by this call (charged to the host as fuel) local total_tokens = (res.tokens and res.tokens.total) or 0 - local consumed_gen = rat.new(total_tokens) * gen_per_token(ctx, call.provider, call.model) + local consumed_gen = (rat.new(total_tokens) * token_price):ceil() res.consumed_gen = consumed_gen return consumed_gen end) From fa02df086acba387ee76bba952a4a2d92ddb4d26 Mon Sep 17 00:00:00 2001 From: kp2pml30 <kp2pml30@gmail.com> Date: Thu, 24 Sep 2026 21:01:24 +0900 Subject: [PATCH 23/45] =?UTF-8?q?fix(fees):=20flatten=20allocation=20input?= =?UTF-8?q?s=20and=20correct=20executor=20funding=20=F0=9F=90=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The host supplies descendant funding and exact subtree transport bytes so pinned storage modes and proofs can be handled without executor ABI encoding. Both executor lines use the same input shape; coordinate the node producer upgrade. --- crates/modules-interfaces/src/domain.rs | 2 +- .../modules-interfaces/src/domain/fees/abi.rs | 170 ------------------ .../modules-interfaces/src/domain/fees/mod.rs | 60 ++----- docs/website/src/impl-spec/04-fees.rst | 44 ++++- .../src/impl-spec/appendix/manager-api.yaml | 31 +++- executors/v0.2.x | 2 +- executors/v0.3.x | 2 +- implementation/src/manager/run.rs | 2 +- .../runner/genvm_tool_plugins/integration.py | 5 +- tests/runner/origin/fees.py | 22 +-- 10 files changed, 103 insertions(+), 237 deletions(-) delete mode 100644 crates/modules-interfaces/src/domain/fees/abi.rs diff --git a/crates/modules-interfaces/src/domain.rs b/crates/modules-interfaces/src/domain.rs index 02e37de6..9b58650e 100644 --- a/crates/modules-interfaces/src/domain.rs +++ b/crates/modules-interfaces/src/domain.rs @@ -45,7 +45,7 @@ pub struct ExecutionData { pub bucket_totals: std::collections::BTreeMap<String, num_bigint::BigInt>, /// Host-provided `node` fee constants (moved off `host_data`). pub gas_data: std::collections::BTreeMap<String, String>, - /// Message-fee allocation tree passed alongside the execution. + /// Flat allocations with host-encoded subtree payloads. pub message_fee_allocation: Vec<fees::MessageAllocationNode>, /// Initial time-unit budget for this execution. pub initial_time_units_allocation: u32, diff --git a/crates/modules-interfaces/src/domain/fees/abi.rs b/crates/modules-interfaces/src/domain/fees/abi.rs deleted file mode 100644 index 778c9cf0..00000000 --- a/crates/modules-interfaces/src/domain/fees/abi.rs +++ /dev/null @@ -1,170 +0,0 @@ -//! EVM ABI encoding of the message-fee allocation tree into the chain's flat -//! parent-pointer `MessageAllocationNode[]` representation. - -use crate::On; -use primitive_types::U256; - -use super::{ - ExternalMessageParams, InternalMessageParams, MessageAllocationNode, - MessageAllocationNodeParams, -}; - -/// Parent pointer used by top-level (root-layer) allocation nodes in the flat -/// chain representation: `type(uint256).max`. Any other value is a 0-based index -/// into the submitted array. -const NODE_ROOT_SENTINEL: U256 = U256::MAX; - -/// Solidity `enum MessageType { External, Internal }`. -const MESSAGE_TYPE_EXTERNAL: u64 = 0; -const MESSAGE_TYPE_INTERNAL: u64 = 1; - -/// Number of head words in the Solidity `MessageAllocationNode` tuple -/// (messageType, onAcceptance, parentIndex, recipient, callKey, budget, feeParams). -const NODE_HEAD_WORDS: usize = 7; - -/// Number of head words in the Solidity `InternalMessageParams` tuple -/// (leader, validator, appealRounds, executionBudgetPerRound, rotations, -/// maxPriceGenPerTimeUnit, storageFeeMaxGasPrice, receiptFeeMaxGasPrice). -/// `rotations` is the only dynamic field; the three price caps (v0.6-dev, -/// CON-549) are static and follow it in the head region. -const INTERNAL_PARAMS_HEAD_WORDS: usize = 8; - -fn push_u256(buf: &mut Vec<u8>, value: U256) { - buf.extend_from_slice(&value.to_big_endian()); -} - -fn push_word(buf: &mut Vec<u8>, value: u64) { - push_u256(buf, U256::from(value)); -} - -fn push_bool(buf: &mut Vec<u8>, value: bool) { - push_word(buf, value as u64); -} - -/// Left-pads a 20-byte address into a 32-byte ABI word. -fn push_address(buf: &mut Vec<u8>, addr: Option<&genlayer_calldata::Address>) { - buf.extend_from_slice(&[0u8; 12]); - match addr { - Some(addr) => buf.extend_from_slice(&addr.raw()), - None => buf.extend_from_slice(&[0u8; 20]), - } -} - -/// `callKey` is already a 32-byte word; `None` is `CALL_KEY_WILDCARD`. -fn push_call_key(buf: &mut Vec<u8>, call_key: Option<&crate::abi_stub::CallKey>) { - match call_key { - Some(ck) => buf.extend_from_slice(&ck.0), - None => buf.extend_from_slice(&super::CALL_KEY_WILDCARD.0), - } -} - -/// Encodes the transport payload consumed by `decodeAllocationSubtree`. -pub(super) fn encode(root: &MessageAllocationNode) -> Vec<u8> { - let mut flat: Vec<(&MessageAllocationNode, U256)> = Vec::new(); - let mut queue = std::collections::VecDeque::from([(root, NODE_ROOT_SENTINEL)]); - while let Some((node, parent_index)) = queue.pop_front() { - let node_index = U256::from(flat.len()); - flat.push((node, parent_index)); - for child in &node.children { - queue.push_back((child, node_index)); - } - } - - let elements: Vec<Vec<u8>> = flat - .iter() - .map(|(node, parent_index)| encode_node(node, *parent_index)) - .collect(); - - let mut buf = Vec::new(); - push_word(&mut buf, 0x20); - - push_word(&mut buf, elements.len() as u64); - let mut offset = elements.len() * 32; - for element in &elements { - push_word(&mut buf, offset as u64); - offset += element.len(); - } - for element in &elements { - buf.extend_from_slice(element); - } - - buf -} - -/// Encodes a single node as the Solidity `MessageAllocationNode` dynamic -/// tuple (offsets are relative to the start of this tuple). -fn encode_node(node: &MessageAllocationNode, parent_index: U256) -> Vec<u8> { - let (message_type, on_acceptance, fee_params) = match &node.fee_params { - MessageAllocationNodeParams::Internal(params) => ( - MESSAGE_TYPE_INTERNAL, - matches!(node.on, On::Decided), - encode_internal_params(params), - ), - // External messages have no acceptance/finalize lifecycle. - MessageAllocationNodeParams::External(params) => { - (MESSAGE_TYPE_EXTERNAL, false, encode_external_params(params)) - } - }; - - let mut buf = Vec::new(); - push_word(&mut buf, message_type); - push_bool(&mut buf, on_acceptance); - push_u256(&mut buf, parent_index); - push_address(&mut buf, node.recipient.as_ref()); - push_call_key(&mut buf, node.call_key.as_ref()); - push_u256(&mut buf, node.budget); - // `feeParams` is the only dynamic field; its offset follows the head words. - push_word(&mut buf, (NODE_HEAD_WORDS * 32) as u64); - // Tail: `bytes feeParams` = length + 32-aligned data. - push_word(&mut buf, fee_params.len() as u64); - buf.extend_from_slice(&fee_params); - let pad = fee_params.len().next_multiple_of(32) - fee_params.len(); - buf.extend(std::iter::repeat_n(0u8, pad)); - - buf -} - -/// `abi.encode(InternalMessageParams)` -- a dynamic tuple (contains -/// `uint256[] rotations`), so it is prefixed with the offset word. `appealRounds` -/// is reconstructed as `len(rotations) - 1` (the chain-derived value). -/// -/// Field order matches the chain's `InternalMessageFeeParams` (v0.6-dev): -/// leader, validator, appealRounds, executionBudgetPerRound, rotations, -/// maxPriceGenPerTimeUnit, storageFeeMaxGasPrice, receiptFeeMaxGasPrice. The -/// three price caps are static and sit in the head *after* the `rotations` -/// offset word, so the rotations tail starts past all 8 head words. -fn encode_internal_params(params: &InternalMessageParams) -> Vec<u8> { - let appeal_rounds = U256::from(params.rotations.len().saturating_sub(1) as u64); - - let mut buf = Vec::new(); - // `abi.encode` of a single dynamic struct prefixes the tuple with its offset. - push_word(&mut buf, 0x20); - push_u256(&mut buf, params.leader_timeunits_allocation); - push_u256(&mut buf, params.validator_timeunits_allocation); - push_u256(&mut buf, appeal_rounds); - push_u256(&mut buf, params.execution_budget_per_round); - // `rotations` is the only dynamic field; its offset follows all head words. - push_word(&mut buf, (INTERNAL_PARAMS_HEAD_WORDS * 32) as u64); - // Static price caps occupy head words 6-8 (between the rotations offset and - // the rotations tail), mirroring the chain struct field order. - push_u256(&mut buf, params.max_price_gen_per_time_unit); - push_u256(&mut buf, params.storage_fee_max_gas_price); - push_u256(&mut buf, params.receipt_fee_max_gas_price); - // Tail: `rotations` length + data, after the 8 head words. - push_word(&mut buf, params.rotations.len() as u64); - for rotation in ¶ms.rotations { - push_u256(&mut buf, *rotation); - } - - buf -} - -/// `abi.encode(ExternalMessageParams)` -- a fully static tuple, so it is just -/// the two inline words with no leading offset. -fn encode_external_params(params: &ExternalMessageParams) -> Vec<u8> { - let mut buf = Vec::new(); - push_u256(&mut buf, params.gas_limit); - push_u256(&mut buf, params.max_gas_price); - - buf -} diff --git a/crates/modules-interfaces/src/domain/fees/mod.rs b/crates/modules-interfaces/src/domain/fees/mod.rs index bea5819b..a533d37e 100644 --- a/crates/modules-interfaces/src/domain/fees/mod.rs +++ b/crates/modules-interfaces/src/domain/fees/mod.rs @@ -1,14 +1,7 @@ -pub mod abi; - use primitive_types::U256; use crate::On; -pub const CALL_KEY_WILDCARD: crate::abi_stub::CallKey = crate::abi_stub::CallKey([ - 0xc5, 0xd2, 0x46, 0x01, 0x86, 0xf7, 0x23, 0x3c, 0x92, 0x7e, 0x7d, 0xb2, 0xdc, 0xc7, 0x03, 0xc0, - 0xe5, 0x00, 0xb6, 0x53, 0xca, 0x82, 0x27, 0x3b, 0x7b, 0xfa, 0xd8, 0x04, 0x5d, 0x85, 0xa4, 0x70, -]); - #[derive( Debug, Clone, @@ -26,12 +19,7 @@ pub struct InternalMessageParams { /// Per-round rotation allocations; `rotations[0]` is the initial round, the /// rest are appeal rounds. Must be non-empty. /// - /// The chain's `InternalMessageFeeParams` carries an explicit `appealRounds` - /// field, but it is not stored here: the chain enforces - /// `appealRounds == rotations.length - 1` (`FeesVerifier.InvalidAppealRounds`), - /// so we derive `appeal_rounds = rotations.len() - 1` instead. The ABI - /// encoder re-inserts it so the encoded bytes (and their `keccak256` fee-param - /// pin) match the chain's field layout. + /// The fee evaluator derives `appeal_rounds = rotations.len() - 1`. pub rotations: Vec<U256>, /// Per-time-unit GEN price cap locked at activation (consensus CON-549, /// v0.6-dev). The chain charges at this cap as the funding multiplier and @@ -75,7 +63,7 @@ pub enum MessageAllocationNodeParams { External(ExternalMessageParams), } -/// One node of the message-fee allocation tree. +/// One allocation available to messages emitted by this execution. #[derive( Debug, Clone, @@ -87,25 +75,19 @@ pub enum MessageAllocationNodeParams { pub struct MessageAllocationNode { /// Target contract address; `None` means wildcard (any recipient). pub recipient: Option<genlayer_calldata::Address>, - /// `None` = wildcard: all call keys for this recipient - /// (chain sentinel: `CALL_KEY_WILDCARD` = `keccak256("")`). + /// `None` means any call key; the node converts the chain's wildcard sentinel. pub call_key: Option<crate::abi_stub::CallKey>, - /// Max budget for matching messages. - pub budget: U256, + /// Available allowance; zero is exhausted, `None` is uncapped. + pub budget: Option<U256>, pub on: On, - /// Same structure as TX-level params. pub fee_params: MessageAllocationNodeParams, - pub children: Vec<MessageAllocationNode>, + /// Sum of the direct descendants' allocation budgets, funded per emission. + pub children_budget: U256, + /// Host-encoded matched subtree, including any proof required by the chain. + pub subtree: bytes::Bytes, } impl MessageAllocationNode { - /// ABI-encodes this matched node and its descendants for transport to the chain. - /// The matched node is element 0 and descendants are in BFS order. - pub fn abi_encode(&self) -> Vec<u8> { - abi::encode(self) - } - - #[allow(clippy::if_same_then_else)] pub fn matches_internal( &self, on: On, @@ -115,18 +97,12 @@ impl MessageAllocationNode { let MessageAllocationNodeParams::Internal(params) = &self.fee_params else { return None; }; - if on != self.on { - None - } else if self.recipient.as_ref().is_some_and(|r| *r != recipient) { - None - } else if self.call_key.as_ref().is_some_and(|ck| *ck != call_key) { - None - } else { - Some(params.clone()) - } + (on == self.on + && self.recipient.is_none_or(|r| r == recipient) + && self.call_key.is_none_or(|ck| ck == call_key)) + .then(|| params.clone()) } - #[allow(clippy::if_same_then_else)] pub fn matches_external( &self, recipient: genlayer_calldata::Address, @@ -135,12 +111,8 @@ impl MessageAllocationNode { let MessageAllocationNodeParams::External(params) = &self.fee_params else { return None; }; - if self.recipient.as_ref().is_some_and(|r| *r != recipient) { - None - } else if self.call_key.as_ref().is_some_and(|ck| *ck != call_key) { - None - } else { - Some(*params) - } + (self.recipient.is_none_or(|r| r == recipient) + && self.call_key.is_none_or(|ck| ck == call_key)) + .then_some(*params) } } diff --git a/docs/website/src/impl-spec/04-fees.rst b/docs/website/src/impl-spec/04-fees.rst index 9b2212af..f27a1898 100644 --- a/docs/website/src/impl-spec/04-fees.rst +++ b/docs/website/src/impl-spec/04-fees.rst @@ -286,21 +286,38 @@ after the exact key; executor-only open-bucket recipient wildcards are less spec than either. List order therefore cannot make a wildcard shadow an exact allocation. For internal messages, ``on`` is checked after the allocation key is resolved, so a -phase mismatch on an exact allocation does not fall through to a wildcard. For +phase mismatch on an exact allocation does not fall through to a wildcard. Chain +keys are unique across phases; only synthetic recipient-wildcard entries may +select different fee parameters by phase. For external messages, an exhausted exact allocation spills to the per-recipient ``call_key`` wildcard. If neither key has an allocation, the external message uses the legacy unallocated path and consumes only its receipt cost. Existing but exhausted candidates yield an allocation-budget error. +Entry presence determines matching, independently of ``budget``. The host supplies +the allowance available at execution start, optionally reduced by prior consumption. +Zero means an exhausted allocation; ``null`` removes the per-allocation cap while +keeping the execution's fee buckets. Local consumption is tracked separately. +An exhausted internal exact key still wins and fails its budget check; it never +falls through. An absent chain allocation (including a chain record whose original +budget is zero) must be omitted, not represented by an exhausted entry. + +The host must preserve every existing pinned key, including exhausted keys, and +must not add recipient wildcards to a pinned tree. An empty list restricts internal +pool-funded emissions. Open-pool and view executions may supply synthetic recipient +wildcards with concrete fee parameters and phase, and optionally uncapped budgets. +The executor conservatively treats each emission as novel; remaining allowances +alone do not identify previously delivered occurrences. + Funding modes ~~~~~~~~~~~~~~ An outgoing internal message is funded one of two ways: - **Allocation-matched (default).** The fee is matched against the allocation - tree as above. After the expression computes the primary reserve, the executor - adds the budgets of the matched node's direct children; deeper budgets are - already contained by their direct parent. A declared budget exceeding the + list as above. After the expression computes the primary reserve, the executor + adds the host-supplied ``children_budget``; deeper budgets are already + contained by their direct parent. A declared budget exceeding the matched node's remaining ``budget`` is rejected with an allocation-budget error; otherwise it consumes ``message_fee`` atomically with the ``message_receipt`` charge. @@ -323,6 +340,25 @@ For balance funding the sum is zero. The primary fee already covers the child's configured lifecycle, including appeals; the remainder becomes the child's message-fee bucket. External messages declare zero +The v0.3 primary reserve includes successful-appellant profit for each configured +appeal slot. The bond is the next normal round's time-unit cost, including its +rotations, multiplied by the child's price cap. Its profit reserve is +``bond + floor(bond / 2)``; the developer/DAO overlay applies only to time-unit +work, never to this profit reserve + +The manager input ``message_fee_allocation`` contains only the allocations matched +by this execution. Each carries a required ``children_budget`` and opaque +``subtree`` bytes. The host sums the direct descendants' budgets and encodes +the matched subtree with any proof required by the transaction's pinned +storage mode. Both executors forward those bytes unchanged; v0.3 charges their +full length against receipt gas, submitted-message bytes and memory. The subtree +includes the matched root; it is not merely an encoding of its descendants + +``children_budget`` funds onward messages per emission; it is +not the unspent allocation allowance or consumption from earlier generations. +All v0.3 internal emissions reject zero time-unit, storage or receipt price caps +with ``Inval`` before charging fees or appending an emission + The parent allocation's aggregate capacity is a separate invariant. If ``L = appealRounds + 1`` novel executions may each emit a child carrying budget ``C``, preserving that descendant capacity for every execution requires:: diff --git a/docs/website/src/impl-spec/appendix/manager-api.yaml b/docs/website/src/impl-spec/appendix/manager-api.yaml index b90ea207..36e5bc79 100644 --- a/docs/website/src/impl-spec/appendix/manager-api.yaml +++ b/docs/website/src/impl-spec/appendix/manager-api.yaml @@ -879,10 +879,37 @@ components: type: array items: type: object + required: [recipient, call_key, budget, on, fee_params, children_budget, subtree] + properties: + recipient: + nullable: true + description: Recipient address, or null for an open allocation. + call_key: + nullable: true + description: Call key bytes, or null for a wildcard. + budget: + nullable: true + description: Available allowance at execution start; zero is exhausted, null is uncapped. Does not affect key matching. + on: + type: string + enum: [decided, finalized] + fee_params: + type: object + description: Internal or External fee parameters. + children_budget: + description: Sum of direct descendant allocation budgets, funded per emitted child. + subtree: + description: Opaque bytes for the matched subtree, including any required Merkle proof. default: [] description: | - Message-fee allocation tree. See `MessageAllocationNode` in - `crates/modules-interfaces/src/domain/fees/mod.rs` for the node shape. + Flat funding allocations. The node supplies the exact subtree transport + bytes for the transaction's pinned storage mode. GenVM forwards these bytes + unchanged and meters their length. External and open allocations without a + committed subtree use empty bytes. children_budget and subtree are required + on every entry, including allocations without descendants. + Include exhausted allocations with budget zero to preserve internal key + precedence; omit allocations absent on-chain. Recipient wildcards are + synthetic entries for open-pool or view execution, not chain allocations. record_actions: type: array items: diff --git a/executors/v0.2.x b/executors/v0.2.x index 75e4d1fd..b08736a2 160000 --- a/executors/v0.2.x +++ b/executors/v0.2.x @@ -1 +1 @@ -Subproject commit 75e4d1fd5945aadea065f6928e60155ffca63797 +Subproject commit b08736a2d9d1292c6a5c50a2d1b4b12530986a4d diff --git a/executors/v0.3.x b/executors/v0.3.x index e813da59..624aecf9 160000 --- a/executors/v0.3.x +++ b/executors/v0.3.x @@ -1 +1 @@ -Subproject commit e813da599329bb848efd058d9e20ae660d740ebe +Subproject commit 624aecf98eb5f1c8f045201e93b88a70f71c1a8c diff --git a/implementation/src/manager/run.rs b/implementation/src/manager/run.rs index 3c2651a4..b1938ce8 100644 --- a/implementation/src/manager/run.rs +++ b/implementation/src/manager/run.rs @@ -1013,7 +1013,7 @@ pub struct Request { #[serde(default)] #[calldata(default = default_gas_data)] pub gas_data: std::collections::BTreeMap<String, String>, - /// Message-fee allocation tree passed alongside the execution. + /// Flat allocations with opaque subtree payloads. #[serde(default)] #[calldata(default = default_message_fee_allocation)] pub message_fee_allocation: Vec<genvm_modules_interfaces::fees::MessageAllocationNode>, diff --git a/tests/runner/genvm_tool_plugins/integration.py b/tests/runner/genvm_tool_plugins/integration.py index d1173785..89d26085 100644 --- a/tests/runner/genvm_tool_plugins/integration.py +++ b/tests/runner/genvm_tool_plugins/integration.py @@ -90,9 +90,8 @@ def _normalize_message_fee_allocation( allocation = allocation.copy() if isinstance(allocation['recipient'], str): allocation['recipient'] = Address(allocation['recipient']) - allocation['children'] = [ - _normalize_message_fee_allocation(child) for child in allocation['children'] - ] + if isinstance(allocation['subtree'], list): + allocation['subtree'] = bytes(allocation['subtree']) return allocation diff --git a/tests/runner/origin/fees.py b/tests/runner/origin/fees.py index 156748f8..3345fdc4 100644 --- a/tests/runner/origin/fees.py +++ b/tests/runner/origin/fees.py @@ -1,8 +1,8 @@ """ -Message-fee allocation tree types. +Message-fee allocation types. -Mirrors the executor's `genvm_common::domain::fees` module: the fee parameters -and the nested `MessageAllocationNode` tree that is passed alongside an +Mirrors the shared `genvm_modules_interfaces::fees` module: the fee parameters +and the flat allocations that are passed alongside an execution and matched against emitted messages. """ @@ -43,13 +43,12 @@ class _ExternalParams(typing.TypedDict): class MessageAllocationNode(typing.TypedDict): recipient: Address | None call_key: bytes | None - budget: int + budget: int | None # Lifecycle the node matches against (only meaningful for internal messages). on: typing.Literal['finalized', 'decided'] fee_params: MessageAllocationNodeParams - # Nested allocation subtree; the chain receives this flattened to - # parent-pointer form. - children: list['MessageAllocationNode'] + children_budget: int + subtree: bytes DEFAULT_EXTERNAL_MESSAGE_ALLOC: MessageAllocationNode = { @@ -64,7 +63,8 @@ class MessageAllocationNode(typing.TypedDict): 'max_gas_price': 0, }, }, - 'children': [], + 'children_budget': 0, + 'subtree': b'', } DEFAULT_INTERNAL_DEC_MESSAGE_ALLOC: MessageAllocationNode = { @@ -83,7 +83,8 @@ class MessageAllocationNode(typing.TypedDict): 'receipt_fee_max_gas_price': 2**200, }, }, - 'children': [], + 'children_budget': 0, + 'subtree': b'', } DEFAULT_INTERNAL_FIN_MESSAGE_ALLOC: MessageAllocationNode = { @@ -102,5 +103,6 @@ class MessageAllocationNode(typing.TypedDict): 'receipt_fee_max_gas_price': 20, }, }, - 'children': [], + 'children_budget': 0, + 'subtree': b'', } From 9bf8e8ff214fa37fb811fda470fee42061c516de Mon Sep 17 00:00:00 2001 From: kp2pml30 <kp2pml30@gmail.com> Date: Sun, 27 Sep 2026 22:54:14 +0900 Subject: [PATCH 24/45] =?UTF-8?q?fix(executor):=20classify=20v0.3=20leader?= =?UTF-8?q?=20nondet=20timeout=20as=20leader=20fault=20=F0=9F=90=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../03-wasi_genlayer_sdk/03-schemas.rst | 2 ++ executors/v0.3.x | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/03-schemas.rst b/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/03-schemas.rst index 51bb6afa..4ff7213a 100644 --- a/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/03-schemas.rst +++ b/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/03-schemas.rst @@ -56,6 +56,8 @@ disagreement without running the comparison stage :ref:`gvm-def-str-trie-vm-error` path, including the canonical spelling of any parameter. Codes outside the trie yield :ref:`gvm-def-str-trie-value-vm-error-leader-fault-nondet-output-malformed`. +#. A proposed :ref:`gvm-def-str-trie-value-vm-error-timeout` yields + :ref:`gvm-def-str-trie-value-vm-error-leader-fault-nondet-output-malformed`. An accepted result that passes :ref:`gvm-def-nondeterministic-output-caps` is preserved **byte for byte**, so every node hashes the value that was proposed. diff --git a/executors/v0.3.x b/executors/v0.3.x index 624aecf9..dc7b4671 160000 --- a/executors/v0.3.x +++ b/executors/v0.3.x @@ -1 +1 @@ -Subproject commit 624aecf98eb5f1c8f045201e93b88a70f71c1a8c +Subproject commit dc7b4671ada529fdf200e91c79e33d9e6a36eca5 From 5ec0d658142624ea0318bce52628ef1a2929cb22 Mon Sep 17 00:00:00 2001 From: kp2pml30 <kp2pml30@gmail.com> Date: Tue, 29 Sep 2026 02:13:18 +0900 Subject: [PATCH 25/45] =?UTF-8?q?docs(executor):=20describe=20startup=20an?= =?UTF-8?q?d=20receipt=20fee=20rules=20=F0=9F=93=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/website/src/impl-spec/04-fees.rst | 4 ++++ docs/website/src/spec/03-vm/05-result.rst | 3 +++ executors/v0.3.x | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/website/src/impl-spec/04-fees.rst b/docs/website/src/impl-spec/04-fees.rst index f27a1898..86f616a0 100644 --- a/docs/website/src/impl-spec/04-fees.rst +++ b/docs/website/src/impl-spec/04-fees.rst @@ -294,6 +294,10 @@ external messages, an exhausted exact allocation spills to the per-recipient the legacy unallocated path and consumes only its receipt cost. Existing but exhausted candidates yield an allocation-budget error. +The default v0.3 fee expressions reject external reservations and receipts with +``fee below_minimum`` unless ``node.lockedReceiptGasPrice`` is positive, including +external messages without a matching allocation + Entry presence determines matching, independently of ``budget``. The host supplies the allowance available at execution start, optionally reduced by prior consumption. Zero means an exhausted allocation; ``null`` removes the per-allocation cap while diff --git a/docs/website/src/spec/03-vm/05-result.rst b/docs/website/src/spec/03-vm/05-result.rst index 39aa88eb..56a186e8 100644 --- a/docs/website/src/spec/03-vm/05-result.rst +++ b/docs/website/src/spec/03-vm/05-result.rst @@ -248,3 +248,6 @@ parameter is the first 6 characters of the :ref:`gvm-def-gvm32` encoding of the otherwise have returned β€” the whole wire buffer as emitted, not a decoded payload or alternate representation. No non-deterministic disagreement is caused by this. + +This check also applies when startup fails before any non-deterministic block, +including when initial fees cannot be paid. An existing fatal result is preserved. diff --git a/executors/v0.3.x b/executors/v0.3.x index dc7b4671..7b5633f8 160000 --- a/executors/v0.3.x +++ b/executors/v0.3.x @@ -1 +1 @@ -Subproject commit dc7b4671ada529fdf200e91c79e33d9e6a36eca5 +Subproject commit 7b5633f86d8bf3edcf1b3d0538673c74e4653d1c From b1f21b8f54846d29fde12edb612cd3618c707142 Mon Sep 17 00:00:00 2001 From: Edgars <edgars@genlayerlabs.com> Date: Sun, 27 Sep 2026 17:57:22 +0100 Subject: [PATCH 26/45] =?UTF-8?q?chore(modules):=20expose=20wrapped=20budg?= =?UTF-8?q?et=20timeout=20=E2=9C=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- implementation/src/llm/handler.rs | 4 +++ implementation/src/llm/handler_test.rs | 42 ++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 implementation/src/llm/handler_test.rs diff --git a/implementation/src/llm/handler.rs b/implementation/src/llm/handler.rs index 76b843ab..ae80a790 100644 --- a/implementation/src/llm/handler.rs +++ b/implementation/src/llm/handler.rs @@ -20,6 +20,10 @@ pub struct Inner { genvm_id: genvm_modules_interfaces::GenVMId, } +#[cfg(test)] +#[path = "handler_test.rs"] +mod tests; + type LlmSubContext = crate::manager::execution_context::LlmSubContext; pub struct Provider { diff --git a/implementation/src/llm/handler_test.rs b/implementation/src/llm/handler_test.rs new file mode 100644 index 00000000..8138cb28 --- /dev/null +++ b/implementation/src/llm/handler_test.rs @@ -0,0 +1,42 @@ +use super::Inner; + +#[tokio::test] +async fn budget_exhaustion_from_lua_callback_becomes_timeout_sentinel() -> anyhow::Result<()> { + let direct = Inner::try_catch_budget_exhausted(anyhow::Error::from(mlua::Error::external( + crate::common::BudgetExhausted, + )))?; + assert_eq!(direct.consumed_gen, primitive_types::U256::MAX); + + let lua = mlua::Lua::new(); + lua.globals().set( + "exhaust", + lua.create_async_function(|_, ()| async { + Err::<(), mlua::Error>(mlua::Error::external(crate::common::BudgetExhausted)) + })?, + )?; + + let call = lua + .load("return function() exhaust() end") + .eval::<mlua::Function>()?; + let err = call + .call_async::<()>(()) + .await + .expect_err("budget must exhaust"); + assert!( + matches!( + err, + mlua::Error::CallbackError { ref cause, .. } + if matches!(cause.as_ref(), mlua::Error::ExternalError(ext) + if ext.downcast_ref::<crate::common::BudgetExhausted>().is_some()) + ), + "expected Lua callback to wrap BudgetExhausted, got {err:?}" + ); + + let answer = Inner::try_catch_budget_exhausted(anyhow::Error::from(err))?; + assert_eq!(answer.consumed_gen, primitive_types::U256::MAX); + assert!(matches!( + answer.data, + genvm_modules_interfaces::llm::PromptAnswerData::Text(ref text) if text.is_empty() + )); + Ok(()) +} From cc35474cbc8055554bd1a4b96f9d2ed0c2c02099 Mon Sep 17 00:00:00 2001 From: kp2pml30 <kp2pml30@gmail.com> Date: Tue, 29 Sep 2026 13:47:44 +0900 Subject: [PATCH 27/45] =?UTF-8?q?fix(modules):=20catch=20budget=20exhausti?= =?UTF-8?q?on=20under=20any=20Lua=20error=20wrapper=20=F0=9F=90=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Walk the whole error chain for `BudgetExhausted`, including `CallbackError`, `BadArgument`, `WithContext` and the `Arc`s `scripting::call_fn` unwraps to - Cover a `pcall` rethrow, both `call_fn` shapes and an unrelated callback error --- implementation/src/llm/handler.rs | 43 +++++++++-------- implementation/src/llm/handler_test.rs | 65 ++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 20 deletions(-) diff --git a/implementation/src/llm/handler.rs b/implementation/src/llm/handler.rs index ae80a790..eb599f11 100644 --- a/implementation/src/llm/handler.rs +++ b/implementation/src/llm/handler.rs @@ -184,28 +184,31 @@ impl Inner { Ok(llm_iface::PromptAnswer { data, consumed_gen }) } - fn try_catch_budget_exhausted(err: anyhow::Error) -> ModuleResult<llm_iface::PromptAnswer> { - if err - .downcast_ref::<crate::common::BudgetExhausted>() - .is_some() - { - return Ok(llm_iface::PromptAnswer { - data: llm_iface::PromptAnswerData::Text(String::new()), - consumed_gen: primitive_types::U256::MAX, - }); + fn is_budget_exhausted(err: &(dyn std::error::Error + 'static)) -> bool { + // `scripting::call_fn` unwraps top-level `ExternalError`/`WithContext` into an `Arc` + if let Some(ext) = err.downcast_ref::<Arc<dyn std::error::Error + Send + Sync>>() { + return Self::is_budget_exhausted(&**ext); } - if let Some(mlua::Error::ExternalError(ext)) = err.downcast_ref::<mlua::Error>() { - if ext - .downcast_ref::<crate::common::BudgetExhausted>() - .is_some() - { - return Ok(llm_iface::PromptAnswer { - data: llm_iface::PromptAnswerData::Text(String::new()), - consumed_gen: primitive_types::U256::MAX, - }); - } + if let Some(cause) = err.downcast_ref::<Arc<mlua::Error>>() { + return Self::is_budget_exhausted(&**cause); + } + match err.downcast_ref::<mlua::Error>() { + // walks `CallbackError`, `BadArgument`, `WithContext` and `ExternalError` wrappers + Some(lua_err) => lua_err + .chain() + .any(|e| e.is::<crate::common::BudgetExhausted>()), + None => err.is::<crate::common::BudgetExhausted>(), + } + } + + fn try_catch_budget_exhausted(err: anyhow::Error) -> ModuleResult<llm_iface::PromptAnswer> { + if !err.chain().any(Self::is_budget_exhausted) { + return Err(err); } - Err(err) + Ok(llm_iface::PromptAnswer { + data: llm_iface::PromptAnswerData::Text(String::new()), + consumed_gen: primitive_types::U256::MAX, + }) } async fn exec_prompt( diff --git a/implementation/src/llm/handler_test.rs b/implementation/src/llm/handler_test.rs index 8138cb28..b3928d88 100644 --- a/implementation/src/llm/handler_test.rs +++ b/implementation/src/llm/handler_test.rs @@ -40,3 +40,68 @@ async fn budget_exhaustion_from_lua_callback_becomes_timeout_sentinel() -> anyho )); Ok(()) } + +#[tokio::test] +async fn budget_exhaustion_survives_lua_rethrow() -> anyhow::Result<()> { + let lua = mlua::Lua::new(); + lua.globals().set( + "exhaust", + lua.create_function(|_, ()| -> mlua::Result<()> { + Err(mlua::Error::external(crate::common::BudgetExhausted)) + })?, + )?; + + let call = lua + .load( + r#" + local function inner() exhaust() end + return function() + local ok, err = pcall(inner) + if not ok then error(err, 0) end + end + "#, + ) + .eval::<mlua::Function>()?; + let err = call + .call_async::<()>(()) + .await + .expect_err("budget must exhaust"); + + let answer = Inner::try_catch_budget_exhausted(anyhow::Error::from(err))?; + assert_eq!(answer.consumed_gen, primitive_types::U256::MAX); + Ok(()) +} + +#[test] +fn budget_exhaustion_unwrapped_by_call_fn_becomes_timeout_sentinel() -> anyhow::Result<()> { + let ext: std::sync::Arc<dyn std::error::Error + Send + Sync> = + std::sync::Arc::new(crate::common::BudgetExhausted); + let answer = Inner::try_catch_budget_exhausted(anyhow::Error::from(ext))?; + assert_eq!(answer.consumed_gen, primitive_types::U256::MAX); + + let cause = std::sync::Arc::new(mlua::Error::CallbackError { + traceback: String::new(), + cause: std::sync::Arc::new(mlua::Error::external(crate::common::BudgetExhausted)), + }); + let answer = + Inner::try_catch_budget_exhausted(anyhow::Error::from(cause).context("exec_prompt"))?; + assert_eq!(answer.consumed_gen, primitive_types::U256::MAX); + Ok(()) +} + +#[tokio::test] +async fn unrelated_lua_callback_error_is_not_budget_exhaustion() -> anyhow::Result<()> { + let lua = mlua::Lua::new(); + lua.globals().set( + "fail", + lua.create_function(|_, ()| -> mlua::Result<()> { Err(mlua::Error::runtime("boom")) })?, + )?; + + let call = lua + .load("return function() fail() end") + .eval::<mlua::Function>()?; + let err = call.call_async::<()>(()).await.expect_err("must fail"); + + assert!(Inner::try_catch_budget_exhausted(anyhow::Error::from(err)).is_err()); + Ok(()) +} From bb912a01e1e707d85cbc83b5f6728661568d488a Mon Sep 17 00:00:00 2001 From: Kanan <93033289+kriss39@users.noreply.github.com> Date: Wed, 16 Sep 2026 23:11:52 +0400 Subject: [PATCH 28/45] =?UTF-8?q?fix(lua):=20substitute=20prompt=20templat?= =?UTF-8?q?e=20placeholders=20in=20one=20pass=20=F0=9F=94=92=EF=B8=8F?= =?UTF-8?q?=F0=9F=90=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- implementation/tests/prompt_templates.rs | 88 ++++++++++++++++++++++++ install/lib/genvm-lua/lib-llm.lua | 12 ++-- 2 files changed, 95 insertions(+), 5 deletions(-) create mode 100644 implementation/tests/prompt_templates.rs diff --git a/implementation/tests/prompt_templates.rs b/implementation/tests/prompt_templates.rs new file mode 100644 index 00000000..e3cc3905 --- /dev/null +++ b/implementation/tests/prompt_templates.rs @@ -0,0 +1,88 @@ +// `exec_prompt_template_transform` in lib-llm.lua fills the equivalence +// principle templates with values that come from other validators (the +// leader's answer, its output). A value must land in the prompt verbatim: it +// may not be rescanned for placeholders, and the result may not depend on the +// iteration order of the variables. + +use mlua::prelude::*; + +const TEMPLATE: &str = "<primary_output>#{leader_answer}</primary_output>\ +<validator>#{validator_answer}</validator><rule>#{principle}</rule>#{unknown}"; + +fn load_lib_llm() -> Lua { + let vm = Lua::new(); + let lib_dir = std::path::PathBuf::from("../install/lib/genvm-lua") + .canonicalize() + .unwrap(); + let package: LuaTable = vm.globals().get("package").unwrap(); + package + .set("path", format!("{}/?.lua", lib_dir.display())) + .unwrap(); + + // Stand-ins for the globals the Rust side installs before the scripts load. + vm.load("__dflt = { log_json = function() end }") + .exec() + .unwrap(); + let templates = vm.create_table().unwrap(); + let eq = vm.create_table().unwrap(); + eq.set("system", "system").unwrap(); + eq.set("user", TEMPLATE).unwrap(); + templates.set("eq_comparative", eq).unwrap(); + let llm = vm.create_table().unwrap(); + llm.set("templates", templates).unwrap(); + llm.set("providers", vm.create_table().unwrap()).unwrap(); + vm.globals().set("__llm", llm).unwrap(); + vm +} + +fn transform(vm: &Lua, leader_answer: &str) -> String { + let code = r#" + local llm = require("lib-llm") + return function(leader_answer) + local mapped = llm.exec_prompt_template_transform({ + template = "EqComparative", + leader_answer = leader_answer, + validator_answer = "VALIDATOR", + principle = "identical", + }) + return mapped.prompt.user_message + end + "#; + let f: LuaFunction = vm.load(code).eval().unwrap(); + f.call(leader_answer).unwrap() +} + +#[test] +fn placeholders_are_substituted_once() { + let vm = load_lib_llm(); + assert_eq!( + transform(&vm, "LEADER"), + "<primary_output>LEADER</primary_output><validator>VALIDATOR</validator>\ +<rule>identical</rule>#{unknown}" + ); +} + +// The leader's answer contains another placeholder. Whatever order `pairs` +// visits the variables in, it must reach the judge as the literal text. +#[test] +fn values_are_not_rescanned_for_placeholders() { + for _ in 0..16 { + let vm = load_lib_llm(); + let out = transform(&vm, "#{validator_answer}"); + assert_eq!( + out, + "<primary_output>#{validator_answer}</primary_output>\ +<validator>VALIDATOR</validator><rule>identical</rule>#{unknown}" + ); + } +} + +#[test] +fn values_with_pattern_characters_are_literal() { + let vm = load_lib_llm(); + let out = transform(&vm, "100% sure, %1 and %%"); + assert!( + out.starts_with("<primary_output>100% sure, %1 and %%</primary_output>"), + "{out}" + ); +} diff --git a/install/lib/genvm-lua/lib-llm.lua b/install/lib/genvm-lua/lib-llm.lua index 1c801f3c..c0a4dc7f 100644 --- a/install/lib/genvm-lua/lib-llm.lua +++ b/install/lib/genvm-lua/lib-llm.lua @@ -274,11 +274,13 @@ M.exec_prompt_template_transform = function(args) local vars = shallow_copy(args) vars.template = nil - local as_user_text = my_template.user - for key, val in pairs(vars) do - local val_escaped = string.gsub(val, "%%", "%%%%") - as_user_text = string.gsub(as_user_text, "#{" .. key .. "}", val_escaped) - end + -- Substitute every placeholder in one pass. A replacement returned from + -- a function is not rescanned, so a value that itself contains `#{key}` + -- (a leader answer, for instance) is inserted literally instead of being + -- expanded by a later substitution. Unknown placeholders stay as they are. + local as_user_text = string.gsub(my_template.user, "#{([%w_]+)}", function(key) + return vars[key] + end) local format = my_data.format From 5519ae604824d507eef92cdb9b53f2b998b72e13 Mon Sep 17 00:00:00 2001 From: Kanan <93033289+kriss39@users.noreply.github.com> Date: Wed, 16 Sep 2026 23:11:41 +0400 Subject: [PATCH 29/45] =?UTF-8?q?fix(executor):=20zero-fill=20the=20code?= =?UTF-8?q?=20blob=20buffer=20instead=20of=20assume=5Finit=20=F0=9F=90=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- executors/v0.3.x | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/executors/v0.3.x b/executors/v0.3.x index 7b5633f8..16b81732 160000 --- a/executors/v0.3.x +++ b/executors/v0.3.x @@ -1 +1 @@ -Subproject commit 7b5633f86d8bf3edcf1b3d0538673c74e4653d1c +Subproject commit 16b81732e12117daf7ef0335cbd32fc7d326b16d From 619dfad4bae51c191ef66c3e8cac6ed996961858 Mon Sep 17 00:00:00 2001 From: kp2pml30 <kp2pml30@gmail.com> Date: Thu, 1 Oct 2026 00:19:18 +0900 Subject: [PATCH 30/45] =?UTF-8?q?fix(fees):=20treat=20zero-budget=20chain?= =?UTF-8?q?=20allocations=20as=20absent=20=F0=9F=90=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Match consensus allocation resolution in the v0.3 executor: an entry with a concrete recipient and a zero budget no longer matches, so a zero-budget exact key falls through to the per-recipient `call_key` wildcard and a zero-budget wildcard to no allocation - Describe the rule in the fee impl-spec and the manager API schema --- .../modules-interfaces/src/domain/fees/mod.rs | 2 +- docs/website/src/impl-spec/04-fees.rst | 27 ++++++++++--------- .../src/impl-spec/appendix/manager-api.yaml | 5 ++-- executors/v0.3.x | 2 +- 4 files changed, 19 insertions(+), 17 deletions(-) diff --git a/crates/modules-interfaces/src/domain/fees/mod.rs b/crates/modules-interfaces/src/domain/fees/mod.rs index a533d37e..9a97fe3e 100644 --- a/crates/modules-interfaces/src/domain/fees/mod.rs +++ b/crates/modules-interfaces/src/domain/fees/mod.rs @@ -77,7 +77,7 @@ pub struct MessageAllocationNode { pub recipient: Option<genlayer_calldata::Address>, /// `None` means any call key; the node converts the chain's wildcard sentinel. pub call_key: Option<crate::abi_stub::CallKey>, - /// Available allowance; zero is exhausted, `None` is uncapped. + /// Stored chain budget, or a synthetic wildcard's allowance; `None` is uncapped. pub budget: Option<U256>, pub on: On, pub fee_params: MessageAllocationNodeParams, diff --git a/docs/website/src/impl-spec/04-fees.rst b/docs/website/src/impl-spec/04-fees.rst index 86f616a0..fe7724a4 100644 --- a/docs/website/src/impl-spec/04-fees.rst +++ b/docs/website/src/impl-spec/04-fees.rst @@ -289,8 +289,8 @@ For internal messages, ``on`` is checked after the allocation key is resolved, s phase mismatch on an exact allocation does not fall through to a wildcard. Chain keys are unique across phases; only synthetic recipient-wildcard entries may select different fee parameters by phase. For -external messages, an exhausted exact allocation spills to the per-recipient -``call_key`` wildcard. If neither key has an allocation, the external message uses +external messages, an exact allocation without room for the reservation spills to +the per-recipient ``call_key`` wildcard. If neither key has a present allocation, the external message uses the legacy unallocated path and consumes only its receipt cost. Existing but exhausted candidates yield an allocation-budget error. @@ -298,16 +298,19 @@ The default v0.3 fee expressions reject external reservations and receipts with ``fee below_minimum`` unless ``node.lockedReceiptGasPrice`` is positive, including external messages without a matching allocation -Entry presence determines matching, independently of ``budget``. The host supplies -the allowance available at execution start, optionally reduced by prior consumption. -Zero means an exhausted allocation; ``null`` removes the per-allocation cap while -keeping the execution's fee buckets. Local consumption is tracked separately. -An exhausted internal exact key still wins and fails its budget check; it never -falls through. An absent chain allocation (including a chain record whose original -budget is zero) must be omitted, not represented by an exhausted entry. - -The host must preserve every existing pinned key, including exhausted keys, and -must not add recipient wildcards to a pinned tree. An empty list restricts internal +Chain entries, those with a concrete ``recipient``, carry the budget stored on +chain. Consensus resolves only keys with a nonzero budget, so a chain entry whose +``budget`` is zero is absent: the message falls through to the per-recipient +``call_key`` wildcard, and with no wildcard left an internal message fails with +``fee no_matching_allocation`` while an external one takes the unallocated path. +Synthetic recipient wildcards match independently of ``budget``; zero there means +an exhausted allocation. ``null`` removes the per-allocation cap while keeping the +execution's fee buckets. Local consumption is tracked separately: an internal key +exhausted by this execution's own emissions still wins and fails its budget check; +it never falls through. + +The host must preserve every existing pinned key and must not add recipient +wildcards to a pinned tree. An empty list restricts internal pool-funded emissions. Open-pool and view executions may supply synthetic recipient wildcards with concrete fee parameters and phase, and optionally uncapped budgets. The executor conservatively treats each emission as novel; remaining allowances diff --git a/docs/website/src/impl-spec/appendix/manager-api.yaml b/docs/website/src/impl-spec/appendix/manager-api.yaml index 36e5bc79..17346a3b 100644 --- a/docs/website/src/impl-spec/appendix/manager-api.yaml +++ b/docs/website/src/impl-spec/appendix/manager-api.yaml @@ -889,7 +889,7 @@ components: description: Call key bytes, or null for a wildcard. budget: nullable: true - description: Available allowance at execution start; zero is exhausted, null is uncapped. Does not affect key matching. + description: Stored chain budget, or the allowance of a synthetic recipient wildcard; null is uncapped. A zero-budget entry with a recipient is absent, as on chain. on: type: string enum: [decided, finalized] @@ -907,8 +907,7 @@ components: unchanged and meters their length. External and open allocations without a committed subtree use empty bytes. children_budget and subtree are required on every entry, including allocations without descendants. - Include exhausted allocations with budget zero to preserve internal key - precedence; omit allocations absent on-chain. Recipient wildcards are + Omit allocations absent on-chain. Recipient wildcards are synthetic entries for open-pool or view execution, not chain allocations. record_actions: type: array diff --git a/executors/v0.3.x b/executors/v0.3.x index 16b81732..f7f95a31 160000 --- a/executors/v0.3.x +++ b/executors/v0.3.x @@ -1 +1 @@ -Subproject commit 16b81732e12117daf7ef0335cbd32fc7d326b16d +Subproject commit f7f95a31dddbef2aa9371e2228644f83dcce9fe3 From c0b53c9d2e3219ac160bbbf62bf3b826e369db78 Mon Sep 17 00:00:00 2001 From: kp2pml30 <kp2pml30@gmail.com> Date: Fri, 2 Oct 2026 14:36:43 +0900 Subject: [PATCH 31/45] =?UTF-8?q?fix(fees):=20reject=20internal=20messages?= =?UTF-8?q?=20with=20a=20zero=20declared=20budget=20=F0=9F=90=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Bump the v0.3 executor: an internal message whose declaredBudget is zero fails with `fee below_minimum` on both allocation and balance funding, so no child transaction is created unfunded - State the rule in the fee impl-spec --- docs/website/src/impl-spec/04-fees.rst | 3 ++- executors/v0.3.x | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/website/src/impl-spec/04-fees.rst b/docs/website/src/impl-spec/04-fees.rst index fe7724a4..e6e6fd3a 100644 --- a/docs/website/src/impl-spec/04-fees.rst +++ b/docs/website/src/impl-spec/04-fees.rst @@ -345,7 +345,8 @@ For either phase, an internal message therefore declares:: For balance funding the sum is zero. The primary fee already covers the child's configured lifecycle, including appeals; the remainder becomes the child's -message-fee bucket. External messages declare zero +message-fee bucket. An internal message whose declared amount is zero is rejected +with ``fee below_minimum`` on both funding paths. External messages declare zero The v0.3 primary reserve includes successful-appellant profit for each configured appeal slot. The bond is the next normal round's time-unit cost, including its diff --git a/executors/v0.3.x b/executors/v0.3.x index f7f95a31..ed72c14e 160000 --- a/executors/v0.3.x +++ b/executors/v0.3.x @@ -1 +1 @@ -Subproject commit f7f95a31dddbef2aa9371e2228644f83dcce9fe3 +Subproject commit ed72c14eae73485a8580181282935e9c0ffa6678 From 6f7e8dead0c2af5ed30330baac973a83cde7ec08 Mon Sep 17 00:00:00 2001 From: kp2pml30 <kp2pml30@gmail.com> Date: Fri, 2 Oct 2026 15:12:13 +0900 Subject: [PATCH 32/45] =?UTF-8?q?fix(calldata):=20decode=20and=20encode=20?= =?UTF-8?q?uleb=20in=20linear=20time=20=F0=9F=90=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Bump the v0.3 executor: calldata uleb decoding and encoding were O(kΒ²) in the integer's byte length, 35s and 23s for a 1 MiB magnitude in release, now 3ms each, with identical values and errors --- executors/v0.3.x | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/executors/v0.3.x b/executors/v0.3.x index ed72c14e..7e0936a1 160000 --- a/executors/v0.3.x +++ b/executors/v0.3.x @@ -1 +1 @@ -Subproject commit ed72c14eae73485a8580181282935e9c0ffa6678 +Subproject commit 7e0936a125f917a8071b66add10dd555c2a07c8f From b660554e02935d6c54c78dd2f7ab3c2cca303d4b Mon Sep 17 00:00:00 2001 From: kp2pml30 <kp2pml30@gmail.com> Date: Fri, 2 Oct 2026 19:04:52 +0900 Subject: [PATCH 33/45] =?UTF-8?q?fix(llm):=20reject=20prompt=20images=20th?= =?UTF-8?q?at=20do=20not=20decode=20=F0=9F=90=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(llm): fully decode prompt images and reject malformed ones as INVALID_IMAGE πŸ› * perf(llm): stream PNG rows instead of holding the bitmap ⚑️ * build(deps): drop default image features βž– * fix(executor): require free memory before exec_prompt πŸ”’οΈ A malformed image used to reach every provider, fail, and end as a fatal NO_PROVIDER_FOR_PROMPT; it is now an error the contract can catch --- .../src/spec/appendix/internal-constants.rst | 7 + executors/v0.3.x | 2 +- implementation/Cargo.lock | 484 +----------------- implementation/Cargo.toml | 7 +- implementation/src/llm/handler.rs | 16 +- implementation/src/llm/prompt.rs | 46 ++ implementation/tests/data/progressive.jpg | Bin 0 -> 545 bytes implementation/tests/image_decode.rs | 192 +++++++ 8 files changed, 268 insertions(+), 486 deletions(-) create mode 100644 implementation/tests/data/progressive.jpg create mode 100644 implementation/tests/image_decode.rs diff --git a/docs/website/src/spec/appendix/internal-constants.rst b/docs/website/src/spec/appendix/internal-constants.rst index 24fa3481..4c7a1fba 100644 --- a/docs/website/src/spec/appendix/internal-constants.rst +++ b/docs/website/src/spec/appendix/internal-constants.rst @@ -134,6 +134,13 @@ web_render_min_space Value: ``134217728`` +.. _gvm-def-consts-value-top-limits-exec-prompt-min-space: + +exec_prompt_min_space +~~~~~~~~~~~~~~~~~~~~~ + +Value: ``134217728`` + .. _gvm-def-consts-value-top-limits-max-fds: max_fds diff --git a/executors/v0.3.x b/executors/v0.3.x index 7e0936a1..84f9db9c 160000 --- a/executors/v0.3.x +++ b/executors/v0.3.x @@ -1 +1 @@ -Subproject commit 7e0936a125f917a8071b66add10dd555c2a07c8f +Subproject commit 84f9db9ca240d19a587558abc0d1c4309b464766 diff --git a/implementation/Cargo.lock b/implementation/Cargo.lock index ee5803e6..d9a5ae6a 100644 --- a/implementation/Cargo.lock +++ b/implementation/Cargo.lock @@ -29,24 +29,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "aligned" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee4508988c62edf04abd8d92897fca0c2995d907ce1dfeaf369dac3716a40685" -dependencies = [ - "as-slice", -] - -[[package]] -name = "aligned-vec" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc890384c8602f339876ded803c97ad529f3842aba97f6392b3dba0dd171769b" -dependencies = [ - "equator", -] - [[package]] name = "allocator-api2" version = "0.2.21" @@ -118,12 +100,6 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" -[[package]] -name = "arbitrary" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" - [[package]] name = "archery" version = "1.2.2" @@ -133,32 +109,12 @@ dependencies = [ "triomphe", ] -[[package]] -name = "arg_enum_proc_macro" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ae92a5119aa49cdbcf6b9f893fe4e1d98b04ccbf82ee0584ad948a44a734dea" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "arrayvec" version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" -[[package]] -name = "as-slice" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "516b6b4f0e40d50dcda9365d53964ec74560ad4284da2e7fc97122cd83174516" -dependencies = [ - "stable_deref_trait", -] - [[package]] name = "async-compression" version = "0.4.41" @@ -194,49 +150,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" -[[package]] -name = "av-scenechange" -version = "0.14.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f321d77c20e19b92c39e7471cf986812cbb46659d2af674adc4331ef3f18394" -dependencies = [ - "aligned", - "anyhow", - "arg_enum_proc_macro", - "arrayvec", - "log", - "num-rational", - "num-traits", - "pastey", - "rayon", - "thiserror", - "v_frame", - "y4m", -] - -[[package]] -name = "av1-grain" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cfddb07216410377231960af4fcab838eaa12e013417781b78bd95ee22077f8" -dependencies = [ - "anyhow", - "arrayvec", - "log", - "nom", - "num-rational", - "v_frame", -] - -[[package]] -name = "avif-serialize" -version = "0.8.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "375082f007bd67184fb9c0374614b29f9aaa604ec301635f72338bb65386a53d" -dependencies = [ - "arrayvec", -] - [[package]] name = "axum" version = "0.8.9" @@ -298,27 +211,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" -[[package]] -name = "bit_field" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e4b40c7323adcfc0a41c4b88143ed58346ff65a288fc144329c5c45e05d70c6" - [[package]] name = "bitflags" version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" -[[package]] -name = "bitstream-io" -version = "4.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eff00be299a18769011411c9def0d827e8f2d7bf0c3dbf53633147a8867fd1f" -dependencies = [ - "no_std_io2", -] - [[package]] name = "bitvec" version = "1.0.1" @@ -350,12 +248,6 @@ dependencies = [ "serde", ] -[[package]] -name = "built" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4ad8f11f288f48ca24471bbd51ac257aaeaaa07adae295591266b792902ae64" - [[package]] name = "bumpalo" version = "3.20.2" @@ -402,8 +294,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20" dependencies = [ "find-msvc-tools", - "jobserver", - "libc", "shlex", ] @@ -473,12 +363,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" -[[package]] -name = "color_quant" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" - [[package]] name = "colorchoice" version = "1.0.5" @@ -787,26 +671,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7f84e12ccf0a7ddc17a6c41c93326024c42920d7ee630d04950e6926645c0fe" -[[package]] -name = "equator" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4711b213838dfee0117e3be6ac926007d7f433d7bbe33595975d4190cb07e6fc" -dependencies = [ - "equator-macro", -] - -[[package]] -name = "equator-macro" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44f23cf4b44bfce11a86ace86f8a73ffdec849c9fd00a386a53d278bd9e81fb3" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "equivalent" version = "1.0.2" @@ -834,41 +698,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "exr" -version = "1.74.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4300e043a56aa2cb633c01af81ca8f699a321879a7854d3896a0ba89056363be" -dependencies = [ - "bit_field", - "half", - "lebe", - "miniz_oxide", - "rayon-core", - "smallvec", - "zune-inflate", -] - -[[package]] -name = "fax" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f05de7d48f37cd6730705cbca900770cab77a89f413d23e100ad7fad7795a0ab" -dependencies = [ - "fax_derive", -] - -[[package]] -name = "fax_derive" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0aca10fb742cb43f9e7bb8467c91aa9bcb8e3ffbc6a6f7389bb93ffc920577d" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "fdeflate" version = "0.3.7" @@ -1099,6 +928,7 @@ dependencies = [ "num-rational", "num-traits", "papaya", + "png", "primitive-types", "rand 0.9.4", "regex", @@ -1180,16 +1010,6 @@ dependencies = [ "wasip3", ] -[[package]] -name = "gif" -version = "0.14.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee8cfcc411d9adbbaba82fb72661cc1bcca13e8bba98b364e62b2dba8f960159" -dependencies = [ - "color_quant", - "weezl", -] - [[package]] name = "h2" version = "0.4.13" @@ -1209,17 +1029,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "half" -version = "2.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" -dependencies = [ - "cfg-if", - "crunchy", - "zerocopy", -] - [[package]] name = "hashbrown" version = "0.12.3" @@ -1569,18 +1378,10 @@ checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" dependencies = [ "bytemuck", "byteorder-lite", - "color_quant", - "exr", - "gif", "image-webp", "moxcms", "num-traits", "png", - "qoi", - "ravif", - "rayon", - "rgb", - "tiff", "zune-core", "zune-jpeg", ] @@ -1595,12 +1396,6 @@ dependencies = [ "quick-error", ] -[[package]] -name = "imgref" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7c5cedc30da3a610cac6b4ba17597bdf7152cf974e8aab3afb3d54455e371c8" - [[package]] name = "impl-codec" version = "0.7.1" @@ -1653,17 +1448,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "interpolate_name" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c34819042dc3d3971c46c2190835914dfbe0c3c13f61449b2997f4e9722dfa60" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "ipconfig" version = "0.3.4" @@ -1699,31 +1483,12 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" -[[package]] -name = "itertools" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" -dependencies = [ - "either", -] - [[package]] name = "itoa" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" -[[package]] -name = "jobserver" -version = "0.1.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" -dependencies = [ - "getrandom 0.3.4", - "libc", -] - [[package]] name = "js-sys" version = "0.3.95" @@ -1766,28 +1531,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" -[[package]] -name = "lebe" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a79a3332a6609480d7d0c9eab957bca6b455b91bb84e66d19f5ff66294b85b8" - [[package]] name = "libc" version = "0.2.185" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" -[[package]] -name = "libfuzzer-sys" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f12a681b7dd8ce12bff52488013ba614b869148d54dd79836ab85aafdd53f08d" -dependencies = [ - "arbitrary", - "cc", -] - [[package]] name = "linux-raw-sys" version = "0.4.15" @@ -1827,15 +1576,6 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" -[[package]] -name = "loop9" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fae87c125b03c1d2c0150c90365d7d6bcc53fb73a9acaef207d2d065860f062" -dependencies = [ - "imgref", -] - [[package]] name = "lru-slab" version = "0.1.2" @@ -1867,16 +1607,6 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" -[[package]] -name = "maybe-rayon" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ea1f30cedd69f0a2954655f7188c6a834246d2bcf1e315e2ac40c4b24dc9519" -dependencies = [ - "cfg-if", - "rayon", -] - [[package]] name = "memchr" version = "2.8.0" @@ -1969,36 +1699,6 @@ dependencies = [ "pxfm", ] -[[package]] -name = "new_debug_unreachable" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" - -[[package]] -name = "no_std_io2" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b51ed7824b6e07d354605f4abb3d9d300350701299da96642ee084f5ce631550" -dependencies = [ - "memchr", -] - -[[package]] -name = "nom" -version = "8.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" -dependencies = [ - "memchr", -] - -[[package]] -name = "noop_proc_macro" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0676bb32a98c1a483ce53e500a81ad9c3d5b3f7c920c28c24e9cb0980d0b5bc8" - [[package]] name = "ntapi" version = "0.4.3" @@ -2025,17 +1725,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" -[[package]] -name = "num-derive" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "num-integer" version = "0.1.46" @@ -2170,18 +1859,6 @@ dependencies = [ "windows-link 0.2.1", ] -[[package]] -name = "paste" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" - -[[package]] -name = "pastey" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35fb2e5f958ec131621fdd531e9fc186ed768cbe395337403ae56c17a74c68ec" - [[package]] name = "percent-encoding" version = "2.3.2" @@ -2283,25 +1960,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "profiling" -version = "1.0.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eb8486b569e12e2c32ad3e204dbaba5e4b5b216e9367044f25f1dba42341773" -dependencies = [ - "profiling-procmacros", -] - -[[package]] -name = "profiling-procmacros" -version = "1.0.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52717f9a02b6965224f95ca2a81e2e0c5c43baacd28ca057577988930b6c3d5b" -dependencies = [ - "quote", - "syn", -] - [[package]] name = "psl-types" version = "2.0.11" @@ -2324,15 +1982,6 @@ version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0c5ccf5294c6ccd63a74f1565028353830a9c2f5eb0c682c355c471726a6e3f" -[[package]] -name = "qoi" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f6d64c71eb498fe9eae14ce4ec935c555749aef511cca85b5568910d6e48001" -dependencies = [ - "bytemuck", -] - [[package]] name = "quick-error" version = "2.0.1" @@ -2480,76 +2129,6 @@ dependencies = [ "getrandom 0.3.4", ] -[[package]] -name = "rav1e" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43b6dd56e85d9483277cde964fd1bdb0428de4fec5ebba7540995639a21cb32b" -dependencies = [ - "aligned-vec", - "arbitrary", - "arg_enum_proc_macro", - "arrayvec", - "av-scenechange", - "av1-grain", - "bitstream-io", - "built", - "cfg-if", - "interpolate_name", - "itertools", - "libc", - "libfuzzer-sys", - "log", - "maybe-rayon", - "new_debug_unreachable", - "noop_proc_macro", - "num-derive", - "num-traits", - "paste", - "profiling", - "rand 0.9.4", - "rand_chacha 0.9.0", - "simd_helpers", - "thiserror", - "v_frame", - "wasm-bindgen", -] - -[[package]] -name = "ravif" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e52310197d971b0f5be7fe6b57530dcd27beb35c1b013f29d66c1ad73fbbcc45" -dependencies = [ - "avif-serialize", - "imgref", - "loop9", - "quick-error", - "rav1e", - "rayon", - "rgb", -] - -[[package]] -name = "rayon" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" -dependencies = [ - "crossbeam-deque", - "crossbeam-utils", -] - [[package]] name = "redox_syscall" version = "0.5.18" @@ -2657,12 +2236,6 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" -[[package]] -name = "rgb" -version = "0.8.53" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b34b781b31e5d73e9fbc8689c70551fd1ade9a19e3e28cfec8580a79290cc4" - [[package]] name = "ring" version = "0.17.14" @@ -3010,15 +2583,6 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" -[[package]] -name = "simd_helpers" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95890f873bec569a0362c235787f3aca6e1e887302ba4840839bcc6459c42da6" -dependencies = [ - "quote", -] - [[package]] name = "slab" version = "0.4.12" @@ -3153,20 +2717,6 @@ dependencies = [ "syn", ] -[[package]] -name = "tiff" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b63feaf3343d35b6ca4d50483f94843803b0f51634937cc2ec519fc32232bc52" -dependencies = [ - "fax", - "flate2", - "half", - "quick-error", - "weezl", - "zune-jpeg", -] - [[package]] name = "time" version = "0.3.47" @@ -3548,17 +3098,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "v_frame" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "666b7727c8875d6ab5db9533418d7c764233ac9c0cff1d469aec8fa127597be2" -dependencies = [ - "aligned-vec", - "num-traits", - "wasm-bindgen", -] - [[package]] name = "version_check" version = "0.9.5" @@ -3716,12 +3255,6 @@ dependencies = [ "rustls-pki-types", ] -[[package]] -name = "weezl" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" - [[package]] name = "which" version = "7.0.3" @@ -4216,12 +3749,6 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fb433233f2df9344722454bc7e96465c9d03bff9d77c248f9e7523fe79585b5" -[[package]] -name = "y4m" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5a4b21e1a62b67a2970e6831bc091d7b87e119e7f9791aef9702e3bef04448" - [[package]] name = "yoke" version = "0.8.2" @@ -4337,15 +3864,6 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb8a0807f7c01457d0379ba880ba6322660448ddebc890ce29bb64da71fb40f9" -[[package]] -name = "zune-inflate" -version = "0.2.54" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73ab332fe2f6680068f3582b16a24f90ad7096d5d39b974d1c0aff0125116f02" -dependencies = [ - "simd-adler32", -] - [[package]] name = "zune-jpeg" version = "0.5.15" diff --git a/implementation/Cargo.toml b/implementation/Cargo.toml index 80a3b907..8e52421e 100644 --- a/implementation/Cargo.toml +++ b/implementation/Cargo.toml @@ -37,13 +37,18 @@ clap = { version = "4.5.35", features = ["derive"] } crossbeam = "0.8.4" futures-util = "0.3.31" hex = "0.4.3" -image = { version = "0.25.8", features = ["png", "jpeg", "webp"] } +image = { version = "0.25.8", default-features = false, features = [ + "png", + "jpeg", + "webp", +] } libc = "0.2" num-bigint = { version = "0.4.6", features = ["serde"] } num-integer = { version = "0.1.46" } num-rational = { version = "0.4.2" } num-traits = { version = "0.2.19" } papaya = "0.2.3" +png = { version = "0.18.1", default-features = false } primitive-types = { version = "0.13", features = ["impl-serde"] } rand = { version = "0.9.1", features = ["std_rng"] } regex = "1.11.1" diff --git a/implementation/src/llm/handler.rs b/implementation/src/llm/handler.rs index eb599f11..afbae746 100644 --- a/implementation/src/llm/handler.rs +++ b/implementation/src/llm/handler.rs @@ -115,7 +115,21 @@ impl crate::common::MessageHandler<llm_iface::Message, llm_iface::PromptAnswer> .into()); } - if prompt::ImageType::sniff(img.as_ref()).is_none() { + let decodes = match prompt::ImageType::sniff(img.as_ref()) { + None => false, + Some(kind) => { + let img = img.clone(); + // a decoder panic is the image's fault, not the manager's + match tokio::task::spawn_blocking(move || kind.decodes(&img)).await { + Ok(decodes) => decodes, + Err(error) => { + log_warn_into!(@user,&LoggerWithId, genvm_id:id = self.0.genvm_id.0, error:err = error; "image decoder failed"); + false + } + } + } + }; + if !decodes { return Err(ModuleError { causes: vec!["INVALID_IMAGE".into()], fatal: false, diff --git a/implementation/src/llm/prompt.rs b/implementation/src/llm/prompt.rs index c81ed358..a8af696f 100644 --- a/implementation/src/llm/prompt.rs +++ b/implementation/src/llm/prompt.rs @@ -52,6 +52,12 @@ impl Internal { } } +/// Matches the largest side the strictest provider (Anthropic) accepts +pub const IMAGE_MAX_SIDE: u32 = 8000; +/// Fits an 8-bit RGB JPEG at [`IMAGE_MAX_SIDE`] squared. Independent of the +/// executor's `EXEC_PROMPT_MIN_SPACE`, which reserves contract memory instead +const IMAGE_MAX_ALLOC: u64 = 256 * 1024 * 1024; + #[derive(Serialize, Deserialize, Clone, Copy)] pub enum ImageType { PNG, @@ -69,6 +75,15 @@ impl ImageType { } } + /// Whether `data` fully decodes as `self` within [`IMAGE_MAX_SIDE`]; it is + /// CPU-bound, so call it off the async runtime + pub fn decodes(self, data: &[u8]) -> bool { + match self { + Self::JPG => jpeg_decodes(data), + Self::PNG => png_decodes(data).unwrap_or(false), + } + } + pub fn media_type(self) -> &'static str { match self { Self::JPG => "image/jpeg", @@ -77,6 +92,37 @@ impl ImageType { } } +/// Holds the whole bitmap in memory, bounded by [`IMAGE_MAX_ALLOC`] +fn jpeg_decodes(data: &[u8]) -> bool { + let mut limits = image::Limits::default(); + limits.max_image_width = Some(IMAGE_MAX_SIDE); + limits.max_image_height = Some(IMAGE_MAX_SIDE); + limits.max_alloc = Some(IMAGE_MAX_ALLOC); + + let mut reader = + image::ImageReader::with_format(std::io::Cursor::new(data), image::ImageFormat::Jpeg); + reader.limits(limits); + reader.decode().is_ok() +} + +/// Streams rows to the end of the file, so memory stays a few rows whatever the +/// declared size +fn png_decodes(data: &[u8]) -> Result<bool, png::DecodingError> { + let mut decoder = png::Decoder::new(std::io::Cursor::new(data)); + // expanding is what rejects a palette image that has no `PLTE` + decoder.set_transformations(png::Transformations::EXPAND); + + let header = decoder.read_header_info()?; + if header.width > IMAGE_MAX_SIDE || header.height > IMAGE_MAX_SIDE { + return Ok(false); + } + + let mut reader = decoder.read_info()?; + while reader.next_row()?.is_some() {} + reader.finish()?; + Ok(true) +} + #[derive(Serialize, Deserialize)] pub struct ImageLua(#[serde(with = "serde_bytes")] pub Vec<u8>); diff --git a/implementation/tests/data/progressive.jpg b/implementation/tests/data/progressive.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3617b8fc60f7c9c0f45edf560964a7d4f8373e84 GIT binary patch literal 545 zcmb7<(MrQG7=_Q@*e0<lrdnl=%&t|K!Wdc$Dok1s2Kp2}kq;%_d*O}F8{fePi0Ou- z3|8?ZS3lqR&-v}U{epg&-OoS-kbAJ7xPu4+9{7TDE~VI3Nu<J19q#V!B!hl>H0T>M zJeiD!`S{qF)2p-b`9)cl>FDP6x|mKb%c5CAQc4x7UKI6;%w)yCWuFlX)TkNd@M2;y z*%$0Ds-i95jStAFJp{G_JBMNfPc!YVd&=KhK<Z41UbEoQS%h0ezSo2$@j6g107g_1 q?*P0NurVU2mJu5Psxtb$0L)d_->=iv?+9<&>A%`7fsSxZ_TvYwI8<2x literal 0 HcmV?d00001 diff --git a/implementation/tests/image_decode.rs b/implementation/tests/image_decode.rs new file mode 100644 index 00000000..437e6d18 --- /dev/null +++ b/implementation/tests/image_decode.rs @@ -0,0 +1,192 @@ +use genvm_modules::llm::prompt; + +fn hex(s: &str) -> Vec<u8> { + (0..s.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&s[i..i + 2], 16).unwrap()) + .collect() +} + +fn encode(img: image::DynamicImage, format: image::ImageFormat) -> Vec<u8> { + let mut out = std::io::Cursor::new(Vec::new()); + img.write_to(&mut out, format).unwrap(); + out.into_inner() +} + +fn rgb() -> image::DynamicImage { + image::DynamicImage::ImageRgb8(image::RgbImage::from_pixel( + 64, + 64, + image::Rgb([200, 30, 30]), + )) +} + +/// Encoded row by row, so the test never holds the bitmap either +fn black_rgba16_png(side: u32) -> Vec<u8> { + let mut data = Vec::new(); + let mut encoder = png::Encoder::new(&mut data, side, side); + encoder.set_color(png::ColorType::Rgba); + encoder.set_depth(png::BitDepth::Sixteen); + encoder.set_compression(png::Compression::Fastest); + let mut writer = encoder.write_header().unwrap(); + let mut stream = writer.stream_writer().unwrap(); + let row = vec![0u8; side as usize * 8]; + for _ in 0..side { + std::io::Write::write_all(&mut stream, &row).unwrap(); + } + stream.finish().unwrap(); + writer.finish().unwrap(); + data +} + +fn accepts(data: &[u8]) -> bool { + let kind = prompt::ImageType::sniff(data).expect("test data must pass the header sniff"); + kind.decodes(data) +} + +// -- Well-formed images -- + +#[test] +fn accepts_png() { + assert!(accepts(&encode(rgb(), image::ImageFormat::Png))); +} + +#[test] +fn accepts_jpeg() { + assert!(accepts(&encode(rgb(), image::ImageFormat::Jpeg))); +} + +#[test] +fn accepts_progressive_jpeg() { + assert!(accepts(include_bytes!("data/progressive.jpg"))); +} + +#[test] +fn accepts_grey_jpeg() { + let img = image::DynamicImage::ImageLuma8(image::GrayImage::new(16, 16)); + assert!(accepts(&encode(img, image::ImageFormat::Jpeg))); +} + +#[test] +fn accepts_16_bit_rgba_png() { + let img = image::DynamicImage::ImageRgba16(image::ImageBuffer::new(16, 16)); + assert!(accepts(&encode(img, image::ImageFormat::Png))); +} + +#[test] +fn accepts_png_with_trailing_junk() { + let mut data = encode(rgb(), image::ImageFormat::Png); + data.extend_from_slice(&[0; 1000]); + assert!(accepts(&data)); +} + +// -- Header passes, body does not -- + +#[test] +fn rejects_png_magic_with_garbage() { + let mut data = b"\x89PNG\r\n\x1a\n".to_vec(); + data.extend(b"garbage".repeat(50)); + assert!(!accepts(&data)); +} + +#[test] +fn rejects_bare_png_magic() { + assert!(!accepts(b"\x89PNG\r\n\x1a\n")); +} + +#[test] +fn rejects_jfif_magic_with_garbage() { + let mut data = b"\xff\xd8\xff\xe0".to_vec(); + data.extend(b"garbage".repeat(50)); + assert!(!accepts(&data)); +} + +#[test] +fn rejects_truncated_png() { + let data = encode(rgb(), image::ImageFormat::Png); + assert!(!accepts(&data[..data.len() / 2])); +} + +#[test] +fn rejects_truncated_jpeg() { + let data = encode(rgb(), image::ImageFormat::Jpeg); + assert!(!accepts(&data[..data.len() / 2])); +} + +#[test] +fn rejects_jpeg_magic_over_png_body() { + let mut data = b"\xff\xd8\xff\xe0".to_vec(); + data.extend(encode(rgb(), image::ImageFormat::Png)); + assert!(!accepts(&data)); +} + +#[test] +fn rejects_png_magic_over_jpeg_body() { + let mut data = b"\x89PNG\r\n\x1a\n".to_vec(); + data.extend(encode(rgb(), image::ImageFormat::Jpeg)); + assert!(!accepts(&data)); +} + +// -- Malformed PNG structure -- + +#[test] +fn rejects_palette_png_without_plte() { + assert!(!accepts(&hex("89504e470d0a1a0a0000000d49484452000000040000000408030000009e2f6e4c0000000e4944415478da636004020654020000b40011197e69d80000000049454e44ae426082"))); +} + +#[test] +fn rejects_palette_png_with_16_bit_depth() { + assert!(!accepts(&hex("89504e470d0a1a0a0000000d4948445200000004000000041003000000cebfb20f00000003504c5445000000a77a3dda0000000b4944415478da6360200c000024000125c2a8e30000000049454e44ae426082"))); +} + +#[test] +fn rejects_png_with_invalid_color_type() { + assert!(!accepts(&hex("89504e470d0a1a0a0000000d4948445200000004000000040805000000bb4431900000000b4944415478da6360c0040000140001ee5a69090000000049454e44ae426082"))); +} + +#[test] +fn rejects_png_with_zero_dimensions() { + assert!(!accepts(&hex("89504e470d0a1a0a0000000d4948445200000000000000000802000000b4e9eb45000000084944415478da0300000000016fddc9910000000049454e44ae426082"))); +} + +#[test] +fn rejects_png_with_bad_header_crc() { + assert!(!accepts(&hex("89504e470d0a1a0a0000000d4948445200000004000000040802000000000000000000000e4944415478da636840020cc47100705218019eaebe6f0000000049454e44ae426082"))); +} + +// -- Dimension and allocation limits -- + +#[test] +fn rejects_png_declaring_huge_dimensions() { + assert!(!accepts(&hex("89504e470d0a1a0a0000000d494844527fffffff7fffffff08020000009bab9c310000000b4944415478da636040050000100001aa19f8820000000049454e44ae426082"))); +} + +#[test] +fn rejects_png_wider_than_the_limit() { + let img = image::DynamicImage::ImageLuma8(image::GrayImage::new(prompt::IMAGE_MAX_SIDE + 1, 1)); + assert!(!accepts(&encode(img, image::ImageFormat::Png))); +} + +#[test] +fn rejects_png_taller_than_the_limit() { + let img = image::DynamicImage::ImageLuma8(image::GrayImage::new(1, prompt::IMAGE_MAX_SIDE + 1)); + assert!(!accepts(&encode(img, image::ImageFormat::Png))); +} + +#[test] +fn rejects_jpeg_wider_than_the_limit() { + let img = image::DynamicImage::ImageLuma8(image::GrayImage::new(prompt::IMAGE_MAX_SIDE + 1, 8)); + assert!(!accepts(&encode(img, image::ImageFormat::Jpeg))); +} + +#[test] +fn accepts_16_bit_rgba_png_at_the_side_limit() { + // 512 MB decoded, which a whole-bitmap decode under the JPEG budget would refuse + assert!(accepts(&black_rgba16_png(prompt::IMAGE_MAX_SIDE))); +} + +#[test] +fn accepts_png_at_the_side_limit() { + let img = image::DynamicImage::ImageLuma8(image::GrayImage::new(prompt::IMAGE_MAX_SIDE, 1)); + assert!(accepts(&encode(img, image::ImageFormat::Png))); +} From 36d78e27a73a1d7fbdb590494ac2298838d17d85 Mon Sep 17 00:00:00 2001 From: kp2pml30 <kp2pml30@gmail.com> Date: Fri, 2 Oct 2026 20:48:00 +0900 Subject: [PATCH 34/45] =?UTF-8?q?feat(ci):=20land=20a=20PR=20by=20fast-for?= =?UTF-8?q?warding=20its=20base=20on=20/genvm-fast-forward=20=E2=9C=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A writer with another writer's approval of the head, or an admin, can move a dev branch to the PR head once E2E and the base's required checks pass. The push uses the manager deploy key, so the tool enforces what the rulesets would: linear history, no fork PRs, release branches admin-only, and unknown rules refused --- .../branch_fast_forward_command.yaml | 68 +++ docs/contributing/howto/pr.md | 18 +- executors/v0.3.x | 2 +- support/ci/__main__.py | 2 + support/ci/tools/fast_forward_command.py | 308 ++++++++++++++ .../unit_tests/test_fast_forward_command.py | 391 ++++++++++++++++++ 6 files changed, 787 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/branch_fast_forward_command.yaml create mode 100644 support/ci/tools/fast_forward_command.py create mode 100644 support/ci/unit_tests/test_fast_forward_command.py diff --git a/.github/workflows/branch_fast_forward_command.yaml b/.github/workflows/branch_fast_forward_command.yaml new file mode 100644 index 00000000..d2f2da66 --- /dev/null +++ b/.github/workflows/branch_fast_forward_command.yaml @@ -0,0 +1,68 @@ +name: branch / fast-forward command + +# `/genvm-fast-forward` lands a PR by moving its base branch to the PR head, +# without a merge or squash commit. The tool gates on approvals, required checks +# and E2E, since the deploy key it pushes with bypasses the base's rulesets. +# The resulting push triggers the usual executor branch sync + +on: + issue_comment: + types: [created] + +permissions: {} + +# One request per PR at a time; across PRs the non-force push keeps landings safe +concurrency: + group: fast-forward-command-${{ github.event.issue.number }} + cancel-in-progress: false + +defaults: + run: + shell: bash -exo pipefail {0} + +jobs: + fast-forward: + # The association only spares strangers the deploy key; the tool authorizes + if: >- + github.event.issue.pull_request && + github.event.comment.body == '/genvm-fast-forward' && + contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association) + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + checks: read + contents: read + issues: write + pull-requests: write + statuses: read + steps: + # issue_comment workflows run from the default branch. Keep that trusted + # checkout: the deploy key is in scope, so no PR code may run here. + # Full history, so git can show the push builds on the base + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Set up manager push access + env: + MANAGER_DEPLOY_KEY: ${{ secrets.GENVM_CI_PRIVATE_KEY }} + run: | + install -m700 -d ~/.ssh + key="${RUNNER_TEMP}/manager_push_key" + set +x + printf '%s\n' "$MANAGER_DEPLOY_KEY" > "$key" + set -x + chmod 600 "$key" + git config --global url."git@github.com:".insteadOf "https://github.com/" + echo "GIT_SSH_COMMAND=ssh -i $key -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new" >> "$GITHUB_ENV" + + - name: Verify and fast-forward + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.issue.number }} + COMMENT_ID: ${{ github.event.comment.id }} + COMMENT_BODY: ${{ github.event.comment.body }} + SENDER: ${{ github.event.sender.login }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: ./support/ci/run.sh tool fast-forward-command diff --git a/docs/contributing/howto/pr.md b/docs/contributing/howto/pr.md index 3bde9bf9..87b52e50 100644 --- a/docs/contributing/howto/pr.md +++ b/docs/contributing/howto/pr.md @@ -30,6 +30,21 @@ reading the diff With `ci-safe` and write access, `/genvm-run-tests` links one run and reacts πŸ‘€ β†’ πŸš€/πŸ˜• without changing the sticky `run-full-tests` label +## Fast-Forward + +`/genvm-fast-forward` lands a PR without the App: it moves the base branch to +the PR head with a non-force push, so the PR's commits land as they are. It +refuses, listing every unmet gate, unless: + +1. The commenter is an admin, or a writer and the PR head carries an approval + from a writer who is neither the author nor the commenter, with no writer's + change request +2. The base is `v<X>-dev`, or `v<X>` for an admin; the PR is not from a fork +3. `E2E Tests` and every check the base requires succeeded on the head +4. The head is strictly ahead of the base and has no merge commits + +The push then syncs executor branches as any manager push does + ## Gates `queue.yaml` always runs the pre-commit hooks. On a `pull_request` event it also @@ -50,7 +65,8 @@ rerun manually against the live manager branch ## Authority -Needs a maintainer: merging, `--admin`, cross-repo E2E runs, and anything +Needs a maintainer: merging (a writer may `/genvm-fast-forward` an approved +PR), `--admin`, cross-repo E2E runs, and anything irreversible outside the PR β€” releases, deploys, messages to other teams. Never force-push a shared branch (`v<X>`, `v<X>-dev`); branch protection refuses it anyway. Force-pushing your own PR branch is routine and needs no one, and so diff --git a/executors/v0.3.x b/executors/v0.3.x index 84f9db9c..c476c2b4 160000 --- a/executors/v0.3.x +++ b/executors/v0.3.x @@ -1 +1 @@ -Subproject commit 84f9db9ca240d19a587558abc0d1c4309b464766 +Subproject commit c476c2b4b8d10f50e79db328db1171b087a5ffdc diff --git a/support/ci/__main__.py b/support/ci/__main__.py index 1ecd9726..e76636fa 100644 --- a/support/ci/__main__.py +++ b/support/ci/__main__.py @@ -8,6 +8,7 @@ from tools import ( branches, deploy_docs, + fast_forward_command, full_tests_command, make_release_notes, open_executor_prs, @@ -30,6 +31,7 @@ *versions.COMMANDS, *branches.COMMANDS, *deploy_docs.COMMANDS, + *fast_forward_command.COMMANDS, *full_tests_command.COMMANDS, *pr_branches.COMMANDS, *make_release_notes.COMMANDS, diff --git a/support/ci/tools/fast_forward_command.py b/support/ci/tools/fast_forward_command.py new file mode 100644 index 00000000..65fc6498 --- /dev/null +++ b/support/ci/tools/fast_forward_command.py @@ -0,0 +1,308 @@ +#!/usr/bin/env python3 +""" +`/genvm-fast-forward`: move a PR's base branch to the PR's exact head. + +The handler runs from the default branch with a write deploy key in scope, so it +reads the PR only through the API and never executes its code. The deploy key +bypasses the base branch's rulesets, which is why every gate lives here: + +1. The commenter is an admin, or a writer and the head carries an approval from + a writer other than the author and the commenter, with no writer's change + request outstanding +2. The base is a dev branch, or a release branch and the commenter is an admin; + the PR is not from a fork +3. `E2E Tests` and every check the base's rulesets require concluded `success` + on the head, and the rulesets hold nothing else the push could violate +4. The head is strictly ahead of the base tip, without merge commits + +The push is a plain non-force push of the verified SHA, so the server enforces +the fast-forward again. Executor branches follow through the usual sync of a +manager branch push +""" + +import argparse +import json +import os +import re +import subprocess +import urllib.parse + +import ci_lib +import gh_common +from gh_common import pr_number, repo +from tools.full_tests_command import best_effort_add_reaction, delete_reaction + +COMMAND = '/genvm-fast-forward' +E2E_CHECK = 'E2E Tests' +E2E_APP = 2929262 +DEV_BRANCH_RE = re.compile(r'^v\d+\.\d+-dev$') +RELEASE_BRANCH_RE = re.compile(r'^v\d+\.\d+$') +# what the push itself must honor; everything else is the bypass's business +UNDERSTOOD_RULES = frozenset( + { + 'creation', + 'deletion', + 'non_fast_forward', + 'required_linear_history', + 'required_status_checks', + 'update', + } +) + + +class Refused(RuntimeError): + pass + + +def gh_json(path: str) -> dict: + return json.loads(gh_common.gh_manager('api', path)) + + +def gh_items(path: str, jq: str = '.[]') -> list[dict]: + out = gh_common.gh_manager('api', '--paginate', path, '--jq', f'{jq} | @json') + return [json.loads(line) for line in out.splitlines()] + + +def permission(login: str) -> str | None: + result = gh_common.gh( + 'api', + f'repos/{repo()}/collaborators/{urllib.parse.quote(login)}/permission', + token=gh_common.manager_token(), + check=False, + ) + if result.returncode == 0: + return json.loads(result.stdout).get('permission') + # anything but "not a collaborator" must not quietly drop a change request + if 'HTTP 403' in result.stderr or 'HTTP 404' in result.stderr: + return None + raise RuntimeError(f'could not read the permission of `{login}`') + + +def base_rules(base: str) -> list[dict]: + return gh_items(f'repos/{repo()}/rules/branches/{urllib.parse.quote(base, safe="")}') + + +def rule_failures(rules: list[dict]) -> list[str]: + return [ + f'the base has a `{rule["type"]}` rule this command does not enforce' + for rule in rules + if rule['type'] not in UNDERSTOOD_RULES + ] + + +def required_checks(rules: list[dict]) -> list[tuple[str, int | None]]: + required = [ + (check['context'], check.get('integration_id')) + for rule in rules + if rule['type'] == 'required_status_checks' + for check in rule['parameters']['required_status_checks'] + ] + if all(context != E2E_CHECK for context, _app in required): + required.append((E2E_CHECK, E2E_APP)) + return required + + +def check_failures( + required: list[tuple[str, int | None]], runs: list[dict], statuses: list[dict] +) -> list[str]: + failures = [] + for context, app in required: + matching = [ + run + for run in runs + if run['name'] == context and (app is None or run['app']['id'] == app) + ] + if matching: + # a rerun is a new check run, and GitHub judges the latest one + run = max(matching, key=lambda run: run['id']) + if run['status'] != 'completed': + failures.append(f'`{context}` is {run["status"].replace("_", " ")}') + elif run['conclusion'] != 'success': + failures.append(f'`{context}` concluded `{run["conclusion"]}`') + continue + # statuses come newest first and carry no app to match against + status = next((s for s in statuses if s['context'] == context), None) + if app is not None or status is None: + failures.append(f'`{context}` has not run') + elif status['state'] != 'success': + failures.append(f'`{context}` is `{status["state"]}`') + return failures + + +def review_failures( + *, + sender: str, + sender_permission: str | None, + author: str, + head_sha: str, + reviews: list[dict], + permission_of, +) -> list[str]: + if sender_permission == 'admin': + return [] + if sender_permission not in gh_common.WRITE_ROLES: + return [f'`{sender}` is neither an admin nor a writer'] + + # reviews come oldest first; a plain comment neither grants nor revokes + latest: dict[str, dict] = {} + for review in reviews: + login = (review.get('user') or {}).get('login') + if login and review['state'] in ('APPROVED', 'CHANGES_REQUESTED', 'DISMISSED'): + latest[login] = review + + failures = [] + approved = False + for login, review in latest.items(): + if ( + review['user'].get('type') != 'User' + or permission_of(login) not in gh_common.WRITE_ROLES + ): + continue + if review['state'] == 'CHANGES_REQUESTED': + failures.append(f'`{login}` requested changes') + elif ( + review['state'] == 'APPROVED' + and review['commit_id'] == head_sha + and login not in (author, sender) + ): + approved = True + if not approved: + failures.append( + f'no writer other than the author and `{sender}` approved `{head_sha[:12]}`' + ) + return failures + + +def gate(pr: dict, sender: str) -> list[str]: + if pr['state'] != 'open': + return ['the PR is not open'] + base = pr['base']['ref'] + head_sha = pr['head']['sha'] + failures = [] + if pr['draft']: + failures.append('the PR is a draft') + # a fork skips the executor precondition, so its gitlinks are unproven + head_repo = pr['head'].get('repo') + if head_repo is None or head_repo['full_name'] != repo(): + failures.append('the PR comes from a fork') + sender_permission = permission(sender) + if RELEASE_BRANCH_RE.fullmatch(base): + if sender_permission != 'admin': + failures.append(f'only an admin may fast-forward the release branch `{base}`') + elif not DEV_BRANCH_RE.fullmatch(base): + failures.append(f'`{base}` is neither a dev nor a release branch') + + rules = base_rules(base) + failures += rule_failures(rules) + failures += review_failures( + sender=sender, + sender_permission=sender_permission, + author=pr['user']['login'], + head_sha=head_sha, + reviews=gh_items(f'repos/{repo()}/pulls/{pr_number()}/reviews?per_page=100'), + permission_of=permission, + ) + failures += check_failures( + required_checks(rules), + gh_items( + f'repos/{repo()}/commits/{head_sha}/check-runs?per_page=100', '.check_runs[]' + ), + gh_items(f'repos/{repo()}/commits/{head_sha}/statuses?per_page=100'), + ) + + compared = gh_json( + f'repos/{repo()}/compare/{urllib.parse.quote(base, safe="")}...{head_sha}' + )['status'] + if compared == 'identical': + failures.append(f'`{base}` is already at `{head_sha[:12]}`') + elif compared != 'ahead': + failures.append(f'`{head_sha[:12]}` is {compared} relative to `{base}`; rebase it') + return failures + + +def git(*args: str) -> subprocess.CompletedProcess: + return ci_lib.run(['git', *args], check=False, capture_output=True) + + +def fast_forward(base: str, head_sha: str) -> None: + fetched = git( + 'fetch', + '--no-tags', + 'origin', + f'+refs/heads/{base}:refs/genvm-ff/base', + f'+refs/pull/{pr_number()}/head:refs/genvm-ff/head', + ) + if fetched.returncode != 0: + raise RuntimeError( + f'could not fetch `{base}` and the PR head: {fetched.stderr.strip()}' + ) + # a head pushed during the checks must not land unverified + if git('rev-parse', 'refs/genvm-ff/head').stdout.strip() != head_sha: + raise Refused('the head moved during verification; comment again') + merges = git('rev-list', '--min-parents=2', 'refs/genvm-ff/base..refs/genvm-ff/head') + if merges.returncode != 0 or merges.stdout.strip(): + raise Refused(f'`{base}` requires linear history, and the PR has merge commits') + pushed = git('push', 'origin', f'{head_sha}:refs/heads/{base}') + if pushed.returncode != 0: + raise Refused(f'`{base}` rejected the push:\n```\n{pushed.stderr.strip()}\n```') + + +def comment(body: str) -> None: + gh_common.gh_manager( + 'api', + '--method', + 'POST', + f'repos/{repo()}/issues/{pr_number()}/comments', + '-f', + f'body={body}', + retry=False, + ) + + +def run() -> int: + if os.environ.get('COMMENT_BODY', '').strip() != COMMAND: + print('comment is not an exact /genvm-fast-forward command; ignoring') + return 0 + request = os.environ['COMMENT_ID'] + sender = os.environ['SENDER'] + run_url = os.environ['RUN_URL'] + eyes = best_effort_add_reaction(request, 'eyes') + + def finish(reaction: str, body: str) -> None: + if eyes is not None: + delete_reaction(request, eyes) + best_effort_add_reaction(request, reaction) + comment(f'{body}\n\n[Open run]({run_url})') + + try: + pr = gh_json(f'repos/{repo()}/pulls/{pr_number()}') + base = pr['base']['ref'] + head_sha = pr['head']['sha'] + if failures := gate(pr, sender): + raise Refused('\n'.join(f'- {failure}' for failure in failures)) + fast_forward(base, head_sha) + except Refused as refusal: + finish('confused', f'πŸ˜• Not fast-forwarding:\n{refusal}') + return 1 + except Exception: + finish('confused', 'πŸ˜• `/genvm-fast-forward` failed unexpectedly') + raise + finish('rocket', f'πŸš€ Fast-forwarded `{base}` to `{head_sha[:12]}`') + return 0 + + +class FastForwardCommand(ci_lib.Tool): + """Fast-forward a PR's base branch to its head on `/genvm-fast-forward`.""" + + def name(self) -> str: + return 'fast-forward-command' + + def add_to(self, parser: argparse.ArgumentParser) -> None: + gh_common.add_args(parser, executor_repo=False, head_ref=False) + + def handler(self, args: argparse.Namespace) -> int: + gh_common.set_ctx(gh_common.Ctx.from_args(args)) + return run() + + +COMMANDS = [FastForwardCommand()] diff --git a/support/ci/unit_tests/test_fast_forward_command.py b/support/ci/unit_tests/test_fast_forward_command.py new file mode 100644 index 00000000..41d063d8 --- /dev/null +++ b/support/ci/unit_tests/test_fast_forward_command.py @@ -0,0 +1,391 @@ +import subprocess + +import gh_common +import pytest +from tools import fast_forward_command as command + +HEAD = 'a' * 40 +OLD = 'b' * 40 +E2E_APP = 2929262 +ACTIONS_APP = 15368 +REQUIRED = [('validate-end', ACTIONS_APP), ('E2E Tests', E2E_APP)] + + +@pytest.fixture(autouse=True) +def context(): + gh_common.set_ctx( + gh_common.Ctx( + manager_repo='org/manager', + executor_repo='org/executor', + _pr_number='42', + _head_ref=None, + ) + ) + + +def check_run(name, app, conclusion='success', *, id=1, status='completed'): + return { + 'id': id, + 'name': name, + 'app': {'id': app}, + 'status': status, + 'conclusion': conclusion if status == 'completed' else None, + } + + +def green_runs(): + return [check_run('validate-end', ACTIONS_APP), check_run('E2E Tests', E2E_APP)] + + +def review(login, state, commit=HEAD, user_type='User'): + return { + 'user': {'login': login, 'type': user_type}, + 'state': state, + 'commit_id': commit, + } + + +PERMISSIONS = { + 'admin': 'admin', + 'writer': 'write', + 'other-writer': 'write', + 'reader': 'read', + 'coderabbitai[bot]': 'none', +} + + +def reviews_gate(reviews, sender='writer', author='author'): + return command.review_failures( + sender=sender, + sender_permission=PERMISSIONS.get(sender), + author=author, + head_sha=HEAD, + reviews=reviews, + permission_of=PERMISSIONS.get, + ) + + +# -- Checks -- + + +def test_green_required_checks_pass(): + assert command.check_failures(REQUIRED, green_runs(), []) == [] + + +def test_missing_e2e_fails(): + runs = [check_run('validate-end', ACTIONS_APP)] + assert command.check_failures(REQUIRED, runs, []) == ['`E2E Tests` has not run'] + + +def test_pending_check_fails(): + runs = [ + check_run('validate-end', ACTIONS_APP), + check_run('E2E Tests', E2E_APP, status='in_progress'), + ] + assert command.check_failures(REQUIRED, runs, []) == ['`E2E Tests` is in progress'] + + +def test_skipped_check_fails(): + runs = [ + check_run('validate-end', ACTIONS_APP, 'skipped'), + check_run('E2E Tests', E2E_APP), + ] + assert command.check_failures(REQUIRED, runs, []) == [ + '`validate-end` concluded `skipped`' + ] + + +def test_latest_rerun_decides(): + runs = [ + check_run('validate-end', ACTIONS_APP, 'failure', id=1), + check_run('validate-end', ACTIONS_APP, 'success', id=2), + check_run('E2E Tests', E2E_APP), + ] + assert command.check_failures(REQUIRED, runs, []) == [] + runs[0]['id'] = 3 + assert command.check_failures(REQUIRED, runs, []) == [ + '`validate-end` concluded `failure`' + ] + + +def test_same_name_from_another_app_does_not_count(): + runs = [check_run('validate-end', ACTIONS_APP), check_run('E2E Tests', ACTIONS_APP)] + assert command.check_failures(REQUIRED, runs, []) == ['`E2E Tests` has not run'] + + +def test_status_satisfies_a_check_without_app(): + required = [('CodeRabbit', None)] + statuses = [ + {'context': 'CodeRabbit', 'state': 'success'}, + {'context': 'CodeRabbit', 'state': 'pending'}, + ] + assert command.check_failures(required, [], statuses) == [] + statuses.reverse() + assert command.check_failures(required, [], statuses) == ['`CodeRabbit` is `pending`'] + + +def test_status_does_not_satisfy_a_check_bound_to_an_app(): + statuses = [{'context': 'E2E Tests', 'state': 'success'}] + runs = [check_run('validate-end', ACTIONS_APP)] + assert command.check_failures(REQUIRED, runs, statuses) == ['`E2E Tests` has not run'] + + +RULES = [ + { + 'type': 'required_status_checks', + 'parameters': { + 'required_status_checks': [ + {'context': 'validate-end', 'integration_id': ACTIONS_APP} + ] + }, + }, + {'type': 'non_fast_forward', 'parameters': None}, +] + + +def test_e2e_is_required_from_its_app_even_when_rules_omit_it(): + assert command.required_checks(RULES) == [ + ('validate-end', ACTIONS_APP), + ('E2E Tests', E2E_APP), + ] + + +def test_unknown_rule_fails_closed(): + rules = [*RULES, {'type': 'pull_request', 'parameters': {}}] + assert command.rule_failures(RULES) == [] + assert command.rule_failures(rules) == [ + 'the base has a `pull_request` rule this command does not enforce' + ] + + +# -- Reviews -- + + +def test_admin_needs_no_approval(): + assert reviews_gate([], sender='admin') == [] + + +def test_reader_is_refused(): + assert reviews_gate([review('other-writer', 'APPROVED')], sender='reader') == [ + '`reader` is neither an admin nor a writer' + ] + + +def test_writer_with_writer_approval_passes(): + assert reviews_gate([review('other-writer', 'APPROVED')]) == [] + + +def test_writer_without_approval_fails(): + assert reviews_gate([]) == [ + f'no writer other than the author and `writer` approved `{HEAD[:12]}`' + ] + + +def test_stale_approval_does_not_count(): + assert reviews_gate([review('other-writer', 'APPROVED', commit=OLD)]) != [] + + +def test_bot_and_reader_approvals_do_not_count(): + reviews = [ + review('coderabbitai[bot]', 'APPROVED', user_type='Bot'), + review('reader', 'APPROVED'), + ] + assert reviews_gate(reviews) != [] + + +def test_author_approval_does_not_count(): + assert reviews_gate([review('writer', 'APPROVED')], author='writer') != [] + + +def test_commenter_approval_does_not_count(): + assert reviews_gate([review('writer', 'APPROVED')], sender='writer') != [] + + +def test_later_comment_keeps_approval(): + assert ( + reviews_gate( + [review('other-writer', 'APPROVED'), review('other-writer', 'COMMENTED')] + ) + == [] + ) + + +def test_dismissed_approval_does_not_count(): + assert ( + reviews_gate( + [review('other-writer', 'APPROVED'), review('other-writer', 'DISMISSED')] + ) + != [] + ) + + +def test_writer_change_request_blocks_despite_approval(): + reviews = [ + review('other-writer', 'APPROVED'), + review('admin', 'CHANGES_REQUESTED', commit=OLD), + ] + assert reviews_gate(reviews) == ['`admin` requested changes'] + + +def test_reader_change_request_does_not_block(): + assert ( + reviews_gate( + [review('other-writer', 'APPROVED'), review('reader', 'CHANGES_REQUESTED')] + ) + == [] + ) + + +def test_deleted_reviewer_is_ignored(): + reviews = [ + {'user': None, 'state': 'APPROVED', 'commit_id': HEAD}, + review('other-writer', 'APPROVED'), + ] + assert reviews_gate(reviews) == [] + + +# -- Command -- + + +def pr(**overrides): + return { + 'state': 'open', + 'draft': False, + 'user': {'login': 'author'}, + 'base': {'ref': 'v0.6-dev', 'repo': {'default_branch': 'main'}}, + 'head': {'sha': HEAD, 'repo': {'full_name': 'org/manager'}}, + **overrides, + } + + +@pytest.fixture +def api(monkeypatch): + state = { + 'compare': 'ahead', + 'pr': pr(), + 'rules': RULES, + 'pushed': [], + 'comments': [], + 'reactions': [], + } + + def gh_json(path): + if '/compare/' in path: + return {'status': state['compare']} + return state['pr'] + + def gh_items(path, jq='.[]'): + if path.endswith('/reviews?per_page=100'): + return [review('other-writer', 'APPROVED')] + if '/check-runs' in path: + return green_runs() + return [] + + monkeypatch.setattr(command, 'gh_json', gh_json) + monkeypatch.setattr(command, 'gh_items', gh_items) + monkeypatch.setattr(command, 'base_rules', lambda _base: state['rules']) + monkeypatch.setattr(command, 'permission', PERMISSIONS.get) + monkeypatch.setattr( + command, 'fast_forward', lambda *args: state['pushed'].append(args) + ) + monkeypatch.setattr(command, 'comment', state['comments'].append) + monkeypatch.setattr( + command, + 'best_effort_add_reaction', + lambda _comment, content: state['reactions'].append(content) or '7', + ) + monkeypatch.setattr( + command, 'delete_reaction', lambda *_args: state['reactions'].append('-') + ) + monkeypatch.setenv('COMMENT_BODY', '/genvm-fast-forward\n') + monkeypatch.setenv('COMMENT_ID', '9001') + monkeypatch.setenv('SENDER', 'writer') + monkeypatch.setenv('RUN_URL', 'https://example.test/run/1') + return state + + +def test_green_pr_is_fast_forwarded(api): + assert command.run() == 0 + assert api['pushed'] == [('v0.6-dev', HEAD)] + assert api['reactions'] == ['eyes', '-', 'rocket'] + assert api['comments'][0].startswith(f'πŸš€ Fast-forwarded `v0.6-dev` to `{HEAD[:12]}`') + + +def test_other_comment_is_ignored(api, monkeypatch): + monkeypatch.setenv('COMMENT_BODY', '/genvm-fast-forward please') + assert command.run() == 0 + assert api['pushed'] == api['reactions'] == [] + + +def test_refusal_lists_every_failure(api): + api['pr'] = pr( + draft=True, + base={'ref': 'main', 'repo': {'default_branch': 'main'}}, + head={'sha': HEAD, 'repo': {'full_name': 'someone/manager'}}, + ) + api['compare'] = 'diverged' + assert command.run() == 1 + assert api['pushed'] == [] + assert api['reactions'] == ['eyes', '-', 'confused'] + body = api['comments'][0] + assert '- the PR is a draft' in body, body + assert '- the PR comes from a fork' in body, body + assert '- `main` is neither a dev nor a release branch' in body, body + assert f'- `{HEAD[:12]}` is diverged relative to `main`; rebase it' in body, body + + +@pytest.mark.parametrize(('sender', 'refused'), [('writer', True), ('admin', False)]) +def test_release_branch_needs_an_admin(api, monkeypatch, sender, refused): + monkeypatch.setenv('SENDER', sender) + api['pr'] = pr(base={'ref': 'v0.6', 'repo': {'default_branch': 'main'}}) + assert command.run() == int(refused) + assert (api['pushed'] == []) == refused + + +def test_unreadable_permission_raises(monkeypatch): + monkeypatch.setattr( + gh_common, + 'gh', + lambda *_args, **_kwargs: subprocess.CompletedProcess([], 1, '', 'HTTP 502'), + ) + with pytest.raises(RuntimeError): + command.permission('writer') + monkeypatch.setattr( + gh_common, + 'gh', + lambda *_args, **_kwargs: subprocess.CompletedProcess([], 1, '', 'HTTP 404'), + ) + assert command.permission('stranger') is None + + +def test_up_to_date_base_is_refused(api): + api['compare'] = 'identical' + assert command.run() == 1 + assert api['pushed'] == [] + + +def test_closed_pr_is_refused(api): + api['pr'] = pr(state='closed') + assert command.run() == 1 + assert '- the PR is not open' in api['comments'][0] + + +def test_rejected_push_is_reported(api, monkeypatch): + def reject(*_args): + raise command.Refused('`v0.6-dev` rejected the push') + + monkeypatch.setattr(command, 'fast_forward', reject) + assert command.run() == 1 + assert api['reactions'][-1] == 'confused' + assert 'rejected the push' in api['comments'][0] + + +def test_unexpected_error_is_reported_and_raised(api, monkeypatch): + def boom(_path): + raise RuntimeError('api down') + + monkeypatch.setattr(command, 'gh_json', boom) + with pytest.raises(RuntimeError): + command.run() + assert api['reactions'][-1] == 'confused' + assert 'failed unexpectedly' in api['comments'][0] From 616fb9de11055bdcb6f75150b7d1f5d987500421 Mon Sep 17 00:00:00 2001 From: kp2pml30 <kp2pml30@gmail.com> Date: Fri, 2 Oct 2026 22:16:53 +0900 Subject: [PATCH 35/45] =?UTF-8?q?build(deps):=20bump=20wasmtime=20to=2048.?= =?UTF-8?q?0.3=20=E2=AC=86=EF=B8=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Bump both executor lines to wasmtime v48.0.3, which fixes GHSA-jqpg-j7w6-42pr and GHSA-m63x-6p34-q65x; neither is reachable, as the executor runs without fuel and outside the component model --- executors/v0.2.x | 2 +- executors/v0.3.x | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/executors/v0.2.x b/executors/v0.2.x index b08736a2..7cba87e0 160000 --- a/executors/v0.2.x +++ b/executors/v0.2.x @@ -1 +1 @@ -Subproject commit b08736a2d9d1292c6a5c50a2d1b4b12530986a4d +Subproject commit 7cba87e0dd5278895b91fea744c75ace3b6bbab1 diff --git a/executors/v0.3.x b/executors/v0.3.x index c476c2b4..be2fe817 160000 --- a/executors/v0.3.x +++ b/executors/v0.3.x @@ -1 +1 @@ -Subproject commit c476c2b4b8d10f50e79db328db1171b087a5ffdc +Subproject commit be2fe81793a27376427a6a42f58f76a5d506e2b4 From 072d9eacb871f307bd68fe2123c425f4a06eb349 Mon Sep 17 00:00:00 2001 From: kp2pml30 <kp2pml30@gmail.com> Date: Fri, 2 Oct 2026 22:17:05 +0900 Subject: [PATCH 36/45] =?UTF-8?q?fix(fees):=20let=20a=20zero-budget=20exac?= =?UTF-8?q?t=20key=20shadow=20internal=20wildcards=20=F0=9F=90=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Consensus resolves internal allocation keys by presence: an exact key with no budget left still wins over a funded `call_key` wildcard, its phase still decides, and emission fails with `out_of message_fee allocation_budget internal` - External messages keep treating zero-budget chain keys as absent - Describe the split in the fee impl-spec and the manager API schema Co-authored-by: Edgars <edgars@genlayerlabs.com> --- docs/website/src/impl-spec/04-fees.rst | 19 ++++++++++--------- .../src/impl-spec/appendix/manager-api.yaml | 2 +- executors/v0.3.x | 2 +- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/docs/website/src/impl-spec/04-fees.rst b/docs/website/src/impl-spec/04-fees.rst index e6e6fd3a..269644c5 100644 --- a/docs/website/src/impl-spec/04-fees.rst +++ b/docs/website/src/impl-spec/04-fees.rst @@ -299,15 +299,16 @@ The default v0.3 fee expressions reject external reservations and receipts with external messages without a matching allocation Chain entries, those with a concrete ``recipient``, carry the budget stored on -chain. Consensus resolves only keys with a nonzero budget, so a chain entry whose -``budget`` is zero is absent: the message falls through to the per-recipient -``call_key`` wildcard, and with no wildcard left an internal message fails with -``fee no_matching_allocation`` while an external one takes the unallocated path. -Synthetic recipient wildcards match independently of ``budget``; zero there means -an exhausted allocation. ``null`` removes the per-allocation cap while keeping the -execution's fee buckets. Local consumption is tracked separately: an internal key -exhausted by this execution's own emissions still wins and fails its budget check; -it never falls through. +chain. For internal messages, presence alone resolves a key: an entry whose +``budget`` is zero still shadows broader keys and fails emission with +``out_of message_fee allocation_budget internal``, as does a key exhausted by this +execution's own emissions; neither falls through. For external messages, +consensus resolves only chain keys with a nonzero budget, so a zero-budget entry is +absent: the message falls through to the per-recipient ``call_key`` wildcard, and +with no wildcard left takes the unallocated path. Synthetic recipient wildcards +match independently of ``budget``; zero there means an exhausted allocation. +``null`` removes the per-allocation cap while keeping the execution's fee buckets. +Local consumption is tracked separately. The host must preserve every existing pinned key and must not add recipient wildcards to a pinned tree. An empty list restricts internal diff --git a/docs/website/src/impl-spec/appendix/manager-api.yaml b/docs/website/src/impl-spec/appendix/manager-api.yaml index 17346a3b..3bbddf34 100644 --- a/docs/website/src/impl-spec/appendix/manager-api.yaml +++ b/docs/website/src/impl-spec/appendix/manager-api.yaml @@ -889,7 +889,7 @@ components: description: Call key bytes, or null for a wildcard. budget: nullable: true - description: Stored chain budget, or the allowance of a synthetic recipient wildcard; null is uncapped. A zero-budget entry with a recipient is absent, as on chain. + description: Stored chain budget, or the allowance of a synthetic recipient wildcard; null is uncapped. A zero-budget entry with a recipient still matches internal messages and is absent for external ones, as on chain. on: type: string enum: [decided, finalized] diff --git a/executors/v0.3.x b/executors/v0.3.x index be2fe817..3cd91d26 160000 --- a/executors/v0.3.x +++ b/executors/v0.3.x @@ -1 +1 @@ -Subproject commit be2fe81793a27376427a6a42f58f76a5d506e2b4 +Subproject commit 3cd91d26d7cf4d1a4e5de096dcb71ad333dd0c07 From 7c1748a7d3a49f8c6f5c95cb53631edf86cc033c Mon Sep 17 00:00:00 2001 From: kp2pml30 <kp2pml30@gmail.com> Date: Fri, 2 Oct 2026 22:17:17 +0900 Subject: [PATCH 37/45] =?UTF-8?q?build(deps):=20bump=20puppeteer-core=20to?= =?UTF-8?q?=2025.12.0=20and=20the=20webdriver=20to=20Node=2024=20=E2=AC=86?= =?UTF-8?q?=EF=B8=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - puppeteer-core 25 no longer pulls `extract-zip` or the `proxy-agent` stack, clearing every open npm advisory in the webdriver lock (`extract-zip`, `ip-address`, `basic-ftp`, `ws`) - It needs Node 22.12 or newer; the image moves from Node 20, which is past end of life, to Node 24 LTS --- webdriver/Dockerfile | 2 +- webdriver/src/prj/package-lock.json | 864 +++++----------------------- webdriver/src/prj/package.json | 4 +- 3 files changed, 149 insertions(+), 721 deletions(-) diff --git a/webdriver/Dockerfile b/webdriver/Dockerfile index 1ad8d177..339f0810 100644 --- a/webdriver/Dockerfile +++ b/webdriver/Dockerfile @@ -36,7 +36,7 @@ RUN \ xdg-utils \ chromium \ chromium-sandbox ; \ - curl -fsSL https://deb.nodesource.com/setup_20.x | bash - ; \ + curl -fsSL https://deb.nodesource.com/setup_24.x | bash - ; \ apt-get install -y nodejs ; \ apt-get clean ; \ rm -rf /var/lib/apt/lists/* ; \ diff --git a/webdriver/src/prj/package-lock.json b/webdriver/src/prj/package-lock.json index 7452abc9..c740a047 100644 --- a/webdriver/src/prj/package-lock.json +++ b/webdriver/src/prj/package-lock.json @@ -9,7 +9,7 @@ "version": "1.0.0", "dependencies": { "commander": "^11.0.0", - "puppeteer-core": "24.16.0" + "puppeteer-core": "25.12.0" }, "devDependencies": { "@types/node": "^20.0.0", @@ -17,7 +17,7 @@ "typescript": "^5.0.0" }, "engines": { - "node": ">=18.0.0" + "node": ">=22.12.0" } }, "node_modules/@cspotcode/source-map-support": { @@ -62,32 +62,33 @@ } }, "node_modules/@puppeteer/browsers": { - "version": "2.10.6", - "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-2.10.6.tgz", - "integrity": "sha512-pHUn6ZRt39bP3698HFQlu2ZHCkS/lPcpv7fVQcGBSzNNygw171UXAKrCUhy+TEMw4lEttOKDgNpb04hwUAJeiQ==", + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-3.2.3.tgz", + "integrity": "sha512-2Bt3m6dDAJqmZehn0wiXSYyQmuhyiHHZ7FjvrVaS1W2RoFGV6eCK71UvKYkNJjkwC1Q9nobjhvTMhLQztFVINA==", "license": "Apache-2.0", "dependencies": { - "debug": "^4.4.1", - "extract-zip": "^2.0.1", - "progress": "^2.0.3", - "proxy-agent": "^6.5.0", - "semver": "^7.7.2", - "tar-fs": "^3.1.0", - "yargs": "^17.7.2" + "modern-tar": "^0.8.4", + "yargs": "^18.0.0" }, "bin": { - "browsers": "lib/cjs/main-cli.js" + "browsers": "lib/main-cli.js" }, "engines": { - "node": ">=18" + "node": ">=22.12.0" + }, + "peerDependencies": { + "proxy-agent": ">=8.0.1", + "yauzl": "^2.10.0 || ^3.4.0" + }, + "peerDependenciesMeta": { + "proxy-agent": { + "optional": true + }, + "yauzl": { + "optional": true + } } }, - "node_modules/@tootallnate/quickjs-emscripten": { - "version": "0.23.0", - "resolved": "https://registry.npmjs.org/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz", - "integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==", - "license": "MIT" - }, "node_modules/@tsconfig/node10": { "version": "1.0.12", "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.12.tgz", @@ -120,22 +121,12 @@ "version": "20.19.27", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.27.tgz", "integrity": "sha512-N2clP5pJhB2YnZJ3PIHFk5RkygRX5WO/5f0WC08tp0wd+sv0rsJk3MqWn3CbNmT2J505a5336jaQj4ph1AdMug==", - "devOptional": true, + "dev": true, "license": "MIT", "dependencies": { "undici-types": "~6.21.0" } }, - "node_modules/@types/yauzl": { - "version": "2.10.3", - "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.3.tgz", - "integrity": "sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==", - "license": "MIT", - "optional": true, - "dependencies": { - "@types/node": "*" - } - }, "node_modules/acorn": { "version": "8.15.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", @@ -162,34 +153,25 @@ "node": ">=0.4.0" } }, - "node_modules/agent-base": { - "version": "7.1.4", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", - "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", - "license": "MIT", - "engines": { - "node": ">= 14" - } - }, "node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "version": "6.4.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.4.0.tgz", + "integrity": "sha512-KzTVk2tCWAHtYrvvvaP8bJKJq2pVinhLcGEQdtLIYPbmNGNyYe8QwNaTUYQp2J7/vIsUKt5QCqAfUkYyG9DkOw==", "license": "MIT", "engines": { - "node": ">=8" + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" } }, "node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", "license": "MIT", - "dependencies": { - "color-convert": "^2.0.1" - }, "engines": { - "node": ">=8" + "node": ">=12" }, "funding": { "url": "https://github.com/chalk/ansi-styles?sponsor=1" @@ -202,186 +184,53 @@ "dev": true, "license": "MIT" }, - "node_modules/ast-types": { - "version": "0.13.4", - "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.13.4.tgz", - "integrity": "sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w==", - "license": "MIT", - "dependencies": { - "tslib": "^2.0.1" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/b4a": { - "version": "1.7.3", - "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.7.3.tgz", - "integrity": "sha512-5Q2mfq2WfGuFp3uS//0s6baOJLMoVduPYVeNmDYxu5OUA1/cBfvr2RIS7vi62LdNj/urk1hfmj867I3qt6uZ7Q==", - "license": "Apache-2.0", - "peerDependencies": { - "react-native-b4a": "*" - }, - "peerDependenciesMeta": { - "react-native-b4a": { - "optional": true - } - } - }, - "node_modules/bare-events": { - "version": "2.8.2", - "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.8.2.tgz", - "integrity": "sha512-riJjyv1/mHLIPX4RwiK+oW9/4c3TEUeORHKefKAKnZ5kyslbN+HXowtbaVEqt4IMUB7OXlfixcs6gsFeo/jhiQ==", - "license": "Apache-2.0", - "peerDependencies": { - "bare-abort-controller": "*" - }, - "peerDependenciesMeta": { - "bare-abort-controller": { - "optional": true - } - } - }, - "node_modules/bare-fs": { - "version": "4.5.2", - "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-4.5.2.tgz", - "integrity": "sha512-veTnRzkb6aPHOvSKIOy60KzURfBdUflr5VReI+NSaPL6xf+XLdONQgZgpYvUuZLVQ8dCqxpBAudaOM1+KpAUxw==", - "license": "Apache-2.0", - "optional": true, - "dependencies": { - "bare-events": "^2.5.4", - "bare-path": "^3.0.0", - "bare-stream": "^2.6.4", - "bare-url": "^2.2.2", - "fast-fifo": "^1.3.2" - }, - "engines": { - "bare": ">=1.16.0" - }, - "peerDependencies": { - "bare-buffer": "*" - }, - "peerDependenciesMeta": { - "bare-buffer": { - "optional": true - } - } - }, - "node_modules/bare-os": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/bare-os/-/bare-os-3.6.2.tgz", - "integrity": "sha512-T+V1+1srU2qYNBmJCXZkUY5vQ0B4FSlL3QDROnKQYOqeiQR8UbjNHlPa+TIbM4cuidiN9GaTaOZgSEgsvPbh5A==", - "license": "Apache-2.0", - "optional": true, - "engines": { - "bare": ">=1.14.0" - } - }, - "node_modules/bare-path": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-3.0.0.tgz", - "integrity": "sha512-tyfW2cQcB5NN8Saijrhqn0Zh7AnFNsnczRcuWODH0eYAXBsJ5gVxAUuNr7tsHSC6IZ77cA0SitzT+s47kot8Mw==", - "license": "Apache-2.0", - "optional": true, - "dependencies": { - "bare-os": "^3.0.1" - } - }, - "node_modules/bare-stream": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.7.0.tgz", - "integrity": "sha512-oyXQNicV1y8nc2aKffH+BUHFRXmx6VrPzlnaEvMhram0nPBrKcEdcyBg5r08D0i8VxngHFAiVyn1QKXpSG0B8A==", - "license": "Apache-2.0", - "optional": true, - "dependencies": { - "streamx": "^2.21.0" - }, - "peerDependencies": { - "bare-buffer": "*", - "bare-events": "*" - }, - "peerDependenciesMeta": { - "bare-buffer": { - "optional": true - }, - "bare-events": { - "optional": true - } - } - }, - "node_modules/bare-url": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/bare-url/-/bare-url-2.3.2.tgz", - "integrity": "sha512-ZMq4gd9ngV5aTMa5p9+UfY0b3skwhHELaDkhEHetMdX0LRkW9kzaym4oo/Eh+Ghm0CCDuMTsRIGM/ytUc1ZYmw==", - "license": "Apache-2.0", - "optional": true, - "dependencies": { - "bare-path": "^3.0.0" - } - }, - "node_modules/basic-ftp": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.3.0.tgz", - "integrity": "sha512-5K9eNNn7ywHPsYnFwjKgYH8Hf8B5emh7JKcPaVjjrMJFQQwGpwowEnZNEtHs7DfR7hCZsmaK3VA4HUK0YarT+w==", - "license": "MIT", - "engines": { - "node": ">=10.0.0" - } - }, - "node_modules/buffer-crc32": { - "version": "0.2.13", - "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", - "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", - "license": "MIT", - "engines": { - "node": "*" - } - }, "node_modules/chromium-bidi": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-7.2.0.tgz", - "integrity": "sha512-gREyhyBstermK+0RbcJLbFhcQctg92AGgDe/h/taMJEOLRdtSswBAO9KmvltFSQWgM2LrwWu5SIuEUbdm3JsyQ==", + "version": "17.0.2", + "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-17.0.2.tgz", + "integrity": "sha512-5v9GQFhTktFvotn/OFNJBmKLKRAb6n9r0bVCwf7sHgWc3/JryK0bj1nn93L3pHFrfgcsu6Be6EWsDi+1XHTGDg==", "license": "Apache-2.0", "dependencies": { "mitt": "^3.0.1", "zod": "^3.24.1" }, + "engines": { + "node": ">=20.19.0 <22.0.0 || >=22.12.0" + }, "peerDependencies": { "devtools-protocol": "*" } }, "node_modules/cliui": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", - "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-9.0.1.tgz", + "integrity": "sha512-k7ndgKhwoQveBL+/1tqGJYNz097I7WOvwbmmU2AR5+magtbjPWQTS1C5vzGkBC8Ym8UWRzfKUzUUqFLypY4Q+w==", "license": "ISC", "dependencies": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.1", - "wrap-ansi": "^7.0.0" + "string-width": "^7.2.0", + "strip-ansi": "^7.1.0", + "wrap-ansi": "^9.0.0" }, "engines": { - "node": ">=12" + "node": ">=20" } }, - "node_modules/color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "node_modules/cliui/node_modules/string-width": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", + "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", "license": "MIT", "dependencies": { - "color-name": "~1.1.4" + "emoji-regex": "^10.3.0", + "get-east-asian-width": "^1.0.0", + "strip-ansi": "^7.1.0" }, "engines": { - "node": ">=7.0.0" + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "license": "MIT" - }, "node_modules/commander": { "version": "11.1.0", "resolved": "https://registry.npmjs.org/commander/-/commander-11.1.0.tgz", @@ -398,50 +247,10 @@ "dev": true, "license": "MIT" }, - "node_modules/data-uri-to-buffer": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-6.0.2.tgz", - "integrity": "sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw==", - "license": "MIT", - "engines": { - "node": ">= 14" - } - }, - "node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/degenerator": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/degenerator/-/degenerator-5.0.1.tgz", - "integrity": "sha512-TllpMR/t0M5sqCXfj85i4XaAzxmS5tVA16dqvdkMwGmzI+dXLXnw3J+3Vdv7VKw+ThlTMboK6i9rnZ6Nntj5CQ==", - "license": "MIT", - "dependencies": { - "ast-types": "^0.13.4", - "escodegen": "^2.1.0", - "esprima": "^4.0.1" - }, - "engines": { - "node": ">= 14" - } - }, "node_modules/devtools-protocol": { - "version": "0.0.1475386", - "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1475386.tgz", - "integrity": "sha512-RQ809ykTfJ+dgj9bftdeL2vRVxASAuGU+I9LEx9Ij5TXU5HrgAQVmzi72VA+mkzscE12uzlRv5/tWWv9R9J1SA==", + "version": "0.0.1687809", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1687809.tgz", + "integrity": "sha512-t7kSb+UKYdyxcmqy98U2P86ne6rCve3DIJ4+lp54i/hhClgFuG35JrLj23rO14Jw3sWDsp7i+22QhNU0+/qk8g==", "license": "BSD-3-Clause" }, "node_modules/diff": { @@ -455,20 +264,11 @@ } }, "node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "version": "10.6.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.6.0.tgz", + "integrity": "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==", "license": "MIT" }, - "node_modules/end-of-stream": { - "version": "1.4.5", - "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", - "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", - "license": "MIT", - "dependencies": { - "once": "^1.4.0" - } - }, "node_modules/escalade": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", @@ -478,102 +278,6 @@ "node": ">=6" } }, - "node_modules/escodegen": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz", - "integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==", - "license": "BSD-2-Clause", - "dependencies": { - "esprima": "^4.0.1", - "estraverse": "^5.2.0", - "esutils": "^2.0.2" - }, - "bin": { - "escodegen": "bin/escodegen.js", - "esgenerate": "bin/esgenerate.js" - }, - "engines": { - "node": ">=6.0" - }, - "optionalDependencies": { - "source-map": "~0.6.1" - } - }, - "node_modules/esprima": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", - "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", - "license": "BSD-2-Clause", - "bin": { - "esparse": "bin/esparse.js", - "esvalidate": "bin/esvalidate.js" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/estraverse": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", - "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", - "license": "BSD-2-Clause", - "engines": { - "node": ">=4.0" - } - }, - "node_modules/esutils": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", - "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", - "license": "BSD-2-Clause", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/events-universal": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/events-universal/-/events-universal-1.0.1.tgz", - "integrity": "sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==", - "license": "Apache-2.0", - "dependencies": { - "bare-events": "^2.7.0" - } - }, - "node_modules/extract-zip": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz", - "integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==", - "license": "BSD-2-Clause", - "dependencies": { - "debug": "^4.1.1", - "get-stream": "^5.1.0", - "yauzl": "^2.10.0" - }, - "bin": { - "extract-zip": "cli.js" - }, - "engines": { - "node": ">= 10.17.0" - }, - "optionalDependencies": { - "@types/yauzl": "^2.9.1" - } - }, - "node_modules/fast-fifo": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", - "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==", - "license": "MIT" - }, - "node_modules/fd-slicer": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", - "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==", - "license": "MIT", - "dependencies": { - "pend": "~1.2.0" - } - }, "node_modules/get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", @@ -583,88 +287,18 @@ "node": "6.* || 8.* || >= 10.*" } }, - "node_modules/get-stream": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", - "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", + "node_modules/get-east-asian-width": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.7.0.tgz", + "integrity": "sha512-XjH1AECxf0giL2V1aU8vKyRR2ppRUb5c0EvT7zuJTokQ74bNo52zOtghqdWIqrhUD79fo3x0WfKZdOqxF6LG1Q==", "license": "MIT", - "dependencies": { - "pump": "^3.0.0" - }, "engines": { - "node": ">=8" + "node": ">=18" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/get-uri": { - "version": "6.0.5", - "resolved": "https://registry.npmjs.org/get-uri/-/get-uri-6.0.5.tgz", - "integrity": "sha512-b1O07XYq8eRuVzBNgJLstU6FYc1tS6wnMtF1I1D9lE8LxZSOGZ7LhxN54yPP6mGw5f2CkXY2BQUL9Fx41qvcIg==", - "license": "MIT", - "dependencies": { - "basic-ftp": "^5.0.2", - "data-uri-to-buffer": "^6.0.2", - "debug": "^4.3.4" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/http-proxy-agent": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", - "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", - "license": "MIT", - "dependencies": { - "agent-base": "^7.1.0", - "debug": "^4.3.4" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/https-proxy-agent": { - "version": "7.0.6", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", - "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", - "license": "MIT", - "dependencies": { - "agent-base": "^7.1.2", - "debug": "4" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/ip-address": { - "version": "10.1.0", - "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.1.0.tgz", - "integrity": "sha512-XXADHxXmvT9+CRxhXg56LJovE+bmWnEWB78LB83VZTprKTmaC5QfruXocxzTZ2Kl0DNwKuBdlIhjL8LeY8Sf8Q==", - "license": "MIT", - "engines": { - "node": ">= 12" - } - }, - "node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/lru-cache": { - "version": "7.18.3", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", - "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==", - "license": "ISC", - "engines": { - "node": ">=12" - } - }, "node_modules/make-error": { "version": "1.3.6", "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", @@ -678,267 +312,61 @@ "integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==", "license": "MIT" }, - "node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, - "node_modules/netmask": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/netmask/-/netmask-2.0.2.tgz", - "integrity": "sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg==", - "license": "MIT", - "engines": { - "node": ">= 0.4.0" - } - }, - "node_modules/once": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "license": "ISC", - "dependencies": { - "wrappy": "1" - } - }, - "node_modules/pac-proxy-agent": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.2.0.tgz", - "integrity": "sha512-TEB8ESquiLMc0lV8vcd5Ql/JAKAoyzHFXaStwjkzpOpC5Yv+pIzLfHvjTSdf3vpa2bMiUQrg9i6276yn8666aA==", + "node_modules/modern-tar": { + "version": "0.8.5", + "resolved": "https://registry.npmjs.org/modern-tar/-/modern-tar-0.8.5.tgz", + "integrity": "sha512-snEhs+6G5Tjd4I7tLCDOaoln2RgE0bD19RzEKgvgK2hZ5VKy3MpLhLTZ2fWpXSTg4K2cyPwp+VHATFJhxfnOeA==", "license": "MIT", - "dependencies": { - "@tootallnate/quickjs-emscripten": "^0.23.0", - "agent-base": "^7.1.2", - "debug": "^4.3.4", - "get-uri": "^6.0.1", - "http-proxy-agent": "^7.0.0", - "https-proxy-agent": "^7.0.6", - "pac-resolver": "^7.0.1", - "socks-proxy-agent": "^8.0.5" - }, "engines": { - "node": ">= 14" - } - }, - "node_modules/pac-resolver": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/pac-resolver/-/pac-resolver-7.0.1.tgz", - "integrity": "sha512-5NPgf87AT2STgwa2ntRMr45jTKrYBGkVU36yT0ig/n/GMAa3oPqhZfIQ2kMEimReg0+t9kZViDVZ83qfVUlckg==", - "license": "MIT", - "dependencies": { - "degenerator": "^5.0.0", - "netmask": "^2.0.2" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/pend": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", - "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", - "license": "MIT" - }, - "node_modules/progress": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", - "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", - "license": "MIT", - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/proxy-agent": { - "version": "6.5.0", - "resolved": "https://registry.npmjs.org/proxy-agent/-/proxy-agent-6.5.0.tgz", - "integrity": "sha512-TmatMXdr2KlRiA2CyDu8GqR8EjahTG3aY3nXjdzFyoZbmB8hrBsTyMezhULIXKnC0jpfjlmiZ3+EaCzoInSu/A==", - "license": "MIT", - "dependencies": { - "agent-base": "^7.1.2", - "debug": "^4.3.4", - "http-proxy-agent": "^7.0.1", - "https-proxy-agent": "^7.0.6", - "lru-cache": "^7.14.1", - "pac-proxy-agent": "^7.1.0", - "proxy-from-env": "^1.1.0", - "socks-proxy-agent": "^8.0.5" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/proxy-from-env": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", - "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", - "license": "MIT" - }, - "node_modules/pump": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.3.tgz", - "integrity": "sha512-todwxLMY7/heScKmntwQG8CXVkWUOdYxIvY2s0VWAAMh/nd8SoYiRaKjlr7+iCs984f2P8zvrfWcDDYVb73NfA==", - "license": "MIT", - "dependencies": { - "end-of-stream": "^1.1.0", - "once": "^1.3.1" + "node": ">=18.0.0" } }, "node_modules/puppeteer-core": { - "version": "24.16.0", - "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-24.16.0.tgz", - "integrity": "sha512-tZ0tJiOYaDGTRzzr2giDpf8O/55JsoqkrafS1Xu4H6S8oP4eeL6RbZzY9OzjShSf5EQvx/zAc55QKpDqzXos/Q==", + "version": "25.12.0", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-25.12.0.tgz", + "integrity": "sha512-z6LQUt5SH7jwGtGA+nMtJuEjZIsqPiwQ+EFHCYuVDQrJpvifNmmGuyDvTzAKPyFzihxgHc5f3aMzu2kn8Ps37A==", "license": "Apache-2.0", "dependencies": { - "@puppeteer/browsers": "2.10.6", - "chromium-bidi": "7.2.0", - "debug": "^4.4.1", - "devtools-protocol": "0.0.1475386", - "typed-query-selector": "^2.12.0", - "ws": "^8.18.3" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/require-directory": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", - "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/semver": { - "version": "7.7.3", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", - "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/smart-buffer": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", - "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", - "license": "MIT", - "engines": { - "node": ">= 6.0.0", - "npm": ">= 3.0.0" - } - }, - "node_modules/socks": { - "version": "2.8.7", - "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.7.tgz", - "integrity": "sha512-HLpt+uLy/pxB+bum/9DzAgiKS8CX1EvbWxI4zlmgGCExImLdiad2iCwXT5Z4c9c3Eq8rP2318mPW2c+QbtjK8A==", - "license": "MIT", - "dependencies": { - "ip-address": "^10.0.1", - "smart-buffer": "^4.2.0" - }, - "engines": { - "node": ">= 10.0.0", - "npm": ">= 3.0.0" - } - }, - "node_modules/socks-proxy-agent": { - "version": "8.0.5", - "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz", - "integrity": "sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==", - "license": "MIT", - "dependencies": { - "agent-base": "^7.1.2", - "debug": "^4.3.4", - "socks": "^2.8.3" + "@puppeteer/browsers": "3.2.3", + "chromium-bidi": "17.0.2", + "devtools-protocol": "0.0.1687809", + "typed-query-selector": "^2.12.2", + "webdriver-bidi-protocol": "0.4.3", + "ws": "^8.21.3" }, "engines": { - "node": ">= 14" - } - }, - "node_modules/source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "license": "BSD-3-Clause", - "optional": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/streamx": { - "version": "2.23.0", - "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.23.0.tgz", - "integrity": "sha512-kn+e44esVfn2Fa/O0CPFcex27fjIL6MkVae0Mm6q+E6f0hWv578YCERbv+4m02cjxvDsPKLnmxral/rR6lBMAg==", - "license": "MIT", - "dependencies": { - "events-universal": "^1.0.0", - "fast-fifo": "^1.3.2", - "text-decoder": "^1.1.0" + "node": ">=22.12.0" } }, "node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-8.3.0.tgz", + "integrity": "sha512-ZbmZM0JCihQN91dWnxoipT2KOEyHqEyfRXUyjuRhW8b/xnqPDoq4gWEVApTVa9db2wN8mmoikgFBbjh71+cGeQ==", "license": "MIT", "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" + "get-east-asian-width": "^1.5.0", + "strip-ansi": "^7.1.2" }, "engines": { - "node": ">=8" + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", + "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", "license": "MIT", "dependencies": { - "ansi-regex": "^5.0.1" + "ansi-regex": "^6.2.2" }, "engines": { - "node": ">=8" - } - }, - "node_modules/tar-fs": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.1.1.tgz", - "integrity": "sha512-LZA0oaPOc2fVo82Txf3gw+AkEd38szODlptMYejQUhndHMLQ9M059uXR+AfS7DNo0NpINvSqDsvyaCrBVkptWg==", - "license": "MIT", - "dependencies": { - "pump": "^3.0.0", - "tar-stream": "^3.1.5" + "node": ">=12" }, - "optionalDependencies": { - "bare-fs": "^4.0.1", - "bare-path": "^3.0.0" - } - }, - "node_modules/tar-stream": { - "version": "3.1.7", - "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.7.tgz", - "integrity": "sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==", - "license": "MIT", - "dependencies": { - "b4a": "^1.6.4", - "fast-fifo": "^1.2.0", - "streamx": "^2.15.0" - } - }, - "node_modules/text-decoder": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.3.tgz", - "integrity": "sha512-3/o9z3X0X0fTupwsYvR03pJ/DjWuqqrfwBgTQzdWDiQSm9KitAyz/9WqsT2JQW7KV2m+bC2ol/zqpW37NHxLaA==", - "license": "Apache-2.0", - "dependencies": { - "b4a": "^1.6.4" + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" } }, "node_modules/ts-node": { @@ -985,16 +413,10 @@ } } }, - "node_modules/tslib": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", - "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "license": "0BSD" - }, "node_modules/typed-query-selector": { - "version": "2.12.0", - "resolved": "https://registry.npmjs.org/typed-query-selector/-/typed-query-selector-2.12.0.tgz", - "integrity": "sha512-SbklCd1F0EiZOyPiW192rrHZzZ5sBijB6xM+cpmrwDqObvdtunOHHIk9fCGsoK5JVIYXoyEp4iEdE3upFH3PAg==", + "version": "2.12.3", + "resolved": "https://registry.npmjs.org/typed-query-selector/-/typed-query-selector-2.12.3.tgz", + "integrity": "sha512-/dtuMZyutHLlrxWhhC1uib00ELUr2dZ8C9B8YzZYsJmeE3V56OxRblrSM2UaF142pcMtak7ehZ6+poWGeehNIg==", "license": "MIT" }, "node_modules/typescript": { @@ -1015,7 +437,7 @@ "version": "6.21.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", - "devOptional": true, + "dev": true, "license": "MIT" }, "node_modules/v8-compile-cache-lib": { @@ -1025,33 +447,50 @@ "dev": true, "license": "MIT" }, + "node_modules/webdriver-bidi-protocol": { + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/webdriver-bidi-protocol/-/webdriver-bidi-protocol-0.4.3.tgz", + "integrity": "sha512-uuN0goWfxP22B7J/uAgBpOYNPttC+XVseYE+rSY5+rQ+YBeVz/VORw8WbmLVcqW78zNg5A4qnjNXYUWR3il2ig==", + "license": "Apache-2.0" + }, "node_modules/wrap-ansi": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-9.0.2.tgz", + "integrity": "sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==", "license": "MIT", "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" + "ansi-styles": "^6.2.1", + "string-width": "^7.0.0", + "strip-ansi": "^7.1.0" }, "engines": { - "node": ">=10" + "node": ">=18" }, "funding": { "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, - "node_modules/wrappy": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "license": "ISC" + "node_modules/wrap-ansi/node_modules/string-width": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", + "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^10.3.0", + "get-east-asian-width": "^1.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } }, "node_modules/ws": { - "version": "8.18.3", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.3.tgz", - "integrity": "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==", + "version": "8.22.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.22.0.tgz", + "integrity": "sha512-Ydggc987+RO0AnWtZ/7Wq9FtNvcrL1b/RO0ud9mWjUPgDrsAAwQSF51sm2hm1XofbU/4jkpGEsLFsZZxU+1DOg==", "license": "MIT", "engines": { "node": ">=10.0.0" @@ -1079,40 +518,29 @@ } }, "node_modules/yargs": { - "version": "17.7.2", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", - "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "version": "18.2.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-18.2.0.tgz", + "integrity": "sha512-9OpKOLeaoNFecEp7P6iYbzze/5CqWoH7N3SMs/6y1XF4nMvFMspEGZzJ6uFi9MmQwXhyzqYoSEKO3tvA3Z/o2w==", "license": "MIT", "dependencies": { - "cliui": "^8.0.1", + "cliui": "^9.0.1", "escalade": "^3.1.1", "get-caller-file": "^2.0.5", - "require-directory": "^2.1.1", - "string-width": "^4.2.3", + "string-width": "^8.2.1", "y18n": "^5.0.5", - "yargs-parser": "^21.1.1" + "yargs-parser": "^22.0.0" }, "engines": { - "node": ">=12" + "node": "^20.19.0 || ^22.12.0 || >=23" } }, "node_modules/yargs-parser": { - "version": "21.1.1", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", - "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "version": "22.0.0", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-22.0.0.tgz", + "integrity": "sha512-rwu/ClNdSMpkSrUb+d6BRsSkLUq1fmfsY6TOpYzTwvwkg1/NRG85KBy3kq++A8LKQwX6lsu+aWad+2khvuXrqw==", "license": "ISC", "engines": { - "node": ">=12" - } - }, - "node_modules/yauzl": { - "version": "2.10.0", - "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", - "integrity": "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==", - "license": "MIT", - "dependencies": { - "buffer-crc32": "~0.2.3", - "fd-slicer": "~1.1.0" + "node": "^20.19.0 || ^22.12.0 || >=23" } }, "node_modules/yn": { diff --git a/webdriver/src/prj/package.json b/webdriver/src/prj/package.json index 7311501a..c9a670b9 100644 --- a/webdriver/src/prj/package.json +++ b/webdriver/src/prj/package.json @@ -12,7 +12,7 @@ "typecheck": "tsc --noEmit" }, "dependencies": { - "puppeteer-core": "24.16.0", + "puppeteer-core": "25.12.0", "commander": "^11.0.0" }, "devDependencies": { @@ -21,6 +21,6 @@ "ts-node": "^10.0.0" }, "engines": { - "node": ">=18.0.0" + "node": ">=22.12.0" } } From e4252e59692d9c6dc2c15bc9f851623a0ad973f5 Mon Sep 17 00:00:00 2001 From: kp2pml30 <kp2pml30@gmail.com> Date: Fri, 2 Oct 2026 22:31:08 +0900 Subject: [PATCH 38/45] =?UTF-8?q?docs(fees):=20say=20when=20a=20zero-budge?= =?UTF-8?q?t=20internal=20key=20fails=20on=20its=20budget=20=F0=9F=93=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - A zero-budget exact key fails with `allocation_budget` only when its phase matches; otherwise no allocation matches - Pin a zero-budget recipient wildcard shadowing a synthetic wildcard on the internal path --- docs/website/src/impl-spec/04-fees.rst | 18 +++++++++--------- executors/v0.3.x | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/website/src/impl-spec/04-fees.rst b/docs/website/src/impl-spec/04-fees.rst index 269644c5..627d1df6 100644 --- a/docs/website/src/impl-spec/04-fees.rst +++ b/docs/website/src/impl-spec/04-fees.rst @@ -300,15 +300,15 @@ external messages without a matching allocation Chain entries, those with a concrete ``recipient``, carry the budget stored on chain. For internal messages, presence alone resolves a key: an entry whose -``budget`` is zero still shadows broader keys and fails emission with -``out_of message_fee allocation_budget internal``, as does a key exhausted by this -execution's own emissions; neither falls through. For external messages, -consensus resolves only chain keys with a nonzero budget, so a zero-budget entry is -absent: the message falls through to the per-recipient ``call_key`` wildcard, and -with no wildcard left takes the unallocated path. Synthetic recipient wildcards -match independently of ``budget``; zero there means an exhausted allocation. -``null`` removes the per-allocation cap while keeping the execution's fee buckets. -Local consumption is tracked separately. +``budget`` is zero still shadows broader keys and, when its phase matches, fails +emission with ``out_of message_fee allocation_budget internal``, as does a key +exhausted by this execution's own emissions, tracked separately; neither falls +through. For external messages, consensus resolves only chain keys with a nonzero +budget, so a zero-budget entry is absent: the message falls through to the +per-recipient ``call_key`` wildcard, and with no wildcard left takes the +unallocated path. Synthetic recipient wildcards match independently of ``budget``; +zero there means an exhausted allocation. ``null`` removes the per-allocation cap +while keeping the execution's fee buckets. The host must preserve every existing pinned key and must not add recipient wildcards to a pinned tree. An empty list restricts internal diff --git a/executors/v0.3.x b/executors/v0.3.x index 3cd91d26..49aa5746 160000 --- a/executors/v0.3.x +++ b/executors/v0.3.x @@ -1 +1 @@ -Subproject commit 3cd91d26d7cf4d1a4e5de096dcb71ad333dd0c07 +Subproject commit 49aa5746603f9a1b41a0144a87698de62a28282a From ec5c8aca3c0f3d7a95183dd79192143aef88c5ba Mon Sep 17 00:00:00 2001 From: Kira <kp2pml30@gmail.com> Date: Tue, 6 Oct 2026 23:28:20 +0900 Subject: [PATCH 39/45] =?UTF-8?q?fix(manager):=20socket=20run=20lifecycle?= =?UTF-8?q?=20and=20API=20spec=20train=20=F0=9F=90=9B=20(#58)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(cross-major): extract the shared manager harness ♻️ * test(cross-major): share the manager, deploy and call plumbing in gvm_extra ♻️ * test(cross-major): keep the loop fixture resolve-recursion needs πŸ› * test(cross-major): run every case through one runner with a deploy hook ♻️ * test(cross-major): stop shadowing the case context and fixture names ♻️ Co-authored-by: Hussein Adeiza <HusseinAdeiza@users.noreply.github.com> * docs(manager): bring the manager API schema in line with the server πŸ“ * docs(manager): type chain_id, value and bucket totals as integers πŸ› * docs(manager): quote the `on` key so YAML does not read it as true πŸ› * docs(manager): describe bodies and handshakes as the handlers encode them πŸ“ * docs(manager): give nullable properties a type for OpenAPI 3.0 πŸ“ * test(manager): say the parity tests compare property names only πŸ§ͺ Co-authored-by: Hussein Adeiza <HusseinAdeiza@users.noreply.github.com> * fix(manager): report a spawned run as finished, not failed_to_start πŸ› * fix(manager): keep the real exit code when an executor closes stdin early πŸ› * fix(manager): report a failure after the spawn as finished πŸ› * test(manager): check stderr capture for an executor exiting mid-write βœ… * docs(manager): say a failure after the spawn has a null exit_code πŸ“ Co-authored-by: Hussein Adeiza <HusseinAdeiza@users.noreply.github.com> * fix(manager): answer run before its semantic validation πŸ› * fix(manager): reject a bad top-level run as failed_to_start before permits πŸ› * fix(manager): take the module locks in supervision, not before the run id πŸ› * fix(manager): read module handlers through the held guards, not a relock πŸ› * fix(manager): ack a run that failed to start on the HTTP path πŸ› * test(manager): check both rejected-run kinds still get a run id βœ… * docs(manager): say a failed run holds its idempotency token until acked πŸ“ Co-authored-by: Hussein Adeiza <HusseinAdeiza@users.noreply.github.com> * fix(manager): ack only self-created failed runs and keep rejected permits intact πŸ› * docs(manager): align manager API and socket spec with the server πŸ“ * fix(manager): release module locks while a run waits for permits πŸ› --------- Co-authored-by: Hussein Adeiza <HusseinAdeiza@users.noreply.github.com> --- .../src/impl-spec/appendix/manager-api.yaml | 410 +++++++++++++----- .../src/impl-spec/appendix/manager-socket.rst | 70 +-- implementation/src/manager/handlers.rs | 24 +- implementation/src/manager/modules.rs | 89 ++-- implementation/src/manager/run.rs | 228 +++++++--- implementation/src/manager/run_test.rs | 193 ++++++++- implementation/src/manager/socket.rs | 45 +- tests/runner/gvm_extra/cross_major.py | 332 ++++++++++++++ tests/runner/origin/base_host.py | 6 +- .../system/cross-major-observability/test.py | 236 ++-------- tests/system/cross-major/test.py | 381 +++------------- tests/system/manager-socket/test.py | 163 ++++++- 12 files changed, 1339 insertions(+), 838 deletions(-) create mode 100644 tests/runner/gvm_extra/cross_major.py diff --git a/docs/website/src/impl-spec/appendix/manager-api.yaml b/docs/website/src/impl-spec/appendix/manager-api.yaml index 3bbddf34..104c85ea 100644 --- a/docs/website/src/impl-spec/appendix/manager-api.yaml +++ b/docs/website/src/impl-spec/appendix/manager-api.yaml @@ -24,8 +24,10 @@ paths: responses: '101': description: Switching protocols; the connection becomes a manager socket + '400': + description: Not a WebSocket handshake; the body is plain text '426': - description: Upgrade required (the request was not a WebSocket handshake) + description: A handshake on a connection that cannot be upgraded, such as HTTP/1.0 /status: get: @@ -42,12 +44,12 @@ paths: properties: llm_module: type: string + enum: [running, stopping, stopped] description: Status of the LLM module - example: running web_module: type: string + enum: [running, stopping, stopped] description: Status of the web module - example: stopped permits: type: object properties: @@ -65,7 +67,28 @@ paths: description: Permits a nondet run costs executions: type: object - description: Status of running executions + description: Known executions, keyed by GenVM id + additionalProperties: + type: object + properties: + result: + type: object + nullable: true + description: >- + Stored finished result; null for pending runs and retained startup failures + properties: + finished_at: + type: string + format: date-time + version: + type: string + description: Executor version the run resolved to, empty until resolved + started_at: + type: string + format: date-time + strict_deadline: + type: string + format: date-time /module/start: post: @@ -77,22 +100,7 @@ paths: content: application/json: schema: - type: object - required: - - module_type - properties: - module_type: - type: string - enum: [llm, web] - description: Type of module to start - config: - type: object - description: Module-specific configuration - example: - module_type: llm - config: - host: localhost - port: 8080 + $ref: '#/components/schemas/ModuleStartRequest' responses: '200': description: Module started successfully @@ -125,10 +133,10 @@ paths: properties: module_type: type: string - enum: [llm, web] + enum: [Llm, Web] description: Type of module to stop example: - module_type: llm + module_type: Llm responses: '200': description: Module stop result @@ -158,22 +166,7 @@ paths: content: application/json: schema: - type: object - required: - - module_type - properties: - module_type: - type: string - enum: [llm, web] - description: Type of module to restart - config: - type: object - description: Module-specific configuration (null to use default config file) - example: - module_type: llm - config: - host: localhost - port: 8080 + $ref: '#/components/schemas/ModuleStartRequest' responses: '200': description: Module restarted successfully @@ -200,17 +193,14 @@ paths: migrate. Unlike the socket `run`, this call responds only after the execution has started (or failed to start). - Both entry points decode the same request type and share the same start - path, so every field of `GenvmRunRequest` behaves identically here -- - including the ones the socket protocol introduced (`deadline`, - `host_hello_data`, `hook_cross_contract_calls`) and the `host_genvm_id` - idempotency token, whose reservation happens below both. That is a - guarantee for as long as this adapter exists, not an accident of shared - types. What differs is only what this call *returns*: an id after the - execution has started, rather than immediately. - - The request body uses calldata binary encoding (not JSON). The schema below - shows the logical structure that must be encoded using the calldata format. + Both entry points decode `GenvmRunRequest` and share the start path and + `host_genvm_id` idempotency token. A failed HTTP start created by this + request is released, freeing its token for a corrected retry. A retry + that finds an existing retained run does not release it. Socket startup + failures remain retained until `ack` or until the retention TTL expires + + The body is calldata, not JSON; `GenvmRunRequest` says how its types map + onto calldata. operationId: runGenvm requestBody: required: true @@ -284,14 +274,38 @@ paths: type: integer status: type: object - description: Current status of the GenVM instance + nullable: true + description: >- + Stored finished result; null for pending runs, retained startup failures, + acknowledged runs, or unknown ids properties: + finished_at: + type: integer + format: int64 + description: Unix time in milliseconds + stdout: + type: string + stderr: + type: string + genvm_log: + type: array + items: + type: object + metrics: + type: object + nullable: true consumed_result: type: string format: byte nullable: true + version_major: + type: integer + version_minor: + type: integer '400': $ref: '#/components/responses/ErrorResponse' + '404': + $ref: '#/components/responses/NotFound' '500': $ref: '#/components/responses/ErrorResponse' delete: @@ -318,6 +332,7 @@ paths: schema: oneOf: - type: object + required: [result, genvm_id] properties: result: type: string @@ -325,6 +340,7 @@ paths: genvm_id: type: integer - type: object + required: [error, genvm_id] properties: error: type: string @@ -333,6 +349,8 @@ paths: type: integer '400': $ref: '#/components/responses/ErrorResponse' + '404': + $ref: '#/components/responses/NotFound' /genvm/{genvm_id}/artifact: get: @@ -391,6 +409,8 @@ paths: format: byte '400': $ref: '#/components/responses/ErrorResponse' + '404': + $ref: '#/components/responses/NotFound' '500': $ref: '#/components/responses/ErrorResponse' @@ -447,8 +467,8 @@ paths: properties: level: type: string - enum: [trace, debug, info, warn, error] - description: Log level to set + enum: [trace, debug, info, warn, warning, error] + description: Log level to set, matched case-insensitively example: level: debug responses: @@ -464,6 +484,7 @@ paths: example: log_level_set level: type: string + enum: [Trace, Debug, Info, Warn, Error] '400': $ref: '#/components/responses/ErrorResponse' '500': @@ -491,11 +512,11 @@ paths: /permits: get: summary: Get permits - description: Get the current number of execution permits available. + description: Get the current maximum number of execution permits; `/status` reports how many are free. operationId: getPermits responses: '200': - description: Current permits + description: Maximum permits content: application/json: schema: @@ -503,7 +524,7 @@ paths: properties: permits: type: integer - description: Number of available permits + description: Maximum number of permits post: summary: Set permits description: >- @@ -574,8 +595,8 @@ paths: example: https://api.openai.com provider: type: string + enum: [ollama, openai-compatible, anthropic, google] description: Provider type - example: openai-compatible model: type: string description: Model name @@ -588,6 +609,12 @@ paths: type: array items: type: object + required: + - user_message + - temperature + - images + - max_tokens + - use_max_completion_tokens properties: system_message: type: string @@ -598,12 +625,34 @@ paths: type: number images: type: array + description: Each image is its raw bytes as an array of integers items: - type: string + type: array + items: + type: integer + minimum: 0 + maximum: 255 max_tokens: type: integer + format: uint32 use_max_completion_tokens: type: boolean + seed: + type: integer + format: uint64 + nullable: true + extra: + type: object + default: {} + description: Provider-specific request fields + extra_merge_strategy: + $ref: '#/components/schemas/MergeStrategy' + timeout: + oneOf: + - type: number + nullable: true + - type: string + description: Seconds as a number, or a string such as `500ms`, `1.5s` or `2m` example: configs: - host: https://api.openai.com @@ -678,14 +727,50 @@ paths: components: schemas: + MergeStrategy: + description: How `extra` merges into the provider request; null means `none`, maps select strategies per key + default: none + oneOf: + - type: string + nullable: true + enum: [null, none, replace, merge_left, merge_right] + - type: object + additionalProperties: + $ref: '#/components/schemas/MergeStrategy' + + ModuleStartRequest: + type: object + required: + - module_type + - config + properties: + module_type: + type: string + enum: [Llm, Web] + config: + type: object + nullable: true + description: Module configuration; null loads the module's default config file + allow_empty_backends: + type: boolean + default: false + description: Start the LLM module even with no valid backend + user_error: + type: boolean + default: false + description: Load the config but answer every module request with a user error + example: + module_type: Llm + config: null + Address: type: string format: hex - description: 20-byte Ethereum-style address as hex string + description: A calldata address (20 bytes), shown here in hex example: "0x1234567890abcdef1234567890abcdef12345678" ExecutorSelector: - description: Which executor line runs a request. + description: Which executor line runs a request oneOf: - $ref: '#/components/schemas/ExecutorSelectorMajor' - $ref: '#/components/schemas/ExecutorSelectorVersion' @@ -697,10 +782,11 @@ components: ExecutorSelectorMajor: type: object + additionalProperties: false description: | Resolved by the manifest's rules, falling back to the newest line when none provides this major. Cannot separate lines that share a semver - major, which every released line does. + major, which every released line does required: - kind - major @@ -714,10 +800,11 @@ components: ExecutorSelectorVersion: type: object + additionalProperties: false description: | Names a line: an executor directory used as it stands, or a `re:`-prefixed regular expression over manifest version keys, the newest match winning. - No match fails the run. + No match fails the run required: - kind - version @@ -729,8 +816,60 @@ components: type: string example: "v0.2.17" + InternalMessageParams: + type: object + additionalProperties: false + required: + - leader_timeunits_allocation + - validator_timeunits_allocation + - execution_budget_per_round + - rotations + - max_price_gen_per_time_unit + - storage_fee_max_gas_price + - receipt_fee_max_gas_price + properties: + leader_timeunits_allocation: + type: integer + format: uint256 + validator_timeunits_allocation: + type: integer + format: uint256 + execution_budget_per_round: + type: integer + format: uint256 + rotations: + type: array + items: + type: integer + format: uint256 + description: Per-round rotation allocations, the initial round first, then appeal rounds + max_price_gen_per_time_unit: + type: integer + format: uint256 + storage_fee_max_gas_price: + type: integer + format: uint256 + receipt_fee_max_gas_price: + type: integer + format: uint256 + + ExternalMessageParams: + type: object + additionalProperties: false + required: + - gas_limit + - max_gas_price + properties: + gas_limit: + type: integer + format: uint256 + max_gas_price: + type: integer + format: uint256 + MessageData: type: object + additionalProperties: false description: Contract execution message data required: - contract_address @@ -750,26 +889,39 @@ components: signer_address: $ref: '#/components/schemas/Address' chain_id: - type: string - description: Chain ID as big integer string - example: "1" + type: integer + example: 1 value: - type: string - description: Transaction value as big integer string - example: "0" + type: integer + description: Transaction value + example: 0 is_init: type: boolean description: Whether this is a contract initialization call transaction_timestamp: type: string format: date-time + default: "2024-11-26T06:42:42.424242Z" description: Transaction timestamp GenvmRunRequest: type: object + additionalProperties: false description: | - GenVM execution request. This structure must be encoded using the calldata - binary format before sending. + GenVM execution request, encoded as calldata (see the spec's Calldata + Encoding page) rather than JSON. This schema and every schema it + references map onto calldata kinds: + + - `integer` is a calldata number, never a decimal string + - `format: binary` is calldata bytes + - `Address` is a calldata address, never a hex string + - `date-time` is an RFC 3339 string + - `format: uint256` is a number in `[0, 2^256)` + + A value of another kind or an unknown key fails decoding, which the + socket answers with `malformed_frame`. A required nullable property must + be present, possibly as null. The range and non-empty-map constraints on + `bucket_totals` are checked when the run starts, not at decode required: - selector - message @@ -796,43 +948,43 @@ components: enum: [disabled, safe, safe-unbounded, unsafe, unsafe-tracing] description: | Executor debug level (see the Executor "Debug modes" section). Each level - adds to the previous one. Replaces the former `capture_output` boolean, - from which the output `Capture` (disabled/bounded/unbounded) is now derived. + adds to the previous one and also sets output capture. - On a consensus network the manager MUST reject (or clamp) `unsafe` and - `unsafe-tracing` from untrusted callers: `unsafe` enables `:latest`/`:test` - resolution (different nodes may diverge) and `unsafe-tracing` exposes real - time (breaks single-machine determinism). An unrecognized value is rejected. + The manager accepts every level as sent, so a host on a consensus network + must not pass `unsafe` or `unsafe-tracing` from untrusted callers: `unsafe` + enables `:latest`/`:test` resolution (different nodes may diverge) and + `unsafe-tracing` exposes real time (breaks single-machine determinism). max_execution_minutes: type: integer format: uint64 default: 20 - description: Maximum execution time in minutes + description: Maximum execution time in minutes, capped at 24 hours bucket_totals: type: object minProperties: 1 additionalProperties: - type: string + type: integer + format: uint256 description: | - Per-bucket data-fee balances, as big integer strings keyed by the - non-empty names the executor's fee config references. No default: - a request that omits it is rejected. + Per-bucket data-fee balances, keyed by the non-empty names the + executor's fee config references host_data: type: string - description: Host-specific data as JSON string + description: JSON object carrying at least string `node_address` and `tx_id` + example: '{"node_address":"0x00","tx_id":"0x00"}' timestamp: type: string format: date-time - description: Execution timestamp in RFC3339 format + description: Execution timestamp host: type: string - description: Host identifier + description: Address of host connection 0, `unix://<path>` or a TCP `host:port` extra_args: type: array items: type: string default: [] - description: Additional arguments for execution + description: Extra arguments for the executor's `run` command calldata: type: string format: binary @@ -841,20 +993,18 @@ components: type: string format: binary nullable: true - description: | - Contract bytecode for a deployment, or null for a call. Required to be - present; only its value is nullable. + description: Contract bytecode for a deployment, or null for a call leader_public_data: type: string format: binary nullable: true description: | Opaque executor-owned leader public data for a validator run, or null - for a leader run. Required to be present; only its value is nullable. + for a leader run initial_time_units_allocation: type: integer format: uint32 - description: Initial time-unit budget for the execution. + description: Initial time-unit budget for the execution permissions: type: string default: wscn @@ -862,44 +1012,68 @@ components: Permission meta-properties granted to the entry VM, as a subset of the characters `w` (write storage), `s` (send messages), `c` (call other contracts) and `n` (spawn nondet). **Defaults to all four**, so a host - that computes narrower permissions must send this field explicitly. + that computes narrower permissions must send this field explicitly no_modules: type: boolean default: false description: | Run without the LLM and web modules even when the permissions would - otherwise require them. + otherwise require them gas_data: type: object additionalProperties: type: string default: {} - description: Host-provided `node` fee constants. + description: Host-provided `node` fee constants message_fee_allocation: type: array items: type: object - required: [recipient, call_key, budget, on, fee_params, children_budget, subtree] + additionalProperties: false + required: [recipient, call_key, budget, 'on', fee_params, children_budget, subtree] properties: recipient: + type: string + format: hex nullable: true - description: Recipient address, or null for an open allocation. + description: Calldata address (20 bytes), or null for an open allocation call_key: + type: string + format: binary + minLength: 32 + maxLength: 32 nullable: true - description: Call key bytes, or null for a wildcard. + description: 32-byte call key, or null for a wildcard budget: + type: integer + format: uint256 nullable: true - description: Stored chain budget, or the allowance of a synthetic recipient wildcard; null is uncapped. A zero-budget entry with a recipient still matches internal messages and is absent for external ones, as on chain. - on: + description: | + Stored chain budget, or the allowance of a synthetic recipient wildcard; null is uncapped. + A zero-budget entry with a recipient still matches internal messages and is absent for + external ones, as on chain + 'on': type: string enum: [decided, finalized] fee_params: type: object - description: Internal or External fee parameters. + additionalProperties: false + minProperties: 1 + maxProperties: 1 + description: Single-key map naming the message kind + properties: + Internal: + $ref: '#/components/schemas/InternalMessageParams' + External: + $ref: '#/components/schemas/ExternalMessageParams' children_budget: - description: Sum of direct descendant allocation budgets, funded per emitted child. + type: integer + format: uint256 + description: Sum of direct descendant allocation budgets, funded per emitted child subtree: - description: Opaque bytes for the matched subtree, including any required Merkle proof. + type: string + format: binary + description: Opaque bytes for the matched subtree, including any required Merkle proof default: [] description: | Flat funding allocations. The node supplies the exact subtree transport @@ -908,14 +1082,14 @@ components: committed subtree use empty bytes. children_budget and subtree are required on every entry, including allocations without descendants. Omit allocations absent on-chain. Recipient wildcards are - synthetic entries for open-pool or view execution, not chain allocations. + synthetic entries for open-pool or view execution, not chain allocations record_actions: type: array items: type: string enum: [runner_load, vm_spawn] default: [] - description: Auditable supervisor action kinds to return in the result. + description: Auditable supervisor action kinds to return in the result host_genvm_id: type: string nullable: true @@ -923,14 +1097,16 @@ components: description: | Client correlation token, echoed in this run's events. Also an idempotency key on the socket protocol; see the `run` method in the - manager socket protocol page. + manager socket protocol page deadline: type: string nullable: true default: null + example: 30s description: | - Duration string that overrides `max_execution_minutes` as the strict - deadline when set. + Duration, a decimal number followed by `ms`, `s`, `m` or `h`, that + overrides `max_execution_minutes` as the strict deadline when set. + Capped at 24 hours host_hello_data: type: array items: @@ -940,19 +1116,21 @@ components: description: | Bytes written verbatim to each host connection, indexed by connection index, before the first method byte. The manager rejects a non-empty - entry for a connection it owns itself. + entry for a connection it owns itself hook_cross_contract_calls: type: boolean default: false description: | Whether the host wants to answer `resolve_call_contract_executor`. When false the manager answers it with a null reply and every `CallContract` - stays in-process. + stays in-process unsafe_overrides: type: object + additionalProperties: false + default: {} description: | Overrides that reach boundaries production traffic cannot. Each member - states the `debug_mode` it needs; with debugging disabled none apply. + states the `debug_mode` it needs; with debugging disabled none apply properties: reroute_to: type: string @@ -960,7 +1138,7 @@ components: description: | Run this version instead of the one `selector` resolves to, in the same two forms `selector`'s version variant accepts. Empty is - no override. Honored from `safe` up. + no override. Honored from `safe` up initial_recursion: type: integer format: uint32 @@ -968,7 +1146,7 @@ components: default: null description: | Seeds the chain's recursion budget, replacing the executor's own - `VM_RECURSION`. Honored from `unsafe` up. + `VM_RECURSION`. Honored from `unsafe` up allow_two_workers: type: boolean nullable: true @@ -976,7 +1154,7 @@ components: description: | Overrides the manager's `allow_two_workers` config. v0.2 warns when false and keeps its existing scheduling. - Honored from `unsafe` up. Null keeps the configured value. + Honored from `unsafe` up. Null keeps the configured value responses: ErrorResponse: @@ -989,3 +1167,13 @@ components: error: type: string description: Error message + NotFound: + description: The `genvm_id` path segment is not a decimal 64-bit unsigned integer + content: + application/json: + schema: + type: object + properties: + error: + type: string + example: Not Found diff --git a/docs/website/src/impl-spec/appendix/manager-socket.rst b/docs/website/src/impl-spec/appendix/manager-socket.rst index d2874ffe..e31ba52e 100644 --- a/docs/website/src/impl-spec/appendix/manager-socket.rst +++ b/docs/website/src/impl-spec/appendix/manager-socket.rst @@ -58,9 +58,9 @@ is **dropped without a close frame** -- a client observes an abnormal closure (1006), not 1009, and no ``error`` payload is sent. Clients that page artifacts within the documented chunk cap never approach the cap. -Payloads are externally-tagged: a single-key map whose key names the variant. -The ``method_id`` header is authoritative for routing; the in-payload tag is -redundant by design (log readability). +Requests and notifications are externally-tagged: a single-key map whose key +names the variant. Replies, including errors, are untagged maps. +The ``method_id`` header is authoritative for routing Method ids ---------- @@ -95,7 +95,9 @@ Request payload: the same logical structure as the deprecated - ``host_genvm_id`` (string, optional) -- client correlation token, echoed in events for this run. Also an idempotency key: a ``run`` repeated with the same token before the retention TTL expires returns the id already - allocated for it instead of starting a second execution. + allocated for it instead of starting a second execution. A run that ended + with ``failed_to_start`` holds its token too, so ``ack`` it before retrying + with the same token. - ``host_hello_data`` (array of bytes, optional, default ``[]``) -- indexed by host connection index; the executor writes entry *i* verbatim to host *i* on connect, before the first method byte (see :doc:`host-loop`). The @@ -142,6 +144,12 @@ and process spawn continue asynchronously and report through ``event`` notifications. The requesting connection is subscribed to the run's events automatically. +A payload that fails to decode is answered with ``malformed_frame``. An +idempotent retry can receive ``unknown_id`` if the retained run is removed +concurrently before subscription. A decoded request that fails a check, such +as a non-empty ``host_hello_data[1]`` or needing modules while they are stopped, +ends with ``failed_to_start`` + ``attach`` ---------- @@ -167,10 +175,10 @@ affect the run (see below). { "cancel": { "genvm_id": u64 } } -Requests termination. Response is an empty map; the outcome arrives as the -run's single terminal event. Cancelling a run still queued on permits aborts -it before spawn (no permit is consumed) and still yields exactly one terminal -event with cause ``cancelled``. +Requests termination. Response is an empty map; the outcome is reported through +``event`` under the `Lifecycle guarantees`_. Cancelling a run still queued on +permits aborts it before spawn (no permit is consumed) and ends in ``finished`` +with cause ``cancelled`` ``ack`` ------- @@ -205,8 +213,13 @@ connections. connection through a low-priority writer queue, so bulk transfers cannot starve lifecycle events. -The terminal event carries each artifact's total size, so clients can skip -the calls entirely when the blobs are empty. +Artifacts are available only for retained ``finished`` runs. Queued, running, +and retained ``failed_to_start`` runs answer ``unknown_id`` even though they +remain attachable. An invalid ``field`` on a retained ``finished`` run answers +``malformed_frame`` + +The ``finished`` event carries each artifact's total size, so clients can skip +the calls entirely when the blobs are empty ``event`` notifications ----------------------- @@ -215,12 +228,13 @@ Externally-tagged; every event carries ``genvm_id`` and, when the run was started with one, ``host_genvm_id``. Variants: ``queued`` - The run is allocated but no executor process exists yet, because it is - waiting for a permit. Non-terminal. Mainly seen as the ``attach`` snapshot of - a run that has not spawned; a client that attaches this early always observes - the current lifecycle state and every terminal event, but intermediate states + The run is allocated but no executor process exists yet; startup validation + (module locks) or permit acquisition may still be pending. Non-terminal. + Mainly seen as the ``attach`` snapshot of + a run that has not spawned. ``attach`` returns the current lifecycle state; + later notifications follow the `Lifecycle guarantees`_. Intermediate states may be coalesced, so a fast run can go straight from ``queued`` to a terminal - event. :: + event :: { "queued": { "genvm_id": u64, "host_genvm_id": str? } } @@ -243,12 +257,8 @@ started with one, ``host_genvm_id``. Variants: can observe without guessing, and a timer that guesses it would both tax every healthy run and still race a slow failure. - Either way the host receives a terminal event promptly, which is what - matters: this is the class of failure that used to leave it waiting on - ``accept()`` forever. - ``finished`` - Terminal, sent exactly once per run. :: + Terminal; retries can replay its notification (see `Lifecycle guarantees`_) :: { "finished": { "genvm_id": u64, "host_genvm_id": str?, @@ -261,6 +271,10 @@ started with one, ``host_genvm_id``. Variants: "artifact_sizes": { "stdout": u64, "stderr": u64, "genvm_log": u64 } } } + An internal manager failure after the spawn also ends in ``finished`` with + a null ``exit_code``. Its cause is ``exited`` unless a termination cause was + already recorded + For a top-level run, ``consumed_result`` is one outer ``ResultCode`` byte followed by a calldata-encoded ``ReportedResult`` map. Before retaining it, the manager checks that: @@ -279,10 +293,14 @@ therefore never receive a top-level ``FatalVmError`` The manager does not decode reported ``leader_public_data``. The bytes remain opaque, executor-line-specific consensus proposals +.. _Lifecycle guarantees: + Lifecycle guarantees: -- Exactly one terminal event (``failed_to_start`` or ``finished``) per run, - delivered to every connection subscribed at the time. +- Each run has 1 terminal state (``failed_to_start`` or ``finished``). + Subscribed connections receive its notification while connected; retrying + ``run`` with the same retained token can replay it. Manager shutdown may + close connections before the terminal notification is delivered - **Disconnect does not kill a run.** Runs terminate only via ``cancel``, the deadline, or manager shutdown. A client may disconnect, reconnect, and ``attach`` by ``(boot_id, genvm_id)`` to recover the state and result. @@ -323,9 +341,11 @@ Error messages See :ref:`gvm-def-enum-errors` for the generated error code enum. Meanings: ``internal`` is a handler failure with a diagnostic message; ``malformed_frame`` is failed calldata decode, a bad payload shape, a message -shorter than the header, or a non-binary message; ``unknown_method`` is a -method id not in the generated table; ``unknown_id`` means the run never -existed, was acked, or expired; ``boot_id_mismatch`` is ``attach`` across a +shorter than the header, a non-binary message, or an invalid artifact field; +``unknown_method`` is a method id absent from the generated table or unavailable +to client requests (``hello``, ``event``, ``error``); ``unknown_id`` means the run +never existed, was acked, expired, or has no finished result for ``get_artifact``; +``boot_id_mismatch`` is ``attach`` across a manager restart; ``bad_request_id`` is a client request with ``request_id == 0``; and ``not_finished`` is ``ack`` before a terminal event or result. diff --git a/implementation/src/manager/handlers.rs b/implementation/src/manager/handlers.rs index 50b20151..858e5aee 100644 --- a/implementation/src/manager/handlers.rs +++ b/implementation/src/manager/handlers.rs @@ -6,10 +6,7 @@ use std::collections::{BTreeMap, HashMap}; use std::str::FromStr; use std::sync::Arc; -use crate::manager::{ - modules::{self, Ctx}, - run, versioning, -}; +use crate::manager::{modules, run, versioning}; use crate::{common, llm, scripting}; use super::AppContext; @@ -78,20 +75,9 @@ pub async fn handle_genvm_run( ) -> Result<Json<serde_json::Value>> { let res: super::run::Request = calldata::decode_obj(data)?; - let modules_lock = if res.needs_modules() { - let lock = Ctx::get_module_locks(ctx.gep(|x| &x.mod_ctx)).await; - if lock.is_none() { - anyhow::bail!( - "modules are required but not running (is_sync=false with 'n' permission)" - ); - } - lock - } else { - None - }; - let run_ctx = ctx.gep(|x| &x.run_ctx); - let id = run_ctx.start(ctx, res, Box::new(modules_lock)).await?; + let start = run_ctx.start(ctx, res); + let id = start.genvm_id; let (snapshot, mut events) = run_ctx.attach(run_ctx.boot_id(), id)?; let start_event = match snapshot { @@ -105,6 +91,10 @@ pub async fn handle_genvm_run( }; if let run::Event::FailedToStart { error, .. } = start_event { + if start.reserved { + // The caller never learns the id of a newly reserved failure + run_ctx.ack(id); + } anyhow::bail!(error); } diff --git a/implementation/src/manager/modules.rs b/implementation/src/manager/modules.rs index 7ed4321e..6adff06d 100644 --- a/implementation/src/manager/modules.rs +++ b/implementation/src/manager/modules.rs @@ -161,23 +161,6 @@ impl Ctx { } } - pub async fn create_execution_context( - &self, - hello: Arc<genvm_modules_interfaces::GenVMHello>, - ) -> anyhow::Result<sync::DArc<ExecutionContext>> { - let llm = if let Some(state) = self.llm_module.read().await.as_ref() { - Some(state.create_sub_context(&hello)?) - } else { - None - }; - let web = if let Some(state) = self.web_module.read().await.as_ref() { - Some(state.create_sub_context(&hello)?) - } else { - None - }; - Ok(sync::DArc::new(ExecutionContext { hello, llm, web })) - } - pub async fn start(&self, req: StartRequest) -> anyhow::Result<()> { self.start_or_restart(req, false).await } @@ -443,42 +426,48 @@ impl Ctx { } } - /// Get the handler for a running module (returns None if module not running) - pub async fn get_handler(&self, module_type: Type) -> Option<StreamHandler> { - match module_type { - Type::Llm => { - let module_lock = self.llm_module.read().await; - module_lock.as_ref().map(|state| state.base.handler.clone()) - } - Type::Web => { - let module_lock = self.web_module.read().await; - module_lock.as_ref().map(|state| state.base.handler.clone()) - } - } + pub async fn get_module_locks(zelf: sync::DArc<Ctx>) -> Option<ModuleLocks> { + let llm = zelf + .clone() + .into_get_sub_async(|x| x.llm_module.read()) + .await; + llm.as_ref()?; + let web = zelf.into_get_sub_async(|x| x.web_module.read()).await; + web.as_ref()?; + Some(ModuleLocks { llm, web }) } +} - /// Get both module handlers if both are running - pub async fn get_handlers(&self) -> Option<(StreamHandler, StreamHandler)> { - let llm = self.get_handler(Type::Llm).await?; - let web = self.get_handler(Type::Web).await?; - Some((llm, web)) +/// Keeps both modules running while a run uses them. Everything a run needs +/// from the modules is read through these guards: taking the module locks again +/// would queue behind a pending stop or restart that waits on these guards. +pub struct ModuleLocks { + llm: sync::DArcStruct<tokio::sync::RwLockReadGuard<'static, Option<ModuleStateLlm>>>, + web: sync::DArcStruct<tokio::sync::RwLockReadGuard<'static, Option<ModuleStateWeb>>>, +} + +impl ModuleLocks { + fn llm(&self) -> &ModuleStateLlm { + self.llm.as_ref().expect("checked in get_module_locks") } - pub async fn get_module_locks(zelf: sync::DArc<Ctx>) -> Option<impl std::any::Any> { - let llm_lock = zelf - .clone() - .into_get_sub_async(|x| x.llm_module.read()) - .await; - if llm_lock.is_none() { - return None; - } - let web_lock = zelf - .clone() - .into_get_sub_async(|x| x.web_module.read()) - .await; - if web_lock.is_none() { - return None; - } - Some((llm_lock, web_lock)) + fn web(&self) -> &ModuleStateWeb { + self.web.as_ref().expect("checked in get_module_locks") + } + + pub fn handlers(&self) -> (StreamHandler, StreamHandler) { + ( + self.llm().base.handler.clone(), + self.web().base.handler.clone(), + ) + } + + pub fn create_execution_context( + &self, + hello: Arc<genvm_modules_interfaces::GenVMHello>, + ) -> anyhow::Result<sync::DArc<ExecutionContext>> { + let llm = Some(self.llm().create_sub_context(&hello)?); + let web = Some(self.web().create_sub_context(&hello)?); + Ok(sync::DArc::new(ExecutionContext { hello, llm, web })) } } diff --git a/implementation/src/manager/run.rs b/implementation/src/manager/run.rs index b1938ce8..1e72a498 100644 --- a/implementation/src/manager/run.rs +++ b/implementation/src/manager/run.rs @@ -305,6 +305,7 @@ struct SingleGenVMContext { consumed_result: tokio::sync::OnceCell<Vec<u8>>, process_handle: tokio::sync::Mutex<Option<tokio::process::Child>>, + spawned: AtomicBool, cancel_requested: AtomicBool, cancel_notify: tokio::sync::Notify, finish_cause: std::sync::Mutex<Option<FinishCause>>, @@ -331,6 +332,14 @@ impl SingleGenVMContext { .unwrap_or_else(|_| String::new()) } + fn mark_spawned(&self) { + self.spawned.store(true, Ordering::SeqCst); + } + + fn was_spawned(&self) -> bool { + self.spawned.load(Ordering::SeqCst) + } + fn request_finish(&self, cause: FinishCause) { let should_notify = !self.cancel_requested.swap(true, Ordering::SeqCst); if let Ok(mut stored_cause) = self.finish_cause.lock() { @@ -532,18 +541,18 @@ impl Ctx { pub async fn set_permits(&self, permits: usize) -> usize { let mut permits_lock = self.max_permits.lock().await; - permits_lock.max += permits_lock.num_throttled; - permits_lock.num_throttled = 0; - permits_lock.throttled = None; - // actually this causes drop of previous one, so we can enter more genvms than we have permits, but it's ok for now - // especially since this method is expected to be called before starting any genvms at all - let min = self.min_permits(); if permits < min { log_warn!(@operator, permits = permits, min = min; "cannot set permits below the most expensive run"); return permits_lock.max; } + permits_lock.max += permits_lock.num_throttled; + permits_lock.num_throttled = 0; + permits_lock.throttled = None; + // actually this causes drop of previous one, so we can enter more genvms than we have permits, but it's ok for now + // especially since this method is expected to be called before starting any genvms at all + if permits_lock.max > permits { let delta = permits_lock.max - permits; let p = self @@ -1927,6 +1936,27 @@ fn spawn_stdin_writer( }) } +fn check_stdin_write( + genvm_id: GenVMId, + result: Result<std::io::Result<()>, tokio::task::JoinError>, +) -> anyhow::Result<()> { + match result { + Ok(Ok(())) => { + log_debug_into!(&LoggerWithId, genvm_id:id = genvm_id.0; "execution data written"); + Ok(()) + } + // The executor stopped reading; its exit status tells how the run ended + Ok(Err(e)) if e.kind() == std::io::ErrorKind::BrokenPipe => { + log_debug_into!(&LoggerWithId, genvm_id:id = genvm_id.0; "executor closed stdin before reading execution data"); + Ok(()) + } + Ok(Err(e)) => Err(anyhow::anyhow!( + "failed to write execution data to child stdin: {e}" + )), + Err(e) => Err(anyhow::anyhow!("stdin write task panicked: {e}")), + } +} + fn strict_deadline_from_request(req: &Request) -> chrono::DateTime<chrono::Utc> { let max = std::time::Duration::from_secs(24 * 60 * 60); let duration = req @@ -1982,7 +2012,7 @@ fn execution_data_from_request( enum RunResources { TopLevel { permits: tokio::sync::OwnedSemaphorePermit, - modules_lock: Box<dyn std::any::Any + Send + Sync>, + modules_lock: Option<super::modules::ModuleLocks>, }, Nested { caller_stream: Arc<ManagerHostStreamState>, @@ -2014,6 +2044,11 @@ enum Reservation { Reserved(sync::DArc<SingleGenVMContext>), } +pub struct StartResult { + pub genvm_id: GenVMId, + pub reserved: bool, +} + async fn wait_for_process_stop( exec_ctx: &SingleGenVMContext, caller_stream: Option<&ManagerHostStreamState>, @@ -2068,12 +2103,11 @@ impl Ctx { Reservation::Reserved(exec_ctx) } - pub async fn start( + pub fn start( &self, full_ctx: sync::DArc<crate::manager::AppContext>, req: Request, - modules_lock: Box<dyn std::any::Any + Send + Sync>, - ) -> anyhow::Result<GenVMId> { + ) -> StartResult { let reservation = self.reserve_execution(req.host_genvm_id.as_deref(), |genvm_id| { let events = tokio::sync::watch::Sender::new(Snapshot::Queued { genvm_id, @@ -2091,6 +2125,7 @@ impl Ctx { id: genvm_id, host_genvm_id: req.host_genvm_id.clone(), process_handle: tokio::sync::Mutex::new(None), + spawned: AtomicBool::new(false), started_at: chrono::Utc::now(), strict_deadline: strict_deadline_from_request(&req), @@ -2113,14 +2148,22 @@ impl Ctx { }); let exec_ctx = match reservation { - Reservation::Existing(genvm_id) => return Ok(genvm_id), + Reservation::Existing(genvm_id) => { + return StartResult { + genvm_id, + reserved: false, + }; + } Reservation::Reserved(exec_ctx) => exec_ctx, }; let genvm_id = exec_ctx.id; - tokio::spawn(supervise_genvm(full_ctx, exec_ctx, req, modules_lock)); + tokio::spawn(supervise_genvm(full_ctx, exec_ctx, req)); - Ok(genvm_id) + StartResult { + genvm_id, + reserved: true, + } } async fn start_nested( @@ -2233,12 +2276,49 @@ impl Ctx { } } +fn check_top_level_request(req: &Request, has_modules_lock: bool) -> anyhow::Result<()> { + anyhow::ensure!( + !req.needs_modules() || has_modules_lock, + "modules are required but not running" + ); + anyhow::ensure!( + req.host_hello_data + .get(1) + .is_none_or(|data| data.is_empty()), + "host_hello_data for manager-owned host index 1 is not allowed" + ); + Ok(()) +} + async fn supervise_genvm( full_ctx: sync::DArc<crate::manager::AppContext>, exec_ctx: sync::DArc<SingleGenVMContext>, req: Request, - modules_lock: Box<dyn std::any::Any + Send + Sync>, ) { + async fn finish_cancelled(exec_ctx: &SingleGenVMContext) { + let cause = exec_ctx.finish_cause().unwrap_or(FinishCause::Cancelled); + let _ = finish_execution(exec_ctx, None, cause).await; + } + + // A queued module stop or restart blocks new read guards + let modules_lock = if req.needs_modules() { + tokio::select! { + _ = exec_ctx.wait_cancelled() => { + finish_cancelled(&exec_ctx).await; + return; + } + lock = super::modules::Ctx::get_module_locks(full_ctx.gep(|x| &x.mod_ctx)) => lock, + } + } else { + None + }; + + if let Err(e) = check_top_level_request(&req, modules_lock.is_some()) { + fail_to_start(&exec_ctx, e); + return; + } + drop(modules_lock); + let ctx = full_ctx.gep(|x| &x.run_ctx); let permit_count = ctx.permits_for(&req); @@ -2246,8 +2326,7 @@ async fn supervise_genvm( tokio::pin!(permit_future); let permits = tokio::select! { _ = exec_ctx.wait_cancelled() => { - let cause = exec_ctx.finish_cause().unwrap_or(FinishCause::Cancelled); - let _ = finish_execution(&exec_ctx, None, cause).await; + finish_cancelled(&exec_ctx).await; return; } permits = &mut permit_future => match permits { @@ -2259,25 +2338,55 @@ async fn supervise_genvm( }, }; + let modules_lock = if req.needs_modules() { + tokio::select! { + _ = exec_ctx.wait_cancelled() => { + drop(permits); + finish_cancelled(&exec_ctx).await; + return; + } + lock = super::modules::Ctx::get_module_locks(full_ctx.gep(|x| &x.mod_ctx)) => lock, + } + } else { + None + }; + + if let Err(e) = check_top_level_request(&req, modules_lock.is_some()) { + drop(permits); + fail_to_start(&exec_ctx, e); + return; + } + if exec_ctx.cancel_requested.load(Ordering::SeqCst) { - let cause = exec_ctx.finish_cause().unwrap_or(FinishCause::Cancelled); drop(permits); - let _ = finish_execution(&exec_ctx, None, cause).await; + drop(modules_lock); + finish_cancelled(&exec_ctx).await; return; } if let Err(e) = supervise_genvm_inner(full_ctx, exec_ctx.clone(), req, modules_lock, permits).await { - fail_to_start(&exec_ctx, e); + report_run_failure(&exec_ctx, e).await; } } +/// A spawned executor always ends in `finished`, even when the manager fails it +async fn report_run_failure(exec: &SingleGenVMContext, error: anyhow::Error) { + if !exec.was_spawned() { + fail_to_start(exec, error); + return; + } + + log_warn_into!(@operator, &LoggerWithId, genvm_id:id = exec.id.0, error:ah = error; "executor was spawned but the run failed before its exit status was known"); + let _ = finish_execution(exec, None, FinishCause::Exited).await; +} + async fn supervise_genvm_inner( full_ctx: sync::DArc<crate::manager::AppContext>, exec_ctx: sync::DArc<SingleGenVMContext>, req: Request, - modules_lock: Box<dyn std::any::Any + Send + Sync>, + modules_lock: Option<super::modules::ModuleLocks>, permits: tokio::sync::OwnedSemaphorePermit, ) -> anyhow::Result<()> { let execution_data = execution_data_from_request(&req, &full_ctx.config); @@ -2372,18 +2481,18 @@ async fn run_genvm_process( ) -> anyhow::Result<std::process::ExitStatus> { let is_top_level = resources.is_top_level(); let caller_stream = resources.caller_stream(); - let module_handlers = if req.needs_modules() { - let (llm, web) = full_ctx - .mod_ctx - .get_handlers() - .await - .ok_or_else(|| anyhow::anyhow!("modules are required but not all are running"))?; - Some((llm, web)) - } else { - None + let modules_lock = match &resources { + RunResources::TopLevel { modules_lock, .. } => modules_lock.as_ref(), + RunResources::Nested { .. } => None, }; + assert_eq!( + modules_lock.is_some(), + req.needs_modules(), + "a run needing modules must hold their locks" + ); + let module_handlers = modules_lock.map(|lock| lock.handlers()); - let execution_context = if req.needs_modules() { + let execution_context = if let Some(modules_lock) = modules_lock { let host_data: genvm_modules_interfaces::HostData = serde_json::from_str(&req.host_data)?; let role = if req.leader_public_data.is_none() { genvm_modules_interfaces::Role::Leader @@ -2397,7 +2506,7 @@ async fn run_genvm_process( gas_data: req.gas_data.clone(), initial_time_units_allocation: req.initial_time_units_allocation, }); - let ctx = full_ctx.mod_ctx.create_execution_context(hello).await?; + let ctx = modules_lock.create_execution_context(hello)?; let _ = exec_ctx.execution_context.set(ctx.clone()); Some(ctx) } else { @@ -2472,14 +2581,6 @@ async fn run_genvm_process( None }; - if req - .host_hello_data - .get(1) - .is_some_and(|data| !data.is_empty()) - { - anyhow::bail!("host_hello_data for manager-owned host index 1 is not allowed"); - } - let execution_data_bytes = bytes::Bytes::from(calldata::encode_obj(&execution_data)); let log_reader = async { @@ -2494,6 +2595,9 @@ async fn run_genvm_process( let process = async { // Spawn child process, then drop child-side FDs let mut child = proc.spawn()?; + if is_top_level { + exec_ctx.mark_spawned(); + } drop(write_fd); log_debug_into!(&LoggerWithId, genvm_id:id = genvm_id.0, pid:? = child.id(); "genvm process started"); std::mem::drop(module_child_fds); @@ -2507,12 +2611,26 @@ async fn run_genvm_process( let stdout = child.stdout.take(); let stderr = child.stderr.take(); let stdout_perm = if stdout.is_some() { - Some(exec_ctx.stdout_stderr_sem.clone().acquire_owned().await?) + Some( + exec_ctx + .stdout_stderr_sem + .clone() + .acquire_owned() + .await + .expect("stdout/stderr semaphore is never closed"), + ) } else { None }; let stderr_perm = if stderr.is_some() { - Some(exec_ctx.stdout_stderr_sem.clone().acquire_owned().await?) + Some( + exec_ctx + .stdout_stderr_sem + .clone() + .acquire_owned() + .await + .expect("stdout/stderr semaphore is never closed"), + ) } else { None }; @@ -2544,31 +2662,21 @@ async fn run_genvm_process( wait_for_process_stop(&exec_ctx, caller_stream.as_deref(), deadline_duration); tokio::pin!(stop); tokio::select! { - result = &mut stdin_task => match result { - Ok(Ok(())) => { - log_debug_into!(&LoggerWithId, genvm_id:id = genvm_id.0; "execution data written"); - } - Ok(Err(e)) => { + result = &mut stdin_task => { + if let Err(e) = check_stdin_write(genvm_id, result) { let _ = child.start_kill(); let _ = child.wait().await; - anyhow::bail!("failed to write execution data to child stdin: {e}"); + return Err(e); } - Err(e) => { + } + // An early exit is reported by the regular wait below, which caches the status + status = child.wait() => { + stdin_task.abort(); + if let Err(e) = status { let _ = child.start_kill(); let _ = child.wait().await; - anyhow::bail!("stdin write task panicked: {e}"); + return Err(e.into()); } - }, - status = child.wait() => { - stdin_task.abort(); - return match status { - Ok(status) => Ok(status), - Err(e) => { - let _ = child.start_kill(); - let _ = child.wait().await; - Err(e.into()) - } - }; } reason = &mut stop => { if reason == ProcessStop::Deadline { diff --git a/implementation/src/manager/run_test.rs b/implementation/src/manager/run_test.rs index f4db869f..50ef7f82 100644 --- a/implementation/src/manager/run_test.rs +++ b/implementation/src/manager/run_test.rs @@ -19,6 +19,56 @@ fn test_ctx(retention: &str, permits: usize) -> Ctx { } } +async fn test_app_ctx() -> sync::DArc<crate::manager::AppContext> { + let manifest_path = + std::env::temp_dir().join(format!("genvm-run-test-{}.yaml", rand::random::<u128>())); + std::fs::write(&manifest_path, "executor_versions: {}\n").unwrap(); + let config: sync::DArc<crate::manager::Config> = sync::DArc::new( + serde_json::from_value(serde_json::json!({ + "threads": 1, + "blocking_threads": 1, + "log_disable": "", + "manifest_path": manifest_path, + })) + .unwrap(), + ); + let ver_ctx = crate::manager::versioning::Ctx::new(config.clone()).await; + std::fs::remove_file(manifest_path).unwrap(); + let (cancel, _) = cancellation::make(); + sync::DArc::new(crate::manager::AppContext { + mod_ctx: crate::manager::modules::Ctx::new(cancel.clone()), + cancel, + config, + run_ctx: test_ctx("5m", 2), + ver_ctx: ver_ctx.unwrap(), + }) +} + +fn module_request() -> Request { + let message = genvm_modules_interfaces::MessageData { + contract_address: calldata::Address::from([0; 20]), + sender_address: calldata::Address::from([0; 20]), + origin_address: calldata::Address::from([0; 20]), + signer_address: calldata::Address::from([0; 20]), + chain_id: 0.into(), + value: 0.into(), + is_init: false, + transaction_timestamp: chrono::Utc::now(), + }; + serde_json::from_value(serde_json::json!({ + "selector": genvm_modules_interfaces::ExecutorSelector::MajorOverride { major: 0 }, + "message": message, + "is_sync": false, + "bucket_totals": {}, + "host_data": "", + "timestamp": message.transaction_timestamp, + "host": "", + "calldata": [], + "initial_time_units_allocation": 0, + })) + .unwrap() +} + fn fake_execution( genvm_id: GenVMId, finished_at: Option<chrono::DateTime<chrono::Utc>>, @@ -54,6 +104,7 @@ fn fake_execution_with_host_id( log_sink: Arc::new(LogSinkInner::new(false)), consumed_result: tokio::sync::OnceCell::new(), process_handle: tokio::sync::Mutex::new(None), + spawned: AtomicBool::new(false), cancel_requested: AtomicBool::new(false), cancel_notify: tokio::sync::Notify::new(), finish_cause: std::sync::Mutex::new(None), @@ -445,6 +496,75 @@ async fn cancel_while_queued_consumes_no_permit() { assert_eq!(permits.available_permits(), 1); } +#[tokio::test] +async fn permit_queued_run_allows_module_stop_and_rechecks_modules() { + use crate::manager::modules::{StartRequest, Type}; + + let ctx = test_app_ctx().await; + let config = serde_json::json!({ + "threads": 1, + "blocking_threads": 1, + "log_disable": "", + "lua_script_path": "", + "vm_count": 1, + "lua_path": "", + "signer_url": "", + "signer_headers": {}, + "data_dir": "", + "backends": {}, + "prompt_templates": { + "eq_comparative": null, + "eq_non_comparative_leader": null, + "eq_non_comparative_validator": null, + }, + "webdriver_host": "", + "extra_tld": [], + "always_allow_hosts": [], + }); + for module_type in [Type::Llm, Type::Web] { + ctx.mod_ctx + .start(StartRequest { + module_type, + config: config.clone(), + allow_empty_backends: true, + user_error: true, + }) + .await + .unwrap(); + } + let occupied = ctx + .run_ctx + .permits + .clone() + .acquire_many_owned(2) + .await + .unwrap(); + let exec = fake_execution(GenVMId(1), None); + let run = supervise_genvm(ctx.clone(), exec.clone(), module_request()); + tokio::pin!(run); + assert!(futures_util::poll!(run.as_mut()).is_pending()); + assert!(matches!(*exec.events.borrow(), Snapshot::Queued { .. })); + + for module_type in [Type::Llm, Type::Web] { + let stop = ctx.mod_ctx.stop(module_type); + tokio::pin!(stop); + assert!(matches!( + futures_util::poll!(stop.as_mut()), + std::task::Poll::Ready(Ok(true)) + )); + } + + drop(occupied); + assert!(futures_util::poll!(run.as_mut()).is_ready()); + assert_eq!(ctx.run_ctx.permits.available_permits(), 2); + let snapshot = exec.events.borrow(); + assert!( + matches!(&*snapshot, Snapshot::Event(Event::FailedToStart { error, .. }) + if error == "modules are required but not running"), + "unexpected snapshot: {snapshot:?}" + ); +} + #[tokio::test] async fn permits_cannot_drop_below_the_most_expensive_run() { let ctx = test_ctx("5m", 8); @@ -457,6 +577,27 @@ async fn permits_cannot_drop_below_the_most_expensive_run() { assert_eq!(ctx.get_current_permits(), 2); } +#[tokio::test] +async fn rejected_permits_preserve_throttling() { + let ctx = test_ctx("5m", 10); + assert_eq!(ctx.set_permits(5).await, 5); + let active = ctx.permits.clone().acquire_many_owned(3).await.unwrap(); + + for requested in [0, 1] { + assert_eq!(ctx.set_permits(requested).await, 5); + assert_eq!(ctx.get_current_permits(), 2); + let permits = ctx.max_permits.lock().await; + assert_eq!(permits.max, 5); + assert_eq!(permits.num_throttled, 5); + assert_eq!(permits.throttled.as_ref().unwrap().num_permits(), 5); + } + + drop(active); + assert_eq!(ctx.get_current_permits(), 5); + assert_eq!(ctx.set_permits(10).await, 10); + assert_eq!(ctx.get_current_permits(), 10); +} + #[tokio::test] async fn nested_run_does_not_consume_parent_permit() { let ctx = test_ctx("5m", 1); @@ -751,6 +892,50 @@ async fn terminal_event_is_published_once() { assert!(matches!(rx.has_changed(), Ok(false))); } +#[tokio::test] +async fn a_failure_after_spawn_is_a_finished_not_a_failed_to_start() { + let error = || anyhow::anyhow!("stdin write task panicked"); + + let spawned = fake_execution(GenVMId(1), None); + spawned.mark_spawned(); + let mut rx = spawned.events.subscribe(); + report_run_failure(&spawned, error()).await; + match rx.borrow_and_update().clone() { + Snapshot::Event(Event::Finished { + exit_code, cause, .. + }) => { + assert_eq!(exit_code, None); + assert_eq!(cause, FinishCause::Exited); + } + other => panic!("spawned run reported {other:?}, expected a finished event"), + } + + let never_spawned = fake_execution(GenVMId(2), None); + let mut rx = never_spawned.events.subscribe(); + report_run_failure(&never_spawned, error()).await; + assert!(matches!( + rx.borrow_and_update().clone(), + Snapshot::Event(Event::FailedToStart { .. }) + )); +} + +#[tokio::test] +async fn epipe_from_an_exited_executor_is_not_an_error() { + let mut child = tokio::process::Command::new("bash") + .args(["-c", "exit 3"]) + .stdin(std::process::Stdio::piped()) + .spawn() + .unwrap(); + // Larger than any default pipe buffer, so the write must hit EPIPE + let data = bytes::Bytes::from(vec![0u8; 4 << 20]); + let writer = spawn_stdin_writer(child.stdin.take().unwrap(), data); + + let write = writer.await; + assert!(matches!(&write, Ok(Err(e)) if e.kind() == std::io::ErrorKind::BrokenPipe)); + check_stdin_write(GenVMId(1), write).unwrap(); + child.wait().await.unwrap(); +} + #[tokio::test] async fn process_completion_waits_for_final_logs_on_success_and_error() { use std::io::Write; @@ -985,10 +1170,8 @@ fn documented_properties(schema: &str) -> Vec<String> { names } -/// A host implementing against the schema alone must see every field the -/// manager accepts, and none it does not. Drift here is not cosmetic: an -/// undocumented `permissions` silently grants the `wscn` default, and a -/// documented-but-absent field makes a host send something that is ignored. +/// An undocumented `permissions` field would silently grant the default `wscn`. +/// Compares property names only; types, defaults and `required` are unchecked. #[test] fn request_schema_matches_the_rust_struct() { let mut accepted = request_field_names() @@ -1000,7 +1183,7 @@ fn request_schema_matches_the_rust_struct() { assert_eq!(documented_properties("GenvmRunRequest"), accepted); } -/// Same contract for the message the host builds by hand. +/// Same name-only check for `MessageData`. #[test] fn message_schema_matches_the_rust_struct() { #[allow(unused_variables)] diff --git a/implementation/src/manager/socket.rs b/implementation/src/manager/socket.rs index fe962544..dcf661d8 100644 --- a/implementation/src/manager/socket.rs +++ b/implementation/src/manager/socket.rs @@ -491,51 +491,8 @@ where return Ok(()); } }; - if req - .host_hello_data - .get(1) - .is_some_and(|data| !data.is_empty()) - { - self.writer - .error( - request_id, - Errors::MalformedFrame, - "host_hello_data for manager-owned host index 1 is not allowed", - ) - .await?; - return Ok(()); - } - - let modules_lock = if req.needs_modules() { - match super::modules::Ctx::get_module_locks(self.ctx.gep(|x| &x.mod_ctx)).await { - Some(lock) => Some(lock), - None => { - self.writer - .error( - request_id, - Errors::Internal, - "modules are required but not running", - ) - .await?; - return Ok(()); - } - } - } else { - None - }; let run_ctx = self.ctx.gep(|x| &x.run_ctx); - let genvm_id = match run_ctx - .start(self.ctx.clone(), req, Box::new(modules_lock)) - .await - { - Ok(id) => id, - Err(e) => { - self.writer - .error(request_id, Errors::Internal, format!("{:#}", e)) - .await?; - return Ok(()); - } - }; + let genvm_id = run_ctx.start(self.ctx.clone(), req).genvm_id; let (snapshot, rx) = match run_ctx.attach(run_ctx.boot_id(), genvm_id) { Ok(result) => result, Err(e) => { diff --git a/tests/runner/gvm_extra/cross_major.py b/tests/runner/gvm_extra/cross_major.py new file mode 100644 index 00000000..5826eac8 --- /dev/null +++ b/tests/runner/gvm_extra/cross_major.py @@ -0,0 +1,332 @@ +import json +import os +import pickle +import typing +from dataclasses import dataclass +from pathlib import Path + +import genvm_tool.io as gvm_io +import genvm_tool.tests +import genvm_tool.tests.exec.step +import genvm_tool.tests.stage.collection +import genvm_tool.tests.test +import origin.calldata as gvm_calldata +from genvm_tool_plugins import runners +from gvm_extra.mock_host import MockHost, MockStorage, ResolveCallContractExecutorHook +from origin import base_host, host_fns, public_abi +from origin.calldata import Address +from origin.leader_public_data import LeaderPublicData + +SENDER = Address('0x' + '01' * 20) +TIMESTAMP = '2024-11-26T06:42:42.424242Z' + +type Fixtures = dict[str, tuple[int, Address, bytes]] + + +def encode_leader_public_data(output: bytes) -> bytes: + return LeaderPublicData([output]).encode() + + +def encode_call(method: str, *args: typing.Any) -> bytes: + return gvm_calldata.encode({'': method, 'args': list(args)}) + + +def resolve_runners(code: bytes | None, root_dir: Path) -> bytes | None: + """ + Turns the `@RUNNER_LATEST_...@` tokens of a contract source into uids. + + Every comparison against a deployed contract's bytes has to go through this + too, because storage holds what the executor was given. + """ + return None if code is None else runners.substitute(code, root_dir) + + +def contract_source(line: int, name: str, assets_dir: Path) -> bytes: + """ + Reads a contract asset written against `line`'s sdk. + """ + return (assets_dir / f'v0.{line}' / f'{name}.py').read_bytes() + + +def message(address: Address, *, is_init: bool) -> base_host.Message: + return { + 'contract_address': address, + 'sender_address': SENDER, + 'origin_address': SENDER, + 'signer_address': SENDER, + 'chain_id': 61999, + 'value': 0, + 'is_init': is_init, + 'transaction_timestamp': TIMESTAMP, + } + + +def apply_storage_deltas( + storage: MockStorage, + address: Address, + changes: list[tuple[bytes, bytes]], +) -> None: + for key, value in changes: + assert len(key) == 36 + storage.write( + address, + key[:32], + int.from_bytes(key[32:], byteorder='big') * 32, + value, + ) + + +def assert_hashes_agree(label: str, runs: list[tuple[str, typing.Any]]) -> None: + """ + A disagreement is only actionable once the failure says who disagreed. + """ + hashes = [(name, run.execution_hash.hex()) for name, run in runs] + assert len({digest for _name, digest in hashes}) == 1, '\n'.join( + [f'{label}: execution hashes disagree'] + + [f' {name}: {digest}' for name, digest in hashes] + ) + + +class TestContext(base_host.Context): + def __init__(self, logger: base_host.Logger): + self.logger = logger + self.stats: dict[str, typing.Any] = {} + + def on_genvm_success(self): ... + def on_genvm_failure(self): ... + + def add_stat(self, key: str, value: typing.Any, /): + self.stats[key] = value + + +@dataclass +class CrossMajorCase(genvm_tool.tests.test.Case): + """ + Deploys `fixtures` against `manager_service`, then calls the step's + `method` with `arguments`. + """ + + step_type: typing.ClassVar[type['CrossMajorStep']] + + description: genvm_tool.tests.test.Description + shared: genvm_tool.tests.SharedContext + manager_service: genvm_tool.tests.stage.collection.Service + method: str + fixtures: tuple[str, ...] + arguments: tuple[typing.Any, ...] = () + + async def into_steps(self) -> list[genvm_tool.tests.exec.step.Step]: + return [self.step_type(self)] + + +class CrossMajorStep(genvm_tool.tests.exec.step.Python): + """ + Drives a real manager over the socket with a mock host. + """ + + host_data_tx_id: typing.ClassVar[str] + fixture_table: typing.ClassVar[Fixtures] + + def __init__(self, case: CrossMajorCase): + self.case = case + self.phase = 'start' + self.notes: list[typing.Any] = [] + + def to_str(self) -> str: + return '<cross-major CallContract system test>' + + async def run( + self, previous_results: list[typing.Any] + ) -> genvm_tool.tests.test.Result: + try: + await self._run_all() + return genvm_tool.tests.test.Result( + passed=True, context={'notes': self.notes}, elapsed_seconds=0 + ) + except BaseException as exc: + return genvm_tool.tests.test.Result( + passed=False, + context={'phase': self.phase, 'error': exc, 'notes': self.notes}, + elapsed_seconds=0, + ) + + async def _run_all(self): + root = self.case.shared.root_dir + build_info = json.loads((root / 'build' / 'info.json').read_text()) + self.versions = { + 2: build_info['executor_versions']['v0.2'], + 3: build_info['executor_versions']['v0.3'], + } + self.work_dir = self.case.shared.case_dir_for(self.case.description.name) + self.storage_path = self.work_dir / 'storage.pickle' + self.work_dir.mkdir(parents=True, exist_ok=True) + await gvm_io.write_file_bytes(self.storage_path, pickle.dumps(MockStorage())) + + manager = self.case.manager_service.handle + assert manager is not None + self.manager = manager + + self.phase = 'deploy fixtures' + for name in self.case.fixtures: + await self._deploy(*self.fixture_table[name]) + await self._after_deploy() + + self.phase = self.case.method + await getattr(self, self.case.method)(*self.case.arguments) + + async def _after_deploy(self) -> None: + pass + + async def _new_host( + self, + name: str, + running_address: Address, + resolve_hook: ResolveCallContractExecutorHook | None = None, + read_log: list[tuple[Address, public_abi.StorageView]] | None = None, + host_fuel: int | None = None, + ) -> MockHost: + ctx = TestContext(self.case.shared.logger) + host_path = ( + Path('/tmp') + / f'gvm-cross-major-{os.getpid()}' + / f'{name[:20]}-{os.urandom(4).hex()}.sock' + ) + host_path.parent.mkdir(parents=True, exist_ok=True) + host = MockHost( + ctx=ctx, + path=str(host_path), + storage_path_pre=self.storage_path, + storage_path_post=self.storage_path, + balances={}, + running_address=running_address, + resolve_call_contract_executor_hook=resolve_hook, + ) + if read_log is not None: + original = host.storage_read + + async def logged(mode, address, slot, offset, le, /): + read_log.append((Address(address), mode)) + return await original(mode, address, slot, offset, le) + + host.storage_read = logged # type: ignore[method-assign] + if host_fuel is not None: + + async def get_remaining_time_fee_gen_wei() -> int: + return host_fuel + + host.get_remaining_time_fee_gen_wei = ( # type: ignore[method-assign] + get_remaining_time_fee_gen_wei + ) + return host + + async def _execute( + self, + *, + name: str, + line: int, + address: Address, + calldata: bytes, + code: bytes | None, + is_init: bool, + resolve_hook: ResolveCallContractExecutorHook | None = None, + timeout: float = 30, + debug_initial_recursion: int | None = None, + is_sync: bool = True, + leader_public_data: bytes | None = None, + apply_changes: bool = True, + read_log: list[tuple[Address, public_abi.StorageView]] | None = None, + host_fuel: int | None = None, + hook_cross_contract_calls: bool = True, + permissions: str | None = None, + ): + host = await self._new_host( + name, address, resolve_hook, read_log=read_log, host_fuel=host_fuel + ) + request_extra: dict[str, typing.Any] = { + 'no_modules': True, + 'hook_cross_contract_calls': hook_cross_contract_calls, + } + if permissions is not None: + request_extra['permissions'] = permissions + with host as mock_host: + try: + async with base_host.ManagerClient(self.manager.uri) as manager_client: + result = await base_host.run_genvm( + mock_host, + manager_uri=self.manager.uri, + manager_client=manager_client, + ctx=host.ctx, + is_sync=is_sync, + leader_public_data=leader_public_data, + message=message(address, is_init=is_init), + host_data=json.dumps( + {'node_address': 'test', 'tx_id': self.host_data_tx_id}, + separators=(',', ':'), + ), + host='unix://' + mock_host.path, + code=resolve_runners(code, self.case.shared.root_dir), + calldata=calldata, + timeout=timeout, + debug_mode='unsafe', + unsafe_overrides=base_host.UnsafeOverrides( + reroute_to=self.versions[line], + initial_recursion=debug_initial_recursion, + ), + request_extra=request_extra, + bucket_totals=base_host.default_bucket_totals(line), + ) + if apply_changes and result.result_kind == host_fns.ResultCode.RETURN: + assert mock_host.storage is not None + apply_storage_deltas( + mock_host.storage, + address, + result.result_storage_deltas, + ) + return result + finally: + await host.stop_connections() + + async def _deploy(self, line: int, address: Address, code: bytes): + result = await self._execute( + name=f'deploy-{line}-{address.as_bytes.hex()}', + line=line, + address=address, + calldata=gvm_calldata.encode({}), + code=code, + is_init=True, + timeout=10 * 60, + ) + assert result.result_kind == host_fns.ResultCode.RETURN, result + assert len(result.execution_hash) == 32, result.execution_hash + + def _route(self, line: int) -> bytes: + """ + Routing payload placing a callee on `line`'s executor. + + It names the directory rather than the major, because a major cannot + pick between the live lines: every manifest key is semver major 0, so + `{'kind': 'major'}` resolves to the newest line whatever it asks for. + """ + return gvm_calldata.encode({'kind': 'version', 'version': self.versions[line]}) + + async def _lvs(self, *, name: str, **kwargs): + """ + Runs the same call as leader, validator and sync. + """ + + async def one(suffix: str, **mode): + return await self._execute( + name=f'{name}-{suffix}', + code=None, + is_init=False, + apply_changes=False, + **kwargs, + **mode, + ) + + leader = await one('leader', is_sync=False) + validator = await one( + 'validator', is_sync=False, leader_public_data=leader.result_leader_public_data + ) + sync = await one('sync', is_sync=True) + return leader, validator, sync diff --git a/tests/runner/origin/base_host.py b/tests/runner/origin/base_host.py index 41d24f4a..817ae1d5 100644 --- a/tests/runner/origin/base_host.py +++ b/tests/runner/origin/base_host.py @@ -679,9 +679,9 @@ def __init__(self, message: str, *, genvm_id: int): super().__init__(message) -# Transient manager refusals worth retrying. The manager reports these with the -# generic `Errors.INTERNAL` code (no dedicated variant yet), so the message is -# the only discriminator -- kept here so callers never have to string-match. +# Transient manager refusals worth retrying. They arrive as a `failed_to_start` +# error string, so the message is the only discriminator -- kept here so callers +# never have to string-match. _RETRYABLE_RUN_REFUSAL_MARKERS: typing.Final = ( 'modules are required but not running', 'modules are required but not all are running', diff --git a/tests/system/cross-major-observability/test.py b/tests/system/cross-major-observability/test.py index ab279f5b..b85adedd 100644 --- a/tests/system/cross-major-observability/test.py +++ b/tests/system/cross-major-observability/test.py @@ -1,39 +1,23 @@ """ Cross-major CallContract observability and determinism integration tests. - -This case reuses the real-manager harness from ``tests/system/cross-major`` -without adding more concerns to that already-large case. """ import functools -import importlib.util -import json -import pickle -import sys -import typing -from dataclasses import dataclass from pathlib import Path -import genvm_tool.io as gvm_io import genvm_tool.tests import genvm_tool.tests.stage.collection -from gvm_extra.mock_host import MockStorage +import origin.calldata as gvm_calldata +from gvm_extra import cross_major +from gvm_extra.cross_major import ( + SENDER, + assert_hashes_agree, + encode_call, + encode_leader_public_data, +) from origin import host_fns, public_abi from origin.calldata import Address - -def _load_cross_major_harness(): - path = Path(__file__).parents[1] / 'cross-major' / 'test.py' - spec = importlib.util.spec_from_file_location('cross_major_system_harness', path) - assert spec is not None and spec.loader is not None - module = importlib.util.module_from_spec(spec) - sys.modules[spec.name] = module - spec.loader.exec_module(module) - return module - - -base = _load_cross_major_harness() - ADDR_CALLER_V02 = Address('0x' + '41' * 20) ADDR_OBSERVER_V03 = Address('0x' + '42' * 20) ADDR_HELPER_V03 = Address('0x' + '43' * 20) @@ -50,7 +34,7 @@ def _load_cross_major_harness(): _contract_source = functools.partial( - base._contract_source, + cross_major.contract_source, assets_dir=Path(__file__).resolve().parent / 'assets', ) @@ -75,147 +59,19 @@ def _load_cross_major_harness(): } -@dataclass -class ObservabilityCase(genvm_tool.tests.test.Case): - description: genvm_tool.tests.test.Description - shared: genvm_tool.tests.SharedContext - manager_service: genvm_tool.tests.stage.collection.Service - method: str - fixtures: tuple[str, ...] - arguments: tuple[typing.Any, ...] = () - - async def into_steps(self) -> list[genvm_tool.tests.exec.step.Step]: - return [ObservabilityStep(self)] - - -class ObservabilityStep(base.CrossMajorStep): - async def _run_all(self): - root = self.case.shared.root_dir - build_info = json.loads((root / 'build' / 'info.json').read_text()) - self.build_dir = Path(build_info['build_dir']) - self.versions = { - 2: build_info['executor_versions']['v0.2'], - 3: build_info['executor_versions']['v0.3'], - } - work_dir = self.case.shared.case_dir_for(self.case.description.name) - self.storage_path = work_dir / 'storage.pickle' - work_dir.mkdir(parents=True, exist_ok=True) - await gvm_io.write_file_bytes(self.storage_path, pickle.dumps(MockStorage())) - - manager = self.case.manager_service.handle - assert manager is not None - self.manager = manager - - self.phase = 'deploy observability fixtures' - for line, address, code in (FIXTURES[name] for name in self.case.fixtures): - await self._deploy(line, address, code) - - self.phase = self.case.method - await getattr(self, self.case.method)(*self.case.arguments) - - async def _execute( - self, - *, - permissions: str = 'wscn', - **kwargs, - ): - name = typing.cast(str, kwargs['name']) - address = typing.cast(Address, kwargs['address']) - resolve_hook = kwargs.get('resolve_hook') - read_log = kwargs.get('read_log') - host_fuel = kwargs.get('host_fuel') - host = await self._new_host( - name, - address, - resolve_hook, - read_log=read_log, - host_fuel=host_fuel, - ) - ctx = host.ctx - with host as mock_host: - try: - async with base.base_host.ManagerClient(self.manager.uri) as manager_client: - result = await base.base_host.run_genvm( - mock_host, - manager_uri=self.manager.uri, - manager_client=manager_client, - ctx=ctx, - is_sync=kwargs.get('is_sync', True), - leader_public_data=kwargs.get('leader_public_data'), - message=base._message( - address, is_init=typing.cast(bool, kwargs['is_init']) - ), - host_data='{"node_address":"test","tx_id":"cross-major-observe"}', - host='unix://' + mock_host.path, - code=base.resolve_runners( - typing.cast('bytes | None', kwargs.get('code')), - self.case.shared.root_dir, - ), - calldata=typing.cast(bytes, kwargs['calldata']), - timeout=kwargs.get('timeout', 30), - debug_mode='unsafe', - unsafe_overrides=base.base_host.UnsafeOverrides( - reroute_to=self.versions[typing.cast(int, kwargs['line'])], - initial_recursion=kwargs.get('debug_initial_recursion'), - ), - request_extra={ - 'permissions': permissions, - 'no_modules': True, - 'hook_cross_contract_calls': kwargs.get( - 'hook_cross_contract_calls', True - ), - }, - bucket_totals=base.base_host.default_bucket_totals( - typing.cast(int, kwargs['line']) - ), - ) - if ( - kwargs.get('apply_changes', True) - and result.result_kind == host_fns.ResultCode.RETURN - ): - assert mock_host.storage is not None - base._apply_storage_deltas( - mock_host.storage, - address, - result.result_storage_deltas, - ) - return result - finally: - await host.stop_connections() - - async def _lvs_extended(self, *, name: str, **kwargs): - async def one(suffix: str, **mode): - return await self._execute( - name=f'{name}-{suffix}', - code=None, - is_init=False, - apply_changes=False, - **kwargs, - **mode, - ) +class ObservabilityStep(cross_major.CrossMajorStep): + host_data_tx_id = 'cross-major-observe' + fixture_table = FIXTURES - leader = await one('leader', is_sync=False) - validator = await one( - 'validator', - is_sync=False, - leader_public_data=leader.result_leader_public_data, - ) - sync = await one('sync', is_sync=True) - assert leader.execution_hash == validator.execution_hash, ( - name, - leader.execution_hash.hex(), - validator.execution_hash.hex(), - ) - assert leader.execution_hash == sync.execution_hash, ( - name, - leader.execution_hash.hex(), - sync.execution_hash.hex(), - ) - for label, result in ( - ('leader', leader), - ('validator', validator), - ('sync', sync), - ): + async def _lvs_agreeing(self, *, name: str, **kwargs): + """ + Like `_lvs`, but also requires the three runs to agree on the execution + hash and none of them to be an internal error. + """ + leader, validator, sync = await self._lvs(name=name, **kwargs) + runs = [('leader', leader), ('validator', validator), ('sync', sync)] + assert_hashes_agree(name, runs) + for label, result in runs: assert result.result_kind != host_fns.ResultCode.INTERNAL_ERROR, ( name, label, @@ -248,13 +104,13 @@ async def _assert_validator_mode(self): name='validator-mode-is-real', line=3, address=ADDR_CHAIN_V03, - calldata=base._calldata('hop', 1, ADDR_CHAIN_V02), + calldata=encode_call('hop', 1, ADDR_CHAIN_V02), code=None, is_init=False, resolve_hook=self._route_chain, apply_changes=False, is_sync=False, - leader_public_data=base._leader_public_data(base.gvm_calldata.encode({})), + leader_public_data=encode_leader_public_data(gvm_calldata.encode({})), ) assert bogus.result_kind == host_fns.ResultCode.VM_ERROR, bogus @@ -265,18 +121,18 @@ def resolve(address, state, major): routes.append(address) return self._route_v03(address, state, major) - leader, validator, sync = await self._lvs_extended( + leader, validator, sync = await self._lvs_agreeing( name='context', line=2, address=ADDR_CALLER_V02, - calldata=base._calldata('context', ADDR_OBSERVER_V03), + calldata=encode_call('context', ADDR_OBSERVER_V03), resolve_hook=resolve, ) expected = { 'contract': ADDR_OBSERVER_V03.as_hex, - 'sender': base.SENDER.as_hex, - 'origin': base.SENDER.as_hex, - 'signer': base.SENDER.as_hex, + 'sender': SENDER.as_hex, + 'origin': SENDER.as_hex, + 'signer': SENDER.as_hex, 'stack': [ADDR_CALLER_V02.as_hex], 'value': 0, 'is_init': False, @@ -288,11 +144,11 @@ def resolve(address, state, major): self.notes.append(('nested context', expected)) async def _assert_permissions(self): - leader, validator, sync = await self._lvs_extended( + leader, validator, sync = await self._lvs_agreeing( name='permissions', line=2, address=ADDR_CALLER_V02, - calldata=base._calldata('permissions', ADDR_OBSERVER_V03, ADDR_HELPER_V03), + calldata=encode_call('permissions', ADDR_OBSERVER_V03, ADDR_HELPER_V03), resolve_hook=self._route_v03, permissions='wscnu', ) @@ -311,7 +167,7 @@ async def _assert_debug_alias(self): name='debug-alias-top-level', line=3, address=ADDR_OBSERVER_V03, - calldata=base._calldata('debug_alias'), + calldata=encode_call('debug_alias'), code=None, is_init=False, resolve_hook=self._route_v03, @@ -327,11 +183,11 @@ async def _assert_debug_alias(self): public_abi.VmError.invalid_contract().runner().malformed() ), top_level - leader, validator, sync = await self._lvs_extended( + leader, validator, sync = await self._lvs_agreeing( name='debug-alias-nested', line=2, address=ADDR_CALLER_V02, - calldata=base._calldata('debug_alias', ADDR_OBSERVER_V03), + calldata=encode_call('debug_alias', ADDR_OBSERVER_V03), resolve_hook=self._route_v03, ) for result in (leader, validator, sync): @@ -348,11 +204,11 @@ async def _assert_debug_alias(self): async def _assert_read_only_storage(self, permissions: str): reads: list[tuple[Address, public_abi.StorageView]] = [] - leader, validator, sync = await self._lvs_extended( + leader, validator, sync = await self._lvs_agreeing( name=f'storage-{permissions}', line=2, address=ADDR_CALLER_V02, - calldata=base._calldata('read', ADDR_OBSERVER_V03), + calldata=encode_call('read', ADDR_OBSERVER_V03), resolve_hook=self._route_v03, permissions=permissions, read_log=reads, @@ -365,11 +221,11 @@ async def _assert_read_only_storage(self, permissions: str): async def _assert_self_recursion(self, budget: int): expected_error = str(public_abi.VmError.out_of().subvm_recursion()) - leader, validator, sync = await self._lvs_extended( + leader, validator, sync = await self._lvs_agreeing( name=f'self-recursion-{budget}', line=3, address=ADDR_SELF_V03, - calldata=base._calldata('observe_recursion', 4), + calldata=encode_call('observe_recursion', 4), resolve_hook=self._route_v03, debug_initial_recursion=budget, ) @@ -399,11 +255,11 @@ async def _assert_self_recursion(self, budget: int): async def _assert_deep_nesting(self, depth: int): # A chain deeper than the manager's cross-major bound is refused rather # than served, and every node refuses it at the same place. - leader, validator, sync = await self._lvs_extended( + leader, validator, sync = await self._lvs_agreeing( name=f'deep-{depth}', line=3, address=ADDR_CHAIN_V03, - calldata=base._calldata('hop', depth, ADDR_CHAIN_V02), + calldata=encode_call('hop', depth, ADDR_CHAIN_V02), resolve_hook=self._route_chain, ) if depth > CROSS_MAJOR_RECURSION: @@ -422,11 +278,11 @@ async def _assert_deep_nesting(self, depth: int): ) async def _assert_error(self, label: str, target: Address): - leader, validator, sync = await self._lvs_extended( + leader, validator, sync = await self._lvs_agreeing( name=f'error-{label}', line=2, address=ADDR_CALLER_V02, - calldata=base._calldata('answer', target), + calldata=encode_call('answer', target), resolve_hook=lambda address, state, major: ( self._route(3) if address == target else self._route_v03(address, state, major) ), @@ -447,7 +303,7 @@ async def _assert_undeployed_error(self): await self._assert_error('undeployed', ABSENT) async def _assert_non_contract_error(self): - await self._assert_error('non-contract', base.SENDER) + await self._assert_error('non-contract', SENDER) async def _assert_trap_error(self): await self._assert_error('trap', ADDR_TRAP_V03) @@ -456,11 +312,11 @@ async def _assert_user_error(self): await self._assert_error('user-error', ADDR_USER_ERROR_V03) async def _assert_fuel(self, fuel: int): - leader, validator, sync = await self._lvs_extended( + leader, validator, sync = await self._lvs_agreeing( name=f'fuel-{fuel}', line=2, address=ADDR_CALLER_V02, - calldata=base._calldata('read', ADDR_OBSERVER_V03), + calldata=encode_call('read', ADDR_OBSERVER_V03), resolve_hook=self._route_v03, host_fuel=fuel, ) @@ -472,6 +328,10 @@ async def _assert_fuel(self, fuel: int): ) +class ObservabilityCase(cross_major.CrossMajorCase): + step_type = ObservabilityStep + + def collect( ctx: genvm_tool.tests.stage.collection.Context, *, diff --git a/tests/system/cross-major/test.py b/tests/system/cross-major/test.py index 9e629581..b494c322 100644 --- a/tests/system/cross-major/test.py +++ b/tests/system/cross-major/test.py @@ -2,23 +2,27 @@ import asyncio import contextlib -import json -import os +import functools import pickle import typing -from dataclasses import dataclass from pathlib import Path import genvm_tool.io as gvm_io import genvm_tool.tests import genvm_tool.tests.stage.collection -import genvm_tool_plugins.runners as runners -import origin.base_host as base_host import origin.calldata as gvm_calldata -from gvm_extra.mock_host import MockHost, MockStorage -from origin import host_fns, public_abi +from gvm_extra import cross_major +from gvm_extra.cross_major import ( + TIMESTAMP, + assert_hashes_agree, + encode_call, + encode_leader_public_data, + message, + resolve_runners, +) +from gvm_extra.mock_host import MockStorage +from origin import base_host, host_fns, public_abi from origin.calldata import Address -from origin.leader_public_data import LeaderPublicData from origin.process_tree import descendants # One directory per line, because a contract is written against its own sdk. @@ -40,33 +44,15 @@ ADDR_CHAIN_V02 = Address('0x' + '28' * 20) ABSENT_CALLEE = Address('0x' + '99' * 20) -SENDER = Address('0x' + '01' * 20) # Larger than the manager cap the test configures below, so the nested envelope # cannot fit through the socketpair. MANAGER_MAX_MESSAGE_BYTES = 1 * 1024 * 1024 BLOB_BYTES = 2 * 1024 * 1024 -TIMESTAMP = '2024-11-26T06:42:42.424242Z' - -def _leader_public_data(output: bytes) -> bytes: - return LeaderPublicData([output]).encode() - - -def resolve_runners(code: bytes | None, root_dir: Path) -> bytes | None: - """ - Turns the `@RUNNER_LATEST_...@` tokens of a contract source into uids. - - Every comparison against a deployed contract's bytes has to go through this - too, because storage holds what the executor was given. - """ - return None if code is None else runners.substitute(code, root_dir) - - -def _contract_source(line: int, name: str, assets_dir: Path = ASSETS_DIR) -> bytes: - """ - Reads a contract asset written against `line`'s sdk. - """ - return (assets_dir / f'v0.{line}' / f'{name}.py').read_bytes() +_contract_source = functools.partial( + cross_major.contract_source, + assets_dir=ASSETS_DIR, +) CALLER_V03 = _contract_source(3, 'caller') @@ -165,61 +151,6 @@ def _loop_wat(target: Address, *, caller_line: typing.Literal[2, 3]) -> str: """ -class _TestContext(base_host.Context): - def __init__(self, logger: base_host.Logger): - self.logger = logger - self.stats: dict[str, typing.Any] = {} - - def on_genvm_success(self): ... - def on_genvm_failure(self): ... - - def add_stat(self, key: str, value: typing.Any, /): - self.stats[key] = value - - -def _message(address: Address, *, is_init: bool) -> base_host.Message: - return { - 'contract_address': address, - 'sender_address': SENDER, - 'origin_address': SENDER, - 'signer_address': SENDER, - 'chain_id': 61999, - 'value': 0, - 'is_init': is_init, - 'transaction_timestamp': TIMESTAMP, - } - - -def _calldata(method: str, *args: typing.Any) -> bytes: - return gvm_calldata.encode({'': method, 'args': list(args)}) - - -def _assert_hashes_agree(label: str, runs: list[tuple[str, typing.Any]]) -> None: - """ - A disagreement is only actionable once the failure says who disagreed. - """ - hashes = [(name, run.execution_hash.hex()) for name, run in runs] - assert len({digest for _name, digest in hashes}) == 1, '\n'.join( - [f'{label}: execution hashes disagree'] - + [f' {name}: {digest}' for name, digest in hashes] - ) - - -def _apply_storage_deltas( - storage: MockStorage, - address: Address, - changes: list[tuple[bytes, bytes]], -) -> None: - for key, value in changes: - assert len(key) == 36 - storage.write( - address, - key[:32], - int.from_bytes(key[32:], byteorder='big') * 32, - value, - ) - - async def _wait_for_descendants( pid: int, expected: int, timeout: float = 10 ) -> set[int]: @@ -237,197 +168,21 @@ async def _wait_for_no_descendants(pid: int, timeout: float = 10) -> None: await asyncio.sleep(0.05) -@dataclass -class CrossMajorCase(genvm_tool.tests.test.Case): - description: genvm_tool.tests.test.Description - shared: genvm_tool.tests.SharedContext - manager_service: genvm_tool.tests.stage.collection.Service - method: str - fixtures: tuple[str, ...] - arguments: tuple[typing.Any, ...] = () - - async def into_steps(self) -> list[genvm_tool.tests.exec.step.Step]: - return [CrossMajorStep(self)] - - -class CrossMajorStep(genvm_tool.tests.exec.step.Python): - def __init__(self, case: CrossMajorCase): - self.case = case - self.phase = 'start' - self.notes: list[typing.Any] = [] - - def to_str(self) -> str: - return '<cross-major CallContract system test>' - - async def run( - self, previous_results: list[typing.Any] - ) -> genvm_tool.tests.test.Result: - try: - await self._run_all() - return genvm_tool.tests.test.Result( - passed=True, context={'notes': self.notes}, elapsed_seconds=0 - ) - except BaseException as exc: - return genvm_tool.tests.test.Result( - passed=False, - context={'phase': self.phase, 'error': exc, 'notes': self.notes}, - elapsed_seconds=0, - ) - - async def _run_all(self): - root = self.case.shared.root_dir - build_info = json.loads((root / 'build' / 'info.json').read_text()) - self.build_dir = Path(build_info['build_dir']) - self.versions = { - 2: build_info['executor_versions']['v0.2'], - 3: build_info['executor_versions']['v0.3'], - } - work_dir = self.case.shared.case_dir_for(self.case.description.name) - self.storage_path = work_dir / 'storage.pickle' - work_dir.mkdir(parents=True, exist_ok=True) - await gvm_io.write_file_bytes(self.storage_path, pickle.dumps(MockStorage())) - - manager = self.case.manager_service.handle - assert manager is not None - self.manager = manager - - self.phase = 'deploy fixtures' - for line, address, code in (FIXTURES[name] for name in self.case.fixtures): - await self._deploy(line, address, code) - if 'loop-v03' in self.case.fixtures: - loop_v03 = await self._compile_wat( - _loop_wat(ADDR_LOOP_V02, caller_line=3), work_dir / 'loop-v03' - ) - loop_v02 = await self._compile_wat( - _loop_wat(ADDR_LOOP_V03, caller_line=2), work_dir / 'loop-v02' - ) - self._replace_code(ADDR_LOOP_V03, LOOP_V03, loop_v03) - self._replace_code(ADDR_LOOP_V02, LOOP_V02, loop_v02) - - self.phase = self.case.method - await getattr(self, self.case.method)(*self.case.arguments) - - async def _assert_cancel_stops_chain(self): - await self._assert_chain_stopped('cancel') - - async def _assert_deadline_stops_chain(self): - await self._assert_chain_stopped('deadline') - - async def _new_host( - self, - name: str, - running_address: Address, - resolve_hook=None, - read_log: list[tuple[Address, public_abi.StorageView]] | None = None, - host_fuel: int | None = None, - ) -> MockHost: - ctx = _TestContext(self.case.shared.logger) - host_path = ( - Path('/tmp') - / f'gvm-cross-major-{os.getpid()}' - / f'{name[:20]}-{os.urandom(4).hex()}.sock' - ) - host_path.parent.mkdir(parents=True, exist_ok=True) - host = MockHost( - ctx=ctx, - path=str(host_path), - storage_path_pre=self.storage_path, - storage_path_post=self.storage_path, - balances={}, - running_address=running_address, - resolve_call_contract_executor_hook=resolve_hook, - ) - if read_log is not None: - original = host.storage_read - - async def logged(mode, address, slot, offset, le, /): - read_log.append((Address(address), mode)) - return await original(mode, address, slot, offset, le) +class CrossMajorStep(cross_major.CrossMajorStep): + host_data_tx_id = 'cross-major' + fixture_table = FIXTURES - host.storage_read = logged # type: ignore[method-assign] - if host_fuel is not None: - - async def get_remaining_time_fee_gen_wei() -> int: - return host_fuel - - host.get_remaining_time_fee_gen_wei = ( # type: ignore[method-assign] - get_remaining_time_fee_gen_wei - ) - return host - - async def _execute( - self, - *, - name: str, - line: int, - address: Address, - calldata: bytes, - code: bytes | None, - is_init: bool, - resolve_hook=None, - timeout: float = 30, - debug_initial_recursion: int | None = None, - is_sync: bool = True, - leader_public_data: bytes | None = None, - apply_changes: bool = True, - read_log: list[tuple[Address, public_abi.StorageView]] | None = None, - host_fuel: int | None = None, - hook_cross_contract_calls: bool = True, - ): - host = await self._new_host( - name, address, resolve_hook, read_log=read_log, host_fuel=host_fuel + async def _after_deploy(self) -> None: + if 'loop-v03' not in self.case.fixtures: + return + loop_v03 = await self._compile_wat( + _loop_wat(ADDR_LOOP_V02, caller_line=3), self.work_dir / 'loop-v03' ) - ctx = host.ctx - with host as mock_host: - try: - async with base_host.ManagerClient(self.manager.uri) as manager_client: - result = await base_host.run_genvm( - mock_host, - manager_uri=self.manager.uri, - manager_client=manager_client, - ctx=ctx, - is_sync=is_sync, - leader_public_data=leader_public_data, - message=_message(address, is_init=is_init), - host_data='{"node_address":"test","tx_id":"cross-major"}', - host='unix://' + mock_host.path, - code=resolve_runners(code, self.case.shared.root_dir), - calldata=calldata, - timeout=timeout, - debug_mode='unsafe', - unsafe_overrides=base_host.UnsafeOverrides( - reroute_to=self.versions[line], - initial_recursion=debug_initial_recursion, - ), - request_extra={ - 'no_modules': True, - 'hook_cross_contract_calls': hook_cross_contract_calls, - }, - bucket_totals=base_host.default_bucket_totals(line), - ) - if apply_changes and result.result_kind == host_fns.ResultCode.RETURN: - assert mock_host.storage is not None - _apply_storage_deltas( - mock_host.storage, - address, - result.result_storage_deltas, - ) - return result - finally: - await host.stop_connections() - - async def _deploy(self, line: int, address: Address, code: bytes): - result = await self._execute( - name=f'deploy-{line}-{address.as_bytes.hex()}', - line=line, - address=address, - calldata=gvm_calldata.encode({}), - code=code, - is_init=True, - timeout=10 * 60, + loop_v02 = await self._compile_wat( + _loop_wat(ADDR_LOOP_V03, caller_line=2), self.work_dir / 'loop-v02' ) - assert result.result_kind == host_fns.ResultCode.RETURN, result - assert len(result.execution_hash) == 32, result.execution_hash + self._replace_code(ADDR_LOOP_V03, LOOP_V03, loop_v03) + self._replace_code(ADDR_LOOP_V02, LOOP_V02, loop_v02) async def _compile_wat(self, wat: str, path: Path) -> bytes: await gvm_io.make_dir(path) @@ -470,15 +225,11 @@ def _replace_code(self, address: Address, old_code: bytes, new_code: bytes) -> N with open(self.storage_path, 'wb') as storage_file: pickle.dump(storage, storage_file) - def _route(self, line: int) -> bytes: - """ - Routing payload placing a callee on `line`'s executor. + async def _assert_cancel_stops_chain(self): + await self._assert_chain_stopped('cancel') - It names the directory rather than the major, because a major cannot - pick between the live lines: every manifest key is semver major 0, so - `{'kind': 'major'}` resolves to the newest line whatever it asks for. - """ - return gvm_calldata.encode({'kind': 'version', 'version': self.versions[line]}) + async def _assert_deadline_stops_chain(self): + await self._assert_chain_stopped('deadline') def _raw_request( self, @@ -490,7 +241,7 @@ def _raw_request( ) -> dict[str, typing.Any]: return { 'selector': {'kind': 'major', 'major': base_host.UNDEPLOYED_MAJOR}, - 'message': _message(address, is_init=False), + 'message': message(address, is_init=False), 'is_sync': True, 'debug_mode': 'unsafe', 'host_data': '{"node_address":"test","tx_id":"cross-major-lifecycle"}', @@ -499,7 +250,7 @@ def _raw_request( 'host': 'unix://' + host_path, 'extra_args': [], 'code': None, - 'calldata': _calldata('call', ADDR_BUSY_V02), + 'calldata': encode_call('call', ADDR_BUSY_V02), 'leader_public_data': None, 'bucket_totals': base_host.default_bucket_totals(line), 'gas_data': base_host.DEFAULT_GAS_DATA, @@ -568,7 +319,7 @@ async def _assert_envelope_ignores_websocket_cap(self): name='large-envelope-v02-to-v03', line=2, address=ADDR_BLOB_CALLER_V02, - calldata=_calldata('send', ADDR_BLOB_V03, BLOB_BYTES), + calldata=encode_call('send', ADDR_BLOB_V03, BLOB_BYTES), code=None, is_init=False, resolve_hook=lambda address, _state, _major: ( @@ -588,7 +339,7 @@ async def _assert_nested_deadline_is_timeout(self): name='deadline-v03-to-v02', line=3, address=ADDR_CALLER_V03, - calldata=_calldata('call', ADDR_BUSY_V02), + calldata=encode_call('call', ADDR_BUSY_V02), code=None, is_init=False, resolve_hook=lambda address, _state, _major: ( @@ -599,36 +350,6 @@ async def _assert_nested_deadline_is_timeout(self): assert result.result_kind == host_fns.ResultCode.VM_ERROR, result assert result.result_data == str(public_abi.VmError.timeout()), result - async def _lvs( - self, *, name: str, line: int, address: Address, calldata: bytes, resolve_hook - ): - """ - Run the same step as leader, validator and sync run. - - The three must agree on the execution hash; anything else is a - determinism violation an honest validator would see as a disagreement. - """ - - async def one(suffix: str, **kwargs): - return await self._execute( - name=f'{name}-{suffix}', - line=line, - address=address, - calldata=calldata, - code=None, - is_init=False, - resolve_hook=resolve_hook, - apply_changes=False, - **kwargs, - ) - - leader = await one('leader', is_sync=False) - validator = await one( - 'validator', is_sync=False, leader_public_data=leader.result_leader_public_data - ) - sync = await one('sync', is_sync=True) - return leader, validator, sync - async def _assert_lvs_validator_mode(self): # A leader/validator run that silently degraded to a sync one would make # the agreement below vacuous, so first pin that the validator mode is @@ -637,7 +358,7 @@ async def _assert_lvs_validator_mode(self): name='lvs-mode-is-real', line=3, address=ADDR_CALLER_V03, - calldata=_calldata('call', ADDR_CALLEE_V02), + calldata=encode_call('call', ADDR_CALLEE_V02), code=None, is_init=False, resolve_hook=lambda address, _state, _major: ( @@ -645,7 +366,7 @@ async def _assert_lvs_validator_mode(self): ), apply_changes=False, is_sync=False, - leader_public_data=_leader_public_data(gvm_calldata.encode({})), + leader_public_data=encode_leader_public_data(gvm_calldata.encode({})), ) assert bogus.result_kind == host_fns.ResultCode.VM_ERROR, bogus @@ -676,7 +397,7 @@ async def _assert_lvs_direction( name=f'lvs-{suffix}', line=line, address=caller, - calldata=_calldata('call', callee), + calldata=encode_call('call', callee), resolve_hook=lambda address, _state, _major: ( self._route(major) if address == callee else None ), @@ -685,7 +406,7 @@ async def _assert_lvs_direction( for label, run in runs: assert run.result_kind == host_fns.ResultCode.RETURN, (suffix, label, run) assert len(run.execution_hash) == 32, (suffix, label, run.execution_hash) - _assert_hashes_agree(f'nested call, {suffix}', runs) + assert_hashes_agree(f'nested call, {suffix}', runs) async def _assert_unhooked_stays_in_process(self): """ @@ -702,7 +423,7 @@ def resolve(address, _state, _major): name='unhooked-cross-major', line=3, address=ADDR_CALLER_V03, - calldata=_calldata('call', ADDR_CALLEE_V02), + calldata=encode_call('call', ADDR_CALLEE_V02), code=None, is_init=False, resolve_hook=resolve, @@ -725,7 +446,7 @@ async def _assert_route_is_hash_invariant(self): name='route-in-process', line=3, address=ADDR_CALLER_V03, - calldata=_calldata('call', ADDR_CALLEE_V03), + calldata=encode_call('call', ADDR_CALLEE_V03), code=None, is_init=False, resolve_hook=lambda _address, _state, _major: None, @@ -735,7 +456,7 @@ async def _assert_route_is_hash_invariant(self): name='route-nested', line=3, address=ADDR_CALLER_V03, - calldata=_calldata('call', ADDR_CALLEE_V03), + calldata=encode_call('call', ADDR_CALLEE_V03), code=None, is_init=False, resolve_hook=lambda address, _state, _major: ( @@ -748,7 +469,7 @@ async def _assert_route_is_hash_invariant(self): assert in_process.result_data == nested.result_data == 303, (in_process, nested) # The parent's own execution hash folds the callee's small hash, which # commits to the outcome and to nothing about where the callee ran. - _assert_hashes_agree( + assert_hashes_agree( 'the route is consensus-visible', [('in process', in_process), ('nested', nested)], ) @@ -777,7 +498,7 @@ def resolve(address, state, _major, nested=nested, resolved=resolved): name=f'state-mode-{"final" if final else "nonfinal"}-{nested}', line=3, address=ADDR_STATE_CALLER_V03, - calldata=_calldata('call', ADDR_CALLEE_V03, final), + calldata=encode_call('call', ADDR_CALLEE_V03, final), code=None, is_init=False, resolve_hook=resolve, @@ -812,7 +533,7 @@ async def _assert_deep_chain(self): name='deep-chain', line=3, address=ADDR_CHAIN_V03, - calldata=_calldata('hop', depth, ADDR_CHAIN_V02), + calldata=encode_call('hop', depth, ADDR_CHAIN_V02), resolve_hook=self._chain_resolve, ) # The chain alternates v0.3, v0.2, ..., so it sums 3 and 2 in turn. @@ -838,7 +559,7 @@ async def _assert_starved_fuel(self, fuel: int): name=f'starved-fuel-{fuel}', line=3, address=ADDR_CALLER_V03, - calldata=_calldata('call', ADDR_CALLEE_V02), + calldata=encode_call('call', ADDR_CALLEE_V02), code=None, is_init=False, resolve_hook=lambda address, _state, _major: ( @@ -852,7 +573,7 @@ async def _assert_starved_fuel(self, fuel: int): name=f'starved-fuel-{fuel}-again', line=3, address=ADDR_CALLER_V03, - calldata=_calldata('call', ADDR_CALLEE_V02), + calldata=encode_call('call', ADDR_CALLEE_V02), code=None, is_init=False, resolve_hook=lambda address, _state, _major: ( @@ -885,7 +606,7 @@ async def _assert_child_failure_is_canonical( name=f'canonical-{label}', line=line, address=caller, - calldata=_calldata('call', callee), + calldata=encode_call('call', callee), code=None, is_init=False, resolve_hook=lambda address, _state, _major: ( @@ -943,6 +664,10 @@ def resolve(address, _state, advisory_major): ], resolved +class CrossMajorCase(cross_major.CrossMajorCase): + step_type = CrossMajorStep + + def collect( ctx: genvm_tool.tests.stage.collection.Context, *, diff --git a/tests/system/manager-socket/test.py b/tests/system/manager-socket/test.py index d6a73ee6..988bebd4 100644 --- a/tests/system/manager-socket/test.py +++ b/tests/system/manager-socket/test.py @@ -402,26 +402,114 @@ async def _oversized_closes_connection(self): aiohttp.WSCloseCode.MESSAGE_TOO_BIG, ), close_code + async def _rejected_run_still_gets_id(self): + async with ( + await self._manager('async-validation') as manager, + self._occupied_permits(manager), + ): + host_path = manager.work_dir / 'unused-host.sock' + cases = [ + ({'host_hello_data': [b'', b'injected']}, 'host index 1'), + # The fixture manager runs no modules + ( + {'no_modules': False, 'is_sync': False, 'permissions': 'n'}, + 'modules are required', + ), + ] + for request_id, (overrides, error) in enumerate(cases, start=1): + async with ManagerWsClient(manager.uri) as client: + await _read_hello(client) + req = _run_request(self.case.shared.root_dir, host_path) + req['run'].update(overrides) + await client.send(Methods.RUN, request_id, req) + method, got_request_id, payload = await client.read_frame() + assert (method, got_request_id) == (Methods.RUN, request_id), payload + variant, event = await _wait_terminal(client, timeout=5) + assert variant == 'failed_to_start', variant + assert event['genvm_id'] == payload['genvm_id'] + assert error in event['error'], event + status, body = await _http_json('GET', manager.uri, '/status') + assert status == 200, body + assert body['permits']['current'] == 0, body + + @contextlib.asynccontextmanager + async def _occupied_permits(self, manager: _ManagerFixture): + status, body = await _http_json('GET', manager.uri, '/status') + assert status == 200, body + permits = body['permits'] + assert permits['current'] == permits['max'], permits + assert permits['max'] % permits['per_sync_run'] == 0, permits + genvm_ids = [] + host_tasks = [] + cancel_host = asyncio.Event() + async with contextlib.AsyncExitStack() as stack: + try: + for i in range(permits['max'] // permits['per_sync_run']): + ctx = _TestContext(self.case.shared.logger) + host, host_path = _make_mock_host(manager, f'busy-{i}', ctx) + mock_host = stack.enter_context(host) + host_tasks.append( + asyncio.create_task(base_host.host_loop(mock_host, cancel_host, ctx=ctx)) + ) + client = await stack.enter_async_context(ManagerWsClient(manager.uri)) + await _read_hello(client) + req = _run_request(self.case.shared.root_dir, host_path) + req['run']['code'] = (self.case.shared.root_dir / BUSY_CONTRACT).read_bytes() + req['run']['deadline'] = '120s' + await client.send(Methods.RUN, 1, req) + genvm_ids.append((await _read_reply(client, Methods.RUN, 1))['genvm_id']) + await _wait_event(client, 'started') + status, body = await _http_json('GET', manager.uri, '/status') + assert status == 200, body + assert body['permits']['current'] == 0, body + yield + finally: + cancel_host.set() + try: + for genvm_id in genvm_ids: + await _http_json('DELETE', manager.uri, f'/genvm/{genvm_id}') + finally: + for task in host_tasks: + task.cancel() + await asyncio.gather(*host_tasks, return_exceptions=True) + async def _startup_failure_events_and_permits(self): async with await self._manager('startup-failures') as manager: async with ManagerWsClient(manager.uri) as client: await _read_hello(client) host_path = manager.work_dir / 'unused-host.sock' - await client.send( - Methods.RUN, - 10, - _run_request(self.case.shared.root_dir, host_path, extra_args=['--bad-flag']), + req = _run_request( + self.case.shared.root_dir, host_path, extra_args=['--bad-flag'] ) + # Outgrows the stdin pipe, so the executor exits while it is being written + req['run']['calldata'] = gvm_calldata.encode({'pad': bytes(4 << 20)}) + await client.send(Methods.RUN, 10, req) method, request_id, payload = await client.read_frame() assert method == Methods.RUN assert request_id == 10 genvm_id = payload['genvm_id'] - # The executor rejects the flag and exits on its own, so this is - # a `finished` with its exit code rather than `failed_to_start`. + # The executor rejects the flag and exits on its own, so this is a + # `finished` with its exit code and its complaint in stderr, whichever + # of the write and the exit the manager notices first. variant, event = await _wait_terminal(client, timeout=5) assert event['genvm_id'] == genvm_id assert variant == 'finished', variant assert event['exit_code'] not in (0, None), event + assert event['artifact_sizes']['stderr'] > 0, event + await client.send( + Methods.GET_ARTIFACT, + 11, + { + 'get_artifact': { + 'genvm_id': genvm_id, + 'field': 'stderr', + 'offset': 0, + 'max_len': 4096, + } + }, + ) + artifact = await _read_reply(client, Methods.GET_ARTIFACT, 11) + assert b'--bad-flag' in artifact['data'], artifact async with ManagerWsClient(manager.uri) as client: await _read_hello(client) @@ -816,6 +904,28 @@ async def _client_never_returns_ttl_drains(self): ) await _read_error(client, 111, Errors.UNKNOWN_ID) + async def _http_retry_preserves_socket_failure(self): + async with await self._manager('http-retry-socket-failure') as manager: + async with ManagerWsClient(manager.uri) as client: + boot_id = await _read_hello(client) + req = _run_request(self.case.shared.root_dir, manager.work_dir / 'unused.sock') + req['run'].update( + host_genvm_id='socket-failure', host_hello_data=[b'', b'injected'] + ) + await client.send(Methods.RUN, 1, req) + genvm_id = (await _read_reply(client, Methods.RUN, 1))['genvm_id'] + variant, failure = await _wait_terminal(client, timeout=5) + assert variant == 'failed_to_start', failure + status, body = await _http_json( + 'POST', manager.uri, '/genvm/run', data=gvm_calldata.encode(req['run']) + ) + assert status == 500, body + await client.send( + Methods.ATTACH, 2, {'attach': {'boot_id': boot_id, 'genvm_id': genvm_id}} + ) + snapshot = (await _read_reply(client, Methods.ATTACH, 2))['snapshot'] + assert snapshot['failed_to_start'] == failure, snapshot + async def _http_and_socket_same_fixture(self): async with await self._manager('http-socket-parity') as manager: http_res = await self._run_fixture_over_http(manager, 'http') @@ -825,7 +935,23 @@ async def _http_and_socket_same_fixture(self): assert http_res['result_kind'] == socket_res.result_kind assert http_res['result_data'] == socket_res.result_data - async def _run_fixture_over_http(self, manager: _ManagerFixture, name: str): + async def _http_failure_releases_token(self): + async with await self._manager('http-failure-cleanup') as manager: + req = _run_request(self.case.shared.root_dir, manager.work_dir / 'unused.sock') + req['run'].update( + host_genvm_id='http-failure', host_hello_data=[b'', b'injected'] + ) + status, body = await _http_json( + 'POST', manager.uri, '/genvm/run', data=gvm_calldata.encode(req['run']) + ) + assert status == 500, body + await self._run_fixture_over_http( + manager, 'corrected', host_genvm_id='http-failure' + ) + + async def _run_fixture_over_http( + self, manager: _ManagerFixture, name: str, *, host_genvm_id: str | None = None + ): ctx = _TestContext(self.case.shared.logger) host, host_path = _make_mock_host(manager, name, ctx) cancel_host = asyncio.Event() @@ -834,6 +960,7 @@ async def _run_fixture_over_http(self, manager: _ManagerFixture, name: str): base_host.host_loop(mock_host, cancel_host, ctx=ctx) ) req = _run_request(self.case.shared.root_dir, host_path)['run'] + req['host_genvm_id'] = host_genvm_id async with aiohttp.request( 'POST', f'{manager.uri}/genvm/run', @@ -958,6 +1085,13 @@ def service( None, genvm.ManagerService, ), + ( + 'async-validation', + '_rejected_run_still_gets_id', + {}, + None, + genvm.ManagerService, + ), ('happy-path', '_happy_path_artifact_ack_attach', {}, None, genvm.ManagerService), ( 'reconnect-mid-run', @@ -1005,6 +1139,20 @@ def service( genvm.ManagerService, ), ('http-parity', '_http_and_socket_same_fixture', {}, None, genvm.ManagerService), + ( + 'http-failure-cleanup', + '_http_failure_releases_token', + {}, + None, + genvm.ManagerService, + ), + ( + 'http-retry-socket-failure', + '_http_retry_preserves_socket_failure', + {}, + None, + genvm.ManagerService, + ), ( 'unix-stale-socket', '_unix_parent_and_stale_socket', @@ -1024,6 +1172,7 @@ def service( name=f'tests/system/manager-socket/{slug}', needed_services=frozenset({manager_service}), tags=frozenset({'integration', 'stable', 'feature-manager-socket'}), + console_pool=True, ) ctx.add_case( ManagerSocketCase( From 42e6693d8d810f5fb75d59508f3a544b7ce4590f Mon Sep 17 00:00:00 2001 From: Kira <kp2pml30@gmail.com> Date: Wed, 14 Oct 2026 23:51:05 +0900 Subject: [PATCH 40/45] =?UTF-8?q?feat(fees):=20fund=20descendants=20of=20b?= =?UTF-8?q?alance-funded=20messages=20=E2=9C=A8=20(#59)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(py-sdk): count extended-slice targets correctly in DynArray πŸ› * feat(fees): fund descendants of balance-funded messages ✨ * feat(fees): bump the v0.3 executor with Closed, Open and Pinned descendant grants ✨ * fix(calldata): bump the v0.3 executor to stop trusting declared sequence lengths πŸ”’οΈπŸ› * docs(spec): specify `descendants` and the `fee descendant_grant` errors πŸ“ * fix(genvm-tool): fail dependents of a service that does not start πŸ› * chore(skills): build through tee instead of a silent ninja wrapper πŸ”¨ * docs(pr): explain a red full run after /genvm-run-tests πŸ“ * fix(py-sdk): harden storage views and descendant inputs πŸ› * fix(py-sdk): accept any non-string sequence as `descendants`, reject str and bytes πŸ› * fix(py-sdk): index DynArray through `__index__` and normalize slice steps once πŸ› * fix(py-sdk): make `assign` from a view of the same DynArray or TreeMap a no-op πŸ› * fix(py-sdk): zero-initialize recycled TreeMap values inserted by default πŸ› * fix(py-sdk): reject mutating a TreeMap from its own supplier callback πŸ› * chore(executor): bump v0.3 to v0.3.0-rc10 πŸ”– * docs(runners): regenerate hashes only after linters pass πŸ“ * fix(tests): hash mock-host socket paths past the macOS sun_path limit πŸ› Paths over 100 bytes bind at /tmp/genvm-<gvm32 sha3>.sock; use_balance_descendants cases reached 105 and failed release-test on arm64-macos --------- Co-authored-by: Kanan <93033289+kriss39@users.noreply.github.com> --- .agents/skills/build/SKILL.md | 5 +- .agents/skills/build/scripts/run-ninja.sh | 12 -- docs/contributing/howto/committing/runners.md | 2 + docs/contributing/howto/pr.md | 1 + docs/website/src/impl-spec/04-fees.rst | 71 +++++++--- .../src/impl-spec/appendix/manager-api.yaml | 9 +- .../03-wasi_genlayer_sdk/02-gl_call.rst | 129 ++++++++++++++++-- docs/website/src/spec/appendix/constants.rst | 15 ++ executors/v0.3.x | 2 +- .../genvm_tool/tests/stage/execution.py | 13 +- .../unit_tests/test_service_startup.py | 51 +++++++ .../runner/genvm_tool_plugins/integration.py | 12 ++ tests/runner/origin/fees.py | 17 +++ tests/runner/origin/public_abi.py | 14 ++ 14 files changed, 303 insertions(+), 50 deletions(-) delete mode 100644 .agents/skills/build/scripts/run-ninja.sh diff --git a/.agents/skills/build/SKILL.md b/.agents/skills/build/SKILL.md index d4d813b0..b37e2184 100644 --- a/.agents/skills/build/SKILL.md +++ b/.agents/skills/build/SKILL.md @@ -10,8 +10,7 @@ read it first). Related: `building/runners.md`, `releasing/release-build.md`, Claude-specific: - Build binaries with - `bash .agents/skills/build/scripts/run-ninja.sh -C build all/bin` instead of - raw ninja β€” it is silent on success and prints output only on failure, which - saves tokens. + `bash -o pipefail -c 'ninja -C build all/bin 2>&1 | tee "$TMPDIR/ninja.log" | tail -n 40'`; + on failure grep the log for `error` instead of rebuilding to see it again - See also: `/submodules` (multi-repo commits, `?submodules=1`), `/test`, `/macos` (never build runners natively on macOS). diff --git a/.agents/skills/build/scripts/run-ninja.sh b/.agents/skills/build/scripts/run-ninja.sh deleted file mode 100644 index 5c7d935e..00000000 --- a/.agents/skills/build/scripts/run-ninja.sh +++ /dev/null @@ -1,12 +0,0 @@ -mkdir -p "$TMPDIR" -FILE_1=$(mktemp) -ninja "$@" > "$FILE_1" 2>&1 -STATUS=$? -if [ $STATUS -ne 0 ]; then - ninja "$@" 2>&1 - STATUS=$? -else - cat "$FILE_1" -fi -rm "$FILE_1" -exit $STATUS diff --git a/docs/contributing/howto/committing/runners.md b/docs/contributing/howto/committing/runners.md index 927cece0..c8d20b36 100644 --- a/docs/contributing/howto/committing/runners.md +++ b/docs/contributing/howto/committing/runners.md @@ -14,6 +14,8 @@ Before committing: 2. Set every `"test"` hash in `current.nix` to `null` 3. Stage the runner changes and run pre-commit; fix and restage them before hash discovery so the hooks and Nix inspect the same source tree + Ensure linters and formatters have passed before regenerating hashes; + any later source edits invalidate the hashes and require regeneration 4. Run `runners/support/versions/hash-updater.py`, from anywhere inside the repo. It builds the umbrella's `#runners-all` with `--keep-going`, writes every reported `got:` value back into `current.nix`, and repeats until a diff --git a/docs/contributing/howto/pr.md b/docs/contributing/howto/pr.md index 87b52e50..8953588b 100644 --- a/docs/contributing/howto/pr.md +++ b/docs/contributing/howto/pr.md @@ -78,6 +78,7 @@ does fixing a defect you find in your own diff | --- | --- | | Panel does nothing | missing `ci-safe` | | `full tests have not run` | tick *Force run full tests* or comment `/genvm-run-tests` | +| *GenVM full* still red or cancelled after `/genvm-run-tests` | expected: the command runs in its own check suite and cancels the automatic run; follow the linked run instead | | `... commit(s) behind base` | rebase, `push --force-with-lease` | | `pinned commit ... is not on the executor repo` | push it, or tick *Provision executor PRs* | | Approval revoked | re-request approval for the current head | diff --git a/docs/website/src/impl-spec/04-fees.rst b/docs/website/src/impl-spec/04-fees.rst index 627d1df6..c0724b85 100644 --- a/docs/website/src/impl-spec/04-fees.rst +++ b/docs/website/src/impl-spec/04-fees.rst @@ -68,7 +68,8 @@ Name ``attrs`` keys ``rotationsCount``, ``calldataLength``, ``codeLength``, ``subtreeLength`` ``nondet_output`` ``outputLength`` -``message_fee`` ``isInternal``, ``matchedFeeParams``: an object with +``message_fee`` ``isInternal``, ``skipBudgetFloor``, ``matchedFeeParams``: + an object with ``leaderTimeunitsAllocation``, ``validatorTimeunitsAllocation``, ``executionBudgetPerRound``, ``rotations`` (array) @@ -290,9 +291,9 @@ phase mismatch on an exact allocation does not fall through to a wildcard. Chain keys are unique across phases; only synthetic recipient-wildcard entries may select different fee parameters by phase. For external messages, an exact allocation without room for the reservation spills to -the per-recipient ``call_key`` wildcard. If neither key has a present allocation, the external message uses -the legacy unallocated path and consumes only its receipt cost. Existing but exhausted -candidates yield an allocation-budget error. +the per-recipient ``call_key`` wildcard. If no candidate matches, the external +message uses the legacy unallocated path and consumes only its receipt cost. +Existing but exhausted candidates yield an allocation-budget error. The default v0.3 fee expressions reject external reservations and receipts with ``fee below_minimum`` unless ``node.lockedReceiptGasPrice`` is positive, including @@ -310,12 +311,20 @@ unallocated path. Synthetic recipient wildcards match independently of ``budget` zero there means an exhausted allocation. ``null`` removes the per-allocation cap while keeping the execution's fee buckets. -The host must preserve every existing pinned key and must not add recipient -wildcards to a pinned tree. An empty list restricts internal -pool-funded emissions. Open-pool and view executions may supply synthetic recipient -wildcards with concrete fee parameters and phase, and optionally uncapped budgets. -The executor conservatively treats each emission as novel; remaining allowances -alone do not identify previously delivered occurrences. +The host MUST preserve every existing pinned key and MUST NOT add recipient +wildcards to a pinned tree, except for the unmatched-external guard. It MUST append +that guard to every pinned tree and send it alone for a closed tree. The guard is an +ordinary external recipient and call-key wildcard with zero budget, ``finalized`` +phase, unit gas limit, maximum gas price, zero ``children_budget`` and an empty +``subtree``. An unmatched external message therefore resolves the guard as an +exhausted candidate and fails with +:ref:`gvm-def-str-trie-detail-vm-error-out-of-message-fee-allocation-budget-external` +instead of reaching the receipt-only path that consensus rejects when +``hasAllocations`` is true. Open-pool and allocation-absent executions MUST NOT +receive this guard. They may use ordinary synthetic recipient wildcards with +concrete fee parameters and phase, and optionally uncapped budgets. The executor +conservatively treats each emission as novel; remaining allowances alone do not +identify previously delivered occurrences. Funding modes ~~~~~~~~~~~~~~ @@ -333,21 +342,47 @@ An outgoing internal message is funded one of two ways: ``EmitInternalDeployMessage`` sets ``use_balance`` (the chain's ``useBalance``, gated on :ref:`gvm-perm-use-balance-for-message-fees`), allocation matching is skipped - entirely. The fee is metered from the guest-supplied ``fee_params`` and that - metered amount is the child's ``declaredBudget``, reserved from the emitting + entirely. The primary fee is metered from the guest-supplied ``fee_params``. + A positive ``descendants`` integer emits an Open grant; a non-empty allocation + list emits a Pinned grant whose budget is the checked sum of root budgets. + The child's ``declaredBudget`` is the primary fee plus that descendant budget, + reserved once from the emitting contract's balance (jointly with ``value``; insufficient balance yields ``InsufficientBalance``). The ``message_fee`` bucket is **not** consumed (the message is excluded from the sender pool on-chain); only ``message_receipt`` is. The emitted - allocation subtree is empty, so nested child messages must each fund themselves. + allocation subtree contains the canonical grant envelope. Absent, null, zero + and empty-list descendants emit an empty subtree and the Closed policy. -For either phase, an internal message therefore declares:: +For either phase, an internal message therefore declares ``P + D``, where +``P`` is its minimum primary fee and ``D`` is its authored descendant budget:: - minPrimaryFees(feeParams) + sum(directChildAllocation.budget) + minPrimaryFees(feeParams) + descendantBudget -For balance funding the sum is zero. The primary fee already covers the child's +For balance funding the sum is the authored descendant grant budget. The +primary fee already covers the child's configured lifecycle, including appeals; the remainder becomes the child's -message-fee bucket. An internal message whose declared amount is zero is rejected -with ``fee below_minimum`` on both funding paths. External messages declare zero +message-fee bucket. A zero declared budget is rejected with +``fee below_minimum`` on both funding paths. External messages declare zero + +The v0.3 executor currently uses these local stand-ins for transaction-pinned +host inputs that are not available yet: + +- The existing local primary-fee calculation supplies each child quote +- The transaction-pinned minimum external gas limit and per-round allocation + budget floor are not available, so both grant-node admission checks are + skipped. The enclosing child's existing local floor check remains active +- The descendant depth cap equals the 24-node cap + +The Python SDK takes ``use_balance: UseBalanceParams | None`` on calls, +transfers and deploys. ``None`` selects sender funding; the dataclass holds +required ``fee_params`` and optional ``descendants``. Its allocation objects +retain ``children`` for authoring. The SDK walks that tree iteratively in +pre-order and sends a flat array with backward ``parent_index`` references + +The Rust SDK decodes that array with ``LenLimitedVec<1024, AllocationNode>`` +before decoding its elements. The Python flattener has the same 1,024-element +limit, including for cyclic input. Grant admission separately enforces the +24-node limit, parent topology, duplicate sibling keys and budget inequalities The v0.3 primary reserve includes successful-appellant profit for each configured appeal slot. The bond is the next normal round's time-unit cost, including its diff --git a/docs/website/src/impl-spec/appendix/manager-api.yaml b/docs/website/src/impl-spec/appendix/manager-api.yaml index 104c85ea..41c80841 100644 --- a/docs/website/src/impl-spec/appendix/manager-api.yaml +++ b/docs/website/src/impl-spec/appendix/manager-api.yaml @@ -1081,8 +1081,13 @@ components: unchanged and meters their length. External and open allocations without a committed subtree use empty bytes. children_budget and subtree are required on every entry, including allocations without descendants. - Omit allocations absent on-chain. Recipient wildcards are - synthetic entries for open-pool or view execution, not chain allocations + Omit allocations absent on-chain. Recipient wildcards are synthetic entries, + not chain allocations. The host MUST append an unmatched-external guard to + every pinned tree and send it alone for a closed tree: recipient and call_key + null, budget zero, on finalized, External gas_limit one and max_gas_price the + maximum uint256, children_budget zero and an empty subtree. It is an ordinary + exhausted candidate. Open-pool and allocation-absent executions MUST omit the + guard; open-pool and view execution may use ordinary wildcards record_actions: type: array items: diff --git a/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/02-gl_call.rst b/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/02-gl_call.rst index 15dee7ae..d6930da2 100644 --- a/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/02-gl_call.rst +++ b/docs/website/src/spec/02-execution-environment/03-wasi_genlayer_sdk/02-gl_call.rst @@ -26,6 +26,29 @@ Requirements #. :ref:`gvm-perm-send-messages` #. Sufficient contract balance for value transfer +Semantics +~~~~~~~~~ + +External messages are emitted for finalization. GenVM derives their call key from +the first four calldata bytes, padded with zeros to 32 bytes; calldata shorter than +four bytes uses the all-zero key + +GenVM orders matching candidates from the non-zero exact ``(address, call key)`` +allocation through the same address's call-key wildcard to host-provided recipient +wildcards. A zero-budget exact allocation is absent and therefore permits broader +candidates. GenVM selects the first candidate that can fund the reservation. If +candidates exist but none can fund it, the :term:`sub-VM` exits with +:ref:`gvm-def-str-trie-detail-vm-error-out-of-message-fee-allocation-budget-external` + +If no candidate matches, the message uses the legacy unallocated path and consumes +only its receipt cost. Hosts MUST append an exhausted external recipient and +call-key wildcard to every pinned allocation tree, and supply that wildcard alone +for a closed tree. It has zero budget, ``finalized`` phase, unit gas limit, maximum +gas price, zero child budget and an empty subtree. This ordinary exhausted candidate +makes an otherwise unmatched external message exit with the allocation-budget error +above, as required when consensus marks the execution as having allocations. Hosts +MUST NOT supply this guard for open-pool or allocation-absent executions + ``ExternalCall`` Message ------------------------ @@ -122,7 +145,8 @@ Payload "value": U256, // Wei to transfer "on": String, // "finalized" or "decided" "use_balance": Bool, // optional (default false), see below - "fee_params": FeeParams // optional (default absent), required iff use_balance + "fee_params": FeeParams, // optional (default absent), required iff use_balance + "descendants": Descendants // optional (default absent), see below } } @@ -133,6 +157,7 @@ Requirements #. :ref:`gvm-perm-send-messages` #. Sufficient contract balance for value transfer #. When ``use_balance`` is set: :ref:`gvm-perm-use-balance-for-message-fees` and ``fee_params`` +#. ``descendants`` is absent unless ``use_balance`` is set #. ``calldata`` satisfies :ref:`gvm-def-contract-call-conv` Allocation-funded fees @@ -175,26 +200,90 @@ transaction's fee configuration and mirrors the chain's "receipt_fee_max_gas_price": U256 // receipt price cap (revert guard) } +``descendants`` configures the emitted child's descendant funding policy: + +.. code-block:: + + Descendants = Null | U256 | [Allocation] + InternalAllocation { + "parent_index": U256, + "recipient": Address, + "call_key": Bytes32, + "budget": U256, + "fee_params": FeeParams, + "on": String + } + ExternalAllocation { + "parent_index": U256, + "recipient": Address, + "call_key": Bytes32, + "budget": U256, + "fee_params": { + "gas_limit": U256, + "max_gas_price": U256 + } + } + +Absent, null, zero and an empty list select Closed and emit no grant bytes. A +positive integer selects Open with that descendant budget. A non-empty list +selects Pinned; its descendant budget is the checked sum of root budgets. +The list is flat and preserves authored order. A root's ``parent_index`` is +the maximum U256 value; other indices must refer to an earlier Internal +allocation. ``children`` is not a wire field. ``"decided"`` maps to acceptance-phase +authorization and ``"finalized"`` to finalization-phase authorization + +These Closed spellings apply to balance-funded messages. On sender-funded +messages, the field must be absent or null; a raw present zero or empty list is +rejected with ``Inval`` + +A Pinned grant obeys all of the following: + +#. Every node has a non-zero ``budget`` +#. The tree contains at most 24 nodes and has depth at most 24. An internal + node's encoded fee parameters are at most 1,024 bytes, which permits at most + 22 ``rotations`` entries +#. External allocations occur only at the root. Their ``gas_limit`` and + ``max_gas_price`` are non-zero, and their budget is a positive multiple of + ``gas_limit * max_gas_price``. They authorize only finalization-phase + emission +#. Siblings have unique ``(message type, recipient, call_key)`` keys. The + ``on`` phase is not part of the key; the same key under different parents is + permitted +#. For each internal node *i*, let *P_i* be its primary fee quote and *C_i* the + sum of its direct children's budgets. Its budget is at least + ``(P_i + C_i) * mult_i``. ``mult_i`` is the parent allocation's appeal-round + count plus 1 when ``on`` is ``"decided"``, and 1 otherwise. For a root, the + emitted child's appeal-round count is used + Semantics: -- The fee is metered from ``fee_params`` and that metered amount becomes the child - transaction's ``declaredBudget`` β€” the contract balance is the only bound. The +- The fee is metered from ``fee_params``. The child transaction's + ``declaredBudget`` is that primary reserve plus the descendant budget; the + contract balance is the only bound. The consensus term is charged at the guest's ``max_price_gen_per_time_unit`` cap (matching the chain's ``minMessagePrimaryFees``), not the node's live ``genPerTimeUnit``, so the fee scales with the cap. The consensus developer and DAO gross-up applies to this time-unit portion but not the execution budget - The message is excluded from allocation matching, so no matching node is required (and none is consulted). -- The contract must be able to cover ``value + metered_fee`` from its balance; +- The contract must be able to cover ``value + declaredBudget`` from its balance; otherwise the call fails with ``InsufficientBalance``. -- The emitted allocation subtree is **empty**: nesting is fail-closed, so a child - message must itself set ``use_balance`` or it fails to fund. +- Closed is fail-closed. Open and Pinned carry a canonical grant in the emitted + allocation subtree +- A zero ``declaredBudget`` is rejected with + :ref:`gvm-def-str-trie-value-vm-error-fee-below-minimum`; this applies exactly + when the primary reserve plus descendant budget is zero ``fee_params`` is validated before metering. The following are rejected with ``Inval``: - ``use_balance`` without ``fee_params``, or ``fee_params`` without ``use_balance``. +- Non-null ``descendants`` without ``use_balance``, malformed allocation shapes, and + internal allocation fee parameters that fail the checks below - Empty ``rotations`` (``appealRounds`` would underflow). +- Exactly one of ``leader_time_units_allocation`` and + ``validator_time_units_allocation`` being zero. Both zero and both non-zero + are valid shapes; later metering still enforces configured timeout bounds - A zero ``max_price_gen_per_time_unit``, ``storage_fee_max_gas_price`` or ``receipt_fee_max_gas_price`` (the chain reverts ``FeeValueMustBeNonZero`` at reveal). @@ -205,14 +294,26 @@ Semantics: ``validator_time_units_allocation``, each ``rotations`` entry) below 2\ :sup:`32`. These bounds keep the metered floor within ``U256``. +Pinned node/depth limits, external-child placement and duplicate sibling keys use +:ref:`gvm-def-str-trie-value-vm-error-fee-descendant-grant-tree`. Zero node +budgets, arithmetic overflow and a failure of the internal-node budget +inequality use +:ref:`gvm-def-str-trie-value-vm-error-fee-descendant-grant-budget`. Invalid +external allocation parameters use +:ref:`gvm-def-str-trie-value-vm-error-fee-descendant-grant-external`. +Validation completes before fee consumption, balance reservation or emission + Metering additionally enforces node-configured bounds, surfaced as ``VMError``\ s: -- :ref:`gvm-def-str-trie-value-vm-error-fee-below-minimum` β€” either a non-zero - ``execution_budget_per_round`` below ``node.messageBudgetFloor`` (the chain's - ``BudgetTooLow``), or β€” unless both time-unit allocations are zero β€” a leader - allocation outside ``node.minProposeTimeout`` through +- :ref:`gvm-def-str-trie-value-vm-error-fee-below-minimum` β€” for the emitted + child's own fee parameters, either a non-zero ``execution_budget_per_round`` + below ``node.messageBudgetFloor`` (the chain's ``BudgetTooLow``), or β€” unless + both time-unit allocations are zero β€” a leader allocation outside + ``node.minProposeTimeout`` through ``node.maxProposeTimeout`` or a validator allocation outside - ``node.minCommitTimeout`` through ``node.maxCommitTimeout``. + ``node.minCommitTimeout`` through ``node.maxCommitTimeout``. Internal grant + nodes use the same local primary quote but skip the unavailable + transaction-pinned budget-floor admission check - :ref:`gvm-def-str-trie-value-vm-error-fee-too-many-rounds` β€” ``rotations`` implies more consensus rounds than the node's validator table supports (on-chain ``MAX_ROUNDS``). @@ -235,7 +336,8 @@ Payload "on": String, // "finalized" or "decided" "salt_nonce": U256, // Salt for CREATE2-style deterministic addressing "use_balance": Bool, // optional (default false) - "fee_params": FeeParams // optional (default absent), required iff use_balance + "fee_params": FeeParams, // optional (default absent), required iff use_balance + "descendants": Descendants // optional (default absent), see balance-funded fees } } @@ -249,7 +351,8 @@ Requirements #. ``calldata`` satisfies :ref:`gvm-def-contract-call-conv` Supports CREATE2-style deployment with salt nonce for deterministic addressing. -``use_balance`` / ``fee_params`` behave as for :ref:`gvm-gl-call-balance-fees`. +``use_balance`` / ``fee_params`` / ``descendants`` behave as for +:ref:`gvm-gl-call-balance-fees`. .. _gvm-def-gl-call-run-nondet: diff --git a/docs/website/src/spec/appendix/constants.rst b/docs/website/src/spec/appendix/constants.rst index 0e7ad72d..e4b4da78 100644 --- a/docs/website/src/spec/appendix/constants.rst +++ b/docs/website/src/spec/appendix/constants.rst @@ -389,6 +389,21 @@ Param: i32 ``fee too_many_rounds`` ~~~~~~~~~~~~~~~~~~~~~~~ +.. _gvm-def-str-trie-value-vm-error-fee-descendant-grant-budget: + +``fee descendant_grant budget`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. _gvm-def-str-trie-value-vm-error-fee-descendant-grant-tree: + +``fee descendant_grant tree`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. _gvm-def-str-trie-value-vm-error-fee-descendant-grant-external: + +``fee descendant_grant external`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + .. _gvm-def-str-trie-value-vm-error-forbidden: ``forbidden`` diff --git a/executors/v0.3.x b/executors/v0.3.x index 49aa5746..c8116fba 160000 --- a/executors/v0.3.x +++ b/executors/v0.3.x @@ -1 +1 @@ -Subproject commit 49aa5746603f9a1b41a0144a87698de62a28282a +Subproject commit c8116fba35243cac378ec9eca3be20c6ebfa88c0 diff --git a/support/tools/genvm-tool/genvm_tool/tests/stage/execution.py b/support/tools/genvm-tool/genvm_tool/tests/stage/execution.py index 5b0d3989..6e024788 100644 --- a/support/tools/genvm-tool/genvm_tool/tests/stage/execution.py +++ b/support/tools/genvm-tool/genvm_tool/tests/stage/execution.py @@ -110,6 +110,7 @@ class _ExecutionContext: success_count: int = 0 skipped: int = 0 running_services: dict[str, Handle] = field(default_factory=dict) + failed_services: dict[str, str] = field(default_factory=dict) # Per-test completion tracking for test-to-test dependencies test_completed: dict[str, asyncio.Event] = field(default_factory=dict) test_passed: dict[str, bool] = field(default_factory=dict) @@ -167,6 +168,11 @@ async def _start_service(ctx: _ExecutionContext, service: Service) -> None: """Start a service and track its handle.""" ctx.shared.logger.info('Starting service', service_name=service.name) try: + for dependency in service.depends_on or []: + if dependency.name in ctx.failed_services: + raise RuntimeError( + f'Service {dependency.name} failed: {ctx.failed_services[dependency.name]}' + ) async with asyncio.timeout(SPAWN_TIMEOUT): handle = await service.manager.start() await handle.await_startup() @@ -174,12 +180,12 @@ async def _start_service(ctx: _ExecutionContext, service: Service) -> None: ctx.running_services[service.name] = handle ctx.shared.logger.info('Service started', service_name=service.name) except Exception as e: + ctx.failed_services[service.name] = str(e) ctx.shared.logger.error( 'Failed to start service', service_name=service.name, error=e, ) - raise async def _stop_service(ctx: _ExecutionContext, service: Service) -> None: @@ -346,6 +352,11 @@ async def _run_case_locked(ctx: _ExecutionContext, case: genvm_tool.tests.test.C ) try: + for service in case.description.needed_services: + if service.name in ctx.failed_services: + raise RuntimeError( + f'Service {service.name} failed: {ctx.failed_services[service.name]}' + ) run_shared.logger.debug( 'Running test case', case_name=case.description.name, diff --git a/support/tools/genvm-tool/unit_tests/test_service_startup.py b/support/tools/genvm-tool/unit_tests/test_service_startup.py index 4d999b09..b3602d33 100644 --- a/support/tools/genvm-tool/unit_tests/test_service_startup.py +++ b/support/tools/genvm-tool/unit_tests/test_service_startup.py @@ -1,9 +1,13 @@ """A service that never comes up must not wedge the run.""" import asyncio +from types import SimpleNamespace +from unittest.mock import AsyncMock, Mock import pytest +from genvm_tool.tests import SharedContext, test from genvm_tool.tests.exec import service +from genvm_tool.tests.stage import collection, execution, scheduling class _Handle(service.Handle): @@ -77,3 +81,50 @@ async def healthy(self) -> bool: with pytest.raises(RuntimeError, match='not healthy within'): asyncio.run(asyncio.wait_for(handle.await_startup(), timeout=5)) assert handle.interrupted + + +@pytest.mark.parametrize('failure', ['spawn', 'dead', 'timeout']) +def test_startup_failure_fails_dependents_and_continues(tmp_path, failure): + handle = _Handle(reason='exited with code 1' if failure == 'dead' else None) + start = AsyncMock(return_value=handle) + if failure == 'spawn': + start.side_effect = RuntimeError('could not spawn') + failed = collection.Service('failed', service.FunctionService(start)) + dependent_start = AsyncMock() + dependent = collection.Service( + 'dependent', service.FunctionService(dependent_start), depends_on=[failed] + ) + healthy = _Handle(healthy_after=1) + unrelated = collection.Service( + 'unrelated', service.FunctionService(AsyncMock(return_value=healthy)) + ) + cases = [] + for name, needed in [ + ('direct-1', failed), + ('direct-2', failed), + ('transitive', dependent), + ('unrelated', unrelated), + ]: + case = test.StepsCase( + test.Description(name, needed_services=frozenset({needed})), + [test.CONST_PASSED], + ) + case.into_steps = AsyncMock(wraps=case.into_steps) + cases.append(case) + shared = SharedContext(tmp_path, logger=Mock(), printer=Mock(), watchdog=Mock()) + plan = scheduling.run( + shared, collection.Env(cases=cases, args=SimpleNamespace(max_concurrent=2)) + ) + result = asyncio.run(asyncio.wait_for(execution.run(shared, plan), timeout=5)) + assert set(result.failed) == {'direct-1', 'direct-2', 'transitive'} + assert result.success_count == 1 + assert len(result.results) == 4 + for record in result.results: + if record.name != 'unrelated': + assert not record.passed + assert 'Service failed failed:' in record.failure_message + for case in cases[:3]: + case.into_steps.assert_not_called() + dependent_start.assert_not_called() + assert healthy.interrupted + assert all(s.handle is None for s in [failed, dependent, unrelated]) diff --git a/tests/runner/genvm_tool_plugins/integration.py b/tests/runner/genvm_tool_plugins/integration.py index 89d26085..7551a5a8 100644 --- a/tests/runner/genvm_tool_plugins/integration.py +++ b/tests/runner/genvm_tool_plugins/integration.py @@ -88,6 +88,12 @@ def _normalize_message_fee_allocation( allocation: fees.MessageAllocationNode, ) -> fees.MessageAllocationNode: allocation = allocation.copy() + raw_allocation = typing.cast(dict[str, typing.Any], allocation) + external_params = raw_allocation['fee_params'].get('External') + if external_params is not None and isinstance(external_params['max_gas_price'], str): + external_params = external_params.copy() + external_params['max_gas_price'] = int(external_params['max_gas_price']) + raw_allocation['fee_params'] = {'External': external_params} if isinstance(allocation['recipient'], str): allocation['recipient'] = Address(allocation['recipient']) if isinstance(allocation['subtree'], list): @@ -657,6 +663,12 @@ async def _run_single_step( # Set up paths rel_path = jsonnet_path.relative_to(self._test_case.cases_dir) mock_sock_path = Path('/tmp', 'genvm-test', rel_path.with_suffix(f'.sock{suff}')) + # sun_path is 104 bytes on macOS, 108 on Linux + if len(os.fsencode(mock_sock_path)) > 100: + path_hash = genvm_tool.gvm32.encode( + hashlib.sha3_256(os.fsencode(mock_sock_path)).digest() + ) + mock_sock_path = Path('/tmp', f'genvm-{path_hash}.sock') mock_sock_path.parent.mkdir(exist_ok=True, parents=True) is_leader = single_conf.get('mode') == 'l' diff --git a/tests/runner/origin/fees.py b/tests/runner/origin/fees.py index 3345fdc4..01ddbc0a 100644 --- a/tests/runner/origin/fees.py +++ b/tests/runner/origin/fees.py @@ -51,6 +51,23 @@ class MessageAllocationNode(typing.TypedDict): subtree: bytes +# Hosts append this to pinned trees and send it alone for closed trees. +UNMATCHED_EXTERNAL_GUARD_ALLOC: MessageAllocationNode = { + 'budget': 0, + 'recipient': None, + 'call_key': None, + 'on': 'finalized', + 'fee_params': { + 'External': { + 'gas_limit': 1, + 'max_gas_price': 2**256 - 1, + }, + }, + 'children_budget': 0, + 'subtree': b'', +} + + DEFAULT_EXTERNAL_MESSAGE_ALLOC: MessageAllocationNode = { 'budget': 2**200, 'recipient': None, diff --git a/tests/runner/origin/public_abi.py b/tests/runner/origin/public_abi.py index 41d7cf49..96a4cb31 100644 --- a/tests/runner/origin/public_abi.py +++ b/tests/runner/origin/public_abi.py @@ -224,6 +224,17 @@ def internal() -> 'VmError': def external() -> 'VmError': return VmError('fee no_matching_allocation # external') +class _VmErrorFeeDescendantGrant: + @staticmethod + def budget() -> 'VmError': + return VmError('fee descendant_grant budget') + @staticmethod + def tree() -> 'VmError': + return VmError('fee descendant_grant tree') + @staticmethod + def external() -> 'VmError': + return VmError('fee descendant_grant external') + class _VmErrorFee: @staticmethod def below_minimum() -> 'VmError': @@ -234,6 +245,9 @@ def too_many_rounds() -> 'VmError': @staticmethod def no_matching_allocation() -> '_VmErrorFeeNoMatchingAllocation': return _VmErrorFeeNoMatchingAllocation() + @staticmethod + def descendant_grant() -> '_VmErrorFeeDescendantGrant': + return _VmErrorFeeDescendantGrant() class _VmErrorEvm: @staticmethod From 223c53866bcdbf36d359b08b213502d21f67f304 Mon Sep 17 00:00:00 2001 From: Kira <kp2pml30@gmail.com> Date: Thu, 15 Oct 2026 01:54:24 +0900 Subject: [PATCH 41/45] =?UTF-8?q?chore(ci):=20publish=20docs=20for=20relea?= =?UTF-8?q?ses=20and=20branch=20updates=20=F0=9F=9A=80=20(#69)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/docs_nightly.yaml | 38 ++----- .github/workflows/release.yaml | 14 +++ docs/contributing/howto/building/docs.md | 32 +++++- support/ci/tools/deploy_docs.py | 48 +++++++- support/ci/unit_tests/test_deploy_docs.py | 129 ++++++++++++++++++++++ 5 files changed, 227 insertions(+), 34 deletions(-) create mode 100644 support/ci/unit_tests/test_deploy_docs.py diff --git a/.github/workflows/docs_nightly.yaml b/.github/workflows/docs_nightly.yaml index 881fc577..e37b06c4 100644 --- a/.github/workflows/docs_nightly.yaml +++ b/.github/workflows/docs_nightly.yaml @@ -1,10 +1,10 @@ -name: docs / nightly deploy +name: docs / deploy # Builds the docs site and publishes it to the sdk.genlayer.com Pages repo, # which serves one directory per version out of `_site/` and lists them in # `_site/versions.json` (the switcher every page links to). Nightly runs -# refresh `main`; `workflow_dispatch` can publish any other directory, which is -# how a release train (`v0.6.x`) gets its own copy. +# refresh `main`; branch pushes refresh their train, and releases dispatch +# this workflow at the exact release tag. # # GitHub only schedules workflows from the DEFAULT branch, so this file has to # reach it before the cron fires β€” merging it into a dev branch is not enough. @@ -14,6 +14,8 @@ name: docs / nightly deploy # deploy-pages workflow, which is what actually publishes and purges the CDN. on: + push: + branches: [v0.6, v0.6-dev] schedule: # 03:17 UTC β€” off the hour, where scheduling backlogs are smaller. - cron: "17 3 * * *" @@ -29,19 +31,16 @@ on: required: false default: false -# Two runs pushing to the site repo would race on the same branch; the loser -# would just fail, so serialize rather than cancel (a half-finished deploy is -# worse than a slow one). concurrency: - group: docs-deploy - cancel-in-progress: false + group: docs-deploy-${{ inputs.version || (github.event_name == 'push' && (github.ref_name == 'v0.6' && 'v0.6.x' || github.ref_name)) || 'main' }} + cancel-in-progress: true permissions: contents: read env: SITE_REPO: genlayerlabs/sdk.genlayer.com - DOCS_VERSION: ${{ inputs.version || 'main' }} + DOCS_VERSION: ${{ inputs.version || (github.event_name == 'push' && (github.ref_name == 'v0.6' && 'v0.6.x' || github.ref_name)) || 'main' }} # GitHub's stock `bash -e {0}` has no `pipefail`. defaults: @@ -54,6 +53,8 @@ jobs: timeout-minutes: 120 steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + ref: ${{ github.event_name == 'schedule' && 'main' || github.sha }} # The docs build spans the manager and every active executor line, each # in its own `gen-docs` nix shell, so it needs submodules and nix. @@ -78,25 +79,10 @@ jobs: path: build/site ssh-key: ${{ secrets.DOCS_DEPLOY_KEY }} - - name: Stage the built docs + - name: Publish the built docs run: | ./support/ci/run.sh tool deploy-docs \ --site build/site \ --version "$DOCS_VERSION" \ + --push \ ${{ inputs.preferred && '--preferred' || '' }} - - # An unchanged nightly must not push an empty commit: the site repo's - # deploy workflow runs on every push, so that would redeploy and purge the - # CDN for nothing. - - name: Commit and push - working-directory: build/site - run: | - git config user.email "ci@genlayerlabs.com" - git config user.name "CI worker" - git add -A _site - if git diff --cached --quiet; then - echo "docs are unchanged; nothing to deploy" - exit 0 - fi - git commit -m "Deploy documentation version $DOCS_VERSION" - git push origin HEAD diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 57b296d0..fbbf6ecd 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -153,3 +153,17 @@ jobs: body_path: artifacts/notes/${{ needs.build-and-test.outputs.notes_file }} draft: false prerelease: ${{ contains(needs.build-and-test.outputs.tag, '-') }} + + docs-dispatch: + needs: [build-and-test, release-publish] + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + actions: write + steps: + - name: Dispatch release docs + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + TAG: ${{ needs.build-and-test.outputs.tag }} + run: gh workflow run docs_nightly.yaml --ref "$TAG" -f version="$TAG" diff --git a/docs/contributing/howto/building/docs.md b/docs/contributing/howto/building/docs.md index d7a48f63..6289a59f 100644 --- a/docs/contributing/howto/building/docs.md +++ b/docs/contributing/howto/building/docs.md @@ -41,10 +41,30 @@ The site is `sdk.genlayer.com` β€” GitHub Pages out of `_site/versions.json` driving the version switcher; pushing that repo deploys it -`.github/workflows/docs_nightly.yaml` republishes `main` nightly and takes a -version/preferred pair on `workflow_dispatch` for a release train. It needs the -`DOCS_DEPLOY_KEY` secret, and β€” since GitHub schedules only from the default -branch β€” the workflow file must reach that branch before the cron can fire +`.github/workflows/docs_nightly.yaml` publishes these versions: + +| Trigger | Source | Site Directory | +|---|---|---| +| Successful release publication | Exact release tag | Exact release tag | +| Push to `v0.6` | Pushed commit | `v0.6.x` | +| Push to `v0.6-dev` | Pushed commit | `v0.6-dev` | +| Nightly schedule | `main` | `main` | +| Manual dispatch | Selected ref | `version` input (default `main`) | + +The release workflow dispatches docs only after its final publishing job +succeeds, using `actions: write` on that dispatch job. A docs failure can be +retried by dispatching the docs workflow at the release tag with the same +`version`, without republishing the release + +Concurrency is per target version and cancels the previous run for that version +Different versions may publish together: `deploy-docs --push` retries a rejected +push up to 5 attempts, each time reapplying the built tree and version entry to +the latest site `main`. This preserves concurrent changes to other versions and +`versions.json`; unchanged output produces no commit + +The workflow needs `DOCS_DEPLOY_KEY` with write access to the site repo +Schedules and manual dispatch require the workflow on the repository's default +branch; push triggers require it on the pushed branch By hand, into a checkout of the site repo: @@ -55,7 +75,9 @@ By hand, into a checkout of the site repo: The version directory is replaced wholesale, so deleted pages do not linger; `_site/versions.json` keeps its curated order, a missing version is inserted -after `main`. Committing and pushing the site repo is left to you +after `main`. Committing and pushing the site repo is left to you unless you +pass `--push`, which requires a clean, disposable checkout and resets it to the +site repo's `origin/main` on each attempt ## ADRs diff --git a/support/ci/tools/deploy_docs.py b/support/ci/tools/deploy_docs.py index 2a78e5a5..a8f25e07 100644 --- a/support/ci/tools/deploy_docs.py +++ b/support/ci/tools/deploy_docs.py @@ -5,8 +5,7 @@ That repo publishes `_site/` to GitHub Pages, one directory per docs version (`_site/main`, `_site/v0.3.x`, ...) plus a root `_site/versions.json` that feeds the version switcher every built page links to. This tool only rewrites those -files; committing and pushing is left to the caller, so a nightly run and a -local `--site ../sdk.genlayer.com` do exactly the same thing. +files by default; `--push` publishes from a disposable, clean site checkout. The version directory is replaced wholesale rather than merged, so pages deleted upstream do not linger. @@ -21,6 +20,8 @@ import pathlib import re import shutil +import subprocess +import time import ci_lib @@ -100,6 +101,11 @@ def add_to(self, parser: argparse.ArgumentParser) -> None: default=str(ci_lib.ROOT_DIR / 'build' / 'doc' / 'html'), help='built docs tree to publish (default: build/doc/html)', ) + parser.add_argument( + '--push', + action='store_true', + help='commit and push to origin/main, retrying races (disposable clean checkout only)', + ) parser.add_argument( '--preferred', action='store_true', @@ -128,6 +134,12 @@ def handler(self, args: argparse.Namespace) -> int: print(f'{site} does not exist; is --site an sdk.genlayer.com checkout?') return 1 + if args.push: + return self.publish(args, html, site) + self.stage(args, html, site) + return 0 + + def stage(self, args, html, site): target = site / args.version if target.exists(): shutil.rmtree(target) @@ -136,7 +148,37 @@ def handler(self, args: argparse.Namespace) -> int: print(f'published {html} -> {target}') update_versions(site / VERSIONS_JSON, args.version, args.preferred) - return 0 + + def publish(self, args, html, site): + def git(*command, **kwargs): + return subprocess.run(['git', *command], cwd=site.parent, check=True, **kwargs) + + if git('status', '--porcelain', capture_output=True, text=True).stdout: + print('--push requires a clean, disposable site checkout') + return 1 + git('config', 'user.email', 'ci@genlayerlabs.com') + git('config', 'user.name', 'CI worker') + for attempt in range(5): + git('fetch', 'origin', 'refs/heads/main') + git('reset', '--hard', 'FETCH_HEAD') + # Reapply to the latest tree so concurrent versions.json edits survive. + self.stage(args, html, site) + git('add', '-A', '_site') + if not git( + 'diff', '--cached', '--name-only', capture_output=True, text=True + ).stdout: + print('docs are unchanged; nothing to deploy') + return 0 + git('commit', '-m', f'Deploy documentation version {args.version}') + try: + git('push', 'origin', 'HEAD:refs/heads/main') + return 0 + except subprocess.CalledProcessError: + if attempt == 4: + print('docs push failed after 5 attempts') + return 1 + print(f'docs push failed; retrying publication (attempt {attempt + 2}/5)') + time.sleep(2**attempt) COMMANDS = [DeployDocs()] diff --git a/support/ci/unit_tests/test_deploy_docs.py b/support/ci/unit_tests/test_deploy_docs.py new file mode 100644 index 00000000..9b9e54bc --- /dev/null +++ b/support/ci/unit_tests/test_deploy_docs.py @@ -0,0 +1,129 @@ +import argparse +import json +import subprocess + +import pytest + +from tools import deploy_docs as deploy + + +@pytest.fixture +def publication(tmp_path, monkeypatch): + run = subprocess.run + + def git(path, *args): + return run( + ['git', '-C', str(path), *args], check=True, capture_output=True, text=True + ).stdout.strip() + + remote = tmp_path / 'remote.git' + run(['git', 'init', '--bare', '--initial-branch=main', str(remote)], check=True) + site = tmp_path / 'site' + run(['git', 'clone', str(remote), str(site)], check=True) + git(site, 'config', 'user.email', 'test@example.com') + git(site, 'config', 'user.name', 'Test') + (site / '_site' / 'v0.6.x').mkdir(parents=True) + (site / '_site' / 'v0.6.x' / 'obsolete.html').write_text('old') + (site / '_site' / 'versions.json').write_text( + json.dumps([deploy.entry_for('main', True)]) + ) + git(site, 'add', '.') + git(site, 'commit', '-m', 'Initial site') + git(site, 'push', 'origin', 'main') + html = tmp_path / 'html' + html.mkdir() + (html / 'index.html').write_text('new docs') + (html / '.doctrees').mkdir() + (html / '.doctrees' / 'cache').write_text('cache') + parser = argparse.ArgumentParser() + tool = deploy.DeployDocs() + tool.add_to(parser) + args = parser.parse_args( + ['--site', str(site), '--html', str(html), '--version', 'v0.6.x', '--push'] + ) + monkeypatch.setattr(deploy.time, 'sleep', lambda _: None) + return tool, args, git, remote, site, run + + +def test_publish_replaces_tree_and_skips_unchanged(publication): + tool, args, git, remote, site, _ = publication + assert tool.handler(args) == 0 + head = git(remote, 'rev-parse', 'main') + assert (site / '_site' / args.version / 'index.html').read_text() == 'new docs' + assert not (site / '_site' / args.version / 'obsolete.html').exists() + assert not (site / '_site' / args.version / '.doctrees').exists() + assert tool.handler(args) == 0 + assert git(remote, 'rev-parse', 'main') == head + + +@pytest.mark.parametrize('preferred', [False, True]) +def test_publish_retries_concurrent_version_update( + publication, monkeypatch, tmp_path, preferred +): + tool, args, git, remote, site, run = publication + args.preferred = preferred + competitor = tmp_path / 'competitor' + run(['git', 'clone', str(remote), str(competitor)], check=True) + git(competitor, 'config', 'user.email', 'other@example.com') + git(competitor, 'config', 'user.name', 'Other') + pushes = 0 + + def race(command, **kwargs): + nonlocal pushes + if command[:2] == ['git', 'push']: + pushes += 1 + if pushes == 1: + (competitor / '_site' / 'v0.6-dev').mkdir() + (competitor / '_site' / 'v0.6-dev' / 'index.html').write_text('concurrent docs') + deploy.update_versions(competitor / '_site' / 'versions.json', 'v0.6-dev', True) + git(competitor, 'add', '.') + git(competitor, 'commit', '-m', 'Concurrent version') + git(competitor, 'push', 'origin', 'main') + return run(command, **kwargs) + + monkeypatch.setattr(deploy.subprocess, 'run', race) + assert tool.handler(args) == 0 + assert pushes == 2 + assert git(remote, 'show', 'main:_site/v0.6-dev/index.html') == 'concurrent docs' + assert git(remote, 'show', 'main:_site/v0.6.x/index.html') == 'new docs' + versions = json.loads(git(remote, 'show', 'main:_site/versions.json')) + assert {entry['version'] for entry in versions} == {'main', 'v0.6-dev', 'v0.6.x'} + assert [entry['version'] for entry in versions if entry['preferred']] == [ + 'v0.6.x' if preferred else 'v0.6-dev' + ] + + +def test_publish_stops_after_repeated_rejection(publication, monkeypatch): + tool, args, git, remote, _, run = publication + head = git(remote, 'rev-parse', 'main') + pushes = 0 + + def reject(command, **kwargs): + nonlocal pushes + if command[:2] == ['git', 'push']: + pushes += 1 + raise subprocess.CalledProcessError(1, command) + return run(command, **kwargs) + + monkeypatch.setattr(deploy.subprocess, 'run', reject) + assert tool.handler(args) == 1 + assert pushes == 5 + assert git(remote, 'rev-parse', 'main') == head + + +def test_publish_refuses_dirty_checkout(publication): + tool, args, git, remote, site, _ = publication + head = git(remote, 'rev-parse', 'main') + (site / 'keep.txt').write_text('local work') + assert tool.handler(args) == 1 + assert (site / 'keep.txt').read_text() == 'local work' + assert git(remote, 'rev-parse', 'main') == head + + +def test_stage_does_not_push(publication): + tool, args, git, remote, site, _ = publication + head = git(remote, 'rev-parse', 'main') + args.push = False + assert tool.handler(args) == 0 + assert (site / '_site' / args.version / 'index.html').exists() + assert git(remote, 'rev-parse', 'main') == head From 9005003db96b03b29621046a4477dc8bcd599cc8 Mon Sep 17 00:00:00 2001 From: Kira <kp2pml30@gmail.com> Date: Fri, 16 Oct 2026 19:37:55 +0900 Subject: [PATCH 42/45] =?UTF-8?q?feat(fees):=20carve=20remaining=20funds?= =?UTF-8?q?=20and=20validate=20grants=20=E2=9C=A8=20(#68)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crates/modules-interfaces/src/domain.rs | 4 +++ .../modules-interfaces/src/domain/fees/mod.rs | 22 +++++++++++++ docs/website/src/impl-spec/04-fees.rst | 32 ++++++++++++++----- .../src/impl-spec/appendix/manager-api.yaml | 29 ++++++++++++++++- .../src/spec/appendix/internal-constants.rst | 7 ++++ executors/v0.3.x | 2 +- implementation/src/manager/run.rs | 11 +++++++ implementation/src/manager/run_test.rs | 2 ++ .../runner/genvm_tool_plugins/integration.py | 3 ++ 9 files changed, 102 insertions(+), 10 deletions(-) diff --git a/crates/modules-interfaces/src/domain.rs b/crates/modules-interfaces/src/domain.rs index 9b58650e..4a679c09 100644 --- a/crates/modules-interfaces/src/domain.rs +++ b/crates/modules-interfaces/src/domain.rs @@ -47,6 +47,10 @@ pub struct ExecutionData { pub gas_data: std::collections::BTreeMap<String, String>, /// Flat allocations with host-encoded subtree payloads. pub message_fee_allocation: Vec<fees::MessageAllocationNode>, + /// Present only for a top-level open-bucket execution + #[serde(default)] + #[calldata(default = default_none)] + pub message_fee_cascade: Option<fees::MessageFeeCascade>, /// Initial time-unit budget for this execution. pub initial_time_units_allocation: u32, /// Auditable supervisor action kinds to return in the execution result. diff --git a/crates/modules-interfaces/src/domain/fees/mod.rs b/crates/modules-interfaces/src/domain/fees/mod.rs index 9a97fe3e..2a97a4cc 100644 --- a/crates/modules-interfaces/src/domain/fees/mod.rs +++ b/crates/modules-interfaces/src/domain/fees/mod.rs @@ -116,3 +116,25 @@ impl MessageAllocationNode { .then_some(*params) } } + +/// Open-bucket cascade: sender-pool internal messages are raised to +/// `min_primary_fee`, then share half the remaining budget after fees and external reserves +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + genlayer_calldata::Encode, + genlayer_calldata::Decode, +)] +pub struct MessageFeeCascade { + /// Chain `minMessagePrimaryFeesForTx` of the open-bucket internal fee params + pub min_primary_fee: U256, + /// Chain `getMessageFeesBudgetRemaining` at the processing block + pub budget_remaining: U256, + /// Full gas envelope times locked receipt price, per external message + pub external_fee_reserve: U256, +} diff --git a/docs/website/src/impl-spec/04-fees.rst b/docs/website/src/impl-spec/04-fees.rst index c0724b85..d4245ff9 100644 --- a/docs/website/src/impl-spec/04-fees.rst +++ b/docs/website/src/impl-spec/04-fees.rst @@ -364,14 +364,8 @@ configured lifecycle, including appeals; the remainder becomes the child's message-fee bucket. A zero declared budget is rejected with ``fee below_minimum`` on both funding paths. External messages declare zero -The v0.3 executor currently uses these local stand-ins for transaction-pinned -host inputs that are not available yet: - -- The existing local primary-fee calculation supplies each child quote -- The transaction-pinned minimum external gas limit and per-round allocation - budget floor are not available, so both grant-node admission checks are - skipped. The enclosing child's existing local floor check remains active -- The descendant depth cap equals the 24-node cap +The transaction-pinned primary quote is not in v0.3 host input yet, so the +existing local primary-fee calculation supplies each child quote The Python SDK takes ``use_balance: UseBalanceParams | None`` on calls, transfers and deploys. ``None`` selects sender funding; the dataclass holds @@ -403,6 +397,28 @@ not the unspent allocation allowance or consumption from earlier generations. All v0.3 internal emissions reject zero time-unit, storage or receipt price caps with ``Inval`` before charging fees or appending an emission +When the manager input carries ``message_fee_cascade``, a top-level execution +rewrites its final emissions before the execution hash. Each sender-pool +internal message (not ``use_balance``) gets ``fee = max(fee, min_primary_fee)``. +With ``N`` such messages and ``spare = budget_remaining - Ξ£ fee - E`` positive, each then +gets ``floor(spare / (2 * N))`` more. Balance-funded and external messages are +untouched, and buckets are not charged for the addition: the chain carves the +child's own pool from its declared budget. A single floor is enough because the +host gives every open-pool internal wildcard the same fee params + +``budget_remaining`` is read at the pinned processing block. It can still include +a prior generation's charge the chain rolls back only later, which makes the +addition smaller, never unbacked. ``E`` is ``external_fee_reserve`` times the +number of external emissions: each reserves the full chain gas envelope at the +locked receipt price. Floors still apply when the remaining budget is insufficient + +Pinned descendant grants enforce the parent's ``messageBudgetFloor``, +``gas_data.maxAllocationTreeDepth`` (root depth 1) and +``gas_data.minExternalGasLimit``. When the host omits the depth limit, the +executor defaults to 5; an omitted gas floor is 0. Excess depth fails at emission +with ``fee descendant_grant tree``; an external node whose ``gas_limit`` is below +the floor fails with ``fee descendant_grant external`` + The parent allocation's aggregate capacity is a separate invariant. If ``L = appealRounds + 1`` novel executions may each emit a child carrying budget ``C``, preserving that descendant capacity for every execution requires:: diff --git a/docs/website/src/impl-spec/appendix/manager-api.yaml b/docs/website/src/impl-spec/appendix/manager-api.yaml index 41c80841..b822d22c 100644 --- a/docs/website/src/impl-spec/appendix/manager-api.yaml +++ b/docs/website/src/impl-spec/appendix/manager-api.yaml @@ -1024,7 +1024,12 @@ components: additionalProperties: type: string default: {} - description: Host-provided `node` fee constants + description: | + Host-provided `node` fee constants. `maxAllocationTreeDepth` is the + parent's pinned grant depth limit as a decimal uint256; v0.3 defaults + to 5 when absent (root depth is 1). `minExternalGasLimit` is the + parent's pinned external-node gas floor as a decimal uint256; 0 when + absent message_fee_allocation: type: array items: @@ -1088,6 +1093,28 @@ components: maximum uint256, children_budget zero and an empty subtree. It is an ordinary exhausted candidate. Open-pool and allocation-absent executions MUST omit the guard; open-pool and view execution may use ordinary wildcards + message_fee_cascade: + type: object + nullable: true + default: null + additionalProperties: false + required: [min_primary_fee, budget_remaining, external_fee_reserve] + properties: + min_primary_fee: + type: integer + format: uint256 + description: Chain `minMessagePrimaryFeesForTx` of the open-pool internal fee params + budget_remaining: + type: integer + format: uint256 + description: Chain `getMessageFeesBudgetRemaining` at the pinned processing block + external_fee_reserve: + type: integer + format: uint256 + description: Full external gas envelope times locked receipt gas price, per external emission + description: | + Open-pool cascade, see the fees impl-spec. Send it only for an open-pool + execution; null leaves declared budgets as metered record_actions: type: array items: diff --git a/docs/website/src/spec/appendix/internal-constants.rst b/docs/website/src/spec/appendix/internal-constants.rst index 4c7a1fba..05f488f4 100644 --- a/docs/website/src/spec/appendix/internal-constants.rst +++ b/docs/website/src/spec/appendix/internal-constants.rst @@ -99,6 +99,13 @@ nondet_blocks Value: ``4096`` +.. _gvm-def-consts-value-top-limits-default-allocation-tree-depth: + +default_allocation_tree_depth +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Value: ``5`` + .. _gvm-def-consts-value-top-limits-locked-slots: locked_slots diff --git a/executors/v0.3.x b/executors/v0.3.x index c8116fba..2e094c6f 160000 --- a/executors/v0.3.x +++ b/executors/v0.3.x @@ -1 +1 @@ -Subproject commit c8116fba35243cac378ec9eca3be20c6ebfa88c0 +Subproject commit 2e094c6fdd7fb3b620706a2d20845e004fb6d298 diff --git a/implementation/src/manager/run.rs b/implementation/src/manager/run.rs index 1e72a498..9284ce9d 100644 --- a/implementation/src/manager/run.rs +++ b/implementation/src/manager/run.rs @@ -1026,6 +1026,10 @@ pub struct Request { #[serde(default)] #[calldata(default = default_message_fee_allocation)] pub message_fee_allocation: Vec<genvm_modules_interfaces::fees::MessageAllocationNode>, + /// Open-bucket cascade inputs; absent means no cascade + #[serde(default)] + #[calldata(default = default_message_fee_cascade)] + pub message_fee_cascade: Option<genvm_modules_interfaces::fees::MessageFeeCascade>, /// Initial time-unit budget for this execution. pub initial_time_units_allocation: u32, /// Auditable supervisor action kinds to return in the execution result. @@ -1062,6 +1066,10 @@ fn default_message_fee_allocation() -> Vec<genvm_modules_interfaces::fees::Messa Vec::new() } +fn default_message_fee_cascade() -> Option<genvm_modules_interfaces::fees::MessageFeeCascade> { + None +} + fn default_record_actions() -> Vec<String> { Vec::new() } @@ -1998,6 +2006,7 @@ fn execution_data_from_request( bucket_totals: req.bucket_totals.clone(), gas_data: req.gas_data.clone(), message_fee_allocation: req.message_fee_allocation.clone(), + message_fee_cascade: req.message_fee_cascade, initial_time_units_allocation: req.initial_time_units_allocation, record_actions: req.record_actions.clone(), remaining_recursion: if req.debug_mode >= genvm_common::DebugMode::Unsafe { @@ -2229,6 +2238,7 @@ impl Ctx { leader_public_data: None, gas_data: parent_req.gas_data.clone(), message_fee_allocation: Vec::new(), + message_fee_cascade: None, initial_time_units_allocation: 0, record_actions: Vec::new(), host_genvm_id: None, @@ -2243,6 +2253,7 @@ impl Ctx { execution_data.leader_public_data = None; execution_data.record_actions.clear(); execution_data.message_fee_allocation.clear(); + execution_data.message_fee_cascade = None; execution_data.remaining_recursion = Some(cross_major_recursion(envelope.remaining_recursion)); execution_data.nested = Some(genvm_modules_interfaces::NestedExecutionData { diff --git a/implementation/src/manager/run_test.rs b/implementation/src/manager/run_test.rs index 50ef7f82..83f5fc21 100644 --- a/implementation/src/manager/run_test.rs +++ b/implementation/src/manager/run_test.rs @@ -1111,6 +1111,7 @@ fn request_field_names() -> Vec<&'static str> { leader_public_data, gas_data, message_fee_allocation, + message_fee_cascade, initial_time_units_allocation, record_actions, host_genvm_id, @@ -1139,6 +1140,7 @@ fn request_field_names() -> Vec<&'static str> { "leader_public_data", "gas_data", "message_fee_allocation", + "message_fee_cascade", "initial_time_units_allocation", "record_actions", "host_genvm_id", diff --git a/tests/runner/genvm_tool_plugins/integration.py b/tests/runner/genvm_tool_plugins/integration.py index 7551a5a8..ac7b0a44 100644 --- a/tests/runner/genvm_tool_plugins/integration.py +++ b/tests/runner/genvm_tool_plugins/integration.py @@ -782,6 +782,9 @@ def resolve_executor(address, state, advisory_major): request_extra['permissions'] = case_permissions if hook_calls: request_extra['hook_cross_contract_calls'] = True + message_fee_cascade = single_conf.get('message_fee_cascade') + if message_fee_cascade is not None: + request_extra['message_fee_cascade'] = message_fee_cascade bucket_totals = base_host.default_bucket_totals(self._test_case.executor_major) bucket_totals.update(single_conf.get('bucket_totals', {})) From f7b3c4f0c77f8572229bb107368316f256b30ec6 Mon Sep 17 00:00:00 2001 From: kp2pml30 <kp2pml30@gmail.com> Date: Fri, 16 Oct 2026 20:48:28 +0900 Subject: [PATCH 43/45] =?UTF-8?q?fix(tests):=20pass=20wat2wasm=20feature?= =?UTF-8?q?=20flags=20only=20when=20supported=20=F0=9F=90=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit wabt 1.0.42 enables tail-call and annotations by default and drops their `--enable-*` flags, which broke the unpinned Homebrew wabt on the macOS release test leg --- .../runner/genvm_tool_plugins/integration.py | 10 ++-------- tests/runner/gvm_extra/wat2wasm.py | 19 +++++++++++++++++++ tests/system/cross-major/test.py | 8 ++------ tests/system/parse_version/test.py | 9 ++------- 4 files changed, 25 insertions(+), 21 deletions(-) create mode 100644 tests/runner/gvm_extra/wat2wasm.py diff --git a/tests/runner/genvm_tool_plugins/integration.py b/tests/runner/genvm_tool_plugins/integration.py index ac7b0a44..72905792 100644 --- a/tests/runner/genvm_tool_plugins/integration.py +++ b/tests/runner/genvm_tool_plugins/integration.py @@ -29,6 +29,7 @@ import genvm_tool.tests import genvm_tool_plugins.runners as runners import gvm_extra.case_inputs as case_inputs +import gvm_extra.wat2wasm as wat2wasm import origin.base_host as base_host import origin.calldata as gvm_calldata import origin.fees as fees @@ -614,14 +615,7 @@ async def _run_single_step( if code_path.endswith('.wat'): out_path = my_tmp_dir.joinpath(Path(code_path).with_suffix('.wasm').name) cmd = Command( - args=[ - 'wat2wasm', - '--enable-tail-call', - '--enable-annotations', - '-o', - str(out_path), - code_path, - ], + args=wat2wasm.command(str(out_path), code_path), cwd=my_tmp_dir, env=default_env, ) diff --git a/tests/runner/gvm_extra/wat2wasm.py b/tests/runner/gvm_extra/wat2wasm.py new file mode 100644 index 00000000..59802fd8 --- /dev/null +++ b/tests/runner/gvm_extra/wat2wasm.py @@ -0,0 +1,19 @@ +import functools +import subprocess + +# wabt 1.0.42 enabled these by default and dropped their `--enable-*` flags +_FEATURES = ('tail-call', 'annotations') + + +@functools.cache +def _flags() -> tuple[str, ...]: + help_text = subprocess.run( + ['wat2wasm', '--help'], check=True, capture_output=True, text=True + ).stdout + return tuple( + f'--enable-{feature}' for feature in _FEATURES if f'--enable-{feature}' in help_text + ) + + +def command(out_path: str, wat_path: str) -> list[str]: + return ['wat2wasm', *_flags(), '-o', out_path, wat_path] diff --git a/tests/system/cross-major/test.py b/tests/system/cross-major/test.py index b494c322..12beed2e 100644 --- a/tests/system/cross-major/test.py +++ b/tests/system/cross-major/test.py @@ -11,7 +11,7 @@ import genvm_tool.tests import genvm_tool.tests.stage.collection import origin.calldata as gvm_calldata -from gvm_extra import cross_major +from gvm_extra import cross_major, wat2wasm from gvm_extra.cross_major import ( TIMESTAMP, assert_hashes_agree, @@ -190,11 +190,7 @@ async def _compile_wat(self, wat: str, path: Path) -> bytes: wasm_path = path / 'contract.wasm' await gvm_io.write_file_text(wat_path, wat) process = await asyncio.create_subprocess_exec( - 'wat2wasm', - '--enable-annotations', - '-o', - str(wasm_path), - str(wat_path), + *wat2wasm.command(str(wasm_path), str(wat_path)), stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) diff --git a/tests/system/parse_version/test.py b/tests/system/parse_version/test.py index cd887492..aac33885 100644 --- a/tests/system/parse_version/test.py +++ b/tests/system/parse_version/test.py @@ -14,6 +14,7 @@ import genvm_tool.tests from genvm_tool.tests.test import Result +from gvm_extra import wat2wasm def _default_env() -> dict[str, str]: @@ -55,13 +56,7 @@ def _add_case( # Compile WAT to WASM steps.append( genvm_tool.tests.exec.step.Run( - args=[ - 'wat2wasm', - '--enable-annotations', - '-o', - str(wasm_file), - str(wat_file), - ], + args=wat2wasm.command(str(wasm_file), str(wat_file)), mode=genvm_tool.tests.exec.command.RunMode.SILENT, ) ) From 13a6c57607931adba2fb38e252ef2ebb53240d92 Mon Sep 17 00:00:00 2001 From: Kira <kp2pml30@gmail.com> Date: Sat, 17 Oct 2026 21:49:30 +0900 Subject: [PATCH 44/45] =?UTF-8?q?perf(ci):=20correctly=20versioned,=20fast?= =?UTF-8?q?er=20release=20pipeline=20=E2=9A=A1=20(#79)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(ci): build releases at the overridden version πŸ› * fix(nix): stop genvm-rust from storing every component twice πŸ› * perf(ci): run release tests in a slim release-tests dev shell ⚑ * fix(nix): keep zig out of genvm-rust built without the zig wrapper πŸ› --- .../workflows/incl_release_build_test.yaml | 4 +- .../incl_release_build_test_cell_build.yaml | 9 ++ .../incl_release_build_test_cell_test.yaml | 9 ++ .github/workflows/release.yaml | 2 +- .../howto/releasing/release-build.md | 6 ++ flake.nix | 17 ++++ support/ci/pipelines/release.py | 47 +++++++++-- support/ci/unit_tests/test_release.py | 84 +++++++++++++++++++ support/rust.nix | 25 +++--- 9 files changed, 186 insertions(+), 17 deletions(-) create mode 100644 support/ci/unit_tests/test_release.py diff --git a/.github/workflows/incl_release_build_test.yaml b/.github/workflows/incl_release_build_test.yaml index 46c85933..c46ac8af 100644 --- a/.github/workflows/incl_release_build_test.yaml +++ b/.github/workflows/incl_release_build_test.yaml @@ -13,7 +13,7 @@ on: workflow_call: inputs: version_override: - description: "Release tag. Defaults to `version` from .genvm-monorepo-root" + description: "Release tag and embedded version. Defaults to `version` from .genvm-monorepo-root" required: false type: string default: "" @@ -117,6 +117,7 @@ jobs: runs_on: ${{ matrix.runs_on }} artifact_name: ${{ matrix.artifact_name }} job_json: ${{ matrix.job_json }} + version_override: ${{ needs.plan.outputs.tag }} secrets: inherit test: @@ -134,4 +135,5 @@ jobs: with_nix: ${{ matrix.with_nix }} setup_python: ${{ matrix.setup_python }} job_json: ${{ matrix.job_json }} + version_override: ${{ needs.plan.outputs.tag }} secrets: inherit diff --git a/.github/workflows/incl_release_build_test_cell_build.yaml b/.github/workflows/incl_release_build_test_cell_build.yaml index 0b25bc5c..77359317 100644 --- a/.github/workflows/incl_release_build_test_cell_build.yaml +++ b/.github/workflows/incl_release_build_test_cell_build.yaml @@ -11,6 +11,10 @@ name: Build one release artifact cell on: workflow_call: inputs: + version_override: + description: release version from the plan + required: true + type: string name: required: true type: string @@ -41,6 +45,11 @@ jobs: steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - name: Apply release version + env: + VERSION_OVERRIDE: ${{ inputs.version_override }} + run: ./support/ci/run.sh pipeline apply-release-version --version-override "$VERSION_OVERRIDE" + # Guards the "Job:" step below: every key must have a step, every step a key. - name: Check job keys env: diff --git a/.github/workflows/incl_release_build_test_cell_test.yaml b/.github/workflows/incl_release_build_test_cell_test.yaml index 1ac09490..aa5adab2 100644 --- a/.github/workflows/incl_release_build_test_cell_test.yaml +++ b/.github/workflows/incl_release_build_test_cell_test.yaml @@ -8,6 +8,10 @@ name: Test one release platform cell on: workflow_call: inputs: + version_override: + description: release version from the plan + required: true + type: string name: required: true type: string @@ -49,6 +53,11 @@ jobs: steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - name: Apply release version + env: + VERSION_OVERRIDE: ${{ inputs.version_override }} + run: ./support/ci/run.sh pipeline apply-release-version --version-override "$VERSION_OVERRIDE" + - name: Check job keys env: JOB_JSON: ${{ inputs.job_json }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index fbbf6ecd..6e137912 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -24,7 +24,7 @@ on: version_override: type: string required: false - description: "Override release tag (e.g., v1.2.3). Defaults to `version` from .genvm-monorepo-root" + description: "Override release tag and embedded version (e.g., v1.2.3). Defaults to `version` from .genvm-monorepo-root" defaults: run: diff --git a/docs/contributing/howto/releasing/release-build.md b/docs/contributing/howto/releasing/release-build.md index 714f1739..d47e6319 100644 --- a/docs/contributing/howto/releasing/release-build.md +++ b/docs/contributing/howto/releasing/release-build.md @@ -39,6 +39,12 @@ one install root, so a full install is one platform asset plus `genvm-universal`. The platform asset already carries every active executor line ([versioning.md](versioning.md)) +`version_override` sets both the release tag and the embedded manager version. +The plan passes the resolved version to every build and test cell, which applies +it to `.genvm-monorepo-root` before source setup. Nix reads that file for +`GENVM_PROFILE`, so an override changes the build inputs and cache keys. +Executor versions remain pinned by their own manifests + ## Verifying an Asset The publisher uploads `SHA256SUMS` next to the tarballs and records a keyless diff --git a/flake.nix b/flake.nix index 6f4f854b..d8a7bef0 100644 --- a/flake.nix +++ b/flake.nix @@ -74,6 +74,13 @@ withZig = false; withWasi = true; }; + # Enough to build the integration tests' wasm contracts, nothing else. + release-test-rust = import ./support/rust.nix { + inherit pkgs deps system; + withZig = false; + withWasi = true; + withCross = false; + }; custom-rust-builder = import ./support/compile-rust.nix { inherit pkgs system deps; zig = import ./support/zig.nix { inherit pkgs deps system; }; @@ -752,6 +759,16 @@ packages-0 ++ packages-rust ++ packages-debug-test ++ [ pre-commit-check.config.package ]; shellHook = shell-hook-base; }; + # Integration tests against prebuilt release tarballs: no build or fuzz tooling. + devShells.release-tests = pkgs.mkShell { + packages = packages-0 ++ [ + release-test-rust + pkgs.python312Packages.jsonnet + pkgs.python312Packages.aiohttp + pkgs.wabt + ]; + shellHook = shell-hook-base; + }; devShells.full = pkgs.mkShell { packages = packages-0 diff --git a/support/ci/pipelines/release.py b/support/ci/pipelines/release.py index aa7dc837..bca76acf 100644 --- a/support/ci/pipelines/release.py +++ b/support/ci/pipelines/release.py @@ -2,6 +2,7 @@ import json import os import re +import subprocess import ci_lib from pipelines.tests import GithubStep, SubStep, step_nix_develop, step_run @@ -159,7 +160,7 @@ def _cell_amd64_linux() -> dict: {'type': 'group-start', 'name': 'smoke test modules startup'}, _smoke_step('timeout'), {'type': 'group-end'}, - *stable_test_steps('.?submodules=1#mock-tests'), + *stable_test_steps('.?submodules=1#release-tests'), ], runs_on='ubuntu-latest', ) @@ -238,6 +239,43 @@ def test_cells() -> list[dict]: return [_cell_amd64_linux(), _cell_arm64_linux(), _cell_arm64_macos()] +def check_release_version(version: str) -> bool: + ident = r'[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*' + semver = re.fullmatch(rf'v[0-9]+\.[0-9]+\.[0-9]+(-{ident})?(\+{ident})?', version) + if ( + not semver + or subprocess.run(['git', 'check-ref-format', f'refs/tags/{version}']).returncode + != 0 + ): + ci_lib.github_error(f'invalid version {version!r}') + return False + return True + + +class ApplyReleaseVersion(ci_lib.Pipeline): + def name(self) -> str: + return 'apply-release-version' + + def add_to(self, parser: argparse.ArgumentParser) -> None: + parser.add_argument( + '--version-override', + default='', + help='release version to write to .genvm-monorepo-root; empty leaves it unchanged', + ) + + def handler(self, args: argparse.Namespace) -> int: + if not args.version_override: + return 0 + if not check_release_version(args.version_override): + return 1 + path = ci_lib.ROOT_DIR / '.genvm-monorepo-root' + monorepo = json.loads(path.read_text()) + if monorepo['version'] != args.version_override: + monorepo['version'] = args.version_override + path.write_text(json.dumps(monorepo, indent=4) + '\n') + return 0 + + class PlanReleaseMatrix(ci_lib.Pipeline): """ Emit the release build/test matrices and the tag they are built at. @@ -253,7 +291,7 @@ def add_to(self, parser: argparse.ArgumentParser) -> None: parser.add_argument( '--version-override', default='', - help='release tag; defaults to `version` from .genvm-monorepo-root', + help='release tag and embedded version; defaults to `version` from .genvm-monorepo-root', ) def handler(self, args: argparse.Namespace) -> int: @@ -263,8 +301,7 @@ def handler(self, args: argparse.Namespace) -> int: # Shape only. Whether the tag is already taken depends on the remote and # is the publisher's business β€” a PR-label run plans the same tag every # time and must not care. - if not re.match(r'^v[0-9]+\.[0-9]+\.[0-9]+', tag): - ci_lib.github_error(f'invalid version {tag!r}') + if not check_release_version(tag): return 1 builds = build_cells() @@ -301,4 +338,4 @@ def handler(self, args: argparse.Namespace) -> int: return 0 -COMMANDS = [PlanReleaseMatrix()] +COMMANDS = [PlanReleaseMatrix(), ApplyReleaseVersion()] diff --git a/support/ci/unit_tests/test_release.py b/support/ci/unit_tests/test_release.py new file mode 100644 index 00000000..d4b714ed --- /dev/null +++ b/support/ci/unit_tests/test_release.py @@ -0,0 +1,84 @@ +import argparse +import json + +import pytest + +from pipelines import release + + +@pytest.fixture +def monorepo(tmp_path, monkeypatch): + path = tmp_path / '.genvm-monorepo-root' + path.write_text( + json.dumps( + { + 'version': 'v0.6.0-rc2', + 'active-versions': ['v0.3', 'v0.2'], + 'support-only-versions': ['v0.2'], + 'artifacts_dir': 'build/test-artifacts', + }, + indent=4, + ) + + '\n' + ) + monkeypatch.setattr(release.ci_lib, 'ROOT_DIR', tmp_path) + monkeypatch.setattr(release, 'release_filter_tag', lambda: 'integration & stable') + return path + + +def run(command, override=None): + parser = argparse.ArgumentParser() + command.add_to(parser) + argv = [] if override is None else ['--version-override', override] + return command.handler(parser.parse_args(argv)) + + +@pytest.mark.parametrize('override', [None, '', 'v0.6.0-rc2']) +def test_apply_unchanged_version_does_not_write(monorepo, override): + before = monorepo.read_bytes() + mtime = monorepo.stat().st_mtime_ns + assert run(release.ApplyReleaseVersion(), override) == 0 + assert monorepo.read_bytes() == before + assert monorepo.stat().st_mtime_ns == mtime + + +@pytest.mark.parametrize('override', ['v1.2.3', 'v0.6.0-rc3', 'v1.2.3+build.4']) +def test_apply_changes_only_version_and_preserves_format(monorepo, override): + before = monorepo.read_text() + assert run(release.ApplyReleaseVersion(), override) == 0 + assert monorepo.read_text() == before.replace('v0.6.0-rc2', override) + + +@pytest.mark.parametrize( + 'override', + [ + '1.2.3', + 'v1.2', + 'release', + ' v1.2.3', + 'v1.2.3 rc', + 'v1.2.3;x', + 'v1.2.3-rc..1', + 'v1.2.3-rc.', + 'v1.2.3-x.lock', + ], +) +def test_plan_and_apply_reject_invalid_version( + monorepo, tmp_path, monkeypatch, override +): + before = monorepo.read_bytes() + output = tmp_path / 'github-output' + monkeypatch.setenv('GITHUB_OUTPUT', str(output)) + assert run(release.PlanReleaseMatrix(), override) == 1 + assert run(release.ApplyReleaseVersion(), override) == 1 + assert monorepo.read_bytes() == before + assert not output.exists() + + +@pytest.mark.parametrize('override', ['', 'v1.2.3-rc1']) +def test_plan_tags_with_override(monorepo, tmp_path, monkeypatch, override): + output = tmp_path / 'github-output' + monkeypatch.setenv('GITHUB_OUTPUT', str(output)) + assert run(release.PlanReleaseMatrix(), override) == 0 + outputs = dict(line.split('=', 1) for line in output.read_text().splitlines()) + assert outputs['tag'] == (override or 'v0.6.0-rc2') diff --git a/support/rust.nix b/support/rust.nix index d3bd212f..6ad6c3b9 100644 --- a/support/rust.nix +++ b/support/rust.nix @@ -5,6 +5,7 @@ withLinters ? false, withZig ? true, withWasi ? false, + withCross ? true, ... }@args: let @@ -44,12 +45,17 @@ let (simpleComponent manifest.pkg.cargo.target.${systemAsRust}) (simpleComponent manifest.pkg.rustc.target.${systemAsRust}) (simpleComponent manifest.pkg.rust-std.target.${systemAsRust}) - - # cross compilation - (simpleComponent manifest.pkg.rust-std.target.x86_64-unknown-linux-musl) - (simpleComponent manifest.pkg.rust-std.target.aarch64-unknown-linux-musl) - (simpleComponent manifest.pkg.rust-std.target.aarch64-apple-darwin) ] + ++ ( + if !withCross then + [ ] + else + [ + (simpleComponent manifest.pkg.rust-std.target.x86_64-unknown-linux-musl) + (simpleComponent manifest.pkg.rust-std.target.aarch64-unknown-linux-musl) + (simpleComponent manifest.pkg.rust-std.target.aarch64-apple-darwin) + ] + ) ++ ( if !withLinters then [ ] @@ -88,9 +94,8 @@ pkgs.stdenvNoCC.mkDerivation rec { buildInputs = [ pkgs.zlib pkgs.bash - - zig ] + ++ pkgs.lib.optionals withZig [ zig ] ++ pkgs.lib.optionals pkgs.stdenv.hostPlatform.isLinux [ pkgs.glibc pkgs.gcc.cc.lib @@ -140,18 +145,18 @@ pkgs.stdenvNoCC.mkDerivation rec { installPhase = '' mkdir -p $out - for i in $(find . -type d -maxdepth 2 -mindepth 1) ; + # Each tarball is `<pkg>/<component>/...`; only the components belong in $out. + for i in $(find . -type d -maxdepth 2 -mindepth 2) ; do cp -r "$i/." $out/. done ls -l "$out" - - cp ${rust-objcopy} $out/bin/rust-objcopy '' + ( if withZig then '' + cp ${rust-objcopy} $out/bin/rust-objcopy wrapProgram $out/bin/cargo \ --set CC_x86_64_unknown_linux_musl zig-cc-amd64-linux \ --set CC_x86_64_unknown_linux_gnu zig-cc-amd64-linux-gnu \ From fdca7b729c08995343ad42a143ca7bf6081a99ef Mon Sep 17 00:00:00 2001 From: Kira <kp2pml30@gmail.com> Date: Sun, 18 Oct 2026 00:49:52 +0900 Subject: [PATCH 45/45] =?UTF-8?q?fix(webdriver):=20cancel=20dropped=20rend?= =?UTF-8?q?ers=20and=20prevent=20DNS=20rebinding=20=F0=9F=94=92=EF=B8=8F?= =?UTF-8?q?=20(#76)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(webdriver): cancel abandoned renders and release permits πŸ› * fix(webdriver): pin validated destinations through an egress proxy πŸ”’οΈ * fix(lua): cancel abandoned requests and validate redirects πŸ”’οΈ * fix(webdriver): catch single-dash and mixed-case guarded Chrome flags πŸ› --- docs/contributing/howto/testing/typescript.md | 11 + .../03-modules/01-web.rst | 29 ++ .../01-core-architecture/03-modules/index.rst | 17 + implementation/src/common/mod.rs | 98 ++++-- implementation/src/llm/ctx.rs | 2 +- implementation/src/llm/handler.rs | 2 +- implementation/src/manager/modules.rs | 4 +- implementation/src/scripting/cancellation.rs | 92 +++++ implementation/src/scripting/ctx/dflt.rs | 16 +- implementation/src/scripting/mod.rs | 26 +- implementation/src/web/cancellation_test.rs | 147 ++++++++ implementation/src/web/handler.rs | 2 +- implementation/src/web/mod.rs | 2 +- implementation/src/web/tests.rs | 9 +- implementation/tests/dflt_requests.rs | 2 +- .../tests/lua_request_cancellation.rs | 92 +++++ implementation/tests/module_disconnect.rs | 189 +++++++++++ implementation/tests/overloaded.rs | 2 +- implementation/tests/policy_dispatch.rs | 2 +- implementation/tests/timeout.rs | 2 +- implementation/tests/web_redirect_policy.rs | 104 ++++++ implementation/tests/web_render_webdriver.rs | 2 +- webdriver/src/prj/src/browser/chrome.ts | 72 ++-- webdriver/src/prj/src/cancellation.test.ts | 85 +++++ webdriver/src/prj/src/cancellation.ts | 29 ++ webdriver/src/prj/src/egress-browser.test.ts | 313 ++++++++++++++++++ webdriver/src/prj/src/egress-proxy.test.ts | 299 +++++++++++++++++ webdriver/src/prj/src/egress-proxy.ts | 220 ++++++++++++ webdriver/src/prj/src/index.ts | 47 +-- webdriver/src/prj/src/render.test.ts | 263 ++++++++++++++- webdriver/src/prj/src/render.ts | 155 +++++---- webdriver/src/prj/src/semaphore.test.ts | 6 + webdriver/src/prj/src/semaphore.ts | 6 +- webdriver/src/prj/src/ssrf.ts | 18 +- 34 files changed, 2177 insertions(+), 188 deletions(-) create mode 100644 implementation/src/scripting/cancellation.rs create mode 100644 implementation/src/web/cancellation_test.rs create mode 100644 implementation/tests/lua_request_cancellation.rs create mode 100644 implementation/tests/module_disconnect.rs create mode 100644 implementation/tests/web_redirect_policy.rs create mode 100644 webdriver/src/prj/src/cancellation.test.ts create mode 100644 webdriver/src/prj/src/cancellation.ts create mode 100644 webdriver/src/prj/src/egress-browser.test.ts create mode 100644 webdriver/src/prj/src/egress-proxy.test.ts create mode 100644 webdriver/src/prj/src/egress-proxy.ts diff --git a/docs/contributing/howto/testing/typescript.md b/docs/contributing/howto/testing/typescript.md index ca29555f..33db39af 100644 --- a/docs/contributing/howto/testing/typescript.md +++ b/docs/contributing/howto/testing/typescript.md @@ -25,6 +25,17 @@ npm test -- src/render.test.ts # omit the file to run all of them npm run typecheck ``` +The egress proxy tests use local TCP fixtures and controlled DNS answers +To also run real Chromium coverage for DNS rebinding, private destinations, +HTTP, HTTPS, redirects and WebSockets, provide a Chromium executable and +`openssl` on `PATH`: + +```bash +GVM_WEBDRIVER_CHROME_EXECUTABLE=/usr/bin/chromium npm test -- src/egress-browser.test.ts +``` + +Without that variable, the Chromium test is reported as skipped + ## Extra Tags Same marker as Rust ([rust.md](rust.md)), parsed by the shared diff --git a/docs/website/src/impl-spec/01-core-architecture/03-modules/01-web.rst b/docs/website/src/impl-spec/01-core-architecture/03-modules/01-web.rst index 11f0dcdd..541a8fed 100644 --- a/docs/website/src/impl-spec/01-core-architecture/03-modules/01-web.rst +++ b/docs/website/src/impl-spec/01-core-architecture/03-modules/01-web.rst @@ -30,3 +30,32 @@ Security and Compliance - Whitelist port filtering - URL pattern matching and validation - Execution time limits and timeouts + +Lua HTTP Redirects +~~~~~~~~~~~~~~~~~~ + +Ordinary Lua web requests enforce the port and TLD checks on each redirect, +including configured extra TLDs. DNS filtering still applies to each connection; +redirects to private IP literals and HTTPS-to-HTTP downgrades are rejected + +Requests starting at an ``always_allow_hosts`` entry intentionally use the +unfiltered client for the whole redirect chain. The WebDriver sidecar connection +also uses the unfiltered client + +Webdriver Egress +~~~~~~~~~~~~~~~~ + +Each Chromium instance uses a local SOCKS5 proxy for HTTP, HTTPS and WebSocket +connections. The proxy rejects a destination if any resolved address fails +the SSRF address filter, then connects to a validated numeric IP without +another DNS lookup. TLS and the original hostname remain end-to-end + +Chromium has no direct proxy fallback or implicit loopback bypass. Direct +DNS resolution, QUIC and non-proxied WebRTC UDP are disabled. A temporary +browser profile sets the WebRTC IP handling policy and is removed after the +browser exits. Extra arguments in ``GVM_WEBDRIVER_CHROME_ARGS`` that override +the routing controls or select another profile are rejected at startup + +Request interception also checks URLs before navigation and subresource +loads; the proxy enforces the address check at connection time, including +when DNS answers change after interception diff --git a/docs/website/src/impl-spec/01-core-architecture/03-modules/index.rst b/docs/website/src/impl-spec/01-core-architecture/03-modules/index.rst index 1c8da3b1..f73b330a 100644 --- a/docs/website/src/impl-spec/01-core-architecture/03-modules/index.rst +++ b/docs/website/src/impl-spec/01-core-architecture/03-modules/index.rst @@ -66,3 +66,20 @@ Module Lifecycle - **Termination**: - Graceful shutdown and resource cleanup + +Caller Disconnection +~~~~~~~~~~~~~~~~~~~~ + +The executor's module stream is monitored during handler initialization, +requests and response writes. EOF or a read error cancels the active operation. +The reader buffers at most one pending request; further pipelining closes the +connection rather than suspending disconnect detection + +Each Lua call owns its pending HTTP, provider and sleep futures. Cancellation +drops these operations even when Lua retains the suspended coroutine, closing +pending WebDriver requests so that the sidecar releases its rendering slot. +Teardown runs synchronous cleanup but is dropped if it would wait asynchronously + +This differs from the manager's reconnectable WebSocket: disconnecting that +control connection preserves executions for later attachment, as described in +:doc:`../../appendix/manager-socket` diff --git a/implementation/src/common/mod.rs b/implementation/src/common/mod.rs index 005fc27b..c69206db 100644 --- a/implementation/src/common/mod.rs +++ b/implementation/src/common/mod.rs @@ -299,23 +299,16 @@ pub(crate) fn module_error_to_wire<R>( async fn loop_one_inner<T, R, S>( handler: &mut impl MessageHandler<T, R>, + messages: &mut tokio::sync::mpsc::Receiver<Vec<u8>>, stream: &mut S, genvm_id: genvm_modules_interfaces::GenVMId, ) -> anyhow::Result<()> where T: calldata::codec::Decode + 'static, R: calldata::codec::Encode<Vec<u8>, Error = std::convert::Infallible> + Send + 'static, - S: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin, + S: tokio::io::AsyncWrite + Unpin, { - loop { - let Some(data) = read_message(stream) - .await - .context("reading module message")? - else { - // Stream closed - return Ok(()); - }; - + while let Some(data) = messages.recv().await { let res = loop_one_inner_handle(handler, &data).await; let res = match res { Ok(res) => genvm_modules_interfaces::Result::Ok(res), @@ -328,6 +321,7 @@ where .await .context("writing message")?; } + Ok(()) } async fn read_hello<S>( @@ -361,17 +355,48 @@ where { let genvm_id = exec_ctx.genvm_id(); - let mut handler = handler_provider - .new_handler(exec_ctx) - .await - .context("creating handler")?; - - let res = loop_one_inner(&mut handler, stream, genvm_id) + let (mut reader, mut writer) = tokio::io::split(stream); + let (messages_tx, mut messages_rx) = tokio::sync::mpsc::channel(1); + let read = async { + while let Some(data) = read_message(&mut reader) + .await + .context("reading module message")? + { + // Never wait for queue space: that would hide EOF during a request. + messages_tx + .try_send(data) + .context("too many pending module requests")?; + } + Ok(()) + }; + let mut handler = None; + let work = async { + handler = Some( + handler_provider + .new_handler(exec_ctx) + .await + .context("creating handler")?, + ); + loop_one_inner( + handler.as_mut().unwrap(), + &mut messages_rx, + &mut writer, + genvm_id, + ) .await - .context("handling"); + .context("handling") + }; + let res = tokio::select! { + result = read => result, + result = work => result, + }; - if let Err(close) = handler.cleanup().await { - log_error_into!(@operator, &LoggerWithId, error:ah = &close, genvm_id:id = genvm_id.0; "cleanup error"); + if let Some(handler) = handler { + // Preserve synchronous Teardown telemetry without starting another wait after disconnect. + use futures_util::FutureExt as _; + if let Some(Err(close)) = handler.cleanup().now_or_never() { + log_error_into!(@operator, &LoggerWithId, error:ah = &close, genvm_id:id = genvm_id.0; "cleanup error"); + } } res @@ -843,7 +868,12 @@ const MAX_REDIRECTS: usize = 10; /// receivers) onto a plaintext hop observable and replayable by on-path /// attackers. pub fn redirect_policy() -> reqwest::redirect::Policy { - reqwest::redirect::Policy::custom(|attempt| { + redirect_policy_with_tlds(&[]) +} + +pub fn redirect_policy_with_tlds(extra_tld: &[Box<str>]) -> reqwest::redirect::Policy { + let extra_tld = extra_tld.to_vec(); + reqwest::redirect::Policy::custom(move |attempt| { if attempt.previous().len() >= MAX_REDIRECTS { return attempt.error("too many redirects"); } @@ -865,6 +895,28 @@ pub fn redirect_policy() -> reqwest::redirect::Policy { log_warn!(@user, url = attempt.url().as_str(); "HTTPS->HTTP downgrade redirect rejected"); return attempt.error(SchemeDowngrade); } + let url = attempt.url(); + let host = url.host_str().unwrap_or_default(); + let tld = host.rsplit('.').next().unwrap_or_default(); + let cause = if !matches!(url.scheme(), "http" | "https") { + Some("SCHEMA_FORBIDDEN") + } else if !matches!(url.port(), None | Some(80 | 443)) { + Some("PORT_FORBIDDEN") + } else if !crate::web::domains::DOMAINS.contains(&tld) + && !extra_tld.iter().any(|allowed| allowed.as_ref() == tld) + { + Some("TLD_FORBIDDEN") + } else { + None + }; + if let Some(cause) = cause { + let url = url.as_str().to_owned(); + return attempt.error(ModuleError { + causes: vec![cause.to_owned()], + fatal: false, + ctx: BTreeMap::from([("url".to_owned(), url.into())]), + }); + } attempt.follow() }) } @@ -886,9 +938,13 @@ impl std::error::Error for SchemeDowngrade {} /// [`FilteringResolver`]) and whose redirect policy rejects non-routable /// IP-literal hops (see [`redirect_policy`]). For contract-controlled web URLs. pub fn create_client_filtered() -> anyhow::Result<reqwest::Client> { + create_client_filtered_with_tlds(&[]) +} + +pub fn create_client_filtered_with_tlds(extra_tld: &[Box<str>]) -> anyhow::Result<reqwest::Client> { base_client_builder() .dns_resolver(Arc::new(FilteringResolver::new()?)) - .redirect(redirect_policy()) + .redirect(redirect_policy_with_tlds(extra_tld)) .build() .map_err(Into::into) } diff --git a/implementation/src/llm/ctx.rs b/implementation/src/llm/ctx.rs index 95272f34..9d1a87a1 100644 --- a/implementation/src/llm/ctx.rs +++ b/implementation/src/llm/ctx.rs @@ -147,7 +147,7 @@ pub fn create_global(vm: &mlua::Lua, config: &Config) -> anyhow::Result<mlua::Va let llm = vm.create_table()?; llm.set( "exec_prompt_in_provider", - vm.create_async_function(exec_prompt_in_provider)?, + scripting::cancellation::create_async_function(vm, exec_prompt_in_provider)?, )?; let all_providers = diff --git a/implementation/src/llm/handler.rs b/implementation/src/llm/handler.rs index afbae746..f6dc5682 100644 --- a/implementation/src/llm/handler.rs +++ b/implementation/src/llm/handler.rs @@ -47,7 +47,7 @@ impl MessageHandlerProvider<genvm_modules_interfaces::llm::Message, llm_iface::P &hello, &self.config.gep(|x| &x.mod_base), metrics.gep(|x| &x.scripting), - false, + None, )?; let module = ctx::CtxPart { providers: self.providers.clone(), diff --git a/implementation/src/manager/modules.rs b/implementation/src/manager/modules.rs index 6adff06d..fb8d49bd 100644 --- a/implementation/src/manager/modules.rs +++ b/implementation/src/manager/modules.rs @@ -42,7 +42,7 @@ impl ModuleStateLlm { hello, &self.config.gep(|x| &x.mod_base), metrics.gep(|x| &x.scripting), - false, + None, )?; let module = crate::llm::ctx::CtxPart { providers: self.providers.clone(), @@ -62,7 +62,7 @@ impl ModuleStateWeb { hello, &self.config.gep(|x| &x.mod_base), metrics.gep(|x| &x.scripting), - true, + Some(&self.config.extra_tld), )?; Ok(WebSubContext { scripting }) } diff --git a/implementation/src/scripting/cancellation.rs b/implementation/src/scripting/cancellation.rs new file mode 100644 index 00000000..196329d5 --- /dev/null +++ b/implementation/src/scripting/cancellation.rs @@ -0,0 +1,92 @@ +use std::{ + future::Future, + pin::Pin, + sync::{Arc, Mutex, Weak}, +}; + +trait Cancel: Send + Sync { + fn cancel(&self); +} + +// Lua 5.3 may retain suspended coroutines; the call scope must own their pending work. +struct Operation<F>(Mutex<Option<Pin<Box<F>>>>); + +impl<F: Send> Cancel for Operation<F> { + fn cancel(&self) { + self.0.lock().unwrap().take(); + } +} + +struct Scope { + operations: Mutex<Option<Vec<Weak<dyn Cancel>>>>, +} + +pub(super) struct Guard<'a>(&'a mlua::Lua, Arc<Scope>); + +impl<'a> Guard<'a> { + pub(super) fn enter(vm: &'a mlua::Lua) -> mlua::Result<Self> { + if vm.app_data_ref::<Arc<Scope>>().is_some() { + return Err(mlua::Error::external("concurrent Lua calls on one VM")); + } + let scope = Arc::new(Scope { + operations: Mutex::new(Some(Vec::new())), + }); + vm.set_app_data(scope.clone()); + Ok(Self(vm, scope)) + } +} + +impl Drop for Guard<'_> { + fn drop(&mut self) { + self.0.remove_app_data::<Arc<Scope>>(); + let operations = self.1.operations.lock().unwrap().take().unwrap_or_default(); + for operation in operations.into_iter().filter_map(|op| op.upgrade()) { + operation.cancel(); + } + } +} + +pub(crate) fn create_async_function<A, R, F, FR>( + vm: &mlua::Lua, + function: F, +) -> mlua::Result<mlua::Function> +where + A: mlua::FromLuaMulti, + R: mlua::IntoLuaMulti, + F: Fn(mlua::Lua, A) -> FR + Send + 'static, + FR: Future<Output = mlua::Result<R>> + Send + 'static, +{ + vm.create_async_function(move |vm, args| { + let scope = vm.app_data_ref::<Arc<Scope>>().map(|scope| scope.clone()); + let future = function(vm, args); + async move { + let Some(scope) = scope else { + return future.await; + }; + let operation = Arc::new(Operation(Mutex::new(Some(Box::pin(future))))); + { + let mut operations = scope.operations.lock().unwrap(); + let Some(operations) = operations.as_mut() else { + return Err(mlua::Error::external("Lua call cancelled")); + }; + operations.retain(|op| op.strong_count() != 0); + let erased: Arc<dyn Cancel> = operation.clone(); + operations.push(Arc::downgrade(&erased)); + } + std::future::poll_fn(|cx| { + let mut future = operation.0.lock().unwrap(); + let Some(inner) = future.as_mut() else { + return std::task::Poll::Ready(Err(mlua::Error::external( + "Lua call cancelled", + ))); + }; + let result = inner.as_mut().poll(cx); + if result.is_ready() { + future.take(); + } + result + }) + .await + } + }) +} diff --git a/implementation/src/scripting/ctx/dflt.rs b/implementation/src/scripting/ctx/dflt.rs index e561bcde..d4028d6d 100644 --- a/implementation/src/scripting/ctx/dflt.rs +++ b/implementation/src/scripting/ctx/dflt.rs @@ -97,12 +97,15 @@ pub fn create_global( dflt.set( "sleep_seconds", - vm.create_async_function(|vm: mlua::Lua, data: mlua::Value| async move { - let as_seconds: f32 = vm.from_value(data)?; - tokio::time::sleep(tokio::time::Duration::from_secs_f32(as_seconds)).await; + scripting::cancellation::create_async_function( + vm, + |vm: mlua::Lua, data: mlua::Value| async move { + let as_seconds: f32 = vm.from_value(data)?; + tokio::time::sleep(tokio::time::Duration::from_secs_f32(as_seconds)).await; - Ok(()) - })?, + Ok(()) + }, + )?, )?; dflt.set( @@ -238,7 +241,8 @@ pub fn create_global( dflt.set( "request", - vm.create_async_function( + scripting::cancellation::create_async_function( + vm, |vm: mlua::Lua, args: (mlua::Table, mlua::Value)| async move { let (zelf, req) = args; diff --git a/implementation/src/scripting/mod.rs b/implementation/src/scripting/mod.rs index ac317676..7ed108a6 100644 --- a/implementation/src/scripting/mod.rs +++ b/implementation/src/scripting/mod.rs @@ -1,6 +1,7 @@ pub mod pool; pub mod rat; +pub(crate) mod cancellation; mod ctx; use anyhow::Context as _; @@ -65,14 +66,12 @@ pub fn anyhow_to_lua_error(e: anyhow::Error) -> mlua::Error { } } -/// `filter_dns`: when true, `client` resolves through the SSRF-filtering -/// resolver (web module). `client_unfiltered` is always a plain client, used by -/// requests that opt out (allowlisted hosts). +/// `web_extra_tld`: enables the web DNS/redirect policy with these extra TLDs. pub fn create_ctx_part( hello: &Arc<genvm_modules_interfaces::GenVMHello>, base_config: &sync::DArc<common::ModuleBaseConfig>, metrics: sync::DArc<Metrics>, - filter_dns: bool, + web_extra_tld: Option<&[Box<str>]>, ) -> anyhow::Result<ctx::CtxPart> { let mut sign_vars = BTreeMap::new(); @@ -88,8 +87,8 @@ pub fn create_ctx_part( } let client_unfiltered = common::create_client_unfiltered()?; - let client = if filter_dns { - common::create_client_filtered()? + let client = if let Some(extra_tld) = web_extra_tld { + common::create_client_filtered_with_tlds(extra_tld)? } else { client_unfiltered.clone() }; @@ -133,7 +132,7 @@ pub fn create_default_ctx( vm: &mlua::Lua, table: &mlua::Table, ) -> anyhow::Result<sync::DArc<ctx::CtxPart>> { - let ctx_part = sync::DArc::new(create_ctx_part(hello, &base_config, metrics, false)?); + let ctx_part = sync::DArc::new(create_ctx_part(hello, &base_config, metrics, None)?); setup_lua_default_ctx(ctx_part.clone(), vm, table)?; Ok(ctx_part) } @@ -202,6 +201,7 @@ impl<T, R, E> UserVM<T, R, E> { where RR: mlua::FromLuaMulti, { + let _operations = cancellation::Guard::enter(&self.vm)?; let res = f.call_async(args).await; match res { @@ -223,6 +223,7 @@ pub async fn load_script<P>(vm: &mlua::Lua, path: P) -> anyhow::Result<()> where P: AsRef<std::path::Path> + Into<String> + std::fmt::Debug, { + let _operations = cancellation::Guard::enter(vm)?; let script_contents = std::fs::read_to_string(&path) .with_context(|| format!("reading script from {:?}", path))?; let chunk = vm.load(script_contents); @@ -257,9 +258,16 @@ pub struct ResponseJSON { /// Map a failed `send()` to a module error. A request rejected because the /// filtering resolver dropped every resolved address (SSRF guard) becomes a -/// **non-fatal** `ADDRESS_FORBIDDEN`, so the contract can recover; anything else -/// is a fatal `SENDING_REQUEST`. +/// **non-fatal** `ADDRESS_FORBIDDEN`; redirect policy errors retain their cause. +/// Other failures become fatal `SENDING_REQUEST` errors. fn map_send_error(url: &str, err: reqwest::Error) -> anyhow::Error { + let mut source: Option<&(dyn std::error::Error + 'static)> = Some(&err); + while let Some(error) = source { + if let Some(module_err) = error.downcast_ref::<ModuleError>() { + return module_err.clone().into(); + } + source = error.source(); + } let (causes, fatal) = if common::is_no_routable_address(&err) { (vec![common::ErrorKind::ADDRESS_FORBIDDEN.into()], false) } else { diff --git a/implementation/src/web/cancellation_test.rs b/implementation/src/web/cancellation_test.rs new file mode 100644 index 00000000..595af587 --- /dev/null +++ b/implementation/src/web/cancellation_test.rs @@ -0,0 +1,147 @@ +use super::*; +use std::time::Duration; +use tokio::io::AsyncReadExt as _; + +async fn disconnect_cancels_request(render: bool, body_started: bool) { + let server = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}", server.local_addr().unwrap()); + let mut config = test_config(url.clone()); + config.always_allow_hosts.push("127.0.0.1".into()); + let (cancel, _canceller) = genvm_common::cancellation::make(); + let (handler, _, _) = create_web_module(cancel, config).await.unwrap(); + let (mut host, module) = tokio::io::duplex(4096); + let task = tokio::spawn(async move { handler(Box::new(module), None).await }); + common::write_message( + &mut host, + &calldata::encode_obj(&*common::tests::get_hello()), + ) + .await + .unwrap(); + let message = if render { + web_iface::Message::Render( + web_iface::RenderPayload { + url: "https://example.com".to_owned(), + mode: web_iface::RenderMode::Text, + wait_after_loaded: web_iface::WaitAfterLoaded::Seconds(60), + }, + 1024, + ) + } else { + web_iface::Message::Request( + web_iface::RequestPayload { + url: url.clone(), + method: web_iface::RequestMethod::GET, + headers: Default::default(), + body: None, + sign: false, + }, + 1024, + ) + }; + common::write_message(&mut host, &calldata::encode_obj(&message)) + .await + .unwrap(); + let (mut outbound, _) = tokio::time::timeout(Duration::from_secs(2), server.accept()) + .await + .unwrap() + .unwrap(); + let mut request = Vec::new(); + while !request.ends_with(b"\r\n\r\n") { + request.push(outbound.read_u8().await.unwrap()); + } + if render { + assert!(String::from_utf8_lossy(&request).starts_with("GET /render?")); + } + if body_started { + outbound + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\nResulting-Status: 200\r\n\r\nx") + .await + .unwrap(); + tokio::task::yield_now().await; + } + drop(host); + tokio::time::timeout(Duration::from_secs(2), task) + .await + .unwrap() + .unwrap(); + let mut byte = [0]; + assert_eq!( + tokio::time::timeout(Duration::from_secs(2), outbound.read(&mut byte)) + .await + .unwrap() + .unwrap(), + 0 + ); +} + +#[tokio::test] +async fn host_disconnect_cancels_lua_http_headers() { + disconnect_cancels_request(false, false).await; +} + +#[tokio::test] +async fn host_disconnect_cancels_lua_http_body() { + disconnect_cancels_request(false, true).await; +} + +#[tokio::test] +async fn host_disconnect_cancels_webdriver_render() { + disconnect_cancels_request(true, false).await; +} + +#[tokio::test] +async fn host_disconnect_cancels_webdriver_body() { + disconnect_cancels_request(true, true).await; +} + +#[tokio::test] +async fn allowlisted_request_keeps_unfiltered_redirects() { + let target = tokio::net::TcpListener::bind("127.0.0.2:0").await.unwrap(); + let location = format!("http://{}/", target.local_addr().unwrap()); + let source = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}/", source.local_addr().unwrap()); + let source_task = tokio::spawn(async move { + let (mut stream, _) = source.accept().await.unwrap(); + let mut header = Vec::new(); + while !header.ends_with(b"\r\n\r\n") { + header.push(stream.read_u8().await.unwrap()); + } + stream.write_all(format!("HTTP/1.1 302 Found\r\nLocation: {location}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n").as_bytes()).await.unwrap(); + }); + let target_task = tokio::spawn(async move { + let (mut stream, _) = target.accept().await.unwrap(); + let mut header = Vec::new(); + while !header.ends_with(b"\r\n\r\n") { + header.push(stream.read_u8().await.unwrap()); + } + stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok") + .await + .unwrap(); + }); + let mut config = test_config(String::new()); + config.always_allow_hosts.push("127.0.0.1".into()); + let config = sync::DArc::new(config); + let vm = create_test_vm(&config).await; + let (_, ctx) = vm.create_ctx(&create_test_ctx(&config)).unwrap(); + let payload = vm + .vm + .to_value(&web_iface::RequestPayload { + url, + method: web_iface::RequestMethod::GET, + headers: Default::default(), + body: None, + sign: false, + }) + .unwrap(); + let response: mlua::Table = tokio::time::timeout( + Duration::from_secs(2), + vm.call_fn(&vm.data.request, (ctx, payload)), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(response.get::<String>("body").unwrap(), "ok"); + source_task.await.unwrap(); + target_task.await.unwrap(); +} diff --git a/implementation/src/web/handler.rs b/implementation/src/web/handler.rs index 7fc5692f..44ed323a 100644 --- a/implementation/src/web/handler.rs +++ b/implementation/src/web/handler.rs @@ -110,7 +110,7 @@ impl common::MessageHandlerProvider<genvm_modules_interfaces::web::Message, Rend &hello, &self.config.gep(|x| &x.mod_base), metrics.gep(|x| &x.scripting), - true, + Some(&self.config.extra_tld), )?; Ok(sync::DArc::new(WebSubContext { scripting })) } diff --git a/implementation/src/web/mod.rs b/implementation/src/web/mod.rs index 301fee08..059fd82f 100644 --- a/implementation/src/web/mod.rs +++ b/implementation/src/web/mod.rs @@ -10,7 +10,7 @@ type WebSubContext = crate::manager::execution_context::WebSubContext; pub(crate) mod config; mod ctx; -mod domains; +pub(crate) mod domains; mod handler; #[derive(serde::Serialize, Debug, Default)] diff --git a/implementation/src/web/tests.rs b/implementation/src/web/tests.rs index a13d8819..d417f617 100644 --- a/implementation/src/web/tests.rs +++ b/implementation/src/web/tests.rs @@ -5,6 +5,9 @@ use genvm_modules_interfaces::web as web_iface; use mlua::LuaSerdeExt as _; use tokio::io::AsyncWriteExt as _; +#[path = "cancellation_test.rs"] +mod cancellation; + type TestVM = scripting::UserVM<ctx::VMData, Arc<ctx::CtxPart>, WebSubContext>; /// Absolute path of `<repo>/<rel>`. Tests run with the crate @@ -207,9 +210,7 @@ async fn create_test_vm(config: &sync::DArc<config::Config>) -> TestVM { .unwrap() } -/// Same context as `HandlerProvider::create_execution_context`, including -/// `filter_dns = true` -- the webdriver host is an IP literal, which reqwest -/// never sends through the resolver, so the SSRF guard stays out of the way. +/// Same filtering context as `HandlerProvider::create_execution_context`. fn create_test_ctx(config: &sync::DArc<config::Config>) -> sync::DArc<WebSubContext> { let hello = common::tests::get_hello(); let metrics = sync::DArc::new(Metrics::default()); @@ -218,7 +219,7 @@ fn create_test_ctx(config: &sync::DArc<config::Config>) -> sync::DArc<WebSubCont &hello, &config.gep(|x| &x.mod_base), metrics.gep(|x| &x.scripting), - true, + Some(&config.extra_tld), ) .unwrap(); diff --git a/implementation/tests/dflt_requests.rs b/implementation/tests/dflt_requests.rs index eda571f0..3cd5a6d5 100644 --- a/implementation/tests/dflt_requests.rs +++ b/implementation/tests/dflt_requests.rs @@ -62,7 +62,7 @@ fn create_test_ctx() -> sync::DArc<TestCtx> { signer_url: Arc::from(""), data_dir: String::new(), }); - let scripting = scripting::create_ctx_part(&hello, &conf, metrics, false).unwrap(); + let scripting = scripting::create_ctx_part(&hello, &conf, metrics, None).unwrap(); sync::DArc::new(TestCtx { scripting }) } diff --git a/implementation/tests/lua_request_cancellation.rs b/implementation/tests/lua_request_cancellation.rs new file mode 100644 index 00000000..c8b96920 --- /dev/null +++ b/implementation/tests/lua_request_cancellation.rs @@ -0,0 +1,92 @@ +use std::{collections::BTreeMap, sync::Arc, time::Duration}; + +use genvm_common::sync::DArc; +use genvm_modules::{common, scripting}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; + +#[tokio::test] +async fn cancellation_drops_http_even_when_lua_retains_the_coroutine() { + let config = DArc::new(common::ModuleBaseConfig { + bind_address: None, + vm_count: 1, + lua_script_path: String::new(), + lua_path: String::new(), + signer_headers: Arc::new(BTreeMap::new()), + signer_url: Arc::from(""), + data_dir: String::new(), + }); + let vm = scripting::UserVM::create( + &config, + |_| async { Ok(()) }, + Box::new(|_, _, _: &DArc<()>| Ok(())), + ) + .await + .unwrap(); + let ctx = vm.vm.create_table().unwrap(); + let scripting_ctx = scripting::create_ctx_part( + &common::tests::get_hello(), + &config, + DArc::new(scripting::Metrics::default()), + None, + ) + .unwrap(); + scripting::setup_lua_default_ctx(DArc::new(scripting_ctx), &vm.vm, &ctx).unwrap(); + let request = vm + .vm + .load( + r#" + return function(ctx, url) + retained = coroutine.running() + return __dflt.request(ctx, {method = "GET", url = url, headers = {}}) + end + "#, + ) + .eval::<mlua::Function>() + .unwrap(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}/", listener.local_addr().unwrap()); + let (started_tx, started_rx) = tokio::sync::oneshot::channel(); + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + let mut request = Vec::new(); + while !request.ends_with(b"\r\n\r\n") { + request.push(stream.read_u8().await.unwrap()); + } + started_tx.send(()).unwrap(); + let mut byte = [0]; + assert_eq!(stream.read(&mut byte).await.unwrap(), 0); + }); + let mut call = Box::pin(vm.call_fn::<mlua::Value>(&request, (ctx.clone(), url))); + tokio::select! { + result = &mut call => panic!("request finished before cancellation: {result:?}"), + _ = started_rx => {} + _ = tokio::time::sleep(Duration::from_secs(2)) => panic!("request never reached server"), + } + drop(call); + tokio::time::timeout(Duration::from_secs(2), server) + .await + .unwrap() + .unwrap(); + assert!(vm.vm.globals().get::<mlua::Thread>("retained").is_ok()); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}/", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + let mut request = Vec::new(); + while !request.ends_with(b"\r\n\r\n") { + request.push(stream.read_u8().await.unwrap()); + } + stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok") + .await + .unwrap(); + }); + let result: mlua::Table = + tokio::time::timeout(Duration::from_secs(2), vm.call_fn(&request, (ctx, url))) + .await + .unwrap() + .unwrap(); + assert_eq!(result.get::<String>("body").unwrap(), "ok"); + server.await.unwrap(); +} diff --git a/implementation/tests/module_disconnect.rs b/implementation/tests/module_disconnect.rs new file mode 100644 index 00000000..71edc8ae --- /dev/null +++ b/implementation/tests/module_disconnect.rs @@ -0,0 +1,189 @@ +use std::{ + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::Duration, +}; + +use genvm_common::sync::DArc; +use genvm_modules::common; +use genvm_modules_interfaces::{GenVMHello, GenVMId}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; + +#[derive(Default)] +struct State { + started: tokio::sync::Notify, + dropped: AtomicUsize, + cleanup: AtomicUsize, +} + +struct DropProbe(Arc<State>); + +impl Drop for DropProbe { + fn drop(&mut self) { + self.0.dropped.fetch_add(1, Ordering::SeqCst); + } +} + +struct Context; + +impl common::WithGenVMId for Context { + fn genvm_id(&self) -> GenVMId { + GenVMId(1) + } +} + +struct Provider { + state: Arc<State>, + stall_setup: bool, + stall_request: bool, +} + +struct Handler(Arc<State>, bool); + +impl common::MessageHandler<u64, u64> for Handler { + async fn handle(&self, value: u64) -> anyhow::Result<u64> { + if self.1 { + let _probe = DropProbe(self.0.clone()); + self.0.started.notify_one(); + std::future::pending::<()>().await; + } + Ok(value) + } + + async fn cleanup(&self) -> anyhow::Result<()> { + self.0.cleanup.fetch_add(1, Ordering::SeqCst); + let _probe = DropProbe(self.0.clone()); + std::future::pending::<()>().await; + Ok(()) + } +} + +impl common::MessageHandlerProvider<u64, u64> for Provider { + type Ctx = Context; + fn create_execution_context(&self, _: GenVMHello) -> anyhow::Result<DArc<Context>> { + Ok(DArc::new(Context)) + } + async fn new_handler( + &self, + _: DArc<Context>, + ) -> anyhow::Result<impl common::MessageHandler<u64, u64>> { + if self.stall_setup { + let _probe = DropProbe(self.state.clone()); + self.state.started.notify_one(); + std::future::pending::<()>().await; + } + Ok(Handler(self.state.clone(), self.stall_request)) + } +} + +async fn send(host: &mut tokio::io::DuplexStream, data: &[u8]) { + host.write_u32(data.len() as u32).await.unwrap(); + host.write_all(data).await.unwrap(); +} + +async fn start( + stall_setup: bool, + stall_request: bool, +) -> ( + tokio::io::DuplexStream, + tokio::task::JoinHandle<()>, + Arc<State>, +) { + let state = Arc::new(State::default()); + let provider = Arc::new(Provider { + state: state.clone(), + stall_setup, + stall_request, + }); + let (mut host, module) = tokio::io::duplex(4096); + let task = tokio::spawn(common::handle_stream(provider, module, "test", None)); + send( + &mut host, + &genlayer_calldata::encode_obj(&*common::tests::get_hello()), + ) + .await; + (host, task, state) +} + +#[tokio::test] +async fn disconnect_drops_pending_setup() { + let (host, task, state) = start(true, false).await; + tokio::time::timeout(Duration::from_secs(2), state.started.notified()) + .await + .unwrap(); + drop(host); + tokio::time::timeout(Duration::from_secs(2), task) + .await + .unwrap() + .unwrap(); + assert_eq!(state.dropped.load(Ordering::SeqCst), 1); + assert_eq!(state.cleanup.load(Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn partial_frame_disconnect_drops_request_and_pending_teardown() { + let (mut host, task, state) = start(false, true).await; + send(&mut host, &genlayer_calldata::encode_obj(&1u64)).await; + tokio::time::timeout(Duration::from_secs(2), state.started.notified()) + .await + .unwrap(); + host.write_all(&[0, 0]).await.unwrap(); + drop(host); + tokio::time::timeout(Duration::from_secs(2), task) + .await + .unwrap() + .unwrap(); + assert_eq!(state.dropped.load(Ordering::SeqCst), 2); + assert_eq!(state.cleanup.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn excess_pipelining_cannot_hide_disconnect() { + let (mut host, task, state) = start(false, true).await; + let message = genlayer_calldata::encode_obj(&1u64); + send(&mut host, &message).await; + tokio::time::timeout(Duration::from_secs(2), state.started.notified()) + .await + .unwrap(); + send(&mut host, &message).await; + send(&mut host, &message).await; + tokio::time::timeout(Duration::from_secs(2), task) + .await + .unwrap() + .unwrap(); + assert_eq!(state.dropped.load(Ordering::SeqCst), 2); +} + +#[tokio::test] +async fn sequential_requests_keep_framing_and_cleanup() { + let (mut host, task, state) = start(false, false).await; + for value in [1u64, 2] { + let data = genlayer_calldata::encode_obj(&value); + let length = (data.len() as u32).to_be_bytes(); + host.write_all(&length[..2]).await.unwrap(); + tokio::task::yield_now().await; + host.write_all(&length[2..]).await.unwrap(); + host.write_all(&data).await.unwrap(); + let length = tokio::time::timeout(Duration::from_secs(2), host.read_u32()) + .await + .unwrap() + .unwrap(); + let mut response = vec![0; length as usize]; + host.read_exact(&mut response).await.unwrap(); + let response = + genlayer_calldata::decode_obj::<genvm_modules_interfaces::Result<u64>>(&response) + .unwrap(); + assert!( + matches!(response, genvm_modules_interfaces::Result::Ok(result) if result == value) + ); + } + drop(host); + tokio::time::timeout(Duration::from_secs(2), task) + .await + .unwrap() + .unwrap(); + assert_eq!(state.cleanup.load(Ordering::SeqCst), 1); + assert_eq!(state.dropped.load(Ordering::SeqCst), 1); +} diff --git a/implementation/tests/overloaded.rs b/implementation/tests/overloaded.rs index fe431994..03c56719 100644 --- a/implementation/tests/overloaded.rs +++ b/implementation/tests/overloaded.rs @@ -174,7 +174,7 @@ async fn test_overloaded() { &hello, &config.gep(|x| &x.mod_base), metrics.gep(|x| &x.scripting), - false, + None, ) .unwrap(); let llm_ctx = ctx::CtxPart { diff --git a/implementation/tests/policy_dispatch.rs b/implementation/tests/policy_dispatch.rs index 4eb44cf6..c04796ff 100644 --- a/implementation/tests/policy_dispatch.rs +++ b/implementation/tests/policy_dispatch.rs @@ -207,7 +207,7 @@ async fn run_priced_prompt( &hello, &config.gep(|x| &x.mod_base), metrics.gep(|x| &x.scripting), - false, + None, ) .unwrap(); let llm_ctx = ctx::CtxPart { diff --git a/implementation/tests/timeout.rs b/implementation/tests/timeout.rs index 3953f561..5526aaa2 100644 --- a/implementation/tests/timeout.rs +++ b/implementation/tests/timeout.rs @@ -99,7 +99,7 @@ async fn test_timeout() { &hello, &config.gep(|x| &x.mod_base), metrics.gep(|x| &x.scripting), - false, + None, ) .unwrap(); let llm_ctx = ctx::CtxPart { diff --git a/implementation/tests/web_redirect_policy.rs b/implementation/tests/web_redirect_policy.rs new file mode 100644 index 00000000..eceb1bc3 --- /dev/null +++ b/implementation/tests/web_redirect_policy.rs @@ -0,0 +1,104 @@ +use std::time::Duration; + +use genvm_common::sync::DArc; +use genvm_modules::{common, scripting}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; + +async fn request( + location: &str, + extra_tld: &[Box<str>], + allowed: bool, +) -> anyhow::Result<scripting::Response> { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let proxy = format!("http://{}", listener.local_addr().unwrap()); + let location = location.to_owned(); + let server = tokio::spawn(async move { + for hop in 0..2 { + let accepted = + tokio::time::timeout(Duration::from_millis(500), listener.accept()).await; + if hop == 1 && !allowed { + assert!(accepted.is_err(), "forbidden redirect reached the proxy"); + return; + } + let (mut stream, _) = accepted.unwrap().unwrap(); + let mut header = Vec::new(); + while !header.ends_with(b"\r\n\r\n") { + header.push(stream.read_u8().await.unwrap()); + } + let response = if hop == 0 { + format!("HTTP/1.1 302 Found\r\nLocation: {location}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") + } else { + "HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok".to_owned() + }; + stream.write_all(response.as_bytes()).await.unwrap(); + } + }); + // A local proxy lets allowed port-80 destinations reach the fixture without public DNS. + let client = reqwest::Client::builder() + .proxy(reqwest::Proxy::all(proxy).unwrap()) + .redirect(common::redirect_policy_with_tlds(extra_tld)) + .timeout(Duration::from_secs(2)) + .build() + .unwrap(); + let url = "http://source.com/start"; + let result = scripting::send_request_get_lua_compatible_response_bytes( + &DArc::new(scripting::Metrics::default()), + url, + client.get(url), + false, + 1024, + ) + .await; + server.await.unwrap(); + result +} + +#[tokio::test] +async fn redirect_rejects_nonstandard_port_before_connecting() { + let err = request("http://example.com:8080/", &[], false) + .await + .unwrap_err(); + let err = scripting::try_unwrap_any_err(err).unwrap(); + assert_eq!(err.causes, ["PORT_FORBIDDEN"]); + assert!(!err.fatal); +} + +#[tokio::test] +async fn redirect_rejects_unknown_tld_before_connecting() { + let err = request("http://example.invalid/", &[], false) + .await + .unwrap_err(); + let err = scripting::try_unwrap_any_err(err).unwrap(); + assert_eq!(err.causes, ["TLD_FORBIDDEN"]); + assert!(!err.fatal); +} + +#[tokio::test] +async fn redirect_accepts_configured_extra_tld() { + assert_eq!( + request("http://example.custom/", &["custom".into()], true) + .await + .unwrap() + .body, + b"ok" + ); +} + +#[tokio::test] +async fn redirect_accepts_public_tld_and_relative_location() { + assert_eq!( + request("http://example.org/", &[], true) + .await + .unwrap() + .body, + b"ok" + ); + assert_eq!(request("/next", &[], true).await.unwrap().body, b"ok"); +} + +#[tokio::test] +async fn redirect_keeps_private_ip_protection() { + let err = request("http://127.0.0.1/", &[], false).await.unwrap_err(); + let err = scripting::try_unwrap_any_err(err).unwrap(); + assert_eq!(err.causes, ["ADDRESS_FORBIDDEN"]); +} diff --git a/implementation/tests/web_render_webdriver.rs b/implementation/tests/web_render_webdriver.rs index 24b5e23e..a5b0d5d5 100644 --- a/implementation/tests/web_render_webdriver.rs +++ b/implementation/tests/web_render_webdriver.rs @@ -124,7 +124,7 @@ fn create_filtered_ctx() -> sync::DArc<TestCtx> { signer_url: Arc::from(""), data_dir: String::new(), }); - let scripting = scripting::create_ctx_part(&hello, &conf, metrics, true).unwrap(); + let scripting = scripting::create_ctx_part(&hello, &conf, metrics, Some(&[])).unwrap(); sync::DArc::new(TestCtx { scripting }) } diff --git a/webdriver/src/prj/src/browser/chrome.ts b/webdriver/src/prj/src/browser/chrome.ts index e56f0bd4..84aeca5c 100644 --- a/webdriver/src/prj/src/browser/chrome.ts +++ b/webdriver/src/prj/src/browser/chrome.ts @@ -11,6 +11,11 @@ import { formatDurationMs, } from '../duration.js'; import type * as browser from './index.js'; +import { + createEgressProfile, + egressProxyArgs, + startEgressProxy, +} from '../egress-proxy.js'; class BrowserHolder { browser: pup.Browser; @@ -89,33 +94,58 @@ const PROTOCOL_TIMEOUT_MS = envDurationMs( ); async function newBrowser(): Promise<BrowserHolder> { - const args = [ - '--no-sandbox', - '--disable-dev-shm-usage', - '--disable-accelerated-2d-canvas', - '--no-first-run', - '--no-zygote', - '--disable-gpu', - // No popup can be SSRF-guarded before its first navigation, so none is - // allowed to exist -- see the `targetcreated` handler in `render.ts`. - '--block-new-web-contents', - '--enable-precise-memory-info', - `--js-flags=--max-old-space-size=${RENDERER_MAX_OLD_SPACE_MB}`, - `--renderer-process-limit=${RENDERER_PROCESS_LIMIT}`, - ...CHROME_EXTRA_ARGS, - ]; - const realBrowser = await puppeteer.launch({ - headless: CHROME_HEADLESS, - args, - executablePath: CHROME_EXECUTABLE, - protocolTimeout: PROTOCOL_TIMEOUT_MS, - }); + const proxy = await startEgressProxy({ timeoutMs: PROTOCOL_TIMEOUT_MS }); + let profile: Awaited<ReturnType<typeof createEgressProfile>> | undefined; + let realBrowser: pup.Browser; + try { + profile = await createEgressProfile(); + const args = [ + '--no-sandbox', + '--disable-dev-shm-usage', + '--disable-accelerated-2d-canvas', + '--no-first-run', + '--no-zygote', + '--disable-gpu', + // No popup can be SSRF-guarded before its first navigation, so none is + // allowed to exist -- see the `targetcreated` handler in `render.ts`. + '--block-new-web-contents', + '--enable-precise-memory-info', + `--js-flags=--max-old-space-size=${RENDERER_MAX_OLD_SPACE_MB}`, + `--renderer-process-limit=${RENDERER_PROCESS_LIMIT}`, + ...egressProxyArgs(proxy.url, CHROME_EXTRA_ARGS), + ]; + realBrowser = await puppeteer.launch({ + headless: CHROME_HEADLESS, + args, + executablePath: CHROME_EXECUTABLE, + protocolTimeout: PROTOCOL_TIMEOUT_MS, + userDataDir: profile.path, + }); + } catch (error) { + await proxy.close(); + await profile?.close(); + throw error; + } logger.log('info', 'created new raw browser', { pid: realBrowser.process()?.pid, }); realBrowser.on('disconnected', () => { + void proxy.close(); + const cleanup = () => { + void profile! + .close() + .catch((error) => + logger.log('warn', 'could not remove browser profile', { error }), + ); + }; + const process = realBrowser.process(); + if (process && process.exitCode === null && process.signalCode === null) { + process.once('exit', cleanup); + } else { + cleanup(); + } logger.log('info', 'browser disconnected'); }); diff --git a/webdriver/src/prj/src/cancellation.test.ts b/webdriver/src/prj/src/cancellation.test.ts new file mode 100644 index 00000000..cc86718b --- /dev/null +++ b/webdriver/src/prj/src/cancellation.test.ts @@ -0,0 +1,85 @@ +// genvm-tool-test-tags: feature-web-render + +import assert from 'node:assert/strict'; +import http from 'node:http'; +import { once } from 'node:events'; +import { test } from 'node:test'; +import { setImmediate as nextTurn } from 'node:timers/promises'; +import { abortable, disconnectSignal } from './cancellation.js'; + +for (const disconnect of [false, true]) { + test( + `response signal ${disconnect ? 'aborts on disconnect' : 'survives normal completion'}`, + { timeout: 2000 }, + async (t) => { + let received!: (value: { + signal: AbortSignal; + res: http.ServerResponse; + }) => void; + const requestEnded = new Promise<{ + signal: AbortSignal; + res: http.ServerResponse; + }>((resolve) => { + received = resolve; + }); + const server = http.createServer((req, res) => { + const signal = disconnectSignal(res); + req.on('end', () => { + received({ signal, res }); + }); + req.resume(); + }); + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); + t.after(() => { + server.closeAllConnections(); + server.close(); + }); + const address = server.address(); + assert.ok(address && typeof address !== 'string'); + const client = http.get(`http://127.0.0.1:${address.port}`, { + agent: false, + }); + client.on('error', () => {}); + t.after(() => client.destroy()); + const { signal, res } = await requestEnded; + await nextTurn(); + assert.equal( + signal.aborted, + false, + 'finishing the incoming request is not a disconnect', + ); + const closed = once(res, 'close'); + if (disconnect) { + client.destroy(); + } else { + const response = once(client, 'response'); + res.end('ok'); + const [incoming] = await response; + incoming.resume(); + await once(incoming, 'end'); + } + await closed; + assert.equal(signal.aborted, disconnect); + }, + ); +} + +test('cancelling a browser startup wait does not start rendering later', async () => { + const controller = new AbortController(); + let ready!: () => void; + const startup = new Promise<void>((resolve) => { + ready = resolve; + }); + let renders = 0; + const result = (async () => { + await abortable(controller.signal, () => startup); + renders++; + })(); + const rejected = assert.rejects(result, { name: 'AbortError' }); + controller.abort(); + await rejected; + ready(); + await nextTurn(); + assert.equal(renders, 0); +}); diff --git a/webdriver/src/prj/src/cancellation.ts b/webdriver/src/prj/src/cancellation.ts new file mode 100644 index 00000000..149c3ff4 --- /dev/null +++ b/webdriver/src/prj/src/cancellation.ts @@ -0,0 +1,29 @@ +import type http from 'node:http'; + +export function disconnectSignal(res: http.ServerResponse): AbortSignal { + const controller = new AbortController(); + res.once('close', () => { + if (!res.writableFinished) controller.abort(); + }); + if (res.destroyed && !res.writableFinished) controller.abort(); + return controller.signal; +} + +// Cancels the wait; callers must dispose resources owned by the operation. +export async function abortable<T>( + signal: AbortSignal | undefined, + fn: () => Promise<T>, +): Promise<T> { + signal?.throwIfAborted(); + if (!signal) return await fn(); + let onAbort!: () => void; + const aborted = new Promise<never>((_, reject) => { + onAbort = () => reject(signal.reason); + signal.addEventListener('abort', onAbort, { once: true }); + }); + try { + return await Promise.race([aborted, fn()]); + } finally { + signal.removeEventListener('abort', onAbort); + } +} diff --git a/webdriver/src/prj/src/egress-browser.test.ts b/webdriver/src/prj/src/egress-browser.test.ts new file mode 100644 index 00000000..81cc011c --- /dev/null +++ b/webdriver/src/prj/src/egress-browser.test.ts @@ -0,0 +1,313 @@ +// genvm-tool-test-tags: feature-web-render + +import assert from 'node:assert/strict'; +import http from 'node:http'; +import https from 'node:https'; +import * as net from 'node:net'; +import * as dgram from 'node:dgram'; +import { once } from 'node:events'; +import { createHash } from 'node:crypto'; +import { execFileSync } from 'node:child_process'; +import { mkdtemp, readFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { test } from 'node:test'; +import puppeteer from 'puppeteer-core'; +import { + createEgressProfile, + egressProxyArgs, + startEgressProxy, +} from './egress-proxy.js'; +import { renderPageWithBrowser } from './render.js'; +import { Semaphore } from './semaphore.js'; + +const executablePath = process.env.GVM_WEBDRIVER_CHROME_EXECUTABLE; + +test( + 'Chromium uses the checked proxy for HTTP, HTTPS, redirects and WebSockets', + { + skip: + !executablePath && + 'set GVM_WEBDRIVER_CHROME_EXECUTABLE to run Chromium coverage', + timeout: 30_000, + }, + async (t) => { + const requests: string[] = []; + const upgrades: string[] = []; + const fixture = http.createServer((req, res) => { + requests.push(`${req.headers.host}${req.url}`); + if (req.url === '/redirect') { + res.writeHead(302, { Location: `http://127.0.0.1:${port}/private` }); + res.end(); + } else { + res.end('<html><body>public fixture</body></html>'); + } + }); + fixture.on('upgrade', (req, socket) => { + upgrades.push(req.headers.host!); + const accept = createHash('sha1') + .update( + `${req.headers['sec-websocket-key']}258EAFA5-E914-47DA-95CA-C5AB0DC85B11`, + ) + .digest('base64'); + socket.end( + `HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: ${accept}\r\n\r\n`, + ); + }); + fixture.listen(0, '127.0.0.1'); + await once(fixture, 'listening'); + const port = (fixture.address() as net.AddressInfo).port; + t.after(() => { + fixture.closeAllConnections(); + fixture.close(); + }); + let ipv6Requests = 0; + const ipv6 = http.createServer((_req, res) => { + ipv6Requests++; + res.end('private IPv6 fixture'); + }); + ipv6.listen(0, '::1'); + await once(ipv6, 'listening'); + const ipv6Port = (ipv6.address() as net.AddressInfo).port; + t.after(() => { + ipv6.closeAllConnections(); + ipv6.close(); + }); + const dir = await mkdtemp(join(tmpdir(), 'webdriver-tls-')); + t.after(() => rm(dir, { recursive: true, force: true })); + execFileSync( + 'openssl', + [ + 'req', + '-x509', + '-newkey', + 'rsa:2048', + '-nodes', + '-keyout', + join(dir, 'key.pem'), + '-out', + join(dir, 'cert.pem'), + '-days', + '1', + '-subj', + '/CN=allowed.test', + ], + { stdio: 'ignore' }, + ); + let serverName = ''; + const secure = https.createServer( + { + key: await readFile(join(dir, 'key.pem')), + cert: await readFile(join(dir, 'cert.pem')), + }, + (req, res) => { + serverName = req.headers.host!; + res.end('secure fixture'); + }, + ); + secure.listen(0, '127.0.0.1'); + await once(secure, 'listening'); + const tlsPort = (secure.address() as net.AddressInfo).port; + t.after(() => { + secure.closeAllConnections(); + secure.close(); + }); + let rebind = false; + const destinations: string[] = []; + const proxy = await startEgressProxy({ + resolve: async (host) => + host === 'allowed.test' || (host === 'changing.test' && !rebind) + ? ['8.8.8.8'] + : ['127.0.0.1'], + connect: (address, targetPort) => { + assert.equal(address, '8.8.8.8'); + assert.ok(targetPort === port || targetPort === tlsPort); + destinations.push(address); + return net.createConnection({ host: '127.0.0.1', port: targetPort }); + }, + }); + t.after(() => proxy.close()); + const profile = await createEgressProfile(); + const browser = await puppeteer.launch({ + userDataDir: profile.path, + executablePath: executablePath!, + headless: true, + acceptInsecureCerts: true, + args: ['--no-sandbox', ...egressProxyArgs(proxy.url)], + }); + t.after(async () => { + await browser.close(); + await profile.close(); + }); + const page = await browser.newPage(); + assert.equal( + (await page.goto(`http://allowed.test:${port}/`))?.status(), + 200, + ); + assert.equal( + await page.evaluate(() => document.body.innerText), + 'public fixture', + ); + assert.equal( + (await page.goto(`https://allowed.test:${tlsPort}/`))?.status(), + 200, + ); + assert.equal( + await page.evaluate(() => document.body.innerText), + 'secure fixture', + ); + assert.equal(serverName, `allowed.test:${tlsPort}`); + await page.goto(`http://allowed.test:${port}/`); + const webSocket = (url: string) => + page.evaluate( + (address) => + new Promise<string>((resolve) => { + const socket = new WebSocket(address); + socket.onopen = () => { + socket.close(); + resolve('open'); + }; + socket.onerror = () => resolve('blocked'); + }), + url, + ); + assert.equal(await webSocket(`ws://allowed.test:${port}/`), 'open'); + assert.equal(await webSocket(`ws://127.0.0.1:${port}/`), 'blocked'); + assert.deepEqual(upgrades, [`allowed.test:${port}`]); + await page.evaluate( + (target) => + new Promise<void>((resolve) => { + const image = new Image(); + image.onload = image.onerror = () => resolve(); + image.src = target; + }), + `http://127.0.0.1:${port}/private-image`, + ); + await assert.rejects(page.goto(`http://allowed.test:${port}/redirect`)); + for (const host of ['127.0.0.1', 'localhost']) { + await assert.rejects(page.goto(`http://${host}:${port}/private`)); + } + await assert.rejects(page.goto(`http://[::1]:${ipv6Port}/private`)); + assert.equal(ipv6Requests, 0); + assert.ok(requests.every((url) => !url.includes('/private'))); + await t.test( + 'WebRTC cannot reach private STUN or TURN servers directly', + async (t) => { + const context = await browser.createBrowserContext(); + t.after(() => context.close()); + const rtcPage = await context.newPage(); + let packets = 0; + const udp = dgram.createSocket('udp4'); + udp.on('message', () => { + packets++; + }); + udp.bind(0, '127.0.0.1'); + await once(udp, 'listening'); + t.after(() => udp.close()); + let connections = 0; + const tcp = net.createServer((socket) => { + connections++; + socket.destroy(); + }); + tcp.listen(0, '127.0.0.1'); + await once(tcp, 'listening'); + t.after(() => tcp.close()); + await rtcPage.goto(`http://allowed.test:${port}/`); + await rtcPage.evaluate( + async ({ udpPort, tcpPort }) => { + const peer = new RTCPeerConnection({ + iceServers: [ + { urls: `stun:127.0.0.1:${udpPort}` }, + { + urls: `turn:127.0.0.1:${tcpPort}?transport=tcp`, + username: 'test', + credential: 'test', + }, + ], + }); + try { + peer.createDataChannel('probe'); + await peer.setLocalDescription(await peer.createOffer()); + await new Promise<void>((resolve) => { + const timeout = setTimeout(resolve, 3000); + peer.onicegatheringstatechange = () => { + if (peer.iceGatheringState === 'complete') { + clearTimeout(timeout); + resolve(); + } + }; + }); + } finally { + peer.close(); + } + }, + { + udpPort: udp.address().port, + tcpPort: (tcp.address() as net.AddressInfo).port, + }, + ); + assert.equal(packets, 0); + assert.equal(connections, 0); + }, + ); + const first = await browser.createBrowserContext(); + await (await first.newPage()).goto(`http://changing.test:${port}/`); + await first.close(); + rebind = true; + const count = destinations.length; + const second = await browser.createBrowserContext(); + await assert.rejects( + (await second.newPage()).goto(`http://changing.test:${port}/`), + ); + assert.equal( + destinations.length, + count, + 'rebinding must not open another socket', + ); + await second.close(); + await t.test( + 'disconnect stops a real render and releases its permit', + async () => { + const controller = new AbortController(); + const slots = new Semaphore({ permits: 1, maxQueued: 1 }); + const url = 'data:text/html,<body>cancel-this-render</body>'; + const contexts = browser.browserContexts().length; + const rendering = slots.withPermit( + 1000, + () => + renderPageWithBrowser( + browser, + url, + 'text', + { waitAfterLoaded: 60 }, + controller.signal, + ), + controller.signal, + ); + const rejected = assert.rejects(rendering, { name: 'AbortError' }); + const target = await browser.waitForTarget( + (candidate) => candidate.url() === url, + ); + const renderingPage = await target.page(); + assert.ok(renderingPage); + assert.equal( + await renderingPage.evaluate(() => document.body.innerText), + 'cancel-this-render', + ); + const next = slots.acquire(1000); + controller.abort(); + await rejected; + await next; + assert.equal(renderingPage.isClosed(), true); + assert.equal(browser.browserContexts().length, contexts); + slots.release(); + assert.equal(slots.inFlight, 0); + }, + ); + await proxy.close(); + await assert.rejects( + page.goto(`http://127.0.0.1:${port}/no-direct-fallback`), + ); + assert.ok(requests.every((url) => !url.endsWith('/no-direct-fallback'))); + }, +); diff --git a/webdriver/src/prj/src/egress-proxy.test.ts b/webdriver/src/prj/src/egress-proxy.test.ts new file mode 100644 index 00000000..8cec9658 --- /dev/null +++ b/webdriver/src/prj/src/egress-proxy.test.ts @@ -0,0 +1,299 @@ +// genvm-tool-test-tags: feature-web-render + +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import * as net from 'node:net'; +import { once } from 'node:events'; +import { setImmediate as nextTurn } from 'node:timers/promises'; +import { egressProxyArgs, startEgressProxy } from './egress-proxy.js'; + +function reader(socket: net.Socket) { + const chunks = socket[Symbol.asyncIterator](); + let buffer: Buffer = Buffer.alloc(0); + return async (length: number) => { + while (buffer.length < length) { + const chunk = await chunks.next(); + assert.equal(chunk.done, false, 'socket closed before its reply'); + buffer = Buffer.concat([buffer, chunk.value]); + } + const result = buffer.subarray(0, length); + buffer = buffer.subarray(length); + return result; + }; +} + +function request(host: string, port = 443) { + const name = Buffer.from(host); + const result = Buffer.concat([ + Buffer.from([5, 1, 0, 3, name.length]), + name, + Buffer.alloc(2), + ]); + result.writeUInt16BE(port, result.length - 2); + return result; +} + +async function client(url: string) { + const socket = net.createConnection({ + host: '127.0.0.1', + port: Number(new URL(url).port), + }); + await once(socket, 'connect'); + const read = reader(socket); + socket.write(Buffer.from([5, 1, 0])); + assert.deepEqual(await read(2), Buffer.from([5, 0])); + return { socket, read }; +} + +test( + 'DNS rebinding cannot change the validated socket destination', + { timeout: 3000 }, + async (t) => { + const echo = net.createServer((socket) => socket.pipe(socket)); + echo.listen(0, '127.0.0.1'); + await once(echo, 'listening'); + t.after(() => echo.close()); + const port = (echo.address() as net.AddressInfo).port; + let lookups = 0; + const destinations: string[] = []; + const proxy = await startEgressProxy({ + resolve: async () => (++lookups === 1 ? ['8.8.8.8'] : ['127.0.0.1']), + connect: (address, requestedPort) => { + assert.equal(net.isIP(address), 4); + assert.equal(requestedPort, port); + destinations.push(address); + // Map the approved public destination to a local fixture. + return net.createConnection({ host: '127.0.0.1', port }); + }, + }); + t.after(() => proxy.close()); + const first = await client(proxy.url); + t.after(() => first.socket.destroy()); + for (const byte of request('changing.test', port)) { + first.socket.write(Buffer.from([byte])); + await nextTurn(); + } + assert.equal((await first.read(10))[1], 0); + first.socket.write('ping'); + assert.equal((await first.read(4)).toString(), 'ping'); + assert.equal(lookups, 1); + assert.deepEqual(destinations, ['8.8.8.8']); + const second = await client(proxy.url); + t.after(() => second.socket.destroy()); + second.socket.write(request('changing.test', port)); + assert.equal((await second.read(10))[1], 2); + assert.equal(lookups, 2); + assert.deepEqual( + destinations, + ['8.8.8.8'], + 'a later private DNS answer must never reach connect', + ); + }, +); + +for (const addresses of [ + [], + ['127.0.0.1'], + ['169.254.169.254'], + ['::1'], + ['::ffff:7f00:1'], + ['8.8.8.8', '10.0.0.1'], + ['not-an-ip'], +]) { + test( + `rejects DNS answers ${JSON.stringify(addresses)}`, + { timeout: 2000 }, + async (t) => { + let connections = 0; + const proxy = await startEgressProxy({ + resolve: async () => addresses, + connect: () => { + connections++; + throw new Error('must not connect'); + }, + }); + t.after(() => proxy.close()); + const { socket, read } = await client(proxy.url); + t.after(() => socket.destroy()); + socket.write(request('blocked.test')); + assert.equal((await read(10))[1], 2); + assert.equal(connections, 0); + }, + ); +} + +for (const address of [ + Buffer.from([1, 127, 0, 0, 1]), + Buffer.from([4, ...new Array(15).fill(0), 1]), +]) { + test( + `blocks private IP literals of address type ${address[0]}`, + { timeout: 2000 }, + async (t) => { + let lookups = 0; + const proxy = await startEgressProxy({ + resolve: async () => { + lookups++; + return ['8.8.8.8']; + }, + }); + t.after(() => proxy.close()); + const { socket, read } = await client(proxy.url); + t.after(() => socket.destroy()); + socket.write( + Buffer.concat([Buffer.from([5, 1, 0]), address, Buffer.from([0, 80])]), + ); + assert.equal((await read(10))[1], 2); + assert.equal(lookups, 0); + }, + ); +} + +test( + 'disconnect during DNS lookup does not open a late connection', + { timeout: 2000 }, + async (t) => { + let resolve!: (addresses: string[]) => void; + const pending = new Promise<string[]>((r) => { + resolve = r; + }); + let entered!: () => void; + const resolving = new Promise<void>((r) => { + entered = r; + }); + let connections = 0; + const proxy = await startEgressProxy({ + resolve: () => { + entered(); + return pending; + }, + connect: () => { + connections++; + throw new Error('must not connect'); + }, + }); + t.after(() => proxy.close()); + const socket = net.createConnection({ + host: '127.0.0.1', + port: Number(new URL(proxy.url).port), + }); + t.after(() => socket.destroy()); + await once(socket, 'connect'); + socket.resume(); + socket.write(Buffer.concat([Buffer.from([5, 1, 0]), request('slow.test')])); + await resolving; + const ended = once(socket, 'end'); + socket.end(); + await ended; + resolve(['8.8.8.8']); + await nextTurn(); + assert.equal(connections, 0); + }, +); + +test( + 'DNS failure is rejected without opening a socket', + { timeout: 2000 }, + async (t) => { + const proxy = await startEgressProxy({ + resolve: async () => { + throw new Error('DNS unavailable'); + }, + connect: () => assert.fail('must not connect'), + }); + t.after(() => proxy.close()); + const { socket, read } = await client(proxy.url); + t.after(() => socket.destroy()); + socket.write(request('missing.test')); + assert.equal((await read(10))[1], 4); + }, +); + +test( + 'UDP association and unsupported authentication fail closed', + { timeout: 2000 }, + async (t) => { + const proxy = await startEgressProxy({ + connect: () => assert.fail('must not connect'), + }); + t.after(() => proxy.close()); + const { socket, read } = await client(proxy.url); + t.after(() => socket.destroy()); + const udp = request('test.invalid'); + udp[1] = 3; + socket.write(udp); + assert.equal((await read(10))[1], 7); + const auth = net.createConnection({ + host: '127.0.0.1', + port: Number(new URL(proxy.url).port), + }); + t.after(() => auth.destroy()); + await once(auth, 'connect'); + const authRead = reader(auth); + auth.write(Buffer.from([5, 1, 2])); + assert.deepEqual(await authRead(2), Buffer.from([5, 255])); + }, +); + +test( + 'connection failure retries only the already validated addresses', + { timeout: 2000 }, + async (t) => { + const echo = net.createServer((socket) => socket.pipe(socket)); + echo.listen(0, '127.0.0.1'); + await once(echo, 'listening'); + t.after(() => echo.close()); + const port = (echo.address() as net.AddressInfo).port; + const addresses = ['2606:4700:4700::1111', '8.8.8.8']; + const attempted: string[] = []; + let lookups = 0; + const proxy = await startEgressProxy({ + resolve: async () => { + lookups++; + return addresses; + }, + connect: (address) => { + attempted.push(address); + if (attempted.length === 1) { + const failed = new net.Socket(); + queueMicrotask(() => failed.destroy(new Error('ENETUNREACH'))); + return failed; + } + return net.createConnection({ host: '127.0.0.1', port }); + }, + }); + t.after(() => proxy.close()); + const { socket, read } = await client(proxy.url); + t.after(() => socket.destroy()); + socket.write(request('dual-stack.test')); + assert.equal((await read(10))[1], 0); + socket.write('ok'); + assert.equal((await read(2)).toString(), 'ok'); + assert.deepEqual(attempted, addresses); + assert.equal(lookups, 1); + }, +); + +test('extra Chrome flags cannot override the egress guard', () => { + for (const flag of [ + '--no-proxy-server', + '--proxy-server=direct://', + '--proxy-bypass-list=*', + '--proxy-pac-url=file:///tmp/proxy.pac', + '--proxy-auto-detect', + '--host-resolver-rules=MAP * 127.0.0.1', + '--force-webrtc-ip-handling-policy=default', + '--enable-quic', + '--user-data-dir=/tmp/unguarded', + '--profile-directory=unguarded', + '--guest', + '-proxy-server=direct://', + '-no-proxy-server', + '--Proxy-Server=direct://', + ]) { + assert.throws( + () => egressProxyArgs('socks5://127.0.0.1:1080', [flag]), + /conflicts with the egress guard/, + ); + } +}); diff --git a/webdriver/src/prj/src/egress-proxy.ts b/webdriver/src/prj/src/egress-proxy.ts new file mode 100644 index 00000000..f99b7898 --- /dev/null +++ b/webdriver/src/prj/src/egress-proxy.ts @@ -0,0 +1,220 @@ +import * as net from 'node:net'; +import { once } from 'node:events'; +import { mkdtemp, mkdir, writeFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { ipIsBad, resolveHost } from './ssrf.js'; +import * as logger from './logging.js'; + +interface ProxyOptions { + resolve?: (host: string) => Promise<string[]>; + connect?: (address: string, port: number) => net.Socket; + timeoutMs?: number; +} + +export async function startEgressProxy({ + resolve = resolveHost, + connect = (host, port) => net.createConnection({ host, port }), + timeoutMs = 90_000, +}: ProxyOptions = {}) { + const clients = new Set<net.Socket>(); + const server = net.createServer((client) => { + clients.add(client); + let upstream: net.Socket | undefined; + let greeted = false; + let buffer: Buffer = Buffer.alloc(0); + client.setTimeout(timeoutMs, () => client.destroy()); + client.on('error', () => client.destroy()); + client.on('close', () => { + clients.delete(client); + upstream?.destroy(); + }); + const reject = (status: number) => { + client.end(Buffer.from([5, status, 0, 1, 0, 0, 0, 0, 0, 0])); + }; + const open = async (host: string, port: number, initial: Buffer) => { + let pending = initial; + const collect = (chunk: Buffer) => { + pending = Buffer.concat([pending, chunk]); + if (pending.length > 65536) client.destroy(); + }; + client.on('data', collect); + try { + const addresses = net.isIP(host) ? [host] : await resolve(host); + if (client.destroyed) return; + if (!port || !addresses.length || addresses.some(ipIsBad)) { + logger.log('debug', 'egress proxy blocked destination', { + host, + port, + }); + reject(2); + return; + } + const dial = (index: number) => { + if (client.destroyed) return; + if (index === addresses.length) { + reject(5); + return; + } + // Every retry uses the same validated answer, never another lookup. + const socket = connect(addresses[index]!, port); + upstream = socket; + let connected = false; + socket.setTimeout(Math.min(10_000, timeoutMs), () => + socket.destroy(new Error('connect timeout')), + ); + socket.on('error', () => { + if (connected) client.destroy(); + else { + upstream = undefined; + socket.destroy(); + dial(index + 1); + } + }); + socket.on('close', () => { + if (upstream === socket) client.destroy(); + }); + socket.once('connect', () => { + if (client.destroyed) { + socket.destroy(); + return; + } + connected = true; + socket.setTimeout(0); + client.off('data', collect); + client.write(Buffer.from([5, 0, 0, 1, 0, 0, 0, 0, 0, 0])); + if (pending.length) socket.write(pending); + client.pipe(socket).pipe(client); + }); + }; + dial(0); + } catch { + if (!client.destroyed) reject(4); + } + }; + const handshake = (chunk: Buffer) => { + buffer = Buffer.concat([buffer, chunk]); + if (buffer.length > 65536) { + client.destroy(); + return; + } + if (!greeted) { + if (buffer.length < 2) return; + const length = 2 + buffer[1]!; + if (buffer[0] !== 5 || buffer[1] === 0) { + client.destroy(); + return; + } + if (buffer.length < length) return; + if (!buffer.subarray(2, length).includes(0)) { + client.end(Buffer.from([5, 255])); + return; + } + buffer = buffer.subarray(length); + greeted = true; + client.write(Buffer.from([5, 0])); + } + if (buffer.length < 5) return; + if (buffer[0] !== 5 || buffer[1] !== 1 || buffer[2] !== 0) { + client.off('data', handshake); + reject(7); + return; + } + const kind = buffer[3]; + const length = + kind === 1 ? 10 : kind === 4 ? 22 : kind === 3 ? 7 + buffer[4]! : 0; + if (!length) { + client.off('data', handshake); + reject(8); + return; + } + if (buffer.length < length) return; + let host: string; + if (kind === 1) { + host = [...buffer.subarray(4, 8)].join('.'); + } else if (kind === 4) { + host = Array.from({ length: 8 }, (_, i) => + buffer.readUInt16BE(4 + i * 2).toString(16), + ).join(':'); + } else { + host = buffer.subarray(5, length - 2).toString('utf8'); + } + client.off('data', handshake); + if (!host || host.includes('\0')) { + reject(4); + return; + } + void open(host, buffer.readUInt16BE(length - 2), buffer.subarray(length)); + buffer = Buffer.alloc(0); + }; + client.on('data', handshake); + }); + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); + const address = server.address() as net.AddressInfo; + return { + url: `socks5://127.0.0.1:${address.port}`, + close: async () => { + const closed = new Promise<void>((resolve) => + server.close(() => resolve()), + ); + for (const client of clients) client.destroy(); + await closed; + }, + }; +} + +export function egressProxyArgs( + url: string, + extraArgs: string[] = [], +): string[] { + const protectedFlags = new Set([ + '--no-proxy-server', + '--proxy-server', + '--proxy-bypass-list', + '--proxy-pac-url', + '--proxy-auto-detect', + '--host-resolver-rules', + '--force-webrtc-ip-handling-policy', + '--enable-quic', + '--user-data-dir', + '--profile-directory', + '--guest', + ]); + for (const arg of extraArgs) { + // Chromium also accepts single-dash switches. + const name = arg.split('=')[0]!.toLowerCase().replace(/^-+/, '--'); + if (protectedFlags.has(name)) { + throw new Error( + `Chrome argument conflicts with the egress guard: ${arg}`, + ); + } + } + return [ + ...extraArgs, + `--proxy-server=${url}`, + '--proxy-bypass-list=<-loopback>', + '--host-resolver-rules=MAP * ~NOTFOUND, EXCLUDE 127.0.0.1', + '--disable-quic', + '--force-webrtc-ip-handling-policy=disable_non_proxied_udp', + ]; +} + +export async function createEgressProfile() { + const path = await mkdtemp(join(tmpdir(), 'genvm-chromium-')); + const close = () => rm(path, { recursive: true, force: true }); + try { + await mkdir(join(path, 'Default')); + // Chrome's profile preference overrides the content-layer command-line flag. + await writeFile( + join(path, 'Default', 'Preferences'), + JSON.stringify({ + webrtc: { ip_handling_policy: 'disable_non_proxied_udp' }, + }), + ); + return { path, close }; + } catch (error) { + await close(); + throw error; + } +} diff --git a/webdriver/src/prj/src/index.ts b/webdriver/src/prj/src/index.ts index 0bfed63f..2d708375 100644 --- a/webdriver/src/prj/src/index.ts +++ b/webdriver/src/prj/src/index.ts @@ -4,6 +4,7 @@ import { Command } from 'commander'; import * as logger from './logging.js'; import * as chromeBrowser from './browser/chrome.js'; +import { abortable, disconnectSignal } from './cancellation.js'; import { renderPageWithBrowser, statusIsGood, @@ -104,7 +105,11 @@ async function renderPage( } try { - const browserManager = await chromeBrowser.INSTANCE; + const browserManager = await abortable( + signal, + () => chromeBrowser.INSTANCE, + ); + signal?.throwIfAborted(); const browserInstance = browserManager.getBrowser(); try { return await renderPageWithBrowser( @@ -112,6 +117,7 @@ async function renderPage( targetUrl, mode, options, + signal, ); } finally { browserInstance.close(); @@ -121,29 +127,13 @@ async function renderPage( } } -/** - * Fires when the caller goes away before it has been answered, so a request - * nobody is waiting for stops occupying a queue slot. - */ -function disconnectSignal( - req: http.IncomingMessage, - res: http.ServerResponse, -): AbortSignal { - const controller = new AbortController(); - req.on('close', () => { - if (!res.writableEnded) { - controller.abort(); - } - }); - return controller.signal; -} - async function handleRenderRequest( parsedUrl: URL, req: http.IncomingMessage, res: http.ServerResponse, ) { const query = parsedUrl.searchParams; + const signal = disconnectSignal(res); try { const targetUrl = query.get('url') ?? ''; @@ -176,12 +166,7 @@ async function handleRenderRequest( : {}), }; - const result = await renderPage( - targetUrl, - mode, - options, - disconnectSignal(req, res), - ); + const result = await renderPage(targetUrl, mode, options, signal); if (statusIsGood(result.status)) { updateLastSuccessfulRenderTime(); @@ -196,8 +181,8 @@ async function handleRenderRequest( } res.end(result.body); } catch (error) { - if (error instanceof SemaphoreAborted) { - // The caller hung up while queued; there is nobody left to answer. + if (signal.aborted || error instanceof SemaphoreAborted) { + // The caller hung up; there is nobody left to answer. return; } // How loaded this host happens to be is not a property of the page, so it @@ -275,13 +260,9 @@ async function handleHealthcheck( sinceLast: formatDurationMs(sinceLastSuccessMs), cacheDuration: formatDurationMs(HEALTHCHECK_CACHE_DURATION_MS), }); + const signal = disconnectSignal(res); try { - const result = await renderPage( - targetUrl, - mode, - {}, - disconnectSignal(req, res), - ); + const result = await renderPage(targetUrl, mode, {}, signal); if (statusIsGood(result.status)) { updateLastSuccessfulRenderTime(); logger.log('info', 'healthcheck ok', { status: result.status }); @@ -293,7 +274,7 @@ async function handleHealthcheck( res.end('unhealthy'); } } catch (error) { - if (error instanceof SemaphoreAborted) { + if (signal.aborted || error instanceof SemaphoreAborted) { return; } // Saturation lands here too, and reporting unhealthy is right: the probe diff --git a/webdriver/src/prj/src/render.test.ts b/webdriver/src/prj/src/render.test.ts index 6c3fc1b0..0533aa76 100644 --- a/webdriver/src/prj/src/render.test.ts +++ b/webdriver/src/prj/src/render.test.ts @@ -2,7 +2,7 @@ * Failures of this sidecar must leave `renderPageWithBrowser` by being thrown, * and must not be dressed up as a page status. * - * `handleRenderRequest` turns anything thrown out of here into a bare `500`, + * Except for disconnected callers, `handleRenderRequest` turns throws into a bare `500`, * and the module classifies a `500` from its own webdriver as a fatal * `WEBDRIVER_UNAVAILABLE`. That is the intended outcome: a validator whose own * sidecar is broken has no observation of the page, so it must abstain (the @@ -35,6 +35,11 @@ import type * as pup from 'puppeteer-core'; import { ProtocolError, TimeoutError } from 'puppeteer-core'; import { renderPageWithBrowser, statusIsGood } from './render.js'; +import { Semaphore } from './semaphore.js'; +import { setImmediate as nextTurn } from 'node:timers/promises'; +import http from 'node:http'; +import { once } from 'node:events'; +import { disconnectSignal } from './cancellation.js'; interface FakeContext { closes: number; @@ -79,6 +84,262 @@ async function render(browser: pup.Browser) { return renderPageWithBrowser(browser, 'https://example.com/', 'text'); } +for (const stage of [ + 'newPage', + 'guard', + 'viewport', + 'navigation', + 'wait', + 'extraction', + 'heap', + 'finalHeap', + 'screenshot', +] as const) { + test( + `disconnect during ${stage} closes the target and releases its permit`, + { timeout: 2000 }, + async () => { + const controller = new AbortController(); + const slots = new Semaphore({ permits: 1, maxQueued: 1 }); + let entered!: () => void; + const ready = new Promise<void>((resolve) => { + entered = resolve; + }); + const pending = () => { + entered(); + return new Promise<never>(() => {}); + }; + const closes = { page: 0, context: 0 }; + let extractions = 0; + const page = { + setRequestInterception: async () => { + if (stage === 'guard') await pending(); + }, + setViewport: async () => { + if (stage === 'viewport') await pending(); + }, + goto: async () => { + if (stage === 'navigation') await pending(); + if (stage === 'wait') entered(); + return { status: () => 200 }; + }, + evaluate: async (_fn: unknown, ...args: unknown[]) => { + if (args.length === 0) { + if (stage === 'heap' || stage === 'finalHeap') await pending(); + return 0; + } + extractions++; + if (stage === 'extraction') await pending(); + return { tooLarge: false, content: 'ok' }; + }, + close: async () => { + closes.page++; + }, + screenshot: pending, + on: () => {}, + }; + const context = { + newPage: async () => { + if (stage === 'newPage') await pending(); + return page; + }, + close: async () => { + closes.context++; + }, + on: () => {}, + }; + const browser = { + createBrowserContext: async () => context, + } as unknown as pup.Browser; + const result = slots.withPermit( + 1000, + () => + renderPageWithBrowser( + browser, + 'https://example.com/', + stage === 'screenshot' ? 'screenshot' : 'text', + { waitAfterLoaded: stage === 'wait' || stage === 'heap' ? 60 : 0 }, + controller.signal, + ), + controller.signal, + ); + const rejected = assert.rejects(result, { name: 'AbortError' }); + await ready; + // Let the navigation continuation enter its post-load delay. + await nextTurn(); + const next = slots.acquire(1000); + assert.equal(slots.queued, 1); + controller.abort(); + await rejected; + await next; + assert.deepEqual(closes, { + page: stage === 'newPage' ? 0 : 1, + context: 1, + }); + assert.equal( + extractions, + stage === 'extraction' || stage === 'finalHeap' ? 1 : 0, + ); + assert.equal(slots.queued, 0); + slots.release(); + assert.equal(slots.inFlight, 0); + }, + ); +} + +test( + 'disconnect during context creation disposes a late context', + { timeout: 2000 }, + async () => { + const controller = new AbortController(); + let created!: (context: pup.BrowserContext) => void; + const pending = new Promise<pup.BrowserContext>((resolve) => { + created = resolve; + }); + const browser = { + createBrowserContext: () => pending, + } as unknown as pup.Browser; + const result = renderPageWithBrowser( + browser, + 'https://example.com/', + 'text', + {}, + controller.signal, + ); + const rejected = assert.rejects(result, { name: 'AbortError' }); + controller.abort(); + await rejected; + let closes = 0; + created({ + close: async () => { + closes++; + }, + newPage: () => assert.fail('cancelled render opened a page'), + } as unknown as pup.BrowserContext); + await nextTurn(); + assert.equal(closes, 1); + }, +); + +test( + 'HTTP disconnect frees an active render slot for the queued request', + { timeout: 3000 }, + async (t) => { + const slots = new Semaphore({ permits: 1, maxQueued: 1 }); + let entered!: () => void; + const navigating = new Promise<void>((resolve) => { + entered = resolve; + }); + let queued!: () => void; + const waiting = new Promise<void>((resolve) => { + queued = resolve; + }); + let contexts = 0; + let closes = 0; + const browser = { + createBrowserContext: async () => { + const first = ++contexts === 1; + if (!first) + assert.equal( + closes, + 1, + 'dispose the first context before reusing its permit', + ); + return { + newPage: async () => ({ + setRequestInterception: async () => {}, + setViewport: async () => {}, + on: () => {}, + goto: async () => { + if (first) { + entered(); + await new Promise(() => {}); + } + return { status: () => 200 }; + }, + evaluate: async (_fn: unknown, ...args: unknown[]) => + args.length ? { tooLarge: false, content: 'ok' } : 0, + close: async () => {}, + }), + close: async () => { + closes++; + }, + on: () => {}, + }; + }, + } as unknown as pup.Browser; + const errors: unknown[] = []; + const server = http.createServer(async (req, res) => { + const signal = disconnectSignal(res); + req.resume(); + try { + const rendering = slots.withPermit( + 2000, + () => + renderPageWithBrowser( + browser, + 'https://example.com/', + 'text', + {}, + signal, + ), + signal, + ); + if (slots.queued) queued(); + const result = await rendering; + res.end(result.body); + } catch (error) { + if (!signal.aborted) errors.push(error); + res.destroy(); + } + }); + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); + t.after(() => { + server.closeAllConnections(); + server.close(); + }); + const address = server.address(); + assert.ok(address && typeof address !== 'string'); + const url = `http://127.0.0.1:${address.port}`; + const first = http.get(url, { agent: false }); + first.on('error', () => {}); + t.after(() => first.destroy()); + await navigating; + const second = http.get(url, { agent: false }); + t.after(() => second.destroy()); + const response = once(second, 'response'); + await waiting; + assert.equal(contexts, 1); + first.destroy(); + const [incoming] = await response; + let body = ''; + for await (const chunk of incoming) body += chunk; + assert.equal(body, 'ok'); + assert.equal(contexts, 2); + assert.equal(closes, 2); + assert.equal(slots.inFlight, 0); + assert.equal(slots.queued, 0); + assert.deepEqual(errors, []); + }, +); + +test('an already disconnected caller never creates a context', async () => { + const browser = { + createBrowserContext: () => assert.fail('created a context'), + } as unknown as pup.Browser; + await assert.rejects( + renderPageWithBrowser( + browser, + 'https://example.com/', + 'text', + {}, + AbortSignal.abort(), + ), + { name: 'AbortError' }, + ); +}); + // -- a broken sidecar raises, it does not report ---------------------- test('a protocol timeout opening a page is thrown, not reported', async () => { diff --git a/webdriver/src/prj/src/render.ts b/webdriver/src/prj/src/render.ts index be2fd77b..bfdcf25d 100644 --- a/webdriver/src/prj/src/render.ts +++ b/webdriver/src/prj/src/render.ts @@ -8,7 +8,8 @@ * The contract this file upholds: everything it *returns* is an observation * about the page, reported as a status the caller puts in `Resulting-Status` * alongside a `200`, which the module surfaces to the contract as a catchable - * `WEBPAGE_LOAD_FAILED`. Everything it *throws* is this sidecar failing. The + * `WEBPAGE_LOAD_FAILED`. Except for cancellation when the caller disconnects, + * everything it *throws* is this sidecar failing. The * caller turns a throw into a `500`, which the module classifies as fatal, so * the validator abstains rather than voting on a page it never observed. "This * sidecar failing" includes the host it runs on: a machine with no network is @@ -21,9 +22,11 @@ */ import type * as pup from 'puppeteer-core'; +import { setTimeout as delay } from 'node:timers/promises'; import * as logger from './logging.js'; import * as ssrf from './ssrf.js'; +import { abortable } from './cancellation.js'; import { envDurationMs, envInt, @@ -244,23 +247,24 @@ async function withHeapMonitor<T>( page: pup.Page, maxHeapMB: number, fn: () => Promise<T>, + signal?: AbortSignal, ): Promise<T> { - let stopped = false; - let stopResolve: () => void; - const stopPromise = new Promise<void>((r) => { - stopResolve = r; + const stop = new AbortController(); + const stopped = new Promise<void>((resolve) => { + stop.signal.addEventListener('abort', () => resolve(), { once: true }); }); const monitor = (async () => { - while (!stopped) { - await new Promise((r) => setTimeout(r, HEAP_CHECK_INTERVAL_MS)); - if (stopped) break; + while (!stop.signal.aborted) { + await delay(HEAP_CHECK_INTERVAL_MS, undefined, { signal: stop.signal }); let totalMB: number; try { - totalMB = await page.evaluate( - () => (performance as any).memory?.totalJSHeapSize / 1024 / 1024, + totalMB = await abortable(stop.signal, () => + page.evaluate( + () => (performance as any).memory?.totalJSHeapSize / 1024 / 1024, + ), ); } catch { - await stopPromise; + await stopped; return; } logger.log('debug', 'heap monitor check', { @@ -273,16 +277,18 @@ async function withHeapMonitor<T>( })(); try { - const result = await Promise.race([ - fn(), - monitor.then(() => undefined as never), - ]); + const result = await abortable(signal, () => + Promise.race([fn(), monitor.then(() => undefined as never)]), + ); let totalMB: number; try { - totalMB = await page.evaluate( - () => (performance as any).memory?.totalJSHeapSize / 1024 / 1024, + totalMB = await abortable(signal, () => + page.evaluate( + () => (performance as any).memory?.totalJSHeapSize / 1024 / 1024, + ), ); } catch { + signal?.throwIfAborted(); return result; } if (totalMB > maxHeapMB) { @@ -290,8 +296,7 @@ async function withHeapMonitor<T>( } return result; } finally { - stopped = true; - stopResolve!(); + stop.abort(); await monitor.catch(() => {}); } } @@ -310,15 +315,26 @@ export function statusIsGood(status: number): boolean { */ async function newRenderTarget( browserInstance: pup.Browser, + signal?: AbortSignal, ): Promise<{ context: pup.BrowserContext; page: pup.Page }> { // Each render runs in its own browser context so cookies, localStorage, // IndexedDB, service workers and HSTS state never leak between tenants // sharing this long-lived browser. - const context = await browserInstance.createBrowserContext(); + signal?.throwIfAborted(); + const pendingContext = browserInstance.createBrowserContext(); + let context: pup.BrowserContext; + try { + context = await abortable(signal, () => pendingContext); + } catch (error) { + // CDP creation cannot be cancelled; dispose a context that arrives late. + void pendingContext.then((late) => late.close()).catch(() => {}); + throw error; + } try { - return { context, page: await context.newPage() }; + return { context, page: await abortable(signal, () => context.newPage()) }; } catch (error) { await context.close().catch(() => {}); + signal?.throwIfAborted(); // Logged here as well as re-raised: this is the last point that still // knows the failure was ours rather than the page's, and the `500` the // caller sends carries only a message. @@ -335,6 +351,7 @@ export async function renderPageWithBrowser( targetUrl: string, mode: 'text' | 'html' | 'screenshot', options: RenderOptions = {}, + signal?: AbortSignal, ): Promise<{ status: number; body: any }> { const { loadTimeout = 30000, @@ -351,10 +368,10 @@ export async function renderPageWithBrowser( // the validator abstains instead of reporting a page outcome it never // observed. It must NOT be folded into a `Resulting-Status`, which is the // channel reserved for what actually happened to the page. - const { context, page } = await newRenderTarget(browserInstance); + const { context, page } = await newRenderTarget(browserInstance, signal); try { - await ssrf.installSsrfGuard(page); + await abortable(signal, () => ssrf.installSsrfGuard(page)); // A render needs exactly one page, and a popup cannot be guarded after // the fact: nothing holds its first navigation while an async handler // installs an interceptor, so it could already have reached an internal @@ -371,53 +388,61 @@ export async function renderPageWithBrowser( } })(); }); - page.setViewport({ width: 1920 / 2, height: 1080 / 2 }); - - return await withHeapMonitor(page, maxPageHeapMB, async () => { - const navigationResult = await navigateToPage(page, targetUrl, { - waitUntil, - timeout: loadTimeout, - }); - - if (navigationResult.error) { - return { - status: navigationResult.status, - body: navigationResult.error, - }; - } + await abortable(signal, () => + page.setViewport({ width: 1920 / 2, height: 1080 / 2 }), + ); - const statusCode = navigationResult.status; + return await withHeapMonitor( + page, + maxPageHeapMB, + async () => { + const navigationResult = await navigateToPage(page, targetUrl, { + waitUntil, + timeout: loadTimeout, + }); + signal?.throwIfAborted(); + + if (navigationResult.error) { + return { + status: navigationResult.status, + body: navigationResult.error, + }; + } - if (statusIsGood(statusCode) && waitAfterLoaded > 0) { - let waitMs = Math.floor(waitAfterLoaded * 1000); - if (waitMs > MAX_WAIT_AFTER_LOADED_MS) { - logger.log('warn', 'waitAfterLoaded clamped to maximum', { - url: targetUrl, - requested: formatDurationMs(waitMs), - max: formatDurationMs(MAX_WAIT_AFTER_LOADED_MS), - }); - waitMs = MAX_WAIT_AFTER_LOADED_MS; + const statusCode = navigationResult.status; + + if (statusIsGood(statusCode) && waitAfterLoaded > 0) { + let waitMs = Math.floor(waitAfterLoaded * 1000); + if (waitMs > MAX_WAIT_AFTER_LOADED_MS) { + logger.log('warn', 'waitAfterLoaded clamped to maximum', { + url: targetUrl, + requested: formatDurationMs(waitMs), + max: formatDurationMs(MAX_WAIT_AFTER_LOADED_MS), + }); + waitMs = MAX_WAIT_AFTER_LOADED_MS; + } + await delay(waitMs, undefined, { signal }); } - await new Promise((resolve) => setTimeout(resolve, waitMs)); - } - let data; - switch (mode) { - case 'text': - data = await asText(page, maxResponseChars); - break; - case 'html': - data = await asHTML(page, maxResponseChars); - break; - case 'screenshot': - data = await asScreenshot(page, maxResponseChars); - break; - default: - data = 'Invalid mode'; - } + let data; + switch (mode) { + case 'text': + data = await asText(page, maxResponseChars); + break; + case 'html': + data = await asHTML(page, maxResponseChars); + break; + case 'screenshot': + data = await asScreenshot(page, maxResponseChars); + break; + default: + data = 'Invalid mode'; + } - return { status: statusCode, body: data }; - }); + return { status: statusCode, body: data }; + }, + signal, + ); } catch (e) { // Both limits are a property of the page rather than of this sidecar -- // the same document would exceed them anywhere -- so they belong on the diff --git a/webdriver/src/prj/src/semaphore.test.ts b/webdriver/src/prj/src/semaphore.test.ts index afc6d3fa..0d4c9b67 100644 --- a/webdriver/src/prj/src/semaphore.test.ts +++ b/webdriver/src/prj/src/semaphore.test.ts @@ -8,6 +8,12 @@ import { SemaphoreTimeout, } from './semaphore.js'; +it('an already aborted caller cannot take a free permit', async () => { + const s = new Semaphore({ permits: 1, maxQueued: 0 }); + await assert.rejects(s.acquire(1000, AbortSignal.abort()), SemaphoreAborted); + assert.equal(s.inFlight, 0); +}); + const NEVER = 60_000; /** Queue depth is irrelevant to most cases, so default it out of the way. */ diff --git a/webdriver/src/prj/src/semaphore.ts b/webdriver/src/prj/src/semaphore.ts index 84197977..7241f150 100644 --- a/webdriver/src/prj/src/semaphore.ts +++ b/webdriver/src/prj/src/semaphore.ts @@ -64,6 +64,9 @@ export class Semaphore { * @throws {SemaphoreAborted} if `signal` fired while waiting. */ acquire(timeoutMs: number, signal?: AbortSignal): Promise<void> { + if (signal?.aborted) { + return Promise.reject(new SemaphoreAborted()); + } if (this.available > 0) { this.available -= 1; return Promise.resolve(); @@ -71,9 +74,6 @@ export class Semaphore { if (this.waiters.length >= this.maxQueued) { return Promise.reject(new SemaphoreQueueFull(this.maxQueued)); } - if (signal?.aborted) { - return Promise.reject(new SemaphoreAborted()); - } return new Promise<void>((resolve, reject) => { const remove = () => { diff --git a/webdriver/src/prj/src/ssrf.ts b/webdriver/src/prj/src/ssrf.ts index 336c53f1..09ed9222 100644 --- a/webdriver/src/prj/src/ssrf.ts +++ b/webdriver/src/prj/src/ssrf.ts @@ -6,19 +6,9 @@ import * as logger from './logging.js'; /** * SSRF guard for the webdriver. * - * Chrome resolves DNS and follows redirects itself, so it never passes through - * any host-side filter. Without this guard a contract could point an otherwise - * valid public URL (or one of the page's sub-resources) at an internal address - * such as cloud metadata (169.254.169.254), loopback or an RFC1918 host. - * - * We intercept every request the page makes (top-level navigation, - * sub-resources and each redirect hop), resolve the host ourselves and abort - * the request when any resolved address is not globally routable. - * - * NOTE: resolution happens here while Chrome connects separately, so a low-TTL - * attacker controlling DNS could in theory return a routable address to us and - * a private one to Chrome. We mitigate by failing closed: if *any* address a - * host resolves to is private, the host is blocked outright. + * Interception rejects unsafe URLs early. The mandatory egress proxy performs + * the authoritative address check and connects to that literal IP, preventing + * a later DNS answer from redirecting the connection to a private address. */ function parseIPv4(ip: string): [number, number, number, number] | null { @@ -171,7 +161,7 @@ export function ipIsBad(ip: string): boolean { } /** Resolve a URL host to its addresses; IP literals resolve to themselves. */ -async function resolveHost(host: string): Promise<string[]> { +export async function resolveHost(host: string): Promise<string[]> { const stripped = host.startsWith('[') && host.endsWith(']') ? host.slice(1, -1) : host; if (net.isIP(stripped) !== 0) {