diff --git a/.github/ci-paths.toml b/.github/ci-paths.toml index 64753f7b4a..f9f2a53bf4 100644 --- a/.github/ci-paths.toml +++ b/.github/ci-paths.toml @@ -1,18 +1,21 @@ -# Shared runtime, dependency, and CI changes can affect every consumer. +# Shared runtime and dependency changes can affect every consumer. # Patterns use Python fnmatch semantics: * also matches path separators. full = [ - "src/**", "openapi/**", "pyproject.toml", "uv.lock", "Makefile", "tox.ini", - ".mise.toml", ".pre-commit-config.yaml", ".github/actions/**", ".github/workflows/**", - ".github/ci-paths.toml", "scripts/ci_changes.py", "tests/test_ci_changes.py", + "Makefile", "tox.ini", + ".mise.toml", ".pre-commit-config.yaml", ".github/actions/setup-python-env/**", + ".github/workflows/ci-changes.yml", ".github/workflows/master.yml", + ".github/ci-paths.toml", "scripts/ci_changes.py", "tests/test_ci_changes.py", "tests/test_ci_pipeline.py", "tests/conftest.py", "tests/**/conftest.py", "tests/fixtures/**", "integrations/capabilities.toml", "integrations/langchain/pyproject.toml", "integrations/langgraph/pyproject.toml", "integrations/opendal/pyproject.toml", ] -# These files still receive the always-running quality and license checks. +# These files still receive the always-running preflight checks. ignore = [ "AGENTS.md", "REVIEW.md", "CONTRIBUTING.md", "GOVERNANCE.md", "LICENSE", "LICENSES/**", ".github/ISSUE_TEMPLATE/**", ".github/pull_request_template.md", ".github/CODEOWNERS", + ".github/dependabot.yml", ".licenserc.yaml", + "scripts/ci_queue_report.py", ] [suites] @@ -22,7 +25,7 @@ tests = [ "scripts/sync_dsh_plugin_tree.py", "scripts/generate_integration_manifest_docs.py", "scripts/run_atomic_memory_clean_migration.py", "scripts/scan_e2e_evidence.sh", "scripts/check_atomic_memory_migration_shards.py", - "scripts/ci_role_smoke.py", + "scripts/ci_role_smoke.py", "scripts/ci_installation_roles.py", # These evaluator suites still live under the root tests/ directory. "evaluation/memory/locomo/**", "evaluation/memory/locomo_plus/**", "evaluation/memory/write_gate/**", ] @@ -34,7 +37,9 @@ database = [ "tests/e2e/*native_code*", "tests/e2e/test_recall_sufficiency_gate.py", "tests/e2e/test_context_text_assembly.py", "tests/e2e/test_scope_search_migration.py", "scripts/run_atomic_memory_clean_migration.py", - "scripts/check_atomic_memory_migration_shards.py", + ".github/workflows/e2e-harness.yml", ".github/actions/setup-upgrade-oceanbase/**", + "tests/native/test_oceanbase_engine_recovery.py", + "tests/*migration*.py", "scripts/check_atomic_memory_migration_shards.py", ] dify = ["integrations/dify/**", "tests/*dify*"] hermes = ["integrations/hermes/**", "tests/*hermes*"] @@ -59,6 +64,7 @@ docs_tests = ["docs/**", "README*.md"] native = ["tests/native/**", "tests/test_service*.py", "tests/test_cursor_secret.py", "tests/test_server.py"] harness = [ "e2e/**", "integrations/bub/**", "scripts/scan_e2e_evidence.sh", "tests/test_e2e_workflow_contract.py", + ".github/workflows/e2e-harness.yml", "tests/e2e/test_atomic_memory_search_authorization_snapshot.py", "tests/e2e/test_atomic_memory_read_snapshot.py", "tests/e2e/test_runtime_server.py", ] @@ -67,12 +73,123 @@ zcode = [ "integrations/zcode/**", "evaluation/zcode_guidance/**", "tests/*zcode*", "integrations/pi/**", "tests/e2e/test_cross_host_work_continuity.py", "tests/e2e/cross_host_work_continuity/**", ] -kimi = ["integrations/kimi-code/**", "tests/*kimi*"] +kimi = ["integrations/kimi-code/**", "tests/*kimi*", ".github/workflows/kimi-code-acceptance.yml"] +upgrade = [ + "scripts/run_atomic_memory_clean_migration.py", "scripts/build_ops_bundle.py", + "release/**", "scripts/upgrade_*.py", "scripts/ci_legacy_upgrade_rejection.py", + "scripts/ci_release_smoke.py", "scripts/ci_ops_bundle_smoke.py", "scripts/capture_release_schema.py", + "tests/release_upgrades/**", "tests/test_upgrade*.py", "tests/test_registered_source_maintenance_cli.py", + "tests/test_managed_*maintenance.py", "tests/e2e/*migration*", "tests/e2e/*managed_startup*", + "tests/test_managed_startup*.py", "tests/builtin/persistence/**", "tests/*migration*.py", + ".github/workflows/upgrade-release.yml", ".github/workflows/release.yml", + ".github/actions/setup-upgrade-oceanbase/**", +] +observability = [ + "tests/*observability*", "tests/*logging*", "tests/*tracing*", "tests/e2e/*worker*", + "tests/e2e/*observability*", "tests/builtin/runtime/*worker*", "tests/builtin/runtime/*model_usage*", +] +packaging = [ + "scripts/build_ops_bundle.py", + "tests/test_installation.py", "tests/test_upgrade_publication.py", "scripts/ci_role_smoke.py", + "scripts/ci_installation_roles.py", "tests/test_ci_installation_roles.py", "release/**", + ".github/workflows/release.yml", ".github/workflows/build-artifacts.yml", + ".github/workflows/build-docker.yml", "docker/**", +] +desktop = [ + "desktop/**", "tests/test_desktop_fixture_qualification.py", "website/assets/**", + ".github/workflows/desktop.yml", +] +managed_resources = [ + "scripts/*managed_resource*", "scripts/build_ops_bundle.py", "scripts/ci_ops_bundle_smoke.py", + "scripts/build_intel_managed_wheel.py", "tests/test_managed_resource_producer.py", "tests/test_ops_resources.py", + ".github/workflows/managed-resource-candidates.yml", +] +candidate_replay = [ + "src/powercontext/builtin/persistence/**", "src/powercontext/builtin/artifacts/**", "pyproject.toml", "uv.lock", + "tests/builtin/persistence/**", "scripts/replay_candidate_migration.py", + ".github/workflows/candidate-schema-replay.yml", ".github/actions/setup-upgrade-oceanbase/**", +] +release_schema = [ + "scripts/capture_release_schema.py", "scripts/upgrade_database_probe.py", "scripts/upgrade_process.py", + "tests/release_upgrades/test_full_server.py", ".github/workflows/capture-release-schema.yml", + ".github/actions/setup-upgrade-oceanbase/**", +] +seekdb_lock = ["scripts/probe_seekdb_migration_lock.py", ".github/workflows/seekdb-lock-probe.yml", ".github/workflows/seekdb-lock-observation.yml"] +skill_guidance = [ + "evaluation/skills/skill-up/**", + "integrations/claude-code/plugins/powercontext/skills/powercontext-project-context/**", + ".github/workflows/skill-guidance.yml", +] # Packaged guidance and its generators are inputs to the native host tests. -[shared] +[shared.guidance] patterns = [ "scripts/integration_guidance*", "scripts/evaluate_integration_guidance.py", "scripts/guidance_templates/**", "tests/test_integration_guidance*", "tests/test_guidance*", "tests/test_setup_rollback.py", ] suites = ["tests", "dify", "hermes", "dsh", "pi", "opencode", "openclaw", "zcode", "kimi"] + +# Shared Python changes still exercise all runtime consumers. Diagnostic source +# capture and lock probes have their own inputs rather than joining every build. +[shared.runtime] +patterns = ["src/**", "openapi/**", "pyproject.toml", "uv.lock"] +suites = [ + "tests", "evaluation", "database", "dify", "hermes", "dsh", "cindy", "pi", "opencode", "openclaw", "opendal", + "website", "native", "harness", "zcode", "kimi", "upgrade", "observability", "packaging", "desktop", + "managed_resources", +] + +[shared.openapi_guidance] +patterns = ["openapi/powercontext.yaml"] +suites = ["skill_guidance"] + +[shared.native] +patterns = [".github/workflows/native-service.yml"] +suites = ["native"] + +[shared.zcode] +patterns = [".github/workflows/zcode-acceptance.yml"] +suites = ["zcode"] + +[shared.node_cache] +patterns = [".github/actions/cache-pnpm/**"] +suites = ["tests", "desktop", "website", "dsh", "cindy", "pi", "opencode", "openclaw"] + +[shared.docs_deploy] +patterns = [".github/workflows/deploy-docs.yml"] +suites = ["website"] + +[shared.release_verify] +patterns = [".github/workflows/release-verify.yml"] +suites = ["upgrade", "packaging"] + +# A selected scope must produce successful root jobs, not silent skips. +# Main's result gate consumes this contract; reusable workflows own their children. +[checks] +helm = ["helm"] +tests = ["tests", "windows-unit-portability"] +evaluation = ["evaluation-tests"] +database = ["oceanbase-engine-recovery", "code-seekdb", "atomic-memory-clean-seekdb", "harness-validation"] +dify = ["dify-tools"] +hermes = ["hermes-skills"] +dsh = ["dsh-package"] +cindy = ["cindy-package"] +pi = ["pi-package"] +opencode = ["opencode-package"] +openclaw = ["openclaw-package"] +opendal = ["opendal-package"] +website = ["check-website", "website-link-validation-windows"] +docs_tests = ["docs-tests"] +native = ["native-service"] +harness = ["harness-validation"] +zcode = ["zcode-acceptance"] +kimi = ["kimi-acceptance"] +upgrade = ["seekdb-managed-startup", "upgrade-acceptance"] +observability = ["worker-observability"] +packaging = ["isolated-installations"] +desktop = ["desktop-validation"] +managed_resources = ["managed-resource-candidates"] +candidate_replay = ["candidate-replay"] +release_schema = ["release-schema"] +seekdb_lock = ["seekdb-lock"] +skill_guidance = ["skill-guidance"] diff --git a/.github/workflows/candidate-schema-replay.yml b/.github/workflows/candidate-schema-replay.yml index dc30de99ff..b7f65fca51 100644 --- a/.github/workflows/candidate-schema-replay.yml +++ b/.github/workflows/candidate-schema-replay.yml @@ -1,33 +1,37 @@ name: Candidate schema replay on: - pull_request: - paths: - - src/powercontext/builtin/persistence/** - - src/powercontext/builtin/artifacts/** - - tests/builtin/persistence/** - - scripts/replay_candidate_migration.py - - .github/workflows/candidate-schema-replay.yml - - .github/actions/setup-upgrade-oceanbase/** - - pyproject.toml - - uv.lock - push: - branches: [master] + workflow_call: + inputs: + suites: + description: Scope already selected by the Main orchestrator + required: true + type: string workflow_dispatch: permissions: + actions: read contents: read concurrency: - group: candidate-replay-${{ github.event.pull_request.number || github.ref }} + group: candidate-replay-${{ github.event.pull_request.number || github.ref }}-${{ github.event_name }} cancel-in-progress: true jobs: + changes: + if: ${{ inputs.suites == '' }} + uses: ./.github/workflows/ci-changes.yml + with: + baseline-workflow: candidate-schema-replay.yml + replay: + needs: changes + if: ${{ !cancelled() && ((inputs.suites == '' && needs.changes.result != 'success') || fromJSON(inputs.suites || needs.changes.outputs.suites || '{}').candidate_replay) }} name: Candidate replay (${{ matrix.backend }}) runs-on: ubuntu-24.04 timeout-minutes: 30 strategy: + max-parallel: 2 fail-fast: false matrix: backend: [sqlite, seekdb, oceanbase] diff --git a/.github/workflows/capture-release-schema.yml b/.github/workflows/capture-release-schema.yml index ddcfda28c7..1f8125013e 100644 --- a/.github/workflows/capture-release-schema.yml +++ b/.github/workflows/capture-release-schema.yml @@ -1,29 +1,37 @@ name: Capture released Server schema on: - pull_request: - paths: - - scripts/capture_release_schema.py - - scripts/upgrade_database_probe.py - - scripts/upgrade_process.py - - tests/release_upgrades/test_full_server.py - - .github/workflows/capture-release-schema.yml - - .github/actions/setup-upgrade-oceanbase/** + workflow_call: + inputs: + suites: + description: Scope already selected by the Main orchestrator + required: true + type: string workflow_dispatch: permissions: + actions: read contents: read concurrency: - group: release-schema-${{ github.event.pull_request.number || github.ref }} + group: release-schema-${{ github.event.pull_request.number || github.ref }}-${{ github.event_name }} cancel-in-progress: true jobs: + changes: + if: ${{ inputs.suites == '' }} + uses: ./.github/workflows/ci-changes.yml + with: + baseline-workflow: capture-release-schema.yml + capture: + needs: changes + if: ${{ !cancelled() && ((inputs.suites == '' && needs.changes.result != 'success') || fromJSON(inputs.suites || needs.changes.outputs.suites || '{}').release_schema) }} name: Capture 1.4.0 (${{ matrix.backend }}, ${{ matrix.configuration }}) runs-on: ubuntu-24.04 timeout-minutes: 45 strategy: + max-parallel: 2 fail-fast: false matrix: backend: [sqlite, seekdb, oceanbase] diff --git a/.github/workflows/ci-changes.yml b/.github/workflows/ci-changes.yml index 3d98a54ee4..d92991954a 100644 --- a/.github/workflows/ci-changes.yml +++ b/.github/workflows/ci-changes.yml @@ -2,12 +2,24 @@ name: Detect CI scope on: workflow_call: + inputs: + baseline-workflow: + description: Calling workflow filename used to retain changes from cancelled pushes + required: true + type: string outputs: suites: description: JSON object of suites selected for this event value: ${{ jobs.detect.outputs.suites }} + required-jobs: + description: Selected root jobs that must succeed + value: ${{ jobs.detect.outputs.required-jobs }} + test-matrix: + description: Python and suite combinations for this validation profile + value: ${{ jobs.detect.outputs.test-matrix }} permissions: + actions: read contents: read jobs: @@ -16,8 +28,10 @@ jobs: timeout-minutes: 5 outputs: suites: ${{ steps.select.outputs.suites }} + test-matrix: ${{ steps.select.outputs.test-matrix }} + required-jobs: ${{ steps.select.outputs.required-jobs }} steps: - - name: Check out the complete PR history + - name: Check out the complete branch history uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-depth: 0 @@ -27,4 +41,7 @@ jobs: env: PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + CI_BASELINE_WORKFLOW: ${{ inputs.baseline-workflow }} + CI_FULL_VALIDATION: ${{ contains(github.event.pull_request.labels.*.name, 'ci:full') }} + GH_TOKEN: ${{ github.token }} run: python3 scripts/ci_changes.py diff --git a/.github/workflows/desktop.yml b/.github/workflows/desktop.yml index eb719dd4f5..372949fef0 100644 --- a/.github/workflows/desktop.yml +++ b/.github/workflows/desktop.yml @@ -1,11 +1,17 @@ name: Desktop validation on: - pull_request: - paths: ['desktop/**', 'src/powercontext/**', 'pyproject.toml', 'uv.lock', 'openapi/powercontext.yaml', 'tests/fixtures/transport_loopback_vectors.json', 'tests/test_desktop_fixture_qualification.py', 'website/assets/**', '.github/workflows/desktop.yml'] + workflow_call: + inputs: + suites: + description: Scope already selected by the Main orchestrator + required: true + type: string + installer_run: + type: string + default: '' push: - branches: [master, codex/desktop-preview] - paths: ['desktop/**', 'src/powercontext/**', 'pyproject.toml', 'uv.lock', 'openapi/powercontext.yaml', 'tests/fixtures/transport_loopback_vectors.json', 'tests/test_desktop_fixture_qualification.py', 'website/assets/**', '.github/workflows/desktop.yml'] + branches: [codex/desktop-preview] workflow_dispatch: inputs: installer_run: @@ -15,15 +21,25 @@ on: default: '' permissions: + actions: read contents: read concurrency: - group: desktop-${{ github.event.pull_request.number || github.ref }}-${{ inputs.installer_run || 'build' }} + group: desktop-${{ github.event.pull_request.number || github.ref }}-${{ inputs.installer_run || 'build' }}-${{ github.event_name }} cancel-in-progress: true jobs: + changes: + if: ${{ inputs.suites == '' }} + uses: ./.github/workflows/ci-changes.yml + with: + baseline-workflow: desktop.yml + windows: - if: ${{ github.event_name != 'workflow_dispatch' || inputs.installer_run == '' }} + needs: changes + if: >- + ${{ !cancelled() && (github.event_name != 'workflow_dispatch' || inputs.installer_run == '') && + ((inputs.suites == '' && needs.changes.result != 'success') || fromJSON(inputs.suites || needs.changes.outputs.suites || '{}').desktop) }} runs-on: windows-2025 timeout-minutes: 40 defaults: @@ -73,9 +89,12 @@ jobs: - name: Verify Windows vault across processes run: cargo run --locked --manifest-path src-tauri/Cargo.toml --example credential_probe - name: Set up Python fixture tooling + timeout-minutes: 5 uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2 with: version: "0.10.12" + # Use the release source directly; a stalled mirror download can exhaust the job budget. + download-from-astral-mirror: false enable-cache: true cache-suffix: desktop-${{ github.job }}-py3.12 cache-dependency-glob: | @@ -148,7 +167,10 @@ jobs: if-no-files-found: ${{ job.status == 'success' && 'error' || 'warn' }} macos: - if: ${{ github.event_name != 'workflow_dispatch' || inputs.installer_run == '' }} + needs: changes + if: >- + ${{ !cancelled() && (github.event_name != 'workflow_dispatch' || inputs.installer_run == '') && + ((inputs.suites == '' && needs.changes.result != 'success') || fromJSON(inputs.suites || needs.changes.outputs.suites || '{}').desktop) }} strategy: fail-fast: false matrix: @@ -202,7 +224,10 @@ jobs: if-no-files-found: error linux: - if: ${{ github.event_name != 'workflow_dispatch' || inputs.installer_run == '' }} + needs: changes + if: >- + ${{ !cancelled() && (github.event_name != 'workflow_dispatch' || inputs.installer_run == '') && + ((inputs.suites == '' && needs.changes.result != 'success') || fromJSON(inputs.suites || needs.changes.outputs.suites || '{}').desktop) }} runs-on: ubuntu-24.04 timeout-minutes: 40 defaults: diff --git a/.github/workflows/e2e-harness.yml b/.github/workflows/e2e-harness.yml index efe6040e3c..733d36c288 100644 --- a/.github/workflows/e2e-harness.yml +++ b/.github/workflows/e2e-harness.yml @@ -1,11 +1,15 @@ name: E2E harness on: - push: - branches: - - master - pull_request: - types: [opened, synchronize, reopened, ready_for_review] + workflow_call: + inputs: + suites: + description: Scope already selected by the Main orchestrator + required: true + type: string + suite: + type: string + default: all workflow_dispatch: inputs: suite: @@ -17,6 +21,7 @@ on: - recall-effort permissions: + actions: read contents: read env: @@ -24,18 +29,21 @@ env: UV_VERSION: "0.10.12" concurrency: - group: e2e-harness-${{ github.event.pull_request.number || github.sha }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + group: e2e-harness-${{ github.event.pull_request.number || github.ref }}-${{ github.event_name }} + cancel-in-progress: true jobs: changes: + if: ${{ inputs.suites == '' }} uses: ./.github/workflows/ci-changes.yml + with: + baseline-workflow: e2e-harness.yml validate: needs: changes if: >- ${{ !cancelled() && (github.event_name != 'workflow_dispatch' || inputs.suite != 'recall-effort') && - (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').harness) }} + ((inputs.suites == '' && needs.changes.result != 'success') || fromJSON(inputs.suites || needs.changes.outputs.suites || '{}').harness) }} runs-on: ubuntu-latest timeout-minutes: 15 steps: @@ -65,7 +73,7 @@ jobs: recall-effort: needs: changes - if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').database) }} + if: ${{ !cancelled() && ((inputs.suites == '' && needs.changes.result != 'success') || fromJSON(inputs.suites || needs.changes.outputs.suites || '{}').database) }} name: Recall effort (OceanBase) runs-on: ubuntu-latest timeout-minutes: 60 @@ -321,10 +329,11 @@ jobs: needs: changes if: >- ${{ !cancelled() && (github.event_name != 'workflow_dispatch' || inputs.suite != 'recall-effort') && - (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').database) }} + ((inputs.suites == '' && needs.changes.result != 'success') || fromJSON(inputs.suites || needs.changes.outputs.suites || '{}').database) }} runs-on: ubuntu-latest timeout-minutes: 145 strategy: + max-parallel: 2 fail-fast: false matrix: shard: [0, 1, 2] @@ -481,7 +490,7 @@ jobs: needs: [changes, atomic-memory-clean-oceanbase-shards] if: >- ${{ !cancelled() && (github.event_name != 'workflow_dispatch' || inputs.suite != 'recall-effort') && - (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').database) }} + ((inputs.suites == '' && needs.changes.result != 'success') || fromJSON(inputs.suites || needs.changes.outputs.suites || '{}').database) }} runs-on: ubuntu-latest timeout-minutes: 5 steps: @@ -518,6 +527,8 @@ jobs: acceptance: needs: validate + # Main supplies scope, so the detector ancestor is intentionally skipped. + if: ${{ !cancelled() && needs.validate.result == 'success' }} runs-on: ubuntu-latest # OceanBase regressions can take 32 minutes before the separate replay scenarios. timeout-minutes: ${{ matrix.database == 'oceanbase' && 60 || 40 }} diff --git a/.github/workflows/kimi-code-acceptance.yml b/.github/workflows/kimi-code-acceptance.yml index 8d70880bf8..a8a195f278 100644 --- a/.github/workflows/kimi-code-acceptance.yml +++ b/.github/workflows/kimi-code-acceptance.yml @@ -1,17 +1,20 @@ name: Kimi Code acceptance on: - pull_request: - types: [opened, synchronize, reopened, ready_for_review] - push: - branches: [master] + workflow_call: + inputs: + suites: + description: Scope already selected by the Main orchestrator + required: true + type: string workflow_dispatch: permissions: + actions: read contents: read concurrency: - group: kimi-code-acceptance-${{ github.event.pull_request.number || github.ref }} + group: kimi-code-acceptance-${{ github.event.pull_request.number || github.ref }}-${{ github.event_name }} cancel-in-progress: true env: @@ -21,11 +24,14 @@ env: jobs: changes: + if: ${{ inputs.suites == '' }} uses: ./.github/workflows/ci-changes.yml + with: + baseline-workflow: kimi-code-acceptance.yml controlled-cli: needs: changes - if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').kimi) }} + if: ${{ !cancelled() && ((inputs.suites == '' && needs.changes.result != 'success') || fromJSON(inputs.suites || needs.changes.outputs.suites || '{}').kimi) }} name: controlled-cli (${{ matrix.os }}) runs-on: ${{ matrix.os }} timeout-minutes: 15 diff --git a/.github/workflows/license-check.yml b/.github/workflows/license-check.yml deleted file mode 100644 index e611461631..0000000000 --- a/.github/workflows/license-check.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: License Check - -on: - pull_request: - push: - branches: [master] - -permissions: - contents: read - -concurrency: - group: license-check-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - license: - name: Check License Header - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - persist-credentials: false - - - name: Check License Header - uses: apache/skywalking-eyes/header@a196742f472feaffafea537ce5a2a4c3c53a8de4 # v0.9.0 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - config: .licenserc.yaml - mode: check diff --git a/.github/workflows/managed-resource-candidates.yml b/.github/workflows/managed-resource-candidates.yml index 4e44da9b57..59bbb90619 100644 --- a/.github/workflows/managed-resource-candidates.yml +++ b/.github/workflows/managed-resource-candidates.yml @@ -1,32 +1,35 @@ name: Managed resource candidates on: - pull_request: - paths: - - 'scripts/*managed_resource*' - - 'scripts/build_ops_bundle.py' - - 'scripts/ci_ops_bundle_smoke.py' - - 'scripts/build_intel_managed_wheel.py' - - 'scripts/managed_resource_intel_build.json' - - 'tests/test_managed_resource_producer.py' - - 'tests/test_ops_resources.py' - - 'src/powercontext/**' - - 'pyproject.toml' - - 'uv.lock' - - '.github/workflows/managed-resource-candidates.yml' + workflow_call: + inputs: + suites: + description: Scope already selected by the Main orchestrator + required: true + type: string workflow_dispatch: permissions: + actions: read contents: read concurrency: - group: managed-resource-candidates-${{ github.event.pull_request.number || github.ref }} + group: managed-resource-candidates-${{ github.event.pull_request.number || github.ref }}-${{ github.event_name }} cancel-in-progress: true jobs: + changes: + if: ${{ inputs.suites == '' }} + uses: ./.github/workflows/ci-changes.yml + with: + baseline-workflow: managed-resource-candidates.yml + candidate: + needs: changes + if: ${{ !cancelled() && ((inputs.suites == '' && needs.changes.result != 'success') || fromJSON(inputs.suites || needs.changes.outputs.suites || '{}').managed_resources) }} name: Candidate ${{ matrix.target }} strategy: + max-parallel: 2 fail-fast: false matrix: include: diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index f37d12de6c..95679a1bbe 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -5,7 +5,10 @@ on: branches: - master pull_request: - types: [opened, synchronize, reopened, ready_for_review] + branches: [master] + types: [opened, synchronize, reopened, ready_for_review, converted_to_draft] + schedule: + - cron: '17 20 * * *' workflow_dispatch: inputs: upgrade_selection: @@ -18,14 +21,17 @@ on: - full permissions: + actions: read contents: read concurrency: - group: master-${{ github.event.pull_request.number || github.ref }} + group: ci-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true jobs: seekdb-managed-startup: + needs: [quality, tests] + if: ${{ !cancelled() && needs.quality.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.tests.result) && (fromJSON(needs.quality.outputs.suites || '{}').upgrade) }} name: SeekDB managed startup regression (priority) timeout-minutes: 20 runs-on: ubuntu-latest @@ -78,33 +84,38 @@ jobs: if-no-files-found: error retention-days: 7 - changes: - uses: ./.github/workflows/ci-changes.yml - upgrade-acceptance: - needs: seekdb-managed-startup - # Collect selected upgrade cases even when the priority regression fails. - if: ${{ !cancelled() }} + needs: [quality, seekdb-managed-startup, tests] + # Keep diagnostics when the selected priority regression fails. + if: ${{ !cancelled() && needs.quality.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.tests.result) && (fromJSON(needs.quality.outputs.suites || '{}').upgrade) }} uses: ./.github/workflows/upgrade-release.yml with: selection: ${{ github.event_name == 'workflow_dispatch' && inputs.upgrade_selection || 'full' }} upgrade-release-gate: name: Upgrade release gate - needs: upgrade-acceptance - if: always() + needs: [quality, upgrade-acceptance] + if: ${{ !cancelled() && (github.event_name != 'pull_request' || !github.event.pull_request.draft) }} runs-on: ubuntu-24.04 timeout-minutes: 5 steps: - name: Require the selected parallel acceptance workflow env: + SCOPE_RESULT: ${{ needs.quality.result }} + UPGRADE_SELECTED: ${{ fromJSON(needs.quality.outputs.suites || '{}').upgrade }} UPGRADE_RESULT: ${{ needs.upgrade-acceptance.result }} - run: test "$UPGRADE_RESULT" = success + run: | + if [ "$SCOPE_RESULT" = success ] && [ "$UPGRADE_SELECTED" = false ]; then + test "$UPGRADE_RESULT" = skipped + echo "Upgrade acceptance is not required by this change." + else + test "$UPGRADE_RESULT" = success + fi oceanbase-engine-recovery: name: OceanBase engine crash recovery - needs: changes - if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').tests) }} + needs: [quality, tests] + if: ${{ !cancelled() && needs.quality.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.tests.result) && (fromJSON(needs.quality.outputs.suites || '{}').database) }} runs-on: ubuntu-24.04 timeout-minutes: 20 env: @@ -162,14 +173,41 @@ jobs: retention-days: 7 quality: - timeout-minutes: 20 + name: CI preflight + if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }} + timeout-minutes: 30 runs-on: ubuntu-latest + outputs: + suites: ${{ steps.select.outputs.suites }} + test-matrix: ${{ steps.select.outputs.test-matrix }} + required-jobs: ${{ steps.select.outputs.required-jobs }} steps: - - name: Check out + - name: Check out the complete branch history uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: + fetch-depth: 0 persist-credentials: false + - name: Select checks + id: select + timeout-minutes: 5 + env: + PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + CI_BASELINE_WORKFLOW: master.yml + CI_FULL_VALIDATION: ${{ contains(github.event.pull_request.labels.*.name, 'ci:full') }} + GH_TOKEN: ${{ github.token }} + run: python3 scripts/ci_changes.py + + - name: Check License Header + timeout-minutes: 10 + uses: apache/skywalking-eyes/header@a196742f472feaffafea537ce5a2a4c3c53a8de4 # v0.9.0 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + config: .licenserc.yaml + mode: check + - name: Set up the environment uses: ./.github/actions/setup-python-env @@ -190,15 +228,15 @@ jobs: run: make contract-test tests: - needs: changes - if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').tests) }} + name: tests (${{ matrix.python-version }}, ${{ matrix.suite }}) + needs: [quality] + if: ${{ !cancelled() && needs.quality.result == 'success' && (fromJSON(needs.quality.outputs.suites || '{}').tests) }} timeout-minutes: 40 runs-on: ubuntu-latest strategy: - matrix: - python-version: ["3.11", "3.12", "3.13", "3.14"] - suite: [unit, e2e] - fail-fast: false + fail-fast: true + max-parallel: 3 + matrix: ${{ fromJSON(needs.quality.outputs.test-matrix || '{"python-version":["3.11","3.12","3.13","3.14"],"suite":["unit","e2e"]}') }} defaults: run: shell: bash @@ -246,15 +284,15 @@ jobs: run: make e2e-test isolated-installations: - needs: changes - if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').tests) }} + needs: [quality, tests] + if: ${{ !cancelled() && needs.quality.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.tests.result) && (fromJSON(needs.quality.outputs.suites || '{}').packaging) }} timeout-minutes: 20 runs-on: ubuntu-latest strategy: - fail-fast: false + fail-fast: true + max-parallel: 2 matrix: python-version: ["3.11", "3.14"] - extras: ["core", "builtin", "cli", "builtin,cli", "server,cli"] steps: - name: Check out uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -271,34 +309,38 @@ jobs: with: version: "0.10.12" - - name: Build and install only the requested wheel extras - env: - ROLE_EXTRAS: ${{ matrix.extras }} + - name: Build one wheel for all installation roles run: | - uv build --wheel --out-dir "$RUNNER_TEMP/role-wheel" - uv venv --python "$(command -v python)" "$RUNNER_TEMP/role-venv" - wheel="$(find "$RUNNER_TEMP/role-wheel" -name '*.whl' -print -quit)" - if [ "$ROLE_EXTRAS" = core ]; then - uv pip install --python "$RUNNER_TEMP/role-venv/bin/python" "$wheel" - else - uv pip install --python "$RUNNER_TEMP/role-venv/bin/python" "$wheel[$ROLE_EXTRAS]" - fi + uv build --wheel --python "$(command -v python)" --out-dir "$RUNNER_TEMP/role-wheel" - - name: Verify installed role behavior without checkout imports - env: - ROLE_EXTRAS: ${{ matrix.extras }} + - name: Verify all roles in independent environments run: | - "$RUNNER_TEMP/role-venv/bin/python" -I scripts/ci_role_smoke.py "$ROLE_EXTRAS" + wheel="$(find "$RUNNER_TEMP/role-wheel" -name '*.whl' -print -quit)" + python scripts/ci_installation_roles.py --wheel "$wheel" \ + --python "$(command -v python)" --output-dir "$RUNNER_TEMP/installation-roles" - - name: Verify unrelated packages do not enable optional CLI roles - if: matrix.extras == 'cli' + - name: Summarize installation roles + if: always() run: | - uv pip install --python "$RUNNER_TEMP/role-venv/bin/python" sqlalchemy fastapi - "$RUNNER_TEMP/role-venv/bin/python" -I scripts/ci_role_smoke.py cli + if [ -f "$RUNNER_TEMP/installation-roles/summary.md" ]; then + cat "$RUNNER_TEMP/installation-roles/summary.md" >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Save installation role reports + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: installation-roles-py${{ matrix.python-version }}-${{ github.sha }} + path: | + ${{ runner.temp }}/installation-roles/summary.json + ${{ runner.temp }}/installation-roles/summary.md + ${{ runner.temp }}/installation-roles/logs/*.log + if-no-files-found: warn + retention-days: 7 docs-tests: - needs: changes - if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').docs_tests) }} + needs: [quality] + if: ${{ !cancelled() && needs.quality.result == 'success' && (fromJSON(needs.quality.outputs.suites || '{}').docs_tests) }} timeout-minutes: 15 runs-on: ubuntu-latest steps: @@ -322,6 +364,8 @@ jobs: tests/e2e/test_mcp_full_profile.py worker-observability: + needs: [quality, tests] + if: ${{ !cancelled() && needs.quality.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.tests.result) && (fromJSON(needs.quality.outputs.suites || '{}').observability) }} timeout-minutes: 15 runs-on: ubuntu-latest steps: @@ -363,8 +407,8 @@ jobs: if-no-files-found: error evaluation-tests: - needs: changes - if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').evaluation) }} + needs: [quality, tests] + if: ${{ !cancelled() && needs.quality.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.tests.result) && (fromJSON(needs.quality.outputs.suites || '{}').evaluation) }} timeout-minutes: 20 runs-on: ubuntu-latest steps: @@ -425,8 +469,8 @@ jobs: echo "::error title=evaluation unit tests failed::${detail}" code-seekdb: - needs: changes - if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').database) }} + needs: [quality, tests] + if: ${{ !cancelled() && needs.quality.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.tests.result) && (fromJSON(needs.quality.outputs.suites || '{}').database) }} timeout-minutes: 30 runs-on: ubuntu-latest strategy: @@ -560,8 +604,8 @@ jobs: if-no-files-found: ignore atomic-memory-clean-seekdb: - needs: [changes, seekdb-managed-startup] - if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').database) }} + needs: [quality, seekdb-managed-startup, tests] + if: ${{ !cancelled() && needs.quality.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.tests.result) && (fromJSON(needs.quality.outputs.suites || '{}').database) }} name: Atomic Memory clean migration (SeekDB) timeout-minutes: 55 runs-on: ubuntu-latest @@ -622,8 +666,8 @@ jobs: retention-days: 7 windows-unit-portability: - needs: changes - if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').tests) }} + needs: [quality, tests] + if: ${{ !cancelled() && needs.quality.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.tests.result) && (fromJSON(needs.quality.outputs.suites || '{}').tests) }} timeout-minutes: 20 runs-on: windows-latest steps: @@ -718,8 +762,8 @@ jobs: tests/test_service_environment.py dify-tools: - needs: changes - if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').dify) }} + needs: [quality, tests] + if: ${{ !cancelled() && needs.quality.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.tests.result) && (fromJSON(needs.quality.outputs.suites || '{}').dify) }} timeout-minutes: 15 runs-on: ubuntu-latest steps: @@ -784,8 +828,8 @@ jobs: run: make dify-test hermes-skills: - needs: changes - if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').hermes) }} + needs: [quality, tests] + if: ${{ !cancelled() && needs.quality.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.tests.result) && (fromJSON(needs.quality.outputs.suites || '{}').hermes) }} timeout-minutes: 15 runs-on: ubuntu-latest steps: @@ -811,8 +855,8 @@ jobs: run: uv run pytest tests/e2e/test_hermes_skills.py dsh-package: - needs: changes - if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').dsh) }} + needs: [quality, tests] + if: ${{ !cancelled() && needs.quality.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.tests.result) && (fromJSON(needs.quality.outputs.suites || '{}').dsh) }} timeout-minutes: 30 runs-on: ubuntu-latest steps: @@ -858,8 +902,8 @@ jobs: run: uv run pytest tests/test_integration_guidance_evaluation.py::test_dsh_controlled_native_handoff_uses_only_mcp_tools_and_preserves_complete_carriers cindy-package: - needs: changes - if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').cindy) }} + needs: [quality, tests] + if: ${{ !cancelled() && needs.quality.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.tests.result) && (fromJSON(needs.quality.outputs.suites || '{}').cindy) }} timeout-minutes: 15 runs-on: ubuntu-latest steps: @@ -901,8 +945,8 @@ jobs: run: uv run pytest tests/e2e/test_cindy_mcp_chain.py tests/e2e/test_cindy_hooks_chain.py -q pi-package: - needs: changes - if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').pi) }} + needs: [quality, tests] + if: ${{ !cancelled() && needs.quality.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.tests.result) && (fromJSON(needs.quality.outputs.suites || '{}').pi) }} timeout-minutes: 20 runs-on: ubuntu-latest steps: @@ -930,8 +974,8 @@ jobs: run: uv run pytest "tests/test_integration_guidance_evaluation.py::test_native_adapter_handoff_uses_real_request_mapping_and_response_envelopes[pi]" opencode-package: - needs: changes - if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').opencode) }} + needs: [quality, tests] + if: ${{ !cancelled() && needs.quality.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.tests.result) && (fromJSON(needs.quality.outputs.suites || '{}').opencode) }} timeout-minutes: 20 runs-on: ubuntu-latest steps: @@ -954,8 +998,8 @@ jobs: run: uv run pytest "tests/test_integration_guidance_evaluation.py::test_native_adapter_handoff_uses_real_request_mapping_and_response_envelopes[opencode]" openclaw-package: - needs: changes - if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').openclaw) }} + needs: [quality, tests] + if: ${{ !cancelled() && needs.quality.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.tests.result) && (fromJSON(needs.quality.outputs.suites || '{}').openclaw) }} timeout-minutes: 20 runs-on: ubuntu-latest steps: @@ -980,8 +1024,8 @@ jobs: run: mise exec node@24.15.0 -- uv run pytest tests/test_integration_guidance_evaluation.py::test_openclaw_handoff_adapter_preserves_generated_source_identity opendal-package: - needs: changes - if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').opendal) }} + needs: [quality, tests] + if: ${{ !cancelled() && needs.quality.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.tests.result) && (fromJSON(needs.quality.outputs.suites || '{}').opendal) }} timeout-minutes: 20 runs-on: ubuntu-latest steps: @@ -1000,8 +1044,8 @@ jobs: website-link-validation-windows: - needs: changes - if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').website) }} + needs: [quality] + if: ${{ !cancelled() && needs.quality.result == 'success' && (fromJSON(needs.quality.outputs.suites || '{}').website) }} timeout-minutes: 20 runs-on: windows-latest steps: @@ -1026,8 +1070,8 @@ jobs: run: pnpm --dir website test:links check-website: - needs: changes - if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').website) }} + needs: [quality] + if: ${{ !cancelled() && needs.quality.result == 'success' && (fromJSON(needs.quality.outputs.suites || '{}').website) }} timeout-minutes: 20 runs-on: ubuntu-latest steps: @@ -1056,10 +1100,174 @@ jobs: POWERCONTEXT_TEST_BLOG_SITE: "1" run: uv run pytest tests/e2e/test_blog_site.py -q + harness-validation: + needs: [quality, tests] + if: >- + ${{ !cancelled() && needs.quality.result == 'success' && + contains(fromJSON('["success","skipped"]'), needs.tests.result) && + (fromJSON(needs.quality.outputs.suites || '{}').database || fromJSON(needs.quality.outputs.suites || '{}').harness) }} + uses: ./.github/workflows/e2e-harness.yml + with: + suites: ${{ needs.quality.outputs.suites }} + + desktop-validation: + needs: [quality, tests] + if: >- + ${{ !cancelled() && needs.quality.result == 'success' && + contains(fromJSON('["success","skipped"]'), needs.tests.result) && + (fromJSON(needs.quality.outputs.suites || '{}').desktop) }} + uses: ./.github/workflows/desktop.yml + with: + suites: ${{ needs.quality.outputs.suites }} + + managed-resource-candidates: + needs: [quality, tests] + if: >- + ${{ !cancelled() && needs.quality.result == 'success' && + contains(fromJSON('["success","skipped"]'), needs.tests.result) && + (fromJSON(needs.quality.outputs.suites || '{}').managed_resources) }} + uses: ./.github/workflows/managed-resource-candidates.yml + with: + suites: ${{ needs.quality.outputs.suites }} + + native-service: + needs: [quality, tests] + if: >- + ${{ !cancelled() && needs.quality.result == 'success' && + contains(fromJSON('["success","skipped"]'), needs.tests.result) && + (fromJSON(needs.quality.outputs.suites || '{}').native) }} + uses: ./.github/workflows/native-service.yml + with: + suites: ${{ needs.quality.outputs.suites }} + + kimi-acceptance: + needs: [quality, tests] + if: >- + ${{ !cancelled() && needs.quality.result == 'success' && + contains(fromJSON('["success","skipped"]'), needs.tests.result) && + (fromJSON(needs.quality.outputs.suites || '{}').kimi) }} + uses: ./.github/workflows/kimi-code-acceptance.yml + with: + suites: ${{ needs.quality.outputs.suites }} + + zcode-acceptance: + needs: [quality, tests] + if: >- + ${{ !cancelled() && needs.quality.result == 'success' && + contains(fromJSON('["success","skipped"]'), needs.tests.result) && + (fromJSON(needs.quality.outputs.suites || '{}').zcode) }} + uses: ./.github/workflows/zcode-acceptance.yml + with: + suites: ${{ needs.quality.outputs.suites }} + + skill-guidance: + needs: [quality, tests] + if: >- + ${{ !cancelled() && needs.quality.result == 'success' && + contains(fromJSON('["success","skipped"]'), needs.tests.result) && + (fromJSON(needs.quality.outputs.suites || '{}').skill_guidance) }} + uses: ./.github/workflows/skill-guidance.yml + with: + suites: ${{ needs.quality.outputs.suites }} + + candidate-replay: + needs: [quality, tests] + if: >- + ${{ !cancelled() && needs.quality.result == 'success' && + contains(fromJSON('["success","skipped"]'), needs.tests.result) && + (fromJSON(needs.quality.outputs.suites || '{}').candidate_replay) }} + uses: ./.github/workflows/candidate-schema-replay.yml + with: + suites: ${{ needs.quality.outputs.suites }} + + release-schema: + needs: [quality, tests] + if: >- + ${{ !cancelled() && needs.quality.result == 'success' && + contains(fromJSON('["success","skipped"]'), needs.tests.result) && + (fromJSON(needs.quality.outputs.suites || '{}').release_schema) }} + uses: ./.github/workflows/capture-release-schema.yml + with: + suites: ${{ needs.quality.outputs.suites }} + + seekdb-lock: + needs: [quality, tests] + if: >- + ${{ !cancelled() && needs.quality.result == 'success' && + contains(fromJSON('["success","skipped"]'), needs.tests.result) && + (fromJSON(needs.quality.outputs.suites || '{}').seekdb_lock) }} + uses: ./.github/workflows/seekdb-lock-probe.yml + with: + suites: ${{ needs.quality.outputs.suites }} + + ci-result: + name: CI result + needs: + - seekdb-managed-startup + - upgrade-acceptance + - upgrade-release-gate + - oceanbase-engine-recovery + - quality + - tests + - isolated-installations + - docs-tests + - worker-observability + - evaluation-tests + - code-seekdb + - atomic-memory-clean-seekdb + - windows-unit-portability + - dify-tools + - hermes-skills + - dsh-package + - cindy-package + - pi-package + - opencode-package + - openclaw-package + - opendal-package + - website-link-validation-windows + - check-website + - harness-validation + - desktop-validation + - managed-resource-candidates + - helm + - native-service + - kimi-acceptance + - zcode-acceptance + - skill-guidance + - candidate-replay + - release-schema + - seekdb-lock + if: ${{ !cancelled() && (github.event_name != 'pull_request' || !github.event.pull_request.draft) }} + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Require successful selected checks + env: + RESULTS: ${{ toJSON(needs) }} + REQUIRED: ${{ needs.quality.outputs.required-jobs }} + run: | + python3 - <<'PYTHON' + import json + import os + results = json.loads(os.environ["RESULTS"]) + failed = {name: job["result"] for name, job in results.items() + if job["result"] not in {"success", "skipped"}} + if failed: + raise SystemExit(f"CI checks did not succeed: {failed}") + required = json.loads(os.environ["REQUIRED"]) + if not {"quality", "upgrade-release-gate"} <= set(required): + raise SystemExit("The required CI result contract is missing") + missing = {name: results.get(name, {}).get("result", "missing") for name in required + if results.get(name, {}).get("result") != "success"} + if missing: + raise SystemExit(f"Selected CI checks did not run successfully: {missing}") + print("All selected CI checks passed.") + PYTHON + helm: name: Helm deployment contracts - needs: changes - if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').helm) }} + needs: [quality, tests] + if: ${{ !cancelled() && needs.quality.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.tests.result) && (fromJSON(needs.quality.outputs.suites || '{}').helm) }} runs-on: ubuntu-latest timeout-minutes: 10 steps: diff --git a/.github/workflows/native-service.yml b/.github/workflows/native-service.yml index 69e1055663..50a1907fdc 100644 --- a/.github/workflows/native-service.yml +++ b/.github/workflows/native-service.yml @@ -1,27 +1,32 @@ name: Native personal service on: - push: - branches: - - master - pull_request: - types: [opened, synchronize, reopened, ready_for_review] + workflow_call: + inputs: + suites: + description: Scope already selected by the Main orchestrator + required: true + type: string workflow_dispatch: permissions: + actions: read contents: read concurrency: - group: native-service-${{ github.event.pull_request.number || github.ref }} + group: native-service-${{ github.event.pull_request.number || github.ref }}-${{ github.event_name }} cancel-in-progress: true jobs: changes: + if: ${{ inputs.suites == '' }} uses: ./.github/workflows/ci-changes.yml + with: + baseline-workflow: native-service.yml linux-systemd-user: needs: changes - if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').native) }} + if: ${{ !cancelled() && ((inputs.suites == '' && needs.changes.result != 'success') || fromJSON(inputs.suites || needs.changes.outputs.suites || '{}').native) }} runs-on: ubuntu-latest timeout-minutes: 30 env: @@ -84,7 +89,7 @@ jobs: macos-launchagent: needs: changes - if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').native) }} + if: ${{ !cancelled() && ((inputs.suites == '' && needs.changes.result != 'success') || fromJSON(inputs.suites || needs.changes.outputs.suites || '{}').native) }} runs-on: macos-latest timeout-minutes: 30 env: @@ -140,7 +145,7 @@ jobs: windows-task-scheduler: needs: changes - if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').native) }} + if: ${{ !cancelled() && ((inputs.suites == '' && needs.changes.result != 'success') || fromJSON(inputs.suites || needs.changes.outputs.suites || '{}').native) }} runs-on: windows-latest timeout-minutes: 30 env: diff --git a/.github/workflows/seekdb-lock-observation.yml b/.github/workflows/seekdb-lock-observation.yml index fc281623cf..8962e0608e 100644 --- a/.github/workflows/seekdb-lock-observation.yml +++ b/.github/workflows/seekdb-lock-observation.yml @@ -1,9 +1,6 @@ name: SeekDB extended lock observation on: - pull_request: - paths: - - .github/workflows/seekdb-lock-observation.yml workflow_dispatch: permissions: diff --git a/.github/workflows/seekdb-lock-probe.yml b/.github/workflows/seekdb-lock-probe.yml index 5172aa0c78..7d5a4ebc70 100644 --- a/.github/workflows/seekdb-lock-probe.yml +++ b/.github/workflows/seekdb-lock-probe.yml @@ -1,21 +1,32 @@ name: SeekDB migration lock diagnostics on: - pull_request: - paths: - - scripts/probe_seekdb_migration_lock.py - - .github/workflows/seekdb-lock-probe.yml + workflow_call: + inputs: + suites: + description: Scope already selected by the Main orchestrator + required: true + type: string workflow_dispatch: permissions: + actions: read contents: read concurrency: - group: seekdb-lock-probe-${{ github.event.pull_request.number || github.ref }} + group: seekdb-lock-probe-${{ github.event.pull_request.number || github.ref }}-${{ github.event_name }} cancel-in-progress: true jobs: + changes: + if: ${{ inputs.suites == '' }} + uses: ./.github/workflows/ci-changes.yml + with: + baseline-workflow: seekdb-lock-probe.yml + reproduce: + needs: changes + if: ${{ !cancelled() && ((inputs.suites == '' && needs.changes.result != 'success') || fromJSON(inputs.suites || needs.changes.outputs.suites || '{}').seekdb_lock) }} runs-on: ubuntu-latest timeout-minutes: 20 steps: diff --git a/.github/workflows/skill-guidance.yml b/.github/workflows/skill-guidance.yml index 743071218f..44566d3c21 100644 --- a/.github/workflows/skill-guidance.yml +++ b/.github/workflows/skill-guidance.yml @@ -1,26 +1,32 @@ name: Skill guidance validation on: - pull_request: - paths: - - evaluation/skills/skill-up/** - - openapi/powercontext.yaml - - integrations/claude-code/plugins/powercontext/skills/powercontext-project-context/** - - .github/workflows/skill-guidance.yml - push: - branches: [master] - paths: - - evaluation/skills/skill-up/** - - openapi/powercontext.yaml - - integrations/claude-code/plugins/powercontext/skills/powercontext-project-context/** - - .github/workflows/skill-guidance.yml + workflow_call: + inputs: + suites: + description: Scope already selected by the Main orchestrator + required: true + type: string workflow_dispatch: permissions: + actions: read contents: read +concurrency: + group: skill-guidance-${{ github.event.pull_request.number || github.ref }}-${{ github.event_name }} + cancel-in-progress: true + jobs: + changes: + if: ${{ inputs.suites == '' }} + uses: ./.github/workflows/ci-changes.yml + with: + baseline-workflow: skill-guidance.yml + validate: + needs: changes + if: ${{ !cancelled() && ((inputs.suites == '' && needs.changes.result != 'success') || fromJSON(inputs.suites || needs.changes.outputs.suites || '{}').skill_guidance) }} runs-on: ubuntu-latest timeout-minutes: 10 steps: diff --git a/.github/workflows/upgrade-release.yml b/.github/workflows/upgrade-release.yml index ad4748593d..f48d8fb3f1 100644 --- a/.github/workflows/upgrade-release.yml +++ b/.github/workflows/upgrade-release.yml @@ -247,6 +247,7 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 60 strategy: + max-parallel: 3 fail-fast: false matrix: ${{ fromJSON(needs.prepare.outputs.matrix) }} env: @@ -297,7 +298,7 @@ jobs: collect: name: Verify the selected upgrade matrix needs: [prepare, cases] - if: always() && needs.prepare.result == 'success' && !inputs.skip_upgrade_acceptance + if: ${{ !cancelled() && needs.prepare.result == 'success' && !inputs.skip_upgrade_acceptance }} runs-on: ubuntu-24.04 timeout-minutes: 15 steps: diff --git a/.github/workflows/zcode-acceptance.yml b/.github/workflows/zcode-acceptance.yml index fc74e32c7d..5ac24ab145 100644 --- a/.github/workflows/zcode-acceptance.yml +++ b/.github/workflows/zcode-acceptance.yml @@ -1,19 +1,20 @@ name: ZCode acceptance on: - pull_request: - types: [opened, synchronize, reopened, ready_for_review] - push: - branches: [master] + workflow_call: + inputs: + suites: + description: Scope already selected by the Main orchestrator + required: true + type: string workflow_dispatch: - schedule: - - cron: '23 3 * * 0' permissions: + actions: read contents: read concurrency: - group: zcode-acceptance-${{ github.event.pull_request.number || github.ref }} + group: zcode-acceptance-${{ github.event.pull_request.number || github.ref }}-${{ github.event_name }} cancel-in-progress: true env: @@ -25,11 +26,14 @@ env: jobs: changes: + if: ${{ inputs.suites == '' }} uses: ./.github/workflows/ci-changes.yml + with: + baseline-workflow: zcode-acceptance.yml controlled-cli: needs: changes - if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').zcode) }} + if: ${{ !cancelled() && ((inputs.suites == '' && needs.changes.result != 'success') || fromJSON(inputs.suites || needs.changes.outputs.suites || '{}').zcode) }} runs-on: windows-latest timeout-minutes: 30 defaults: diff --git a/docs/en/development/ci-scope.md b/docs/en/development/ci-scope.md new file mode 100644 index 0000000000..6e62b13786 --- /dev/null +++ b/docs/en/development/ci-scope.md @@ -0,0 +1,126 @@ +# GitHub CI design and operation + +CI must validate affected behavior while controlling duplicate work, wasted work and bursts of runner demand. The upstream trunk branch is `master`, and automatic trunk validation listens only to that branch. Workflow policy, path ownership and required outcomes are reviewed with code. Release acceptance remains independent. + +## Entry points and admission + +Ordinary PRs targeting `master` and pushes to `master` start only `Main`. Its `CI preflight` job checks out the repository once and runs scope selection, license validation, quality checks and API-contract validation as separate steps on one runner. It then calls existing reusable workflows with the complete change scope. Called workflows reuse this selection without allocating another detector runner; directly invoked manual workflows retain their own scope detection. The Desktop `codex/desktop-preview` branch and workflow-specific manual diagnostics retain independent entry points. + +```mermaid +flowchart TD + E[PR / trunk push / daily / manual] --> Q[CI preflight: scope, license, quality and API contract] + Q --> T[Selected Python tests] + Q --> W[Selected website and documentation checks] + T --> H[Selected backend, upgrade, platform and host acceptance] + H --> U[Upgrade release gate] + Q --> R[CI result] + T --> R + W --> R + H --> R + U --> R +``` + +Preflight failure prevents downstream tests and heavy work from starting. Python-test failure prevents dependent backend, upgrade, platform and host acceptance from expanding. Website and documentation checks can run alongside Python tests after preflight succeeds. Skipping downstream work never turns the original failure green. Grouping the four preflight checks in one job removes two runner allocations per ordinary validation run and avoids a separate runner wait between license and quality checks; each check retains its own step result and logs. + +Draft PRs allocate no automatic CI runners. Marking a PR ready validates its complete diff; conversion back to draft cancels the old run. Only the latest automatic run survives for each PR. Schedule, push and manual entry points use separate concurrency groups so diagnostics do not cancel trunk or daily validation. These groups replace obsolete work for the same change; they are not cross-PR resource queues. + +## Validation profiles + +| Entry point or input | Python validation | Other acceptance | +| --- | --- | --- | +| Ordinary ready PR | 3.11/3.14 unit tests and 3.12 E2E: 3 combinations | Complete selected backend, upgrade, installation-role, platform and host acceptance | +| PR changes dependencies, shared fixtures, CI policy or unclassified files | 3.11–3.14 × unit/E2E: 8 combinations | Conservative affected or full selection | +| PR update/reopen already carrying `ci:full` | 8 combinations | Complete PR scope, without forcing unrelated diagnostics | +| `master` push | 8 combinations | Compare with a successful trunk baseline, retaining unverified changes | +| Daily at 20:17 UTC | 8 combinations | Full ordinary acceptance, excluding historical schema generation and lock experiments | +| Manual Main | 8 combinations | Full scope; upgrade selection defaults to `full`, with an explicit diagnostic subset available | +| Published release | Independent installed-package and publication checks | Complete upgrade evidence and validated publication bytes; PR scope cannot bypass acceptance | + +Ordinary PR Python combinations drop from eight to three, a 62.5% reduction in combination count. This is not a measured reduction in runner minutes or queue latency. Dependency and CI-policy changes keep the full matrix, so a PR editing CI itself may still run eight combinations. + +When packaging is selected, installation validation uses two jobs, one each for Python 3.11 and 3.14. Each job builds its wheel once and verifies five roles: `core` without extras, plus `builtin`, `cli`, `builtin,cli` and `server,cli`. Each role gets its own virtual environment, application state and temporary directory, and runs against the installed wheel with checkout imports excluded. The core role checks public imports without optional dependencies. The CLI role also verifies that adding unrelated SQLAlchemy and FastAPI packages does not enable optional roles. Every role produces a result; any failed role fails its job. All ten Python/role combinations remain covered while eight runner allocations and eight duplicate wheel builds are removed. The job-count reduction does not establish a queue-latency or throughput improvement without measurements. + +`ci:full` is read from opened/synchronize/reopened/ready PR events. Adding the label to a running PR does not dispatch another workflow: update the branch or reopen the PR afterward. Manual Main uses the repository ref selected in Actions. For a fork PR, select its branch in the source repository; a successful default-branch run is not evidence for the PR head. + +## Ownership and comparison baseline + +`.github/ci-paths.toml` declares independent scopes in `[suites]`, shared inputs in `[shared.*]`, and successful Main root jobs required for each selected scope in `[checks]`. A new scope must declare its required terminal jobs. + +- PRs compare their merge base to the complete head, covering all commits and excluding unrelated base changes. +- Trunk pushes compare with a successful ancestor from the same Main workflow and branch, examining up to 100 successful pushes. Failed or cancelled commits never become baselines; their unverified changes remain in the next comparison. +- Missing history, unavailable APIs or failed diffs select every scope and the full Python matrix. Unknown paths also fall back conservatively. +- Git supplies the full diff, without native path-filter truncation. Disabled rename detection includes both removed and added paths. +- Runtime, API and root dependencies conservatively cover consumers. Websites and isolated integrations retain their ownership rules. Extended lock observation is manual. Historical schema capture and short lock probes run for their own changed inputs or manual requests, outside ordinary daily validation. + +For example, documentation selects website and documentation-contract checks; Cindy changes select Python regression and Cindy-package checks; migrations and shared runtime changes retain relevant real-database and upgrade acceptance. A smaller PR Python matrix does not replace selected real backends with simulations. + +Helm charts select deployment-contract checks. Managed resource producers select the Windows x86-64, macOS arm64 and macOS x86-64 candidate builds through Main, after quality and any selected Python tests succeed. Their native build matrix has no independent PR trigger. Shared runtime and dependency changes retain these candidate builds; unrelated website or Desktop UI changes do not start them. Candidate artifacts remain unqualified until the separate resource qualification process accepts them. + +Shared helpers select their actual consumers: the clean-migration runner also selects managed-startup and upgrade acceptance; the pnpm cache action also selects the Python test setup and Cindy package. A path filter must follow dependencies, not only directory names. + +## Parallelism and cancellation budgets + +| Matrix | Maximum parallel jobs per run | Failure behavior | +| --- | --- | --- | +| Python tests | 3 | Stop remaining combinations after the first failure | +| Isolated installations, one job per Python version | 2 | Attempt every role within a job; a failed job cancels the other Python job if still running | +| Installed-release upgrades | 3 | Preserve all cases and complete evidence collection | +| OceanBase clean-migration shards | 2 | Preserve all shards and completeness checks | +| Candidate replay | 2 | Preserve every backend result | +| Historical schema capture | 2 | Preserve every configuration result | +| Managed resource candidates | 2 | Preserve all three native targets | + +`max-parallel` bounds one matrix run, not the repository or organization. Do not place unrelated PRs in a normal concurrency group as a global semaphore: the default group retains only one pending run and replaces other waiting work. Release publication must not share a cancellable PR group. + +A concurrency cap does not remove work and can lengthen one PR's completion time. Queue reduction comes from skipping unrelated scopes, eliminating duplicate entries, grouping short checks on one runner, reusing installation builds, using the smaller PR compatibility matrix and cancelling superseded runs. Runtime and CI-policy changes still require broad acceptance, so their completion time remains sensitive to organization capacity. + +Organization capacity should provide separate pools: reserve capacity for basic checks and Python tests, isolate database/upgrade workloads in a heavy pool, and budget Windows/macOS separately. GitHub larger-runner groups or appropriately isolated ephemeral runners can provide these pools. Create and verify actual group names and capacity before changing `runs-on`; nonexistent labels would only queue indefinitely. Account for other repositories sharing organization capacity. + +These rollout tasks require organization administration. Workflow code does not change paid plans, budgets, runner groups or organization concurrency: + +1. Inspect organization Settings → Actions → Runners for active jobs, total/OS limits and other repositories' usage. Check billing, budgets and service availability as well. +2. Create and qualify runner groups before migrating workflows. A starting allocation of roughly 20% basic checks, 40% ordinary tests and 40% heavy acceptance can be evaluated against measured minutes; it is not a capacity configuration already applied by this change. +3. After merging and validating the workflows, require the stable `CI result` check while preserving existing review requirements. The quality job's displayed check name is `CI preflight`; license and scope validation are its steps, rather than separate status checks. Replace required-check rules that still name the old quality, license or scope checks. Reusable workflow check names acquire Main caller prefixes, so external status-check rules must be reconciled. + +See GitHub's [concurrency semantics](https://docs.github.com/en/actions/how-tos/write-workflows/choose-when-workflows-run/control-workflow-concurrency) and [organization job and capacity monitoring](https://docs.github.com/en/actions/how-tos/manage-runners/github-hosted-runners/view-current-jobs). + +## Terminal gates + +`CI result` depends on every Main root job. Any failure or cancellation blocks it. Root jobs required by selected scopes must succeed: unexpected skipping or missing results also block. Only explicitly unrelated jobs may skip. + +`Upgrade release gate` accepts skipped acceptance only after successful preflight explicitly excludes upgrades. Selected acceptance must succeed. Cancelled runs allocate no new summary-gate runners; necessary in-job cleanup and diagnostics retain `always()`. Publication still checks the complete upgrade matrix and evidence files independently of path selection. + +The upgrade evidence collector runs after ordinary case failures so missing or failed evidence cannot pass; it stops on workflow cancellation. Harness acceptance explicitly checks successful validation with a cancellation-aware condition, so skipping the redundant child scope detector cannot silently skip its SQLite/OceanBase matrix. + +## Clearing an existing backlog + +Inspect jobs, rather than treating a workflow awaiting its final summary as an entirely unstarted matrix: + +```sh +python3 scripts/ci_queue_report.py --repo oceanbase/powercontext \ + --max-runs 40 --output /data/codex-tmp/ci-queue.json +``` + +This bounded read-only report records sample coverage and actual job states. It never cancels, retries or changes a run. `workflow_age_minutes` is the workflow's age, not an individual job's queue latency. Incomplete or failed queries are not evidence of zero backlog. + +Operational sequence: + +1. Compare queued work with current PR heads and state. Remove only obsolete runs for closed PRs or superseded heads. Preserve current-head checks, latest trunk verification, daily full verification and releases. Updated concurrency policy does not establish that historical runs have already been cleared. +2. For summary jobs remaining after cancellation, confirm the parent run and current PR head before removing the old run. The new cancellation predicates prevent recurrence. +3. If jobs queue while no actual jobs run, investigate organization capacity, budgets and runner availability first. A smaller matrix cannot substitute for capacity troubleshooting. +4. Rerun only failures supported by evidence of nondeterminism. Fix deterministic assertions, contract drift and incompatible database schemas first. Retain failure logs and real-backend evidence; do not add unbounded automatic retries. + +Observe at least one full working day and one daily full validation after rollout. Track ready-PR to basic-check completion, selected jobs' queue p50/p95, actual runner minutes, consumed minutes of cancelled jobs, OS usage and failure/rerun rate. Suggested targets are basic-check p95 under five minutes and ordinary PR test-start queue p95 under ten minutes. These require capacity and runtime measurements; they are not guarantees from a code change. + +To roll back coverage policy, restore the full PR matrix while retaining orchestration, scope selection and cancellation fixes. Never simultaneously enable automatic child triggers and Main calls, which would duplicate executions. + +## Validation + +```sh +uv run pytest -q tests/test_ci_changes.py tests/test_ci_pipeline.py \ + tests/test_upgrade_release_gate.py tests/test_workflow_actions.py tests/test_e2e_workflow_contract.py +uv run python scripts/check_workflow_actions.py .github/workflows .github/actions +actionlint -shellcheck= -pyflakes= +``` + +Local checks use real Git histories, simulated Actions metadata and executable terminal-gate scripts. They validate selection, budgets and outcome contracts, not the GitHub scheduler, native operating systems or real-database acceptance. Measure throughput improvement after deployment. diff --git a/docs/en/development/meta.json b/docs/en/development/meta.json index 69ad9aa294..7f844f2074 100644 --- a/docs/en/development/meta.json +++ b/docs/en/development/meta.json @@ -4,6 +4,7 @@ "defaultOpen": false, "pages": [ "releasing", + "ci-scope", "managed-resource-production", "core-protocol", "memory-layer", diff --git a/docs/zh/development/ci-scope.md b/docs/zh/development/ci-scope.md new file mode 100644 index 0000000000..0bb70c2047 --- /dev/null +++ b/docs/zh/development/ci-scope.md @@ -0,0 +1,126 @@ +# GitHub CI 设计与运行 + +CI 的目标是让每次变更获得必要的验证,同时控制重复工作、无效工作和瞬时 runner 占用。上游主分支为 `master`,主干自动验证只监听该分支。工作流、路径归属与终态要求均随代码审查,发布验收独立保留。 + +## 入口和执行顺序 + +面向 `master` 的普通 PR 和 `master` push 只启动 `Main`。其中 `CI preflight` 任务只检出一次仓库,在同一 runner 上分别执行范围检测、许可证、质量和 API 合约检查,再将完整变更范围传给现有可复用工作流。被调用工作流复用结果,不再分别分配 runner 检测变更;直接手动运行子工作流时仍自行检测范围。Desktop 的 `codex/desktop-preview` 分支和各工作流手动诊断入口仍可独立运行。 + +```mermaid +flowchart TD + E[PR / 主干 push / 每日 / 手动] --> Q[CI preflight: 范围、许可证、质量和 API 合约] + Q --> T[选中的 Python 测试] + Q --> W[选中的网站与文档检查] + T --> H[选中的后端、升级、平台和宿主验收] + H --> U[Upgrade release gate] + Q --> R[CI result] + T --> R + W --> R + H --> R + U --> R +``` + +前置检查失败后,不展开后续测试和重型任务;Python 测试失败后,不展开依赖它们的后端、升级、平台和宿主任务。网站和文档检查可以在前置检查成功后与 Python 测试并行。失败保留在最终状态中,不因跳过后续任务而变绿。四项前置检查合并为一个任务,每次常规验证减少两次 runner 分配,许可证和质量检查之间也无需再次排队;每项检查仍有独立的步骤结果和日志。 + +草稿 PR 不分配自动 CI runner。标记为 ready 时验证完整 PR;转回 draft 会取消旧运行。每个 PR 只保留最新一次自动运行。定时、push、手动入口使用不同的并发组,避免手动诊断取消主干或每日验证。并发组中的取消机制用于淘汰同一变更的旧运行,不作为跨 PR 的资源队列。 + +## 验证层级 + +| 入口或输入 | Python 验证 | 其他验收 | +| --- | --- | --- | +| 普通 ready PR | 3.11/3.14 单元测试,3.12 E2E,共 3 个组合 | 运行变更选中的完整后端、升级、安装角色、平台和宿主验收 | +| PR 修改依赖、共享夹具、CI 策略,或出现未分类文件 | 3.11–3.14 × unit/E2E,共 8 个组合 | 保守选择相关范围或全部范围 | +| 已带 `ci:full` 标签的 PR 更新或重新打开 | 8 个组合 | 按完整 PR 变更范围选择,不强行运行无关诊断 | +| `master` push | 8 个组合 | 从成功主干基线选择,保留此前未验证变更 | +| 每日 UTC 20:17(北京时间次日 04:17) | 8 个组合 | 完整常规验收;不自动生成历史 schema 或执行锁实验 | +| 手动运行 Main | 8 个组合 | 全量范围;升级默认 `full`,可选专门诊断子集 | +| 正式发布 | 独立的安装包与发布验收 | 完整升级证据、汇总与发布字节绑定;不受 PR 路径跳过影响 | + +普通 PR 的 Python 测试组合数由 8 降至 3,减少 62.5%。这是组合数量的变化,不是已经测得的 runner 分钟或等待时间降幅。依赖和 CI 策略变更保持全矩阵,因此修改 CI 的 PR 本身可能仍运行 8 个组合。 + +安装验证在 packaging 范围选中时运行两个任务,分别使用 Python 3.11 和 3.14。每个任务只构建一次 wheel,依次验证不带 extras 的 `core`,以及 `builtin`、`cli`、`builtin,cli`、`server,cli`,共五种角色。每个角色使用独立的虚拟环境、应用状态和临时目录,针对已安装 wheel 验证,排除 checkout 源码导入。core 角色检查未安装可选依赖时的公共导入。CLI 角色还检查安装无关的 SQLAlchemy、FastAPI 包后不会启用可选角色。各角色分别产生结果,任何角色失败都会使所属任务失败。全部十个 Python/角色组合仍受验证,同时减少八次 runner 分配和八次重复 wheel 构建。任务数量减少不代表已测得等待时间或吞吐量改善。 + +`ci:full` 在 PR 的 opened/synchronize/reopened/ready 事件中读取。给一个正在运行的 PR 添加标签不会再触发一套流水线;需要随后更新提交或重新打开 PR。手动 Main 使用 Actions 界面选择的仓库内 ref;fork PR 应在其来源仓库选择对应分支,不能把默认分支的通过结果当成 PR head 的通过结果。 + +## 变更归属与基线 + +`.github/ci-paths.toml` 是归属表:`[suites]` 声明独立范围,`[shared.*]` 声明共同输入,`[checks]` 声明该范围选中后必须成功的 Main 根任务。新增范围必须同时声明终态任务。 + +- PR 使用 merge base 到完整 head 的三点差异,覆盖全部提交,不混入无关基线变更。 +- 主干 push 比较同一 Main 工作流、同一分支上最近的成功祖先,最多检查 100 次成功 push。失败或取消的提交不会成为基线,其变更保留到后续比较。 +- 无可信基线、API 不可用、历史缺失或 diff 失败时,选择全部范围和完整 Python 矩阵。未知路径也保守回退,不能静默漏检。 +- 通过 Git 读取完整差异,禁用重命名检测,同时考虑迁移前后的路径,不受原生 paths 文件数量截断影响。 +- 运行时、API、根依赖变更保守覆盖消费者;网站和独立集成仍按归属选择。长时间锁观察是手动操作。历史 schema 捕获和短锁探针只有自身输入变化或手动请求才执行,不加入每日常规验证。 + +例如:纯文档变更只选择网站和文档合约;Cindy 改动选择 Python 回归和 Cindy 包检查;迁移与共享运行时改动仍选择相关真实数据库和升级验收。调小 PR Python 矩阵不会将被选中的真实后端换成模拟测试。 + +Helm chart 改动选择部署合约检查。托管资源生产器通过 Main 选择 Windows x86-64、macOS arm64 和 macOS x86-64 候选构建,在质量检查和被选中的 Python 测试通过后执行,不再独立响应 PR。共享运行时和依赖改动仍覆盖这些构建;无关的网站或桌面 UI 改动不启动它们。候选产物仍需通过独立的资源资格验证流程才能用于交付。 + +共享辅助文件按实际消费者选择:干净迁移 runner 脚本同时触发 managed-startup 和升级验收;pnpm 缓存 action 同时覆盖 Python 测试环境及 Cindy 包。路径规则需要沿依赖关系判定,不能只看目录名称。 + +## 并发预算与取消 + +| 矩阵 | 每次运行的最大并发 | 失败策略 | +| --- | --- | --- | +| Python 测试 | 3 | 首个失败后停止其余组合 | +| 隔离安装,每个 Python 版本一个任务 | 2 | 单个任务尝试全部角色;任务失败后取消尚未结束的另一个 Python 任务 | +| 安装发布升级 | 3 | 保留全部用例与证据汇总 | +| OceanBase 干净迁移分片 | 2 | 保留全部分片与完整性检查 | +| Candidate replay | 2 | 保留所有后端结果 | +| 历史 schema 捕获 | 2 | 保留所有配置结果 | +| 托管资源候选构建 | 2 | 保留全部三个原生平台 | + +`max-parallel` 只限制一条矩阵运行,不限制整个仓库或组织。不能把多个 PR 放进一个普通 concurrency 组来模拟全局 semaphore:默认组只保留一个 pending 项,会替换其他 PR 的等待任务。正式发布也不能与普通 PR 共用取消组。 + +并发上限不会减少工作总量,还可能延长单个 PR 的完成时间。减少排队主要依靠跳过无关范围、消除重复入口、让短检查共用 runner、复用安装构建、缩小普通 PR 的兼容矩阵,以及取消被新提交替代的运行。运行时和 CI 策略改动仍需广泛验收,完成时间依然受组织可用容量影响。 + +组织层应按实际 runner 并发和 OS 配额建立独立容量池:基础检查与 Python 测试优先保留容量,数据库/升级使用隔离的重型池,Windows/macOS 使用独立平台预算。需要时采用 GitHub larger runner 的 runner group 或有明确隔离边界的临时 runner;具体名称和容量要在组织侧创建后才能写入 `runs-on`,不能在工作流里填写不存在的标签。容量不足应按组织全体仓库核算,不只观察本仓库。 + +以下配置需要组织管理员在上线时操作,代码不会修改付费计划、预算、runner group 或组织并发额度: + +1. 在组织 Settings → Actions → Runners 检查实际活跃任务、并发上限、OS 上限及其他仓库占用;同时核查预算、账单和服务状态。 +2. 若需要容量隔离,先创建并验证 runner group,再迁移工作流。初始配比可按基础检查/普通测试/重型后端约 20%/40%/40% 试运行,根据实际分钟数调整;这不是当前已配置的容量。 +3. 工作流合并并验证后,将稳定的 `CI result` 配置为合并必需检查,同时保留现有审批要求。quality 任务显示的检查名称为 `CI preflight`,许可证和范围检测是其内部步骤,不再提供独立状态检查;仍要求旧 quality、许可证或范围检查名称的规则需要替换。工作流改为可复用后,部分检查名称会带 Main 调用任务前缀,外部规则必须同步核对。 + +GitHub 语义参见 [concurrency](https://docs.github.com/en/actions/how-tos/write-workflows/choose-when-workflows-run/control-workflow-concurrency) 和 [当前任务与组织容量](https://docs.github.com/en/actions/how-tos/manage-runners/github-hosted-runners/view-current-jobs)。 + +## 汇总门禁 + +`CI result` 依赖 Main 的全部根任务:任何失败或取消都会阻断;被选中范围对应的根任务必须成功,意外 skipped 或缺失同样阻断。仅明确无关的任务允许跳过。 + +`Upgrade release gate` 只在前置检查成功且明确未选择升级时接受 skipped。选中的升级必须成功。旧运行被取消后,汇总门禁不会再分配 runner;任务内部的必要清理和诊断仍保留 `always()`。升级发布继续检查全矩阵和证据文件,不能用 CI 路径判定替代发布验收。 + +升级证据收集在普通 case 失败后仍执行,缺失或失败的证据无法通过;整个运行被取消后停止。Harness Acceptance 显式检查 validate 成功及取消状态,跳过重复的子流程范围检测不会连带跳过 SQLite/OceanBase 验收矩阵。 + +## 处理当前积压 + +先按 job 检查,不把一个等待汇总的 workflow 算成整套矩阵尚未运行: + +```sh +python3 scripts/ci_queue_report.py --repo oceanbase/powercontext \ + --max-runs 40 --output /data/codex-tmp/ci-queue.json +``` + +该命令只读,限制查询数量,输出采样覆盖范围和每条运行的真实 job 状态;不会取消、重试或修改任何任务。`workflow_age_minutes` 是整条运行的年龄,不能当成具体 job 的等待时间。查询不完整或出错不能解释成零积压。 + +处理顺序: + +1. 按 PR 当前 head 和状态核对积压;仅清理已关闭或已被新 head 替代的旧运行。保留当前 head、主干最新验证、每日完整验证和发布任务。并发策略更新不应被当成已清理所有历史运行。 +2. 如只剩被取消运行的汇总任务,核对父运行和 PR head 后清理旧运行;新工作流的取消条件用于防止重现。 +3. 若大量 job 排队而没有实际运行任务,先处理组织容量、预算或 runner 可用性;继续降低矩阵并不能替代容量排障。 +4. 仅对确认具有非确定性的失败进行定向重跑。固定断言、合约漂移、数据库 schema 不兼容应先修复。失败日志和真实后端证据保留,不自动无限重试。 + +部署后连续观察至少一个完整工作日及一次每日完整验证:记录 ready PR 到基础检查完成、选中任务 job 级排队 p50/p95、实际 runner 分钟、取消但已消耗的分钟、每个 OS 的占用与失败重跑率。建议目标是基础检查 p95 小于 5 分钟、常规 PR 开始测试的等待 p95 小于 10 分钟;目标需要容量与运行数据验证,不是代码修改后的保证。 + +回滚时可将 PR profile 恢复为完整矩阵,同时保留统一入口、范围选择和取消修复。不要同时启用子工作流的自动触发和 Main 调用,否则会重新产生重复执行。 + +## 验证 + +```sh +uv run pytest -q tests/test_ci_changes.py tests/test_ci_pipeline.py \ + tests/test_upgrade_release_gate.py tests/test_workflow_actions.py tests/test_e2e_workflow_contract.py +uv run python scripts/check_workflow_actions.py .github/workflows .github/actions +actionlint -shellcheck= -pyflakes= +``` + +本地测试使用真实 Git 历史、模拟 Actions 元数据和实际执行的门禁脚本;它们验证选择、预算与终态合约,不能代替 GitHub 调度器、各原生 OS 和真实数据库验收。吞吐量改善应在上线后测量。 diff --git a/docs/zh/development/meta.json b/docs/zh/development/meta.json index 0b8998c4bd..f03b2a55ab 100644 --- a/docs/zh/development/meta.json +++ b/docs/zh/development/meta.json @@ -4,6 +4,7 @@ "defaultOpen": false, "pages": [ "releasing", + "ci-scope", "managed-resource-production", "core-protocol", "memory-layer", diff --git a/release/upgrade-support.json b/release/upgrade-support.json index aea14df8d0..fd88309f29 100644 --- a/release/upgrade-support.json +++ b/release/upgrade-support.json @@ -4,7 +4,7 @@ "mode": "upgrade-required", "schema_review": { "persistence_sha256": "3c7c16fc489d6f03038cbad4dd82e14c01370761cedcb5620027551894cd7a6f", - "rationale": "The reusable Server migration framework registers actual PowerContext 1.3.0 through s1300 -> s1400 -> s1401 -> s1402 and actual 1.4.0 through the separate immutable r1400 -> s1400 -> s1401 -> s1402 source bridge. Original 1.3 resources and checksums remain byte-identical. Real 1.4.0 SQLite, embedded seekDB and OceanBase structures were captured from the official wheel in all twelve vector/Code configurations; provenance and exact source layouts are packaged with the bridge. The 1.4 bridge preserves current Candidate, Artifact and task data, adds the missing Topic budget/block structure, revokes old lease fences and joins the existing archive/portable-storage path. Empty retained tables align the final schema without inventing 1.3 Candidate history. Source histories retain their identity through subsequent upgrades and recovery; shared head names never authorize adopting unknown layouts. Vector profiles remain bound to historical model, profile ID, dimension, unit normalization and L2 distance, with no provider calls during maintenance. Complete schema, domain, projection, profile and retained-history checks precede startup and partial maintenance. Local SQLite and embedded seekDB startup reject an observable active maintenance lock through a read-only probe. SQLite runtime admission validates the default adjacent journal, rejecting unfinished full or partial maintenance, changed recorded revision checksums and missing required retained tables. Absent journals do not invent retention obligations; custom evidence directories require explicit operator verification and remote coordinator locks remain subject to deployment orchestration. Native read-only readiness uses a consistent snapshot and does not acquire Atomic Memory head locks; normal business transaction isolation is unchanged. Offline projection repairs retain the original locked state comparison and write-side CAS protection. Target locks, backup choices, original recovery evidence and explicit remote acknowledgments remain enforced. The installed-artifact catalog retains twelve 1.3.0 source/backend/vector/Code combinations. Embedded seekDB uses pylibseekdb 1.4.0.post2 or newer with the recovered-transaction lock fix. Main and publication require all twelve configurations, including embedded seekDB full-server-code and full-server-vector-code. No configuration is waived; incomplete, failed or skipped evidence rejects acceptance. The seekdb-code diagnostic selection cannot authorize publication. The registered 1.4.0 bridge remains implemented but is outside the CI upgrade acceptance matrix; passing this gate does not certify that path. The gate preserves source-specific Processing counters and isolates OceanBase source databases. Any failed, skipped or missing automatic case rejects publication. The policy remains upgrade-required until the exact target artifact passes the required automatic matrix and the reviewed support declaration is updated. Captured source structure, local tests and checkpoint replay are distinct from installed-release acceptance, native interruption/observer recovery and backup restoration. The Candidate generation prototype observes actual SQLite/seekDB/OceanBase migration checkpoints outside the production bundle; it does not replace frozen registrations or weaken recovery checks. Existing target schema and data formats, including batched persistence interfaces, remain governed by the shared registered path. The synchronized master adds blank-text validation at Source creation and managed worker logging. It changes neither registered table layout nor migration data encodings; existing stored Source values are not rewritten. The source bridges and frozen history remain unchanged. Compatibility commands identify Server-owned storage before opening a writable legacy profile and delegate source admission to the shared framework, including actual 1.4.0 and rejection of unknown layouts. Partial Processing and Atomic Memory projection maintenance verify the original recorded revision checksums; native admission also validates typed journal identity and its completed position against the database revision. Persisted vector profiles reject unsupported partial maintenance before plans, backups or operation evidence, including when the supplied configuration omits embeddings. These are admission-only changes; frozen migration resources, registered layouts, historical checksums and data transformations remain unchanged. PowerContext 1.4.1 assigns release-based presentation names v1_4_1_001 through v1_4_1_003 to the existing shared upgrade steps; the 1.3.0 and 1.4.0 source baselines remain independently registered. Existing revision IDs, bundle and revision checksums, accepted plan identities and journal schemas are unchanged. New Server revisions use software-version-plus-sequence names directly; software releases without storage changes may reuse the same database revision. Naming does not grant acceptance or replace source registration. Full Server status, plan, apply and verify reject unfinished Processing configuration and Atomic Memory projection journals through the same typed admission check used by partial maintenance and SQLite startup. The owning domain runner can still resume its own journal. This only changes admission; frozen revisions, journal formats, database layouts and data transformations remain unchanged. Revision v1_4_1_004 registers the pc_access_owner_recovery outbox on the shared Server chain for both installed-release sources and all registered vector/Code profiles. It creates an empty table through explicit maintenance for existing managed databases and through the registered bootstrap for fresh databases. Earlier revision resources, fingerprints, checksums and existing domain row encodings remain unchanged. External relationship writers use version-1 Artifact and Candidate intents committed with domain data; replay retains the original deployment, principal, resource or target, and idempotency identity. Stable operation and audit event identities prevent lost acknowledgements from duplicating ownership or audit. Conflicting intents are rejected and unknown historical ownership is not inferred. Unregistered outbox layouts are not adopted merely from a table name. This registration does not establish installed-release or external-Provider upgrade acceptance. The policy remains upgrade-required with the full automatic matrix and publication restrictions described above. Database selection now defaults to powercontext for seekDB and for OceanBase URLs without a database, while explicit database names remain authoritative. An implicit seekDB selection is rejected before initialization when the legacy test database contains pc_ tables; operators can explicitly select database=test to retain those records, and no database is moved or deleted automatically. Bootstrap reuses an accessible selected database without CREATE privilege and creates only a confirmed missing database, with idempotent creation and explicit permission errors. Read-only maintenance on an initialized MySQL-compatible engine requires the selected database and never provisions it; an absent or empty seekDB instance remains a no-engine initial plan. MySQL initialization creates only missing requested tables and indexes, and sorts only those requested tables; existing registered tables are preserved. Code graph connections use the selected database, retain historical graph identities for test, and distinguish other database identities. Server identity initialization avoids redundant MySQL DDL on an existing table. Frozen migration resources, revision identifiers and checksums, registered table definitions, journal formats, and existing row encodings are unchanged. These runtime selection and admission changes do not establish an accepted upgrade path; the policy remains upgrade-required and the installed-release acceptance gate still applies. MySQL recovery rejects different registered table sets before normalization. Backend-default normalization copies only the column and index maps it changes and compares each table without cloning whole inventories. Admitted layouts, registered transitions, recovery checks, schema equality and caller-owned inventories remain preserved. SeekDB status and verify distinguish a missing configured database from an uninitialized engine and denied access. Read-only plan can represent a missing selected database as an uninitialized target without creating it; apply provisions that selected database only after the operator accepts its bound plan, then rechecks the target under the maintenance lock. Installed-release seekDB acceptance explicitly retains the historical test database and separately verifies that omitted selection is refused. Registered schema resources, revision checksums, row encodings and write-side provisioning remain unchanged. Server startup can apply an uninitialized plan to provision a missing selected seekDB database; plan, status and verify still never provision it. OceanBase 4.6 SHOW CREATE TABLE can reflect FTS_INDEX_TYPE=MATCH or PHRASE_MATCH on FULLTEXT indexes. Reflection accepts these known types without changing the logical inventory; unknown values remain rejected and indexed columns, parser identity and token-size properties remain significant. PHRASE_MATCH additionally supports phrase matching; the registered layouts require term matching and do not declare a phrase-only contract. Frozen migration resources, checksums, recovery evidence and persisted layouts remain unchanged. Native 4.6.3.1 DDL was captured from a real installed 1.3.0 database; replay of this fixture does not establish complete native upgrade acceptance. The CI database image and master full automatic upgrade matrix remain unchanged. Revision v1_4_1_005 adds nullable Atomic Memory description projection metadata after the registered owner-recovery revision, through explicit SQLite and MySQL schema transitions for both source chains and all vector/Code profiles. Earlier revision resources, fingerprints and checksums remain unchanged. The additive change neither rewrites authoritative Artifact history nor generates summaries or scans stored bodies. Existing managed databases require explicit migration; existing descriptions remain null until a new write or explicit projection repair copies authoritative metadata. Intermediate projection validation uses only columns available at that revision and still rejects inconsistent existing fields. Unmanaged SDK initialization retains its nullable-column compatibility path. The installed-release acceptance matrix and publication restrictions remain required; local regression checks do not establish native upgrade acceptance. Artifact plugin storage extends the shared Server chain from v1_4_1_006 to v1_5_0_001 with registration columns, family declarations, search state and projections, index coverage and changes, and vector-space/index registrations. SQLite also registers portable vectors and FTS auxiliary structures. Native registration columns are added in one ALTER with a matching recovery checkpoint; final column definitions and constraints are unchanged. Native vector tables are created online from registered spaces and are admitted to later maintenance inventories only after exact layout validation. Historical migration resources, retained objects, and existing data are preserved. This review does not expand the supported upgrade matrix or certify installed-release acceptance. Source write plugin registrations use the additive v1_4_1_006 revision after v1_4_1_005 on both source chains. It creates only pc_source_write_plugins; all earlier master revision scripts, schema resources and checksums remain unchanged." + "rationale": "The reusable Server migration framework registers actual PowerContext 1.3.0 through s1300 -> s1400 -> s1401 -> s1402 and actual 1.4.0 through the separate immutable r1400 -> s1400 -> s1401 -> s1402 source bridge. Original 1.3 resources and checksums remain byte-identical. Real 1.4.0 SQLite, embedded seekDB and OceanBase structures were captured from the official wheel in all twelve vector/Code configurations; provenance and exact source layouts are packaged with the bridge. The 1.4 bridge preserves current Candidate, Artifact and task data, adds the missing Topic budget/block structure, revokes old lease fences and joins the existing archive/portable-storage path. Empty retained tables align the final schema without inventing 1.3 Candidate history. Source histories retain their identity through subsequent upgrades and recovery; shared head names never authorize adopting unknown layouts. Vector profiles remain bound to historical model, profile ID, dimension, unit normalization and L2 distance, with no provider calls during maintenance. Complete schema, domain, projection, profile and retained-history checks precede startup and partial maintenance. Local SQLite and embedded seekDB startup reject an observable active maintenance lock through a read-only probe. SQLite runtime admission validates the default adjacent journal, rejecting unfinished full or partial maintenance, changed recorded revision checksums and missing required retained tables. Absent journals do not invent retention obligations; custom evidence directories require explicit operator verification and remote coordinator locks remain subject to deployment orchestration. Native read-only readiness uses a consistent snapshot and does not acquire Atomic Memory head locks; normal business transaction isolation is unchanged. Offline projection repairs retain the original locked state comparison and write-side CAS protection. Target locks, backup choices, original recovery evidence and explicit remote acknowledgments remain enforced. The installed-artifact catalog retains twelve 1.3.0 source/backend/vector/Code combinations. Embedded seekDB uses pylibseekdb 1.4.0.post2 or newer with the recovered-transaction lock fix. Main and publication require all twelve configurations, including embedded seekDB full-server-code and full-server-vector-code. No configuration is waived; incomplete, failed or skipped evidence rejects acceptance. The seekdb-code diagnostic selection cannot authorize publication. The registered 1.4.0 bridge remains implemented but is outside the CI upgrade acceptance matrix; passing this gate does not certify that path. The gate preserves source-specific Processing counters and isolates OceanBase source databases. Any failed, skipped or missing automatic case rejects publication. The policy remains upgrade-required until the exact target artifact passes the required automatic matrix and the reviewed support declaration is updated. Captured source structure, local tests and checkpoint replay are distinct from installed-release acceptance, native interruption/observer recovery and backup restoration. The Candidate generation prototype observes actual SQLite/seekDB/OceanBase migration checkpoints outside the production bundle; it does not replace frozen registrations or weaken recovery checks. Existing target schema and data formats, including batched persistence interfaces, remain governed by the shared registered path. The synchronized master adds blank-text validation at Source creation and managed worker logging. It changes neither registered table layout nor migration data encodings; existing stored Source values are not rewritten. The source bridges and frozen history remain unchanged. Compatibility commands identify Server-owned storage before opening a writable legacy profile and delegate source admission to the shared framework, including actual 1.4.0 and rejection of unknown layouts. Partial Processing and Atomic Memory projection maintenance verify the original recorded revision checksums; native admission also validates typed journal identity and its completed position against the database revision. Persisted vector profiles reject unsupported partial maintenance before plans, backups or operation evidence, including when the supplied configuration omits embeddings. These are admission-only changes; frozen migration resources, registered layouts, historical checksums and data transformations remain unchanged. PowerContext 1.4.1 assigns release-based presentation names v1_4_1_001 through v1_4_1_003 to the existing shared upgrade steps; the 1.3.0 and 1.4.0 source baselines remain independently registered. Existing revision IDs, bundle and revision checksums, accepted plan identities and journal schemas are unchanged. New Server revisions use software-version-plus-sequence names directly; software releases without storage changes may reuse the same database revision. Naming does not grant acceptance or replace source registration. Full Server status, plan, apply and verify reject unfinished Processing configuration and Atomic Memory projection journals through the same typed admission check used by partial maintenance and SQLite startup. The owning domain runner can still resume its own journal. This only changes admission; frozen revisions, journal formats, database layouts and data transformations remain unchanged. Revision v1_4_1_004 registers the pc_access_owner_recovery outbox on the shared Server chain for both installed-release sources and all registered vector/Code profiles. It creates an empty table through explicit maintenance for existing managed databases and through the registered bootstrap for fresh databases. Earlier revision resources, fingerprints, checksums and existing domain row encodings remain unchanged. External relationship writers use version-1 Artifact and Candidate intents committed with domain data; replay retains the original deployment, principal, resource or target, and idempotency identity. Stable operation and audit event identities prevent lost acknowledgements from duplicating ownership or audit. Conflicting intents are rejected and unknown historical ownership is not inferred. Unregistered outbox layouts are not adopted merely from a table name. This registration does not establish installed-release or external-Provider upgrade acceptance. The policy remains upgrade-required with the full automatic matrix and publication restrictions described above. Database selection now defaults to powercontext for seekDB and for OceanBase URLs without a database, while explicit database names remain authoritative. An implicit seekDB selection is rejected before initialization when the legacy test database contains pc_ tables; operators can explicitly select database=test to retain those records, and no database is moved or deleted automatically. Bootstrap reuses an accessible selected database without CREATE privilege and creates only a confirmed missing database, with idempotent creation and explicit permission errors. Read-only maintenance on an initialized MySQL-compatible engine requires the selected database and never provisions it; an absent or empty seekDB instance remains a no-engine initial plan. MySQL initialization creates only missing requested tables and indexes, and sorts only those requested tables; existing registered tables are preserved. Code graph connections use the selected database, retain historical graph identities for test, and distinguish other database identities. Server identity initialization avoids redundant MySQL DDL on an existing table. Frozen migration resources, revision identifiers and checksums, registered table definitions, journal formats, and existing row encodings are unchanged. These runtime selection and admission changes do not establish an accepted upgrade path; the policy remains upgrade-required and the installed-release acceptance gate still applies. MySQL recovery rejects different registered table sets before normalization. Backend-default normalization copies only the column and index maps it changes and compares each table without cloning whole inventories. Admitted layouts, registered transitions, recovery checks, schema equality and caller-owned inventories remain preserved. SeekDB status and verify distinguish a missing configured database from an uninitialized engine and denied access. Read-only plan can represent a missing selected database as an uninitialized target without creating it; apply provisions that selected database only after the operator accepts its bound plan, then rechecks the target under the maintenance lock. Installed-release seekDB acceptance explicitly retains the historical test database and separately verifies that omitted selection is refused. Registered schema resources, revision checksums, row encodings and write-side provisioning remain unchanged. Server startup can apply an uninitialized plan to provision a missing selected seekDB database; plan, status and verify still never provision it. OceanBase 4.6 SHOW CREATE TABLE can reflect FTS_INDEX_TYPE=MATCH or PHRASE_MATCH on FULLTEXT indexes. Reflection accepts these known types without changing the logical inventory; unknown values remain rejected and indexed columns, parser identity and token-size properties remain significant. PHRASE_MATCH additionally supports phrase matching; the registered layouts require term matching and do not declare a phrase-only contract. Frozen migration resources, checksums, recovery evidence and persisted layouts remain unchanged. Native 4.6.3.1 DDL was captured from a real installed 1.3.0 database; replay of this fixture does not establish complete native upgrade acceptance. The CI database image and master full automatic upgrade matrix remain unchanged. Revision v1_4_1_005 adds nullable Atomic Memory description projection metadata after the registered owner-recovery revision, through explicit SQLite and MySQL schema transitions for both source chains and all vector/Code profiles. Earlier revision resources, fingerprints and checksums remain unchanged. The additive change neither rewrites authoritative Artifact history nor generates summaries or scans stored bodies. Existing managed databases require explicit migration; existing descriptions remain null until a new write or explicit projection repair copies authoritative metadata. Intermediate projection validation uses only columns available at that revision and still rejects inconsistent existing fields. Unmanaged SDK initialization retains its nullable-column compatibility path. The installed-release acceptance matrix and publication restrictions remain required; local regression checks do not establish native upgrade acceptance. Artifact plugin storage extends the shared Server chain from v1_4_1_006 to v1_5_0_001 with registration columns, family declarations, search state and projections, index coverage and changes, and vector-space/index registrations. SQLite also registers portable vectors and FTS auxiliary structures. Native registration columns are added in one ALTER with a matching recovery checkpoint; final column definitions and constraints are unchanged. Native vector tables are created online from registered spaces and are admitted to later maintenance inventories only after exact layout validation. Historical migration resources, retained objects, and existing data are preserved. This review does not expand the supported upgrade matrix or certify installed-release acceptance. Source write plugin registrations use the additive v1_4_1_006 revision after v1_4_1_005 on both source chains. It creates only pc_source_write_plugins; all earlier master revision scripts, schema resources and checksums remain unchanged. Code publication recovery adds an optional per-binding publication.json journal that records a generation identifier before its database commit and final directory rename. Caches created before this journal existed remain readable without synthesizing historical publication records; identifiers for already orphaned rows are not inferred. New seekDB generations retain the existing identifier, graph descriptor, manifest and row encodings, while reclamation validates the recorded identifier and scopes deletion to its binding. Live or reader-retained generation directories and the in-flight publication are protected from journal-window reclamation. SQLite graph rows remain owned by their generation directory. This additive local cache journal changes recovery behavior but does not change registered SQL table layouts, migration resources, revision identifiers, historical checksums or the Server migration journal. The policy remains upgrade-required, and the complete installed-release acceptance matrix remains mandatory; this compatibility review is not evidence of completed native upgrade acceptance." }, "cases": [], "automatic_sources": { diff --git a/scripts/ci_changes.py b/scripts/ci_changes.py index 59817c341f..061c64d755 100644 --- a/scripts/ci_changes.py +++ b/scripts/ci_changes.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Select PR checks from the complete Git diff; other events run every suite.""" +"""Select checks from complete PR changes or unverified branch changes.""" from __future__ import annotations @@ -26,6 +26,7 @@ from fnmatch import fnmatchcase from pathlib import Path from typing import Any +from urllib.parse import urlencode ROOT = Path(__file__).resolve().parents[1] @@ -40,8 +41,9 @@ def select_suites(paths: list[str], rules: dict[str, Any]) -> dict[str, bool]: for suite, patterns in rules["suites"].items() if any(fnmatchcase(path, pattern) for pattern in patterns) } - if any(fnmatchcase(path, pattern) for pattern in rules["shared"]["patterns"]): - matches.update(rules["shared"]["suites"]) + for group in rules["shared"].values(): + if any(fnmatchcase(path, pattern) for pattern in group["patterns"]): + matches.update(group["suites"]) if not matches and not any(fnmatchcase(path, pattern) for pattern in rules["ignore"]): # New directories must not silently bypass checks before they have an owner. return dict.fromkeys(selected, True) @@ -50,14 +52,34 @@ def select_suites(paths: list[str], rules: dict[str, Any]) -> dict[str, bool]: return selected -def changed_paths(base: str, head: str) -> list[str]: +def test_matrix(event: str, paths: list[str], rules: dict[str, Any], *, full: bool = False) -> dict[str, Any]: + """Retain full compatibility for dependencies, CI policy and unclassified inputs.""" + shared = any(fnmatchcase(path, pattern) for path in paths for pattern in rules["full"]) + dependencies = any(path in {"pyproject.toml", "uv.lock"} for path in paths) + unknown = any(all(select_suites([path], rules).values()) for path in paths) + if event != "pull_request" or full or shared or dependencies or unknown: + return {"python-version": ["3.11", "3.12", "3.13", "3.14"], "suite": ["unit", "e2e"]} + return { + "include": [ + {"python-version": "3.11", "suite": "unit"}, + {"python-version": "3.14", "suite": "unit"}, + {"python-version": "3.12", "suite": "e2e"}, + ] + } + + +def changed_paths(base: str, head: str, *, pull_request: bool = True) -> list[str]: if not all(re.fullmatch(r"[0-9a-fA-F]{40}", commit) for commit in (base, head)): message = "Expected full base and head commit SHAs" raise ValueError(message) + if not pull_request: + # A rewritten branch must not silently drop changes from the old baseline. + subprocess.run(["git", "merge-base", "--is-ancestor", base, head], check=True, cwd=ROOT) # noqa: S603, S607 + comparison = f"{base}...{head}" if pull_request else f"{base}..{head}" # Three-dot comparison covers the whole PR, excluding unrelated base changes. # Disabling rename detection includes both sides of moves across suite boundaries. result = subprocess.run( # noqa: S603 -- validated SHAs, no shell - ["git", "diff", "--name-only", "--no-renames", "-z", f"{base}...{head}", "--"], # noqa: S607 -- runner tool + ["git", "diff", "--name-only", "--no-renames", "-z", comparison, "--"], # noqa: S607 -- runner tool check=True, capture_output=True, cwd=ROOT, @@ -65,31 +87,95 @@ def changed_paths(base: str, head: str) -> list[str]: return [os.fsdecode(path) for path in result.stdout.split(b"\0") if path] +def successful_baseline(workflow: str, repository: str, branch: str, head: str) -> str: + """Find a successful ancestor for this workflow, retaining cancelled changes.""" + if ( + not re.fullmatch(r"[A-Za-z0-9_-]+\.ya?ml", workflow) + or not re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", repository) + or not re.fullmatch(r"[0-9a-fA-F]{40}", head) + ): + raise ValueError("Expected a workflow filename and repository owner/name") # noqa: TRY003 + query = urlencode({"branch": branch, "event": "push", "status": "success", "per_page": 100}) + result = subprocess.run( # noqa: S603 -- validated API path, encoded query, no shell + ["gh", "api", f"repos/{repository}/actions/workflows/{workflow}/runs?{query}"], # noqa: S607 + check=True, + capture_output=True, + text=True, + timeout=30, + cwd=ROOT, + ) + for run in json.loads(result.stdout)["workflow_runs"]: + candidate = run["head_sha"] + if not re.fullmatch(r"[0-9a-fA-F]{40}", candidate): + continue + ancestor = subprocess.run( # noqa: S603 -- validated commit SHAs, no shell + ["git", "merge-base", "--is-ancestor", candidate, head], # noqa: S607 + capture_output=True, + cwd=ROOT, + check=False, + ) + if ancestor.returncode == 0: + return candidate + raise ValueError("No successful workflow ancestor found") # noqa: TRY003 + + def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--event-name", default=os.environ.get("GITHUB_EVENT_NAME", "")) parser.add_argument("--base", default=os.environ.get("PR_BASE_SHA", "")) - parser.add_argument("--head", default=os.environ.get("PR_HEAD_SHA", "")) + parser.add_argument("--head", default=os.environ.get("PR_HEAD_SHA") or os.environ.get("GITHUB_SHA", "")) + parser.add_argument("--baseline-workflow", default=os.environ.get("CI_BASELINE_WORKFLOW", "")) + parser.add_argument("--repository", default=os.environ.get("GITHUB_REPOSITORY", "")) + parser.add_argument("--branch", default=os.environ.get("GITHUB_REF_NAME", "")) + parser.add_argument( + "--full-validation", action="store_true", default=os.environ.get("CI_FULL_VALIDATION") == "true" + ) arguments = parser.parse_args() rules = tomllib.loads((ROOT / ".github/ci-paths.toml").read_text(encoding="utf-8")) - if arguments.event_name == "pull_request": + reason = "Full validation requested by this event." + paths: list[str] = [] + full = arguments.full_validation + if arguments.event_name in {"pull_request", "push"}: try: - selected = select_suites(changed_paths(arguments.base, arguments.head), rules) - except (OSError, subprocess.CalledProcessError, ValueError): - print("::warning::PR diff unavailable; running every CI suite.") + base = arguments.base + if arguments.event_name == "push" and not base: + base = successful_baseline( + arguments.baseline_workflow, arguments.repository, arguments.branch, arguments.head + ) + paths = changed_paths(base, arguments.head, pull_request=arguments.event_name == "pull_request") + selected = select_suites(paths, rules) + reason = f"Compared {base} to {arguments.head} ({len(paths)} changed paths)." + except (OSError, subprocess.SubprocessError, ValueError, KeyError, TypeError): + reason = "A trustworthy change baseline is unavailable; running every CI suite." + print(f"::warning::{reason}") selected = dict.fromkeys(rules["suites"], True) + full = True else: selected = dict.fromkeys(rules["suites"], True) + if arguments.event_name == "schedule": + # Archive generation and lock experiments have explicit manual entries. + for suite in ("release_schema", "seekdb_lock"): + selected[suite] = False + matrix = test_matrix(arguments.event_name, paths, rules, full=full) + if rules["checks"].keys() != selected.keys(): + raise ValueError("Every CI scope must declare its required result jobs") # noqa: TRY003 + required = sorted( + {"quality", "upgrade-release-gate"} + | {job for suite, jobs in rules["checks"].items() if selected[suite] for job in jobs} + ) print(json.dumps(selected, indent=2)) if output_path := os.environ.get("GITHUB_OUTPUT"): with Path(output_path).open("a", encoding="utf-8") as output: output.write(f"suites={json.dumps(selected)}\n") + output.write(f"test-matrix={json.dumps(matrix)}\n") + output.write(f"required-jobs={json.dumps(required)}\n") if summary_path := os.environ.get("GITHUB_STEP_SUMMARY"): with Path(summary_path).open("a", encoding="utf-8") as summary: - summary.write("## CI scope\n\n| Suite | Run |\n| --- | --- |\n") + summary.write(f"## CI scope\n\n{reason}\n\n| Suite | Run |\n| --- | --- |\n") for suite, enabled in selected.items(): summary.write(f"| {suite} | {'yes' if enabled else 'no'} |\n") + summary.write(f"\nPython matrix: `{json.dumps(matrix)}`\n") if __name__ == "__main__": diff --git a/scripts/ci_installation_roles.py b/scripts/ci_installation_roles.py new file mode 100644 index 0000000000..b57c38bda7 --- /dev/null +++ b/scripts/ci_installation_roles.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Validate every wheel installation role in a fresh environment on one runner.""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Any + +ROLES = ("core", "builtin", "cli", "builtin,cli", "server,cli") + + +def write_summary(output: Path, report: dict[str, Any]) -> None: + (output / "summary.json").write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + rows = ["## Isolated wheel installations", "", "| Role | Result | Stages |", "| --- | --- | --- |"] + for role in report["roles"]: + stages = ", ".join(f"{stage['name']}: {stage['status']}" for stage in role["stages"]) + rows.append(f"| `{role['extras']}` | {role['status']} | {stages} |") + (output / "summary.md").write_text("\n".join(rows) + "\n", encoding="utf-8") + + +def run_roles(*, wheel: Path, python: Path, smoke_script: Path, output: Path, uv: str) -> int: + # Refuse an existing directory so retries cannot inherit an earlier role's state. + output.mkdir(parents=True, exist_ok=False) + (output / "logs").mkdir() + roles: list[dict[str, Any]] = [{"extras": extras, "status": "pending", "stages": []} for extras in ROLES] + report: dict[str, Any] = { + "python": str(python), + "wheel": wheel.name, + "status": "running", + "roles": roles, + } + write_summary(output, report) + for role in roles: + extras = role["extras"] + slug = extras.replace(",", "-") + root = output / "work" / slug + root.mkdir(parents=True) + environment = dict(os.environ) + # Share dependency downloads only; installed packages and application state stay per role. + environment.setdefault("UV_CACHE_DIR", str(output / "uv-cache")) + for name in ("PYTHONPATH", "PYTHONHOME", "VIRTUAL_ENV", "UV_PROJECT_ENVIRONMENT"): + environment.pop(name, None) + for name in ("POWERCONTEXT_HOME", "XDG_CONFIG_HOME", "XDG_DATA_HOME", "XDG_CACHE_HOME", "XDG_STATE_HOME"): + directory = root / name.lower() + directory.mkdir() + environment[name] = str(directory) + temporary = root / "tmp" + temporary.mkdir() + environment.update(TMPDIR=str(temporary), TEMP=str(temporary), TMP=str(temporary), NO_COLOR="1") + executable = root / "venv" / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + requirement = str(wheel) if extras == "core" else f"{wheel}[{extras}]" + commands = [ + ("create environment", [uv, "--no-config", "venv", "--python", str(python), str(root / "venv")]), + ("install wheel", [uv, "--no-config", "pip", "install", "--python", str(executable), requirement]), + ("role smoke", [str(executable), "-I", str(smoke_script), extras]), + ] + if extras == "cli": + commands.extend([ + ( + "install unrelated packages", + [uv, "--no-config", "pip", "install", "--python", str(executable), "sqlalchemy", "fastapi"], + ), + ("optional CLI isolation", [str(executable), "-I", str(smoke_script), extras]), + ]) + stages: list[dict[str, Any]] = [{"name": name, "status": "pending"} for name, _ in commands] + role["stages"] = stages + role["status"] = "running" + write_summary(output, report) + log_path = output / "logs" / f"{slug}.log" + with log_path.open("w", encoding="utf-8") as log: + for stage, (name, command) in zip(stages, commands, strict=True): + print(f"[{extras}] {name}", flush=True) + log.write(f"\n=== {name} ===\n") + log.flush() + try: + result = subprocess.run( # noqa: S603 -- fixed commands with explicit wheel/interpreter paths + command, + cwd=root, + env=environment, + stdout=log, + stderr=subprocess.STDOUT, + check=False, + timeout=600, + ) + except (OSError, subprocess.TimeoutExpired) as error: + log.write(f"{type(error).__name__}: {error}\n") + stage["status"] = "failure" + else: + stage["returncode"] = result.returncode + if result.returncode < 0: + raise SystemExit(128 - result.returncode) + stage["status"] = "success" if result.returncode == 0 else "failure" + write_summary(output, report) + if stage["status"] != "success": + break + role["status"] = "success" if all(stage["status"] == "success" for stage in role["stages"]) else "failure" + write_summary(output, report) + print(f"[{extras}] {role['status']}; log: {log_path}", flush=True) + report["status"] = "success" if all(role["status"] == "success" for role in report["roles"]) else "failure" + write_summary(output, report) + return 0 if report["status"] == "success" else 1 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--wheel", type=Path, required=True) + parser.add_argument("--python", type=Path, default=Path(sys.executable)) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--smoke-script", type=Path, default=Path(__file__).with_name("ci_role_smoke.py")) + arguments = parser.parse_args() + uv = shutil.which("uv") + if uv is None: + parser.error("uv must be installed on PATH") + for value in (arguments.wheel, arguments.python, arguments.smoke_script): + if not value.is_file(): + parser.error(f"Required file does not exist: {value}") + return run_roles( + wheel=arguments.wheel.resolve(), + python=arguments.python.absolute(), + smoke_script=arguments.smoke_script.resolve(), + output=arguments.output_dir.resolve(), + uv=uv, + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ci_queue_report.py b/scripts/ci_queue_report.py new file mode 100644 index 0000000000..3f74057c70 --- /dev/null +++ b/scripts/ci_queue_report.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Read a bounded sample of Actions jobs without cancelling or changing any run.""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +from collections import Counter +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + + +def api(path: str) -> Any: + response = subprocess.run( # noqa: S603 -- validated repository and encoded fixed API queries, no shell + ["gh", "api", path], # noqa: S607 -- standard authenticated GitHub CLI + check=True, + capture_output=True, + text=True, + timeout=60, + ) + return json.loads(response.stdout) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo", default="oceanbase/powercontext") + parser.add_argument("--max-runs", type=int, default=40) + parser.add_argument("--max-job-pages", type=int, default=3) + parser.add_argument("--output", type=Path, required=True) + arguments = parser.parse_args() + if not re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", arguments.repo): + parser.error("--repo must be owner/name") + if not 1 <= arguments.max_runs <= 100 or not 1 <= arguments.max_job_pages <= 3: + parser.error("Use 1..100 runs and 1..3 job pages per run") + root = f"repos/{arguments.repo}/actions" + pending: dict[int, Any] = {} + totals = {} + for status in ("queued", "in_progress", "waiting", "pending"): + data = api(f"{root}/runs?status={status}&per_page=100") + totals[status] = data["total_count"] + pending.update((run["id"], run) for run in data["workflow_runs"]) + # The queue bottleneck is visible in the oldest jobs. A capped sample is + # explicit; unqueried workflows must never be reported as having no jobs. + runs = sorted(pending.values(), key=lambda run: run["created_at"])[: arguments.max_runs] + now = datetime.now(UTC) + records = [] + counts: Counter[str] = Counter() + incomplete = [] + for run in runs: + jobs = [] + total = 0 + for page in range(1, arguments.max_job_pages + 1): + data = api(f"{root}/runs/{run['id']}/jobs?filter=latest&per_page=100&page={page}") + total = data["total_count"] + jobs.extend(data["jobs"]) + if len(jobs) >= total: + break + if len(jobs) < total: + incomplete.append(run["id"]) + states = Counter(job["status"] for job in jobs) + counts.update(states) + records.append({ + "id": run["id"], + "workflow": run["name"], + "status": run["status"], + "event": run["event"], + "head_sha": run["head_sha"], + "branch": run["head_branch"], + "url": run["html_url"], + "created_at": run["created_at"], + "workflow_age_minutes": round( + (now - datetime.fromisoformat(run["created_at"].replace("Z", "+00:00"))).total_seconds() / 60, 1 + ), + "job_counts": dict(states), + "job_total": total, + "unfinished_jobs": [ + { + "id": job["id"], + "name": job["name"], + "status": job["status"], + "labels": job["labels"], + "runner": job["runner_name"], + "url": job["html_url"], + } + for job in jobs + if job["status"] != "completed" + ], + }) + report = { + "captured_at": now.isoformat(), + "repository": arguments.repo, + "workflow_totals": totals, + "sampled_workflows": len(runs), + "workflow_sample_complete": len(runs) == len(pending) and all(value <= 100 for value in totals.values()), + "job_samples_incomplete": incomplete, + "sampled_job_counts": dict(counts), + "runs": records, + } + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(f"Workflow totals: {totals}; sampled {len(runs)} workflows") + print(f"Actual job states in sample: {dict(counts)}") + if not report["workflow_sample_complete"] or incomplete: + print("Partial sample: increase the run limit or query the remaining workflows separately.") + for record in records: + print( + f"{record['id']} {record['workflow']}: {record['job_counts']} " + f"(workflow age {record['workflow_age_minutes']} min; this is not job queue latency)" + ) + + +if __name__ == "__main__": + main() diff --git a/tests/builtin/persistence/test_mysql_vector_admission.py b/tests/builtin/persistence/test_mysql_vector_admission.py index 14a92a7721..330dd3ea2d 100644 --- a/tests/builtin/persistence/test_mysql_vector_admission.py +++ b/tests/builtin/persistence/test_mysql_vector_admission.py @@ -69,7 +69,9 @@ def runner(selected=bundle, configured=configuration): monkeypatch.setattr(migration, "mysql_inventory", lambda _connection: copy.deepcopy(inventory)) monkeypatch.setattr(mysql_schema, "mysql_inventory", lambda _connection: copy.deepcopy(inventory)) - def run(operation, *, writable=False): + def run(operation, *, writable=False, allow_missing_database=False): + # This fixture always has a database, so missing-database admission has no effect. + del allow_missing_database assert not writable, "Admission must fail before requesting a write connection" engine = create_engine(f"sqlite:///{database}") try: diff --git a/tests/builtin/test_code_publication.py b/tests/builtin/test_code_publication.py index 078203a5c7..f97d158449 100644 --- a/tests/builtin/test_code_publication.py +++ b/tests/builtin/test_code_publication.py @@ -308,7 +308,7 @@ def cursor(self) -> Cursor: return Cursor() monkeypatch.setattr(pymysql, "connect", lambda **options: Connection()) - store = SeekDBGraphStore(tmp_path / "seekdb", {}) + store = SeekDBGraphStore(tmp_path / "seekdb", {}, database="test") store.remove_generation("a" * 32, "binding", time.monotonic() + 30) @@ -490,7 +490,7 @@ def cursor(self) -> Cursor: return Cursor() monkeypatch.setattr(pymysql, "connect", lambda **options: Connection()) - store = SeekDBGraphStore(tmp_path / "seekdb", {}) + store = SeekDBGraphStore(tmp_path / "seekdb", {}, database="test") with pytest.raises(CodeError): store.remove_generation("not-hex", "binding", time.monotonic() + 30) @@ -538,7 +538,7 @@ def cursor(self) -> Cursor: return Cursor() monkeypatch.setattr(pymysql, "connect", lambda **options: Connection()) - store = SeekDBGraphStore(tmp_path / "seekdb", {}) + store = SeekDBGraphStore(tmp_path / "seekdb", {}, database="test") store.remove_generation("b" * 32, "binding", time.monotonic() + 30) diff --git a/tests/test_ci_changes.py b/tests/test_ci_changes.py index 02c8b6b0e0..9695998396 100644 --- a/tests/test_ci_changes.py +++ b/tests/test_ci_changes.py @@ -22,6 +22,7 @@ import tomllib from pathlib import Path +import ci_changes import pytest from ci_changes import select_suites @@ -53,13 +54,44 @@ (["integrations/dsh/plugins/powercontext/src/client.ts"], {"tests", "dsh", "cindy"}), (["integrations/dsh/plugins/powercontext/pnpm-lock.yaml"], {"tests", "dsh", "cindy"}), (["scripts/generate_js_operations.py"], {"tests", "dsh", "cindy"}), - (["tests/builtin/persistence/test_mysql_schema.py"], {"tests", "database"}), + (["tests/builtin/persistence/test_mysql_schema.py"], {"tests", "database", "upgrade", "candidate_replay"}), (["tests/native/test_personal_service_lifecycle.py"], {"tests", "native"}), (["e2e/bub/compose.yaml"], {"harness"}), (["scripts/scan_e2e_evidence.sh"], {"tests", "harness"}), - (["scripts/run_atomic_memory_clean_migration.py"], {"tests", "database"}), + (["scripts/run_atomic_memory_clean_migration.py"], {"tests", "database", "upgrade"}), (["scripts/check_atomic_memory_migration_shards.py"], {"tests", "database"}), (["scripts/sync_dsh_plugin_tree.py"], {"tests", "dsh"}), + (["desktop/ui/src/app/App.tsx"], {"desktop"}), + (["release/upgrade-support.json"], {"upgrade", "packaging"}), + ([".github/workflows/desktop.yml"], {"desktop"}), + ([".github/workflows/managed-resource-candidates.yml"], {"managed_resources"}), + (["scripts/produce_managed_resources.py"], {"managed_resources"}), + (["scripts/build_intel_managed_wheel.py"], {"managed_resources"}), + (["scripts/build_ops_bundle.py"], {"managed_resources", "upgrade", "packaging"}), + (["scripts/ci_ops_bundle_smoke.py"], {"managed_resources", "upgrade"}), + (["tests/test_managed_resource_producer.py"], {"tests", "managed_resources"}), + (["tests/test_ops_resources.py"], {"tests", "managed_resources"}), + ( + [".github/actions/cache-pnpm/action.yml"], + {"tests", "desktop", "website", "dsh", "cindy", "pi", "opencode", "openclaw"}, + ), + ([".github/workflows/native-service.yml"], {"native"}), + ([".github/workflows/kimi-code-acceptance.yml"], {"kimi"}), + ([".github/workflows/zcode-acceptance.yml"], {"zcode"}), + ([".github/workflows/upgrade-release.yml"], {"upgrade"}), + ( + [".github/actions/setup-upgrade-oceanbase/action.yml"], + {"database", "upgrade", "candidate_replay", "release_schema"}, + ), + (["tests/e2e/test_worker_otlp.py"], {"tests", "observability"}), + (["scripts/ci_role_smoke.py"], {"tests", "packaging"}), + (["scripts/ci_installation_roles.py"], {"tests", "packaging"}), + (["tests/test_ci_installation_roles.py"], {"tests", "packaging"}), + (["scripts/capture_release_schema.py"], {"upgrade", "release_schema"}), + (["scripts/probe_seekdb_migration_lock.py"], {"seekdb_lock"}), + (["tests/test_database_migration_cli.py"], {"tests", "database", "upgrade"}), + ([".github/workflows/skill-guidance.yml"], {"skill_guidance"}), + ([".licenserc.yaml"], set()), (["integrations/bub/src/powercontext_bub/__init__.py"], {"tests", "harness"}), ( ["docs/en/docs/quickstart.md", "integrations/opencode/package.json"], @@ -79,11 +111,8 @@ def test_pr_selects_affected_suites(paths: list[str], expected: set[str]) -> Non @pytest.mark.parametrize( "path", [ - "src/powercontext/client.py", - "pyproject.toml", - "uv.lock", - "openapi/powercontext.yaml", ".github/actions/setup-python-env/action.yml", + ".github/workflows/ci-changes.yml", ".github/workflows/master.yml", ".github/ci-paths.toml", "tests/e2e/conftest.py", @@ -94,6 +123,26 @@ def test_shared_or_unclassified_changes_run_every_suite(path: str) -> None: assert all(select_suites(["docs/en/docs/quickstart.md", path], RULES).values()) +@pytest.mark.parametrize("path", ["src/powercontext/server/authz/owner_recovery.py", "uv.lock", "pyproject.toml"]) +def test_shared_runtime_inputs_keep_backend_upgrade_and_consumer_coverage(path: str) -> None: + selected = select_suites([path], RULES) + assert { + "tests", + "database", + "upgrade", + "observability", + "packaging", + "native", + "desktop", + "pi", + "zcode", + "cindy", + "managed_resources", + } <= {suite for suite, enabled in selected.items() if enabled} + assert not selected["release_schema"] + assert not selected["seekdb_lock"] + + def git(repository: Path, *arguments: str) -> str: return subprocess.check_output(["git", "-C", str(repository), *arguments], text=True).strip() # noqa: S607 @@ -141,7 +190,25 @@ def run_selector(repository: Path, event: str, base: str = "", head: str = "") - text=True, env={**os.environ, "GITHUB_OUTPUT": str(output), "GITHUB_STEP_SUMMARY": str(repository / "summary.md")}, ) - return json.loads(output.read_text(encoding="utf-8").removeprefix("suites=")) + outputs = dict(line.split("=", 1) for line in output.read_text(encoding="utf-8").splitlines()) + return json.loads(outputs["suites"]) + + +@pytest.mark.parametrize( + ("path", "required_jobs"), + [ + ("scripts/run_atomic_memory_clean_migration.py", {"seekdb-managed-startup", "upgrade-acceptance"}), + (".github/actions/cache-pnpm/action.yml", {"tests", "windows-unit-portability", "cindy-package"}), + ("deploy/helm/powercontext/values.yaml", {"helm"}), + ("scripts/produce_managed_resources.py", {"managed-resource-candidates"}), + ], +) +def test_changed_shared_input_requires_its_consumers_in_result_gate(repository, path, required_jobs): + base = git(repository, "rev-parse", "HEAD") + head = commit(repository, path) + run_selector(repository, "pull_request", base, head) + outputs = dict(line.split("=", 1) for line in (repository / "outputs.txt").read_text().splitlines()) + assert required_jobs <= set(json.loads(outputs["required-jobs"])) @pytest.mark.parametrize( @@ -167,6 +234,15 @@ def test_legacy_evaluation_and_cross_host_changes_select_behavioral_suites( assert {suite for suite, enabled in selected.items() if enabled} == expected +def test_harness_workflow_change_selects_acceptance_and_database_suites(repository: Path) -> None: + base = git(repository, "rev-parse", "HEAD") + head = commit(repository, ".github/workflows/e2e-harness.yml") + + selected = run_selector(repository, "pull_request", base, head) + + assert {suite for suite, enabled in selected.items() if enabled} == {"harness", "database"} + + def test_diff_covers_all_pr_commits_and_excludes_base_branch_changes(repository: Path) -> None: git(repository, "checkout", "-b", "feature") commit(repository, "integrations/pi/plugin.ts") @@ -209,15 +285,121 @@ def test_moves_and_deletions_keep_the_original_suite(repository: Path) -> None: def test_large_diff_includes_code_after_hundreds_of_docs(repository: Path) -> None: base = git(repository, "rev-parse", "HEAD") head = commit(repository, *(f"docs/page-{index}.md" for index in range(350)), "src/powercontext/client.py") - assert all(run_selector(repository, "pull_request", base, head).values()) + selected = run_selector(repository, "pull_request", base, head) + assert selected["tests"] and selected["database"] and selected["upgrade"] + assert selected["desktop"] and selected["website"] -@pytest.mark.parametrize("event", ["push", "workflow_dispatch", "schedule"]) +@pytest.mark.parametrize("event", ["workflow_dispatch", "merge_group"]) def test_non_pr_events_run_every_suite(repository: Path, event: str) -> None: head = commit(repository, "docs/README.md") assert all(run_selector(repository, event, head, head).values()) +def test_nightly_keeps_full_acceptance_without_archive_generation_or_lock_experiments(repository: Path) -> None: + head = commit(repository, "docs/README.md") + selected = run_selector(repository, "schedule", head, head) + assert {suite for suite, enabled in selected.items() if not enabled} == {"release_schema", "seekdb_lock"} + + @pytest.mark.parametrize("base", ["", "not-a-commit", "0" * 40]) def test_unavailable_diff_runs_every_suite(repository: Path, base: str) -> None: assert all(run_selector(repository, "pull_request", base, git(repository, "rev-parse", "HEAD")).values()) + + +def test_push_uses_two_dot_range_from_the_verified_baseline(repository: Path) -> None: + base = git(repository, "rev-parse", "HEAD") + commit(repository, "scripts/upgrade_release_gate.py") + head = commit(repository, "docs/zh/说明.md") + + selected = run_selector(repository, "push", base, head) + + assert {suite for suite, enabled in selected.items() if enabled} == {"upgrade", "website", "docs_tests"} + + +def test_rewritten_push_baseline_runs_every_suite(repository: Path) -> None: + initial = git(repository, "rev-parse", "HEAD") + old_head = commit(repository, "integrations/pi/index.ts") + git(repository, "checkout", "-b", "rewritten", initial) + head = commit(repository, "docs/new.md") + + assert all(run_selector(repository, "push", old_head, head).values()) + + +def test_successful_push_baseline_retains_changes_from_cancelled_runs(repository: Path, monkeypatch) -> None: + base = git(repository, "rev-parse", "HEAD") + # The migration commit has no successful run; the docs push must still test it. + commit(repository, "scripts/upgrade_release_gate.py") + head = commit(repository, "docs/new.md") + real_run = subprocess.run + + def run(command, **kwargs): + if command[0] == "gh": + return subprocess.CompletedProcess(command, 0, json.dumps({"workflow_runs": [{"head_sha": base}]})) + return real_run(command, **kwargs) + + monkeypatch.setattr(ci_changes, "ROOT", repository) + monkeypatch.setattr(subprocess, "run", run) + output = repository / "outputs.txt" + monkeypatch.setenv("GITHUB_OUTPUT", str(output)) + monkeypatch.setattr( + sys, + "argv", + [ + "ci_changes.py", + "--event-name", + "push", + "--base", + "", + "--head", + head, + "--baseline-workflow", + "master.yml", + "--repository", + "example/project", + "--branch", + "master", + ], + ) + + ci_changes.main() + + outputs = dict(line.split("=", 1) for line in output.read_text().splitlines()) + selected = json.loads(outputs["suites"]) + assert {suite for suite, enabled in selected.items() if enabled} == {"upgrade", "website", "docs_tests"} + + +def test_no_successful_push_baseline_runs_every_suite(repository: Path) -> None: + head = commit(repository, "docs/new.md") + assert all(run_selector(repository, "push", "", head).values()) + + +def test_successful_baseline_ignores_a_newer_success_on_rewritten_history(repository: Path, monkeypatch) -> None: + base = git(repository, "rev-parse", "HEAD") + unrelated = commit(repository, "integrations/pi/index.ts") + git(repository, "checkout", "-b", "rewritten", base) + head = commit(repository, "docs/new.md") + real_run = subprocess.run + + def run(command, **kwargs): + if command[0] == "gh": + history = {"workflow_runs": [{"head_sha": unrelated}, {"head_sha": base}]} + return subprocess.CompletedProcess(command, 0, json.dumps(history)) + return real_run(command, **kwargs) + + monkeypatch.setattr(ci_changes, "ROOT", repository) + monkeypatch.setattr(subprocess, "run", run) + + assert ci_changes.successful_baseline("master.yml", "example/project", "master", head) == base + + +def test_selector_cli_exposes_full_matrix_and_required_jobs_on_missing_baseline(repository: Path) -> None: + run_selector(repository, "pull_request", "", git(repository, "rev-parse", "HEAD")) + outputs = dict(line.split("=", 1) for line in (repository / "outputs.txt").read_text().splitlines()) + matrix = json.loads(outputs["test-matrix"]) + assert set(matrix["python-version"]) == {"3.11", "3.12", "3.13", "3.14"} + assert set(matrix["suite"]) == {"unit", "e2e"} + assert set(json.loads(outputs["required-jobs"])) == {job for jobs in RULES["checks"].values() for job in jobs} | { + "quality", + "upgrade-release-gate", + } diff --git a/tests/test_ci_installation_roles.py b/tests/test_ci_installation_roles.py new file mode 100644 index 0000000000..64760d0760 --- /dev/null +++ b/tests/test_ci_installation_roles.py @@ -0,0 +1,151 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Exercise batch isolation and result collection with real disposable virtualenvs.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +SCRIPT = Path(__file__).parents[1] / "scripts/ci_installation_roles.py" +pytestmark = pytest.mark.skipif(os.name == "nt", reason="The Ubuntu batch job probe uses a POSIX uv executable") + + +@pytest.fixture +def installation_probe(tmp_path): + # Simulate only uv's downloads/installations, so the orchestration contract + # runs offline; environment creation and isolated Python smoke processes are real. + uv = tmp_path / "uv" + uv.write_text( + f"#!{sys.executable}\n" + """ +import pathlib +import subprocess +import sys +import venv + +args = sys.argv[1:] +if 'venv' in args: + venv.EnvBuilder(with_pip=False).create(args[-1]) +else: + python = args[args.index('--python') + 1] + site = pathlib.Path(subprocess.check_output( + [python, '-I', '-c', 'import sysconfig; print(sysconfig.get_path("purelib"))'], text=True).strip()) + if args[-1].endswith((']', '.whl')): + extras = args[-1].split('[')[1].removesuffix(']') if args[-1].endswith(']') else 'core' + assert extras != 'core' or args[-1].endswith('.whl') + (site / 'installed_role.py').write_text('ROLE = ' + repr(extras)) + else: + for name in ('sqlalchemy', 'fastapi'): + (site / (name + '.py')).write_text('') +""", + encoding="utf-8", + ) + uv.chmod(0o755) + smoke = tmp_path / "smoke.py" + smoke.write_text( + """ +import importlib.util +import json +import os +import pathlib +import sys +import tempfile + +import installed_role + +role = sys.argv[1] +assert sys.flags.isolated +assert sys.prefix != sys.base_prefix +assert installed_role.ROLE == role +root = pathlib.Path.cwd() +names = ('POWERCONTEXT_HOME', 'XDG_CONFIG_HOME', 'XDG_DATA_HOME', 'XDG_CACHE_HOME', 'XDG_STATE_HOME') +for name in names: + directory = pathlib.Path(os.environ[name]) + assert directory.parent == root + marker = directory / 'owner' + if marker.exists(): + assert marker.read_text() == role + marker.write_text(role) +assert pathlib.Path(tempfile.gettempdir()).parent == root +counter = pathlib.Path(os.environ['POWERCONTEXT_HOME']) / 'smoke-count' +attempt = int(counter.read_text()) + 1 if counter.exists() else 1 +counter.write_text(str(attempt)) +assert (importlib.util.find_spec('fastapi') is not None) == (role == 'cli' and attempt == 2) +with open(os.environ['PROBE_REPORT'], 'a') as report: + report.write(json.dumps({'role': role, 'venv': sys.prefix, 'cwd': str(root), 'attempt': attempt}) + '\\n') +if role == os.environ.get('FAIL_ROLE') and attempt == int(os.environ.get('FAIL_ATTEMPT', '1')): + raise RuntimeError('deliberate role failure') +""", + encoding="utf-8", + ) + wheel = tmp_path / "fixture.whl" + wheel.touch() + return [sys.executable, str(SCRIPT), "--wheel", str(wheel), "--smoke-script", str(smoke)] + + +@pytest.mark.parametrize(("failed_role", "failed_attempt"), [("", 1), ("core", 1), ("builtin", 1), ("cli", 2)]) +def test_batch_keeps_all_roles_isolated_and_reports_every_failure( + installation_probe, tmp_path, failed_role, failed_attempt +): + output = tmp_path / "results" + evidence = tmp_path / "smokes.jsonl" + environment = { + **os.environ, + "PATH": str(tmp_path) + os.pathsep + os.environ["PATH"], + "PROBE_REPORT": str(evidence), + "FAIL_ROLE": failed_role, + "FAIL_ATTEMPT": str(failed_attempt), + } + result = subprocess.run( + [*installation_probe, "--output-dir", str(output)], + env=environment, + text=True, + capture_output=True, + check=False, + timeout=60, + ) + assert result.returncode == (1 if failed_role else 0), result.stdout + result.stderr + report = json.loads((output / "summary.json").read_text()) + assert report["status"] == ("failure" if failed_role else "success") + roles = {row["extras"]: row for row in report["roles"]} + assert set(roles) == {"core", "builtin", "cli", "builtin,cli", "server,cli"} + for role, row in roles.items(): + assert row["status"] == ("failure" if role == failed_role else "success") + assert all(stage["status"] == "success" for stage in row["stages"]) == (role != failed_role) + assert (output / "logs" / f"{role.replace(',', '-')}.log").is_file() + records = [json.loads(line) for line in evidence.read_text().splitlines()] + assert {record["role"] for record in records} == set(roles) + assert len({record["venv"] for record in records}) == 5 + assert len({record["cwd"] for record in records}) == 5 + assert [record["attempt"] for record in records if record["role"] == "cli"] == [1, 2] + assert all(role in (output / "summary.md").read_text() for role in roles) + + # A repeated invocation must not accidentally pass with environments from the first run. + rerun = subprocess.run( + [*installation_probe, "--output-dir", str(output)], + env=environment, + text=True, + capture_output=True, + check=False, + timeout=60, + ) + assert rerun.returncode != 0 + assert [json.loads(line) for line in evidence.read_text().splitlines()] == records diff --git a/tests/test_ci_pipeline.py b/tests/test_ci_pipeline.py new file mode 100644 index 0000000000..ea808a474c --- /dev/null +++ b/tests/test_ci_pipeline.py @@ -0,0 +1,199 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""CI admission, coverage and terminal-result budgets.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import tomllib +from pathlib import Path + +import ci_changes +import pytest +import yaml + +ROOT = Path(__file__).parents[1] +RULES = tomllib.loads((ROOT / ".github/ci-paths.toml").read_text()) + + +def workflow(name): + result = yaml.safe_load((ROOT / ".github/workflows" / name).read_text()) + result["on"] = result.pop(True) + return result + + +def combinations(matrix): + if "include" in matrix: + return {(row["python-version"], row["suite"]) for row in matrix["include"]} + return {(version, suite) for version in matrix["python-version"] for suite in matrix["suite"]} + + +def test_ordinary_pr_has_three_compatibility_jobs(): + assert combinations(ci_changes.test_matrix("pull_request", ["src/powercontext/server/settings.py"], RULES)) == { + ("3.11", "unit"), + ("3.14", "unit"), + ("3.12", "e2e"), + } + + +@pytest.mark.parametrize("event", ["push", "schedule", "workflow_dispatch"]) +def test_full_validation_keeps_all_supported_python_suite_pairs(event): + assert combinations(ci_changes.test_matrix(event, [], RULES)) == { + (version, suite) for version in ("3.11", "3.12", "3.13", "3.14") for suite in ("unit", "e2e") + } + + +@pytest.mark.parametrize("path", ["uv.lock", "pyproject.toml", "tox.ini", "tests/conftest.py", "new-component/x.py"]) +def test_dependency_and_unclassified_pr_changes_keep_full_compatibility(path): + assert len(combinations(ci_changes.test_matrix("pull_request", [path], RULES))) == 8 + + +def test_explicit_full_pr_profile_overrides_the_small_matrix(): + assert ( + len( + combinations( + ci_changes.test_matrix("pull_request", ["src/powercontext/server/settings.py"], RULES, full=True) + ) + ) + == 8 + ) + + +def test_selected_scopes_have_a_complete_terminal_result_contract(): + main = workflow("master.yml")["jobs"] + assert RULES["checks"].keys() == RULES["suites"].keys() + covered = {job for jobs in RULES["checks"].values() for job in jobs} + assert covered | {"quality", "upgrade-release-gate", "ci-result"} == set(main) + assert set(main["ci-result"]["needs"]) == set(main) - {"ci-result"} + + +def test_preflight_uses_one_runner_and_keeps_all_admission_checks_mandatory(): + jobs = workflow("master.yml")["jobs"] + assert {name for name, job in jobs.items() if not job.get("needs")} == {"quality"} + preflight = jobs["quality"] + assert "!github.event.pull_request.draft" in preflight["if"] + assert not preflight.get("continue-on-error") + steps = preflight["steps"] + checkouts = [step for step in steps if step.get("uses", "").startswith("actions/checkout@")] + assert len(checkouts) == 1 + assert checkouts[0]["with"]["fetch-depth"] == 0 + selection = next(step for step in steps if step.get("id") == "select") + license_check = next(step for step in steps if step.get("uses", "").startswith("apache/skywalking-eyes/header@")) + quality = next(step for step in steps if step.get("run") == "make check") + contract = next(step for step in steps if step.get("run") == "make contract-test") + assert selection["env"]["CI_BASELINE_WORKFLOW"] == "master.yml" + assert license_check["with"] == {"config": ".licenserc.yaml", "mode": "check"} + for step in (selection, license_check, quality, contract): + assert not step.get("continue-on-error") + assert "if" not in step # A preceding failure prevents further admission work. + for output in ("suites", "test-matrix", "required-jobs"): + assert preflight["outputs"][output] == "${{ steps.select.outputs." + output + " }}" + for name, job in jobs.items(): + if name not in {"quality", "upgrade-release-gate", "ci-result"}: + assert "quality" in job["needs"] + assert "needs.quality.result == 'success'" in job["if"] + + +def test_installation_roles_use_only_two_python_runners(): + job = workflow("master.yml")["jobs"]["isolated-installations"] + assert job["strategy"]["matrix"] == {"python-version": ["3.11", "3.14"]} + build = [step for step in job["steps"] if "uv build --wheel" in step.get("run", "")] + assert len(build) == 1 + verify = next(step for step in job["steps"] if "ci_installation_roles.py" in step.get("run", "")) + assert not verify.get("continue-on-error") + assert "if" not in verify + reports = next(step for step in job["steps"] if step.get("uses", "").startswith("actions/upload-artifact@")) + assert reports["if"] == "always()" + assert "summary.json" in reports["with"]["path"] + assert "logs/*.log" in reports["with"]["path"] + + +def test_automatic_host_workflows_share_one_scope_and_admission_gate(): + assert workflow("master.yml")["on"]["push"]["branches"] == ["master"] + assert workflow("master.yml")["on"]["pull_request"]["branches"] == ["master"] + main = workflow("master.yml")["jobs"] + for path in (ROOT / ".github/workflows").glob("*.yml"): + if path.name != "master.yml": + events = workflow(path.name)["on"] + assert "pull_request" not in events, path.name + assert "master" not in events.get("push", {}).get("branches", []), path.name + for job in main.values(): + if not job.get("uses", "").endswith(".yml") or "ci-changes" in job["uses"] or "upgrade-release" in job["uses"]: + continue + child = workflow(Path(job["uses"]).name) + assert "pull_request" not in child["on"] + assert "schedule" not in child["on"] + assert "master" not in child["on"].get("push", {}).get("branches", []) + assert job["with"]["suites"] == "${{ needs.quality.outputs.suites }}" + assert {"quality", "tests"} <= set(job["needs"]) + assert "needs.quality.result == 'success'" in job["if"] + assert "needs.tests.result" in job["if"] + assert child["jobs"]["changes"]["if"] == "${{ inputs.suites == '' }}" + for child_job in child["jobs"].values(): + if "needs.changes.result" in child_job.get("if", ""): + assert "(inputs.suites == '' && needs.changes.result != 'success')" in child_job["if"] + assert "fromJSON(inputs.suites || needs.changes.outputs.suites || '{}')" in child_job["if"] + + +def test_compatibility_and_upgrade_matrices_have_bounded_runner_bursts(): + assert int(workflow("master.yml")["jobs"]["tests"]["strategy"]["max-parallel"]) <= 3 + assert int(workflow("master.yml")["jobs"]["isolated-installations"]["strategy"]["max-parallel"]) <= 2 + upgrade = workflow("upgrade-release.yml")["jobs"]["cases"]["strategy"] + assert int(upgrade["max-parallel"]) <= 3 + assert upgrade["fail-fast"] is False # Complete upgrade evidence remains mandatory. + native = workflow("managed-resource-candidates.yml")["jobs"]["candidate"]["strategy"] + assert int(native["max-parallel"]) <= 2 + assert {row["target"] for row in native["matrix"]["include"]} == {"windows-x86_64", "macos-arm64", "macos-x86_64"} + + +def test_cancelled_or_draft_runs_cannot_start_terminal_runner_jobs(): + jobs = workflow("master.yml")["jobs"] + for name in ("upgrade-release-gate", "ci-result"): + assert "!cancelled()" in jobs[name]["if"] + assert "!github.event.pull_request.draft" in jobs[name]["if"] + assert "converted_to_draft" in workflow("master.yml")["on"]["pull_request"]["types"] + + +@pytest.mark.parametrize("selected_job", ["quality", "harness-validation"]) +@pytest.mark.parametrize( + ("selected_result", "unselected_result", "passes"), + [ + ("success", "skipped", True), + ("skipped", "skipped", False), + ("cancelled", "skipped", False), + ("failure", "skipped", False), + ("success", "failure", False), + ], +) +def test_actual_ci_result_script_rejects_missing_selected_acceptance( + selected_job, selected_result, unselected_result, passes +): + script = workflow("master.yml")["jobs"]["ci-result"]["steps"][0]["run"] + script = script.split("<<'PYTHON'\n", 1)[1].rsplit("PYTHON", 1)[0] + required = ["quality", "upgrade-release-gate", "harness-validation"] + results = {job: {"result": "success"} for job in required} + results[selected_job]["result"] = selected_result + results["desktop-validation"] = {"result": unselected_result} + run = subprocess.run( + [sys.executable, "-c", script], + env={**os.environ, "RESULTS": json.dumps(results), "REQUIRED": json.dumps(required)}, + capture_output=True, + text=True, + check=False, + ) + assert (run.returncode == 0) == passes, run.stdout + run.stderr diff --git a/tests/test_e2e_workflow_contract.py b/tests/test_e2e_workflow_contract.py index a48290c62f..2aa2322152 100644 --- a/tests/test_e2e_workflow_contract.py +++ b/tests/test_e2e_workflow_contract.py @@ -14,9 +14,22 @@ from pathlib import Path +import yaml + WORKFLOW = (Path(__file__).parents[1] / ".github" / "workflows" / "e2e-harness.yml").read_text(encoding="utf-8") +def test_supplied_scope_keeps_acceptance_runnable_after_the_detector_is_skipped() -> None: + jobs = yaml.safe_load(WORKFLOW)["jobs"] + assert jobs["changes"]["if"] == "${{ inputs.suites == '' }}" + assert "!cancelled()" in jobs["validate"]["if"] + assert jobs["acceptance"]["needs"] == "validate" + # A successful validate job does not clear the skipped ancestor for Actions' + # implicit success(). Acceptance needs its own status function and success gate. + assert jobs["acceptance"]["if"] == "${{ !cancelled() && needs.validate.result == 'success' }}" + assert set(jobs["acceptance"]["strategy"]["matrix"]["database"]) == {"sqlite", "oceanbase"} + + def test_e2e_scan_keeps_raw_scanner_output_out_of_artifacts() -> None: assert "--json > /dev/null 2>&1" not in WORKFLOW assert "run: bash scripts/scan_e2e_evidence.sh" in WORKFLOW diff --git a/tests/test_upgrade_release_gate.py b/tests/test_upgrade_release_gate.py index d98e3f9ea2..0bf96eeb91 100644 --- a/tests/test_upgrade_release_gate.py +++ b/tests/test_upgrade_release_gate.py @@ -447,9 +447,46 @@ def test_publishing_jobs_require_upgrade_acceptance_unless_manually_bypassed(): main = yaml.safe_load((root / ".github/workflows/master.yml").read_text())["jobs"] assert main["upgrade-acceptance"]["uses"] == build["uses"] verdict = main["upgrade-release-gate"] - assert verdict["name"] == "Upgrade release gate" and verdict["if"] == "always()" - assert verdict["needs"] == "upgrade-acceptance" - assert verdict["steps"][0]["run"] == 'test "$UPGRADE_RESULT" = success' + assert verdict["name"] == "Upgrade release gate" + assert "!cancelled()" in verdict["if"] + dependencies = verdict["needs"] + assert {"quality", "upgrade-acceptance"} <= set([dependencies] if isinstance(dependencies, str) else dependencies) + assert not verdict.get("continue-on-error") + assert all(not step.get("continue-on-error") for step in verdict["steps"]) + + +@pytest.mark.skipif(shutil.which("bash") is None, reason="the Actions gate runs in Bash") +@pytest.mark.parametrize( + ("scope_result", "selected", "upgrade_result", "passes"), + [ + ("success", "false", "skipped", True), + ("success", "false", "failure", False), + ("success", "true", "success", True), + ("success", "true", "skipped", False), + ("success", "true", "failure", False), + ("success", "true", "cancelled", False), + ("failure", "", "skipped", False), + ("success", "", "skipped", False), + ], +) +def test_upgrade_gate_distinguishes_not_required_from_missing_acceptance( + scope_result: str, selected: str, upgrade_result: str, passes: bool +) -> None: + root = Path(__file__).parents[1] + workflow = yaml.safe_load((root / ".github/workflows/master.yml").read_text()) + script = workflow["jobs"]["upgrade-release-gate"]["steps"][0]["run"] + result = subprocess.run( + ["bash", "-e", "-c", script], # noqa: S607 -- execute the checked-in Actions gate + env={ + **os.environ, + "SCOPE_RESULT": scope_result, + "UPGRADE_SELECTED": selected, + "UPGRADE_RESULT": upgrade_result, + }, + capture_output=True, + check=False, + ) + assert (result.returncode == 0) == passes @pytest.mark.parametrize( @@ -477,6 +514,21 @@ def test_repository_baseline_covers_all_three_databases(configuration, monkeypat } +def test_upgrade_collection_keeps_failure_evidence_without_starting_after_cancellation(): + root = Path(__file__).parents[1] + jobs = yaml.safe_load((root / ".github/workflows/upgrade-release.yml").read_text())["jobs"] + collect = jobs["collect"] + assert set(collect["needs"]) == {"prepare", "cases"} + # Explicit !cancelled() overrides the implicit success() check, so failed + # cases still reach evidence collection while superseded runs stop here. + assert collect["if"] == ( + "${{ !cancelled() && needs.prepare.result == 'success' && !inputs.skip_upgrade_acceptance }}" + ) + assert any(step.get("if") == "always()" for step in jobs["cases"]["steps"]) + verdict = next(step for step in collect["steps"] if "CASE_JOBS_RESULT" in step.get("env", {})) + assert verdict["run"] == 'test "$CASE_JOBS_RESULT" = success' + + def test_upgrade_cases_require_isolated_backends_and_always_clean_up(): root = Path(__file__).parents[1] workflow = yaml.safe_load((root / ".github/workflows/upgrade-release.yml").read_text())