Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
329288a
Add BC PR-Review engine arm with BCQuality ref override
Jul 10, 2026
c168e56
Wire engine arm into copilot-evaluation via committed BCQuality-ref t…
Jul 10, 2026
ad0f237
Add BCQuality plugin arm with conditional code-review prompt
Jul 10, 2026
fb5161e
Slim engine adapter to single Invoke-LocalReview.ps1 call
Jul 10, 2026
b3aed49
Drop plugin arm; keep engine-only
Jul 10, 2026
596214b
Trim engine adapter: drop token resolution, env plumbing, unused doma…
Jul 10, 2026
77e0b16
Drop redundant bcquality override plumbing; engine reads BCQUALITY_RE…
Jul 10, 2026
b19d112
Fold engine arm into 'evaluate copilot --engine-scripts-dir' switch; …
Jul 10, 2026
ce21dce
Move engine runner out of agent namespace into evaluate/codereview_en…
Jul 10, 2026
fe46384
Import engine runner directly from its module; keep evaluate __all__ …
Jul 10, 2026
cb4c81a
Rename engine repo reference to microsoft/BC-ALAgents
Jul 10, 2026
6b5d577
Merge branch 'main' into engine-review-arm
gggdttt Jul 24, 2026
ec1e5a1
fix(code-review engine arm): correct Invoke-LocalReview contract
Jul 24, 2026
3ca1717
Harden engine-review arm for BC-Bench conventions
Jul 24, 2026
4dff419
Default engine arm to the latest engine release
Jul 24, 2026
34548ac
Move engine-arm config into code; pin engine, unify severity, sanitiz…
Jul 24, 2026
848dc0d
Parameterize BCQuality ref/repo as workflow_dispatch inputs
Jul 24, 2026
42b8ad0
Merge branch 'main' into engine-review-arm
gggdttt Jul 24, 2026
9325de8
Add publish-results toggle to keep experiment runs off the dashboard
Jul 27, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 33 additions & 3 deletions .github/workflows/copilot-evaluation.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,11 @@ on:
required: false
default: true
type: boolean
publish-results:
description: "Publish results to the shared dashboard (Braintrust/Kusto + leaderboard). Turn off to run a full experiment (e.g. your own BCQuality) without touching the dashboard."
required: false
default: true
type: boolean
al-mcp:
description: "Enable AL MCP server"
required: false
Expand All @@ -62,6 +67,16 @@ on:
required: false
default: ""
type: string
bcquality-ref:
description: "Code-review only: BCQuality ref (branch/tag/SHA) to review against. Blank uses the pinned default (microsoft/BCQuality@main)."
required: false
default: ""
type: string
bcquality-repo:
description: "Code-review only: BCQuality repo as owner/name, for testing a fork. Blank uses microsoft/BCQuality."
required: false
default: ""
type: string

concurrency:
group: copilot-evaluation-${{ inputs.test-run && 'test' || 'full' }}-${{ inputs.category }}
Expand All @@ -82,6 +97,7 @@ jobs:
needs: get-entries
outputs:
results-dir: ${{ env.EVALUATION_RESULTS_DIR }}
agent-label: ${{ steps.agent-label.outputs.label }}
if: needs.get-entries.outputs.entries != '[]'
environment:
name: ado-read
Expand All @@ -100,6 +116,13 @@ jobs:
- name: Checkout repository
uses: actions/checkout@v5

- name: Resolve agent label
id: agent-label
shell: pwsh
run: |
$label = if ('${{ inputs.category }}' -eq 'code-review') { 'BC PR-Review Engine' } else { 'GitHub Copilot CLI' }
"label=$label" | Out-File -FilePath $env:GITHUB_OUTPUT -Append

- name: Setup BC Container and Repository
id: setup-env
timeout-minutes: 40
Expand Down Expand Up @@ -134,6 +157,13 @@ jobs:
shell: pwsh
env:
COPILOT_GITHUB_TOKEN: ${{ github.token }}
GH_TOKEN: ${{ github.token }}
# Code-review only. The engine reads these directly; blank refs use the pinned
# defaults (microsoft/BCQuality@main, engine _DEFAULT_ENGINE_REF). A tester sets
# bcquality-ref (and bcquality-repo for a fork) at dispatch to review their own.
BCQUALITY_REF: ${{ inputs.bcquality-ref }}
BCQUALITY_REPO: ${{ inputs.bcquality-repo }}
ENGINE_REPO_TOKEN: ${{ secrets.ENGINE_REPO_TOKEN }}
run: |
Write-Output "::add-mask::$env:COPILOT_GITHUB_TOKEN"

Expand Down Expand Up @@ -162,8 +192,8 @@ jobs:
with:
results-dir: ${{ needs.evaluate-with-copilot-cli.outputs.results-dir }}
model: ${{ inputs.model }}
agent: "GitHub Copilot CLI"
mock: ${{ inputs.test-run }}
agent: ${{ needs.evaluate-with-copilot-cli.outputs.agent-label }}
mock: ${{ inputs.test-run || !inputs.publish-results }}
category: ${{ inputs.category }}
git-ref: ${{ inputs.git-ref || github.ref_name }}
secrets: inherit
Expand All @@ -179,4 +209,4 @@ jobs:
workflow-file: copilot-evaluation.yml
repeat: ${{ inputs.repeat }}
workflow-inputs: |
{"model": "${{ inputs.model }}", "category": "${{ inputs.category }}", "test-run": "${{ inputs.test-run }}", "al-mcp": "${{ inputs.al-mcp }}", "al-lsp": "${{ inputs.al-lsp }}", "git-ref": "${{ inputs.git-ref || github.ref_name }}"}
{"model": "${{ inputs.model }}", "category": "${{ inputs.category }}", "test-run": "${{ inputs.test-run }}", "al-mcp": "${{ inputs.al-mcp }}", "al-lsp": "${{ inputs.al-lsp }}", "git-ref": "${{ inputs.git-ref || github.ref_name }}", "bcquality-ref": "${{ inputs.bcquality-ref }}", "bcquality-repo": "${{ inputs.bcquality-repo }}", "publish-results": "${{ inputs.publish-results }}"}
10 changes: 9 additions & 1 deletion src/bcbench/agent/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,13 @@
from bcbench.agent.bcal import BCalBackendConfig, run_bcal_agent
from bcbench.agent.claude import run_claude_code
from bcbench.agent.copilot import run_copilot_agent
from bcbench.agent.engine import code_review_uses_engine, run_engine_review

__all__ = ["BCalBackendConfig", "run_bcal_agent", "run_claude_code", "run_copilot_agent"]
__all__ = [
"BCalBackendConfig",
"code_review_uses_engine",
"run_bcal_agent",
"run_claude_code",
"run_copilot_agent",
"run_engine_review",
]
Loading
Loading