radius-cli/.github/workflows/plugin-evals.yml at main · radiustechsystems/radius-cli · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
name: Claude plugin evals

on:
  pull_request:
    paths:
      - .claude-plugin/**
      - plugins/radius/**
      - scripts/validate_plugin.py
      - .github/workflows/plugin-evals.yml
  push:
    branches: [main]
    paths:
      - .claude-plugin/**
      - plugins/radius/**
      - scripts/validate_plugin.py
      - .github/workflows/plugin-evals.yml

permissions: {}

jobs:
  validate:
    runs-on: ubuntu-latest
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
      - name: Validate plugin structure and scenarios
        run: python3 scripts/validate_plugin.py

  eval:
    needs: validate
    # Forked PR code never receives model credentials.
    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository
    runs-on: ubuntu-latest
    timeout-minutes: 45
    permissions:
      contents: read
    env:
      CLAUDE_EVAL_PROVIDER: ${{ vars.CLAUDE_EVAL_PROVIDER }}
      CLAUDE_EVAL_MODEL: ${{ vars.CLAUDE_EVAL_MODEL }}
      CLAUDE_EVAL_JUDGE_MODEL: ${{ vars.CLAUDE_EVAL_JUDGE_MODEL }}
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
      - name: Install Claude Code
        run: |
          curl -fsSL https://claude.ai/install.sh | bash -s 2.1.272
          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
      - name: Validate Claude plugin
        run: claude plugin validate plugins/radius
      - name: Run Claude plugin evals
        working-directory: plugins/radius
        shell: bash
        env:
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
        run: |
          set -euo pipefail
          if [[ "$CLAUDE_EVAL_PROVIDER" == "openrouter" ]]; then
            if [[ -z "$OPENROUTER_API_KEY" ]]; then
              echo '::error::Set OPENROUTER_API_KEY to run plugin evals.'
              exit 1
            fi
            export ANTHROPIC_BASE_URL=https://openrouter.ai/api
            export ANTHROPIC_AUTH_TOKEN="$OPENROUTER_API_KEY"
            export ANTHROPIC_API_KEY=''
            if [[ -z "$CLAUDE_EVAL_MODEL" || -z "$CLAUDE_EVAL_JUDGE_MODEL" ]]; then
              echo '::error::Set CLAUDE_EVAL_MODEL and CLAUDE_EVAL_JUDGE_MODEL for OpenRouter.'
              exit 1
            fi
          elif [[ -z "$CLAUDE_EVAL_PROVIDER" || "$CLAUDE_EVAL_PROVIDER" == "anthropic" ]]; then
            if [[ -z "$ANTHROPIC_API_KEY" ]]; then
              echo '::error::Set ANTHROPIC_API_KEY to run plugin evals, or configure OpenRouter.'
              exit 1
            fi
          else
            echo '::error::CLAUDE_EVAL_PROVIDER must be anthropic or openrouter.'
            exit 1
          fi
          claude plugin eval . \
            --trust-plugin \
            --json results.json \
            --threshold 0.75 \
            --runs 2 \
            --ablation none \
            --model "${CLAUDE_EVAL_MODEL:-claude-sonnet-5}" \
            --judge-model "${CLAUDE_EVAL_JUDGE_MODEL:-claude-haiku-4-5}" \
            --no-publish \
            --max-cost-usd 10
      - name: Upload eval report
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: claude-plugin-evals
          path: |
            plugins/radius/results.json
            plugins/radius/evals/results/
          if-no-files-found: ignore
          retention-days: 14