Repository navigation
Expand file tree
/
Copy pathtest-coderabbit-config-validate.yml
More file actions
220 lines (201 loc) · 8.78 KB
/
Copy pathtest-coderabbit-config-validate.yml
File metadata and controls
220 lines (201 loc) · 8.78 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
name: Test coderabbit-config-validate script
# Runs the unit tests for the .coderabbit.yaml validator plus CLI smoke tests
# that exercise each severity path end-to-end. The checker gates every consumer
# repo's CI, so a regression here silently breaks their required status check —
# and, worse for this particular check, a regression that makes it pass
# everything is invisible: a validator that never fails looks exactly like a
# fleet with no broken configs. Hence the smoke tests assert the FAILING
# directions too, not just that a good config passes.
on:
pull_request:
paths:
- '.github/coderabbit-config/**'
- '.github/workflows/test-coderabbit-config-validate.yml'
- '.github/workflows/coderabbit-config-validate.yml'
push:
branches: [main]
paths:
- '.github/coderabbit-config/**'
- '.github/workflows/test-coderabbit-config-validate.yml'
- '.github/workflows/coderabbit-config-validate.yml'
permissions:
contents: read
concurrency:
group: test-coderabbit-config-validate-${{ github.ref }}
cancel-in-progress: true
jobs:
test:
name: unittest + smoke
runs-on: ubuntu-latest
timeout-minutes: 10
env:
CHECK: .github/coderabbit-config/check_coderabbit_config.py
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: '3.12'
- name: Install the pinned validator dependencies
# The same `--require-hashes` install the reusable workflow performs, so
# a requirements.txt whose hashes or transitive set are wrong fails HERE,
# on the PR that broke it, rather than in every consumer's CI after merge.
run: |
python3 -m pip install --disable-pip-version-check --no-input \
--require-hashes --only-binary=:all: \
-r .github/coderabbit-config/requirements.txt
- name: Run coderabbit-config unit tests
run: python3 -m unittest discover -s .github/coderabbit-config/tests -p 'test_*.py' -v
- name: Smoke test — a valid config passes
run: |
root="$(mktemp -d)"
cat > "$root/.coderabbit.yaml" <<'YAML'
language: en-US
tone_instructions: Be concise and concrete.
reviews:
profile: chill
tools:
golangci-lint:
enabled: false
YAML
python3 "$CHECK" --root "$root"
- name: Smoke test — an absent config passes and says so
run: |
root="$(mktemp -d)"
out="$(python3 "$CHECK" --root "$root")"
echo "$out"
# The log line is the contract: "no config here" must never read the
# same as "config validated clean".
echo "$out" | grep -q 'nothing to validate'
echo "$out" | grep -q 'absent'
- name: Smoke test — the other spelling is validated, not reported absent
run: |
# CodeRabbit honours .coderabbit.yml as well. A repo on that spelling
# that leaves config_file at its default must not get a permanently
# green check over a config nobody looked at.
root="$(mktemp -d)"
python3 -c "open('$root/.coderabbit.yml','w').write('tone_instructions: ' + 'x'*300 + chr(10))"
if out="$(python3 "$CHECK" --root "$root")"; then
echo "::error::checker reported a pass while an invalid .coderabbit.yml sat beside the default name"
echo "$out"
exit 1
fi
echo "$out"
echo "$out" | grep -q '::warning::'
echo "$out" | grep -q '.coderabbit.yml'
- name: Smoke test — a config symlinked out of the tree is refused, not read
run: |
# abspath does not resolve symlinks but open() follows them, so without
# a realpath containment check the TARGET's content reaches this public
# log through the YAML error message.
root="$(mktemp -d)"
outside="$(mktemp -d)"
printf 'this: [is not: valid yaml\n' > "$outside/secret.yaml"
ln -s "$outside/secret.yaml" "$root/.coderabbit.yaml"
set +e
out="$(python3 "$CHECK" --root "$root")"
code=$?
set -e
echo "$out"
if [ "$code" != "2" ]; then
echo "::error::expected exit 2 for a config resolving outside the repo root, got ${code}"
exit 1
fi
echo "$out" | grep -q 'outside the checked-out repo root'
- name: Smoke test — an over-long tone_instructions fails
run: |
root="$(mktemp -d)"
python3 -c "open('$root/.coderabbit.yaml','w').write('tone_instructions: ' + 'x'*300 + chr(10))"
if out="$(python3 "$CHECK" --root "$root")"; then
echo "::error::checker passed an over-long tone_instructions — expected a non-zero exit"
echo "$out"
exit 1
fi
echo "$out"
# Actual vs permitted length, both named — that is what makes the
# annotation actionable without opening the schema.
echo "$out" | grep -q '300 characters'
echo "$out" | grep -q '250'
- name: Smoke test — a misplaced top-level `tools` block warns, then fails under strict
run: |
root="$(mktemp -d)"
cat > "$root/.coderabbit.yaml" <<'YAML'
reviews:
profile: chill
tools:
# Project uses go vet + staticcheck, not golangci-lint
golangci-lint:
enabled: false
YAML
out="$(python3 "$CHECK" --root "$root")" # must NOT fail by default
echo "$out"
echo "$out" | grep -q '::warning '
echo "$out" | grep -q 'unknown key `tools`'
echo "$out" | grep -q 'reviews.tools'
if out="$(python3 "$CHECK" --root "$root" --strict-unknown-keys)"; then
echo "::error::strict_unknown_keys did not fail a misplaced top-level tools block"
echo "$out"
exit 1
fi
echo "$out"
echo "$out" | grep -q '::error '
- name: Smoke test — a typo INSIDE an object warns, then fails under strict
run: |
# The closed-by-omission half. `reviews.tools.github-checks` names only
# `enabled` and says nothing about the rest, so jsonschema has no
# keyword to fire on — `timeout_ms` is a key upstream removed that is
# still live in several org repos, doing nothing. A regression that
# loses this path is invisible from the default-mode exit code alone,
# which is why the strict direction is asserted too.
root="$(mktemp -d)"
cat > "$root/.coderabbit.yaml" <<'YAML'
reviews:
tools:
github-checks:
enabled: true
timeout_ms: 90000
YAML
out="$(python3 "$CHECK" --root "$root")" # must NOT fail by default
echo "$out"
echo "$out" | grep -q '::warning '
echo "$out" | grep -q 'unknown key `timeout_ms`'
echo "$out" | grep -q 'reviews.tools.github-checks'
if out="$(python3 "$CHECK" --root "$root" --strict-unknown-keys)"; then
echo "::error::strict_unknown_keys did not fail a stale key inside a tool object"
echo "$out"
exit 1
fi
echo "$out"
echo "$out" | grep -q '::error '
- name: Smoke test — malformed YAML fails
run: |
root="$(mktemp -d)"
printf 'reviews:\n profile: chill\n path_filters:\n - "x"\n' > "$root/.coderabbit.yaml"
if out="$(python3 "$CHECK" --root "$root")"; then
echo "::error::checker passed a malformed YAML file — expected a non-zero exit"
echo "$out"
exit 1
fi
echo "$out"
echo "$out" | grep -q 'not valid YAML'
- name: Smoke test — an unusable schema exits 2, not 0
run: |
# The redirect-stub case: the schema URL 301-redirects and a `curl`
# without -L writes a 167-byte HTML stub. "I could not run the check"
# must never exit like "the config is fine".
root="$(mktemp -d)"
stub="$(mktemp)"
printf '<html><a href="/x">Moved</a></html>' > "$stub"
printf 'language: en-US\n' > "$root/.coderabbit.yaml"
set +e
python3 "$CHECK" --root "$root" --schema "$stub"
code=$?
set -e
if [ "$code" != "2" ]; then
echo "::error::expected exit 2 for an unusable schema, got ${code}"
exit 1
fi
echo "checker correctly refused to validate against a non-schema"