Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 54 additions & 6 deletions .github/workflows/pr-quality.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ jobs:
outputs:
merge-base: ${{ steps.refs.outputs.merge-base }}
coverage-needed: ${{ steps.refs.outputs.coverage-needed }}
frontend-needed: ${{ steps.refs.outputs.frontend-needed }}
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
Expand All @@ -37,21 +38,29 @@ jobs:
else
echo "coverage-needed=true" >> "$GITHUB_OUTPUT"
fi
if git diff --quiet "$merge_base" "$PR_SHA" -- frontend/ \
'package*.json' eslint.complexity.config.mjs \
'scripts/*complexity*' 'scripts/test/*complexity*' '.github/workflows/pr-quality*'; then
echo "frontend-needed=false" >> "$GITHUB_OUTPUT"
else
echo "frontend-needed=true" >> "$GITHUB_OUTPUT"
fi

coverage:
name: coverage-${{ matrix.revision }}
needs: revisions
if: needs.revisions.outputs.coverage-needed == 'true'
if: needs.revisions.outputs.coverage-needed == 'true' || needs.revisions.outputs.frontend-needed == 'true'
runs-on: ubuntu-latest
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
revision: [base, pr]
steps:
# Use the workflow's merge commit for tools: older PR heads may predate them.
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
ref: ${{ github.event.pull_request.head.sha }}
ref: ${{ github.sha }}
path: tooling
persist-credentials: false
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
Expand All @@ -66,8 +75,10 @@ jobs:
cache-dependency-path: |
source/package-lock.json
source/lambda/yjs-server/package-lock.json
source/frontend/package-lock.json
- name: Install revision dependencies
id: install
if: needs.revisions.outputs.coverage-needed == 'true'
continue-on-error: true
working-directory: source
run: |
Expand All @@ -80,7 +91,7 @@ jobs:
working-directory: source
run: npm test -- --coverage --coverage.reporter=json --coverage.reportOnFailure
- name: Collect revision-bound coverage
if: always()
if: always() && needs.revisions.outputs.coverage-needed == 'true'
env:
TESTS_PASSED: ${{ steps.tests.outcome == 'success' }}
run: |
Expand All @@ -89,11 +100,42 @@ jobs:
node tooling/scripts/complexity-coverage.mjs --cwd "$PWD/source" \
--input "$PWD/source/coverage/coverage-final.json" \
--output "$PWD/coverage.json" "${args[@]}"
- name: Install frontend revision dependencies
id: frontend-install
if: always() && needs.revisions.outputs.frontend-needed == 'true'
continue-on-error: true
working-directory: source/frontend
run: |
npm ci
# Older revisions may not declare the optional coverage provider.
vitest_version=$(node -p "require('vitest/package.json').version")
npm install --no-save --ignore-scripts "@vitest/coverage-v8@$vitest_version"
- name: Measure frontend coverage
id: frontend-tests
if: always() && steps.frontend-install.outcome == 'success'
continue-on-error: true
working-directory: source/frontend
run: >-
npm test -- --coverage --coverage.provider=v8 --coverage.reporter=json
--coverage.reportOnFailure --coverage.include='src/**/*.{ts,tsx}'
--coverage.exclude='src/**/*.test.*' --coverage.exclude='src/test/**'
- name: Collect frontend coverage
if: always() && needs.revisions.outputs.frontend-needed == 'true'
env:
TESTS_PASSED: ${{ steps.frontend-tests.outcome == 'success' }}
run: |
args=()
if [ "$TESTS_PASSED" = true ]; then args+=(--successful); fi
node tooling/scripts/complexity-coverage.mjs --cwd "$PWD/source" --scope frontend \
--input "$PWD/source/frontend/coverage/coverage-final.json" \
--output "$PWD/coverage-frontend.json" "${args[@]}"
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
if: always()
with:
name: coverage-${{ matrix.revision }}-${{ github.run_attempt }}
path: coverage.json
path: |
coverage.json
coverage-frontend.json
if-no-files-found: error
retention-days: 7

Expand All @@ -117,7 +159,7 @@ jobs:
# Dependabot use the default-branch publisher in pr-quality-comment.yml.
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
ref: ${{ github.event.pull_request.head.sha }}
ref: ${{ github.sha }}
persist-credentials: false
sparse-checkout-cone-mode: false
sparse-checkout: |
Expand Down Expand Up @@ -161,7 +203,7 @@ jobs:
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
ref: ${{ github.event.pull_request.head.sha }}
ref: ${{ github.sha }}
fetch-depth: 0
persist-credentials: false
- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
Expand Down Expand Up @@ -209,6 +251,12 @@ jobs:
if [ -f "coverage-input/pr/coverage.json" ]; then
args+=(--pr-coverage "coverage-input/pr/coverage.json")
fi
if [ -f "coverage-input/base/coverage-frontend.json" ]; then
args+=(--base-frontend-coverage "coverage-input/base/coverage-frontend.json")
fi
if [ -f "coverage-input/pr/coverage-frontend.json" ]; then
args+=(--pr-frontend-coverage "coverage-input/pr/coverage-frontend.json")
fi
npm run complexity:compare -- --base "$BASE_SHA" --base-ref "$BASE_REF" --pr "$PR_SHA" "${args[@]}"
cat reports/complexity/comparison.md >> "$GITHUB_STEP_SUMMARY"
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -465,7 +465,7 @@ npm run complexity # advisory warnings for function scores above 15
npm run complexity:compare # compare origin/main's merge base with HEAD
```

CI combines complexity changes, backend branch coverage, and importer counts in one PR comment,
CI combines complexity changes, backend/frontend branch coverage, and importer counts in one PR comment,
helping reviewers find new or worsening complex functions and coverage regressions.
These signals are advisory. See [PR quality reports](docs/development/pr-quality.md)
for coverage details, thresholds, matching limits, and CI behavior.
Expand Down
25 changes: 17 additions & 8 deletions docs/development/pr-quality.md
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
# PR quality reports

CI updates one PR comment with cognitive-complexity, dependency counts, and backend branch-coverage changes
CI updates one PR comment with cognitive-complexity, dependency counts, and branch-coverage changes
for source files changed between the merge base and PR commit. Findings are advisory: functions above 15 appear
when complexity or coverage worsens, or when the function is new/unmatched.
Unchanged debt is omitted; improvements appear separately.
Unchanged debt is omitted; improvements appear in the same table.
When no source files are analyzed, CI posts no comment and removes its previous report if present.

## Reading the table

Expand All @@ -15,21 +16,29 @@ Unchanged debt is omitted; improvements appear separately.
| Uncovered branches | Outcomes not exercised by tests, before → after |
| Icon | 🟢 improving, 🟠 mixed/small regression, 🔴 large regression, 🟣 both metrics worsen substantially |

Without comparable measurements, icons use current complexity: orange above 15,
Existing functions use complexity deltas even when coverage is unavailable.
Complexity increases turn red at `max(5, 20% of base)` for backend and `max(10, 20% of base)` for frontend,
adjusted by coverage changes.
New/unmatched functions use current complexity: orange above 15,
red above 50, purple above 100. Coverage below 50% raises this one level.
More than 10 depending files raises an existing warning one level, capped at purple,
when coverage is below 50% or regresses. Green improvements stay green.
These are review heuristics; use the numbers to judge each finding.

**Unknown** means coverage is missing, incompatible, or unsupported (frontend).
**n/a** means no branch outcomes; **\*** marks coverage from failing/incomplete tests.
**—** marks unavailable coverage; **unknown** marks unavailable backend dependency counts.
**n/a** means no branch outcomes or frontend dependency counts; **\*** marks coverage from failing/incomplete tests.
Coverage comparisons allow Node minor/patch differences within the same major; Vitest, provider, and configuration must match.
**New or unmatched** means no reliable previous function match—not necessarily new code.
The comment shows up to 20 findings and 20 improvements; artifacts contain all results.
Rows are sorted 🟣 → 🔴 → 🟠 → 🟢. The first 10 are visible, or all purple rows if there are more;
the remaining rows appear under “Show more”. GitHub comment size limits still apply; artifacts contain all results.
Coverage diagnostics stay in the artifacts; the comment shows one notice if measurements are unavailable or incomplete.

## CI

Backend/test/configuration/tooling changes run coverage at both revisions. Docs-only,
Terraform-only and frontend-only PRs skip these suites: “not needed for this change.”
Backend and frontend suites run independently at both revisions when their source, tests, or configuration change.
Tools come from the workflow's merge commit so older PR branches need not contain them; tests still use the exact base and PR revisions.
Docs-only and Terraform-only PRs skip coverage. Frontend-only PRs skip the backend suite.
Each `coverage-base-*` / `coverage-pr-*` artifact contains the requested suites' measurements.
Dependency analysis failures leave only those counts unknown. Partial reruns reuse artifacts only when their producers
were not rerun; missing or stale results are reported explicitly.

Expand Down
90 changes: 60 additions & 30 deletions scripts/compare-complexity.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,12 @@ export { renderComparison } from './complexity-report.mjs';

const require = createRequire(import.meta.url);
const loadCoverage = async (path) => (path ? JSON.parse(await readFile(path, 'utf8')) : undefined);
// This advisory comparison tolerates Node minor/patch changes within a major.
const instrumenterKey = (snapshot) =>
JSON.stringify({
...snapshot.instrumenter,
node: snapshot.instrumenter?.node?.split('.')[0],
});

function git(cwd, args) {
return execFileSync('git', ['--literal-pathspecs', ...args], {
Expand Down Expand Up @@ -162,49 +168,69 @@ export async function compareRevisions({
baseRef,
baseCoverage,
prCoverage,
baseFrontendCoverage,
prFrontendCoverage,
coverageNotNeeded = false,
}) {
const baseSha = resolveCommit(cwd, base);
const prSha = resolveCommit(cwd, pr);
const mergeBase = git(cwd, ['merge-base', baseSha, prSha]).trim();
const analyzer = createAnalyzer(cwd);
const candidates = changedFiles(cwd, mergeBase, prSha);
const coverageNotes = [];
for (const [label, snapshot, revision] of [
['Merge-base', baseCoverage, mergeBase],
['PR', prCoverage, prSha],
]) {
if (!snapshot) {
coverageNotes.push(
`${label} coverage: ${coverageNotNeeded ? 'not needed for this change' : 'unknown (no report supplied)'}.`,
);
} else if (!hasCoverage(snapshot)) {
coverageNotes.push(
`${label} coverage: unknown (test run or coverage collection failed to produce a report).`,
);
} else if (snapshot.revision !== revision) {
coverageNotes.push(`${label} coverage: unknown (report belongs to a different revision).`);
} else if (snapshot.status === 'partial') {
const pairs = {
backend: [baseCoverage, prCoverage],
frontend: [baseFrontendCoverage, prFrontendCoverage],
};
for (const [scope, pair] of Object.entries(pairs)) {
const relevant =
scope === 'backend' ||
pair.some(Boolean) ||
candidates.some((file) => file.path.startsWith('frontend/src/'));
if (!relevant) continue;
const prefix = scope === 'frontend' ? 'Frontend ' : '';
for (const [index, label, revision] of [
[0, 'Merge-base', mergeBase],
[1, 'PR', prSha],
]) {
const snapshot = pair[index];
if (!snapshot) {
coverageNotes.push(
`${prefix}${label} coverage: ${coverageNotNeeded && scope === 'backend' ? 'not needed for this change' : 'unknown (no report supplied)'}.`,
);
} else if (!hasCoverage(snapshot)) {
coverageNotes.push(
`${prefix}${label} coverage: unknown (test run or coverage collection failed to produce a report).`,
);
} else if (snapshot.revision !== revision) {
coverageNotes.push(
`${prefix}${label} coverage: unknown (report belongs to a different revision).`,
);
} else if (snapshot.status === 'partial') {
coverageNotes.push(
`${prefix}${label} coverage: measured with failing or interrupted tests (*). Counts and deltas may reflect incomplete execution.`,
);
}
}
if (pair.every(hasCoverage) && instrumenterKey(pair[0]) !== instrumenterKey(pair[1])) {
coverageNotes.push(
`${label} coverage: measured with failing or interrupted tests (*). Counts and deltas may reflect incomplete execution.`,
`${prefix}Coverage is unknown: Node major, Vitest, provider, or coverage configuration differs.`,
);
pair.fill(undefined);
}
}
if (
hasCoverage(baseCoverage) &&
hasCoverage(prCoverage) &&
JSON.stringify(baseCoverage.instrumenter) !== JSON.stringify(prCoverage.instrumenter)
) {
coverageNotes.push(
'Coverage is unknown: Node, Vitest, provider, or coverage configuration differs.',
);
baseCoverage = undefined;
prCoverage = undefined;
}
const candidates = changedFiles(cwd, mergeBase, prSha);
const coverageFor = (path, index) =>
pairs[path?.startsWith('frontend/') ? 'frontend' : 'backend'][index];
const files = [];
for (const file of candidates) {
const before = await measure(cwd, analyzer, mergeBase, file.previousPath, baseCoverage);
const after = await measure(cwd, analyzer, prSha, file.prPath, prCoverage);
const before = await measure(
cwd,
analyzer,
mergeBase,
file.previousPath,
coverageFor(file.previousPath, 0),
);
const after = await measure(cwd, analyzer, prSha, file.prPath, coverageFor(file.prPath, 1));
if (before === null && after === null) continue;
files.push({
path: file.path,
Expand Down Expand Up @@ -272,6 +298,8 @@ async function main() {
'output-dir': { type: 'string', default: 'reports/complexity' },
'base-coverage': { type: 'string' },
'pr-coverage': { type: 'string' },
'base-frontend-coverage': { type: 'string' },
'pr-frontend-coverage': { type: 'string' },
'coverage-not-needed': { type: 'boolean', default: false },
},
});
Expand All @@ -283,6 +311,8 @@ async function main() {
baseRef: values['base-ref'],
baseCoverage: await loadCoverage(values['base-coverage']),
prCoverage: await loadCoverage(values['pr-coverage']),
baseFrontendCoverage: await loadCoverage(values['base-frontend-coverage']),
prFrontendCoverage: await loadCoverage(values['pr-frontend-coverage']),
coverageNotNeeded: values['coverage-not-needed'],
});
const markdown = renderComparison(report);
Expand Down
19 changes: 15 additions & 4 deletions scripts/complexity-coverage.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,8 @@ export function coverageRegressed(before, after) {
return change !== null && (change.percentagePoints < 0 || change.uncovered > 0);
}

export async function collectCoverage({ cwd, input, successful }) {
export async function collectCoverage({ cwd, input, successful, scope = 'backend' }) {
if (!['backend', 'frontend'].includes(scope)) throw new Error('Invalid coverage scope');
cwd = await realpath(cwd);
const revision = execFileSync('git', ['rev-parse', 'HEAD'], { cwd, encoding: 'utf8' }).trim();
const snapshot = { schemaVersion: 1, revision, status: 'unavailable', files: {} };
Expand All @@ -75,16 +76,24 @@ export async function collectCoverage({ cwd, input, successful }) {
if (!successful && error.code === 'ENOENT') return snapshot;
throw error;
}
const require = createRequire(resolve(cwd, 'package.json'));
const frontend = scope === 'frontend';
const require = createRequire(resolve(cwd, frontend ? 'frontend/package.json' : 'package.json'));
snapshot.instrumenter = {
node: process.versions.node,
vitest: require('vitest/package.json').version,
provider: require('@vitest/coverage-v8/package.json').version,
config: sourceHash(await readFile(resolve(cwd, 'vitest.config.js'))),
config: sourceHash(
await readFile(resolve(cwd, frontend ? 'frontend/vitest.config.ts' : 'vitest.config.js')),
),
};
for (const [path, coverage] of Object.entries(raw)) {
const name = relative(cwd, path).split('\\').join('/');
if (name.startsWith('../') || isAbsolute(name) || !name.startsWith('lambda/')) continue;
if (
name.startsWith('../') ||
isAbsolute(name) ||
!name.startsWith(frontend ? 'frontend/src/' : 'lambda/')
)
continue;
const branches = [];
for (const [id, branch] of Object.entries(coverage.branchMap)) {
branch.locations.forEach((_location, index) => {
Expand Down Expand Up @@ -113,13 +122,15 @@ if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.ur
input: { type: 'string' },
output: { type: 'string' },
successful: { type: 'boolean', default: false },
scope: { type: 'string', default: 'backend' },
},
});
try {
const snapshot = await collectCoverage({
cwd: resolve(values.cwd),
input: values.input,
successful: values.successful,
scope: values.scope,
});
await writeFile(values.output, `${JSON.stringify(snapshot)}\n`);
} catch (error) {
Expand Down
Loading
Loading