diff --git a/src/reporters/github/github.test.ts b/src/reporters/github/github.test.ts index aef7607..e9e6686 100644 --- a/src/reporters/github/github.test.ts +++ b/src/reporters/github/github.test.ts @@ -502,3 +502,74 @@ describe("baseline absent vs. temporarily unavailable (Site#3287)", () => { expect(output).not.toContain("add a `push` trigger"); }); }); + +describe("unset-baseline callout (Site #3297 / #3312)", () => { + const unsetBaseline = { + comparisonBranchConfigured: false, + resolvedBranch: "feature-x", + headVsHead: true, + unset: true, + mcpCall: 'get_repo_config({ repo: "owner/repo" })', + }; + + test("warns when the baseline is unset, even though a comparison was produced", () => { + const ctx = makeContext({ + comparison: makeComparison(), + runMetadata: makeMetadata({ baseline: unsetBaseline }), + }); + const output = renderTemplate(ctx); + + // Rendered as a GFM warning alert. + expect(output).toContain("> [!WARNING]"); + expect(output).toContain("No comparison branch configured"); + // Names the fallback branch and the acute head-vs-head consequence... + expect(output).toContain("`feature-x`"); + expect(output).toContain("this PR's own branch"); + expect(output).toContain('0 new'); + // ...and surfaces the MCP call to inspect/fix it. + expect(output).toContain('get_repo_config({ repo: "owner/repo" })'); + }); + + test("frames a base-branch fallback as a divergence/non-PR risk, not head-vs-head", () => { + const ctx = makeContext({ + comparison: makeComparison(), + runMetadata: makeMetadata({ + baseline: { ...unsetBaseline, resolvedBranch: "main", headVsHead: false }, + }), + }); + const output = renderTemplate(ctx); + + expect(output).toContain("No comparison branch configured"); + expect(output).toContain("`main`"); + expect(output).toContain("breaks on non-PR runs"); + expect(output).not.toContain("this PR's own branch"); + }); + + test("renders no callout when a comparison branch is configured", () => { + const ctx = makeContext({ + comparison: makeComparison(), + runMetadata: makeMetadata({ + baseline: { + comparisonBranchConfigured: true, + resolvedBranch: "staging", + headVsHead: false, + unset: false, + mcpCall: 'get_repo_config({ repo: "owner/repo" })', + }, + }), + }); + const output = renderTemplate(ctx); + + expect(output).not.toContain("No comparison branch configured"); + }); + + test("renders no callout when the baseline state is absent (older API / degraded read)", () => { + const ctx = makeContext({ + comparison: makeComparison(), + runMetadata: makeMetadata({ baseline: null }), + }); + const output = renderTemplate(ctx); + + expect(output).not.toContain("No comparison branch configured"); + }); +}); diff --git a/src/reporters/github/success.md.j2 b/src/reporters/github/success.md.j2 index 97a059e..f84584c 100644 --- a/src/reporters/github/success.md.j2 +++ b/src/reporters/github/success.md.j2 @@ -16,6 +16,11 @@ {{ runMetadata.rollupText }} {% endif %} +{% if hasComparison and runMetadata and runMetadata.baseline and runMetadata.baseline.unset %} + +> [!WARNING] +> **No comparison branch configured** — regressions and new queries are compared against `{{ runMetadata.baseline.resolvedBranch }}`{% if runMetadata.baseline.headVsHead %}, this PR's own branch, so the counts above can read "0 new" on a PR full of new queries{% else %}, a fallback that can diverge from the dashboard and breaks on non-PR runs{% endif %}. Set a comparison branch (typically your default branch) in the project's CI settings — or inspect it with {{ runMetadata.baseline.mcpCall }}. +{% endif %} {% if displayImproved.length > 0 %} #### This PR improves queries diff --git a/src/reporters/site-api.ts b/src/reporters/site-api.ts index bb18024..71a9ede 100644 --- a/src/reporters/site-api.ts +++ b/src/reporters/site-api.ts @@ -102,6 +102,23 @@ export interface CiRunMetadata { signalKeys: { new: string; regressed: string; improved: string; index: string }; /** Per-query run-scoped detail links, keyed by query hash. Empty when the repo isn't linked. */ queries: Array<{ hash: string; link: string }>; + /** + * Comparison-baseline state (Site #3297). Optional: absent on a Site API that + * predates it (deploy skew — render nothing), `null` when the API couldn't + * resolve the baseline (a degraded read — unknown, not "unset"). When `unset` + * is true the project has no comparison branch configured (or it collapsed to + * a head-vs-head comparison), so the counts can be inaccurate (#3292) and the + * comment should warn. `resolvedBranch` is what the comparison fell back to; + * `headVsHead` is the acute all-zeros case; `mcpCall` is the MCP call to + * inspect/fix the config. + */ + baseline?: { + comparisonBranchConfigured: boolean; + resolvedBranch: string; + headVsHead: boolean; + unset: boolean; + mcpCall: string; + } | null; } /**