From 868ec6ff582ad65b53f133663b55297caf85cf47 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Sirois Date: Tue, 16 Jun 2026 15:53:33 -0400 Subject: [PATCH 1/2] feat: render the unset-baseline callout on the PR comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the Site API reports `metadata.baseline.unset` (no comparison branch configured, or it collapsed to a head-vs-head comparison), the PR comment now warns — even when a comparison was produced. That's the silent case the existing "no baseline found" copy misses: an unconfigured baseline falls back to head (or the PR base), so `hasComparison` is true and the comment renders counts with no warning, e.g. "0 new" on a PR full of new queries (Site #3292). The callout names the branch the comparison fell back to, distinguishes the acute head-vs-head case from a base-branch fallback, and surfaces the `get_repo_config` MCP call to inspect/fix it. Driven off the Site signal (`metadata.baseline`) rather than re-deriving the rule; the field is optional so an older Site API (deploy skew) and a degraded `null` read both render nothing. Renderer half of Site #3297; closes Site #3312. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/reporters/github/github.test.ts | 69 +++++++++++++++++++++++++++++ src/reporters/github/success.md.j2 | 4 ++ src/reporters/site-api.ts | 17 +++++++ 3 files changed, 90 insertions(+) diff --git a/src/reporters/github/github.test.ts b/src/reporters/github/github.test.ts index aef7607..310778a 100644 --- a/src/reporters/github/github.test.ts +++ b/src/reporters/github/github.test.ts @@ -502,3 +502,72 @@ describe("baseline absent vs. temporarily unavailable (Site#3287)", () => { expect(output).not.toContain("add a `push` trigger"); }); }); + +describe("unset-baseline callout (Site #3297 / #3312)", () => { + const unsetBaseline = { + comparisonBranchConfigured: false, + resolvedBranch: "feature-x", + headVsHead: true, + unset: true, + mcpCall: 'get_repo_config({ repo: "owner/repo" })', + }; + + test("warns when the baseline is unset, even though a comparison was produced", () => { + const ctx = makeContext({ + comparison: makeComparison(), + runMetadata: makeMetadata({ baseline: unsetBaseline }), + }); + const output = renderTemplate(ctx); + + expect(output).toContain("No comparison branch configured"); + // Names the fallback branch and the acute head-vs-head consequence... + expect(output).toContain("`feature-x`"); + expect(output).toContain("this PR's own branch"); + expect(output).toContain('0 new'); + // ...and surfaces the MCP call to inspect/fix it. + expect(output).toContain('get_repo_config({ repo: "owner/repo" })'); + }); + + test("frames a base-branch fallback as a divergence/non-PR risk, not head-vs-head", () => { + const ctx = makeContext({ + comparison: makeComparison(), + runMetadata: makeMetadata({ + baseline: { ...unsetBaseline, resolvedBranch: "main", headVsHead: false }, + }), + }); + const output = renderTemplate(ctx); + + expect(output).toContain("No comparison branch configured"); + expect(output).toContain("`main`"); + expect(output).toContain("breaks on non-PR runs"); + expect(output).not.toContain("this PR's own branch"); + }); + + test("renders no callout when a comparison branch is configured", () => { + const ctx = makeContext({ + comparison: makeComparison(), + runMetadata: makeMetadata({ + baseline: { + comparisonBranchConfigured: true, + resolvedBranch: "staging", + headVsHead: false, + unset: false, + mcpCall: 'get_repo_config({ repo: "owner/repo" })', + }, + }), + }); + const output = renderTemplate(ctx); + + expect(output).not.toContain("No comparison branch configured"); + }); + + test("renders no callout when the baseline state is absent (older API / degraded read)", () => { + const ctx = makeContext({ + comparison: makeComparison(), + runMetadata: makeMetadata({ baseline: null }), + }); + const output = renderTemplate(ctx); + + expect(output).not.toContain("No comparison branch configured"); + }); +}); diff --git a/src/reporters/github/success.md.j2 b/src/reporters/github/success.md.j2 index 97a059e..11ef9e7 100644 --- a/src/reporters/github/success.md.j2 +++ b/src/reporters/github/success.md.j2 @@ -16,6 +16,10 @@ {{ runMetadata.rollupText }} {% endif %} +{% if hasComparison and runMetadata and runMetadata.baseline and runMetadata.baseline.unset %} + +> ⚠️ **No comparison branch configured** — regressions and new queries are compared against `{{ runMetadata.baseline.resolvedBranch }}`{% if runMetadata.baseline.headVsHead %}, this PR's own branch, so the counts above can read "0 new" on a PR full of new queries{% else %}, a fallback that can diverge from the dashboard and breaks on non-PR runs{% endif %}. Set a comparison branch (typically your default branch) in the project's CI settings — or inspect it with {{ runMetadata.baseline.mcpCall }}. +{% endif %} {% if displayImproved.length > 0 %} #### This PR improves queries diff --git a/src/reporters/site-api.ts b/src/reporters/site-api.ts index bb18024..71a9ede 100644 --- a/src/reporters/site-api.ts +++ b/src/reporters/site-api.ts @@ -102,6 +102,23 @@ export interface CiRunMetadata { signalKeys: { new: string; regressed: string; improved: string; index: string }; /** Per-query run-scoped detail links, keyed by query hash. Empty when the repo isn't linked. */ queries: Array<{ hash: string; link: string }>; + /** + * Comparison-baseline state (Site #3297). Optional: absent on a Site API that + * predates it (deploy skew — render nothing), `null` when the API couldn't + * resolve the baseline (a degraded read — unknown, not "unset"). When `unset` + * is true the project has no comparison branch configured (or it collapsed to + * a head-vs-head comparison), so the counts can be inaccurate (#3292) and the + * comment should warn. `resolvedBranch` is what the comparison fell back to; + * `headVsHead` is the acute all-zeros case; `mcpCall` is the MCP call to + * inspect/fix the config. + */ + baseline?: { + comparisonBranchConfigured: boolean; + resolvedBranch: string; + headVsHead: boolean; + unset: boolean; + mcpCall: string; + } | null; } /** From 8a254cd06cd94d5c30dc5d732d6a82f0844ccfb7 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Sirois Date: Tue, 16 Jun 2026 16:00:38 -0400 Subject: [PATCH 2/2] feat: use a GFM warning alert for the unset-baseline callout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Render the callout with GitHub's `> [!WARNING]` alert instead of a manual ⚠️ blockquote, so it shows as the native styled warning box (and drops the hand-placed emoji). Content unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/reporters/github/github.test.ts | 2 ++ src/reporters/github/success.md.j2 | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/reporters/github/github.test.ts b/src/reporters/github/github.test.ts index 310778a..e9e6686 100644 --- a/src/reporters/github/github.test.ts +++ b/src/reporters/github/github.test.ts @@ -519,6 +519,8 @@ describe("unset-baseline callout (Site #3297 / #3312)", () => { }); const output = renderTemplate(ctx); + // Rendered as a GFM warning alert. + expect(output).toContain("> [!WARNING]"); expect(output).toContain("No comparison branch configured"); // Names the fallback branch and the acute head-vs-head consequence... expect(output).toContain("`feature-x`"); diff --git a/src/reporters/github/success.md.j2 b/src/reporters/github/success.md.j2 index 11ef9e7..f84584c 100644 --- a/src/reporters/github/success.md.j2 +++ b/src/reporters/github/success.md.j2 @@ -18,7 +18,8 @@ {% endif %} {% if hasComparison and runMetadata and runMetadata.baseline and runMetadata.baseline.unset %} -> ⚠️ **No comparison branch configured** — regressions and new queries are compared against `{{ runMetadata.baseline.resolvedBranch }}`{% if runMetadata.baseline.headVsHead %}, this PR's own branch, so the counts above can read "0 new" on a PR full of new queries{% else %}, a fallback that can diverge from the dashboard and breaks on non-PR runs{% endif %}. Set a comparison branch (typically your default branch) in the project's CI settings — or inspect it with {{ runMetadata.baseline.mcpCall }}. +> [!WARNING] +> **No comparison branch configured** — regressions and new queries are compared against `{{ runMetadata.baseline.resolvedBranch }}`{% if runMetadata.baseline.headVsHead %}, this PR's own branch, so the counts above can read "0 new" on a PR full of new queries{% else %}, a fallback that can diverge from the dashboard and breaks on non-PR runs{% endif %}. Set a comparison branch (typically your default branch) in the project's CI settings — or inspect it with {{ runMetadata.baseline.mcpCall }}. {% endif %} {% if displayImproved.length > 0 %}