From 6f0ff5570463e558596d67b1b22f3ba605b249c8 Mon Sep 17 00:00:00 2001 From: RaghwendraSingh Date: Fri, 13 Nov 2026 13:23:09 +0530 Subject: [PATCH 1/5] feat(review): optional `dbt_prepare` in the action builds the head, merge-base and catalog artifacts - new action inputs: `dbt_prepare` (`none` default | `compile` | `full`), `dbt_command`, `dbt_profiles_dir` - `compile` runs `dbt compile` on the head and compiles the PR's merge-base into `target-base/` in a temporary worktree (fetching and deepening a shallow checkout when needed); `full` adds `dbt docs generate` - a base compile that fails is a warning: the review still runs and marks those models unverified - inputs reach the script only through env, never interpolated - docs: the manual base-compile snippet now compiles the merge-base, not the base branch's tip - the example workflow is renamed `examples/dbt-pr-review.yml` (was named after one internal repo) Co-Authored-By: Claude Opus 5.5 --- docs/docs/usage/dbt-pr-review.md | 25 ++++++-- github/review/action.yml | 59 +++++++++++++++++++ ...timate-ingestion.yml => dbt-pr-review.yml} | 2 +- .../test/altimate/review-action.test.ts | 24 ++++++++ .../skill/release-v0.8.5-adversarial.test.ts | 2 +- 5 files changed, 106 insertions(+), 6 deletions(-) rename github/review/examples/{altimate-ingestion.yml => dbt-pr-review.yml} (97%) create mode 100644 packages/opencode/test/altimate/review-action.test.ts diff --git a/docs/docs/usage/dbt-pr-review.md b/docs/docs/usage/dbt-pr-review.md index bd96bd5485..c36e62aeda 100644 --- a/docs/docs/usage/dbt-pr-review.md +++ b/docs/docs/usage/dbt-pr-review.md @@ -209,13 +209,30 @@ jobs: | `target-base/compiled/…` | Compile the base ref into `target-base/` (below) | Equivalence proofs and column-breakage for modified models. | | `target/catalog.json` | `dbt docs generate` | Complete warehouse columns for column-lineage breakage. | -Compiling the base (optional, for equivalence and column-breakage): +The action can build these for you with `dbt_prepare` (off by default): + +```yaml + - uses: AltimateAI/altimate-code/github/review@main + with: + dbt_prepare: compile # none | compile | full + dbt_command: dbt # or "uv run dbt", "poetry run dbt" + dbt_profiles_dir: . # relative to project_dir +``` + +- **`compile`:** runs `dbt compile` on the head, then compiles the PR's merge-base + into `target-base/` in a temporary worktree. This enables equivalence and + column-breakage proofs. It usually works offline. +- **`full`:** also runs `dbt docs generate` for `catalog.json` (complete column + types). That needs warehouse credentials in the profile. +- A base compile that fails is a warning: the review still runs, and the + affected models are listed as not verified. + +To build them yourself, compile the **merge-base**, not the base branch's tip: ```yaml - name: Compile the base into target-base/ run: | - # The review diffs against the merge-base, not the target branch tip. - git worktree add ../base "$(git merge-base "origin/${{ github.base_ref }}" HEAD)" + git worktree add ../base "$(git merge-base origin/${{ github.base_ref }} HEAD)" (cd ../base && dbt deps && dbt compile --target-path "$GITHUB_WORKSPACE/target-base") ``` @@ -246,7 +263,7 @@ ways, or neither (it self-disables, leaving a deterministic-only review): Always pass keys as repo **secrets**. Warehouse credentials are consumed by the `dbt compile` step (via your `profiles.yml`), **not** by the review step. A complete, copy-paste workflow lives at -[`github/review/examples/altimate-ingestion.yml`](https://github.com/AltimateAI/altimate-code/blob/main/github/review/examples/altimate-ingestion.yml). +[`github/review/examples/dbt-pr-review.yml`](https://github.com/AltimateAI/altimate-code/blob/main/github/review/examples/dbt-pr-review.yml). Re-pushing commits updates the same summary comment in place; fixed findings are dropped on the next run. `--post` targets **GitHub** PRs (it reads `GITHUB_TOKEN` / diff --git a/github/review/action.yml b/github/review/action.yml index 09c4bc1538..f39818fc6b 100644 --- a/github/review/action.yml +++ b/github/review/action.yml @@ -31,6 +31,17 @@ inputs: description: "Post the verdict to the PR (summary comment + inline review)." required: false default: "true" + dbt_prepare: + description: "Build the dbt artifacts the full review uses before reviewing. `none` (default): you build them in earlier steps. `compile`: `dbt compile` on the head, and on the PR's merge-base into `target-base/` (equivalence and lineage proofs). `full`: also `dbt docs generate` for `catalog.json` (complete column types). `compile` usually works offline; `full` needs warehouse credentials in the profile." + required: false + default: "none" + dbt_command: + description: "How to invoke dbt for `dbt_prepare`, e.g. `dbt`, `uv run dbt`, `poetry run dbt`." + required: false + default: "dbt" + dbt_profiles_dir: + description: "`--profiles-dir` for `dbt_prepare`, relative to `project_dir`. Empty = dbt's default lookup." + required: false signing_key: description: "Optional HMAC key to sign the verdict envelope (ALTIMATE_REVIEW_SIGNING_KEY)." required: false @@ -167,6 +178,54 @@ runs: echo "Advisory lane: no model/credentials provided — running deterministic-only." fi + # Optional: build the head and base artifacts the full-tier review needs. + # Inputs are passed as ENV, never interpolated into the script. + - name: Prepare dbt artifacts + if: inputs.dbt_prepare != 'none' + shell: bash + env: + IN_PREPARE: ${{ inputs.dbt_prepare }} + IN_DBT: ${{ inputs.dbt_command }} + IN_PROFILES: ${{ inputs.dbt_profiles_dir }} + IN_PROJECT_DIR: ${{ inputs.project_dir }} + IN_BASE: ${{ inputs.base }} + EVENT_BASE: ${{ github.event.pull_request.base.ref }} + run: | + set -euo pipefail + case "$IN_PREPARE" in compile|full) ;; *) echo "::error::dbt_prepare must be none, compile or full (got '$IN_PREPARE')"; exit 1 ;; esac + read -ra DBT <<< "$IN_DBT" + PROJECT="$(cd "${IN_PROJECT_DIR:-.}" && pwd)" + REL="$(git -C "$PROJECT" rev-parse --show-prefix)" + PROFILES=() + [[ -n "${IN_PROFILES:-}" ]] && PROFILES=(--profiles-dir "$(cd "$PROJECT" && cd "$IN_PROFILES" && pwd)") + build() { # $1 = project dir, $2 = target path + ( cd "$1" + [[ -f packages.yml || -f dependencies.yml ]] && "${DBT[@]}" deps ${PROFILES[@]+"${PROFILES[@]}"} + "${DBT[@]}" compile --target-path "$2" ${PROFILES[@]+"${PROFILES[@]}"} ) + } + echo "Compiling the head…" + build "$PROJECT" "$PROJECT/target" + if [[ "$IN_PREPARE" == "full" ]]; then + echo "Generating catalog.json…" + ( cd "$PROJECT" && "${DBT[@]}" docs generate --target-path "$PROJECT/target" ${PROFILES[@]+"${PROFILES[@]}"} ) + fi + # The base side: the PR's merge-base, compiled into target-base/. + BASE="${IN_BASE:-$EVENT_BASE}" + BASE="${BASE#origin/}" + if [[ -z "$BASE" ]]; then echo "::warning::No base branch known; skipping the base compile."; exit 0; fi + git fetch --no-tags origin "+refs/heads/$BASE:refs/remotes/origin/$BASE" >/dev/null 2>&1 || true + MB="$(git merge-base "origin/$BASE" HEAD 2>/dev/null || git merge-base "$BASE" HEAD 2>/dev/null || true)" + if [[ -z "$MB" && "$(git rev-parse --is-shallow-repository)" == "true" ]]; then + git fetch --no-tags --deepen=1000 origin "+refs/heads/$BASE:refs/remotes/origin/$BASE" "$(git rev-parse HEAD)" >/dev/null 2>&1 || true + MB="$(git merge-base "origin/$BASE" HEAD 2>/dev/null || true)" + fi + if [[ -z "$MB" ]]; then echo "::warning::Could not find the merge-base with $BASE; skipping the base compile."; exit 0; fi + WT="$RUNNER_TEMP/altimate-review-base" + git worktree add --detach "$WT" "$MB" >/dev/null + trap 'git worktree remove --force "$WT" >/dev/null 2>&1 || true' EXIT + echo "Compiling the base ($BASE @ ${MB:0:12})…" + build "$WT/$REL" "$PROJECT/target-base" || echo "::warning::The base compile failed; equivalence will be marked unverified." + - name: Run dbt PR review shell: bash # Inputs are passed as ENV (data), never interpolated into the command — diff --git a/github/review/examples/altimate-ingestion.yml b/github/review/examples/dbt-pr-review.yml similarity index 97% rename from github/review/examples/altimate-ingestion.yml rename to github/review/examples/dbt-pr-review.yml index dc52867909..799d424424 100644 --- a/github/review/examples/altimate-ingestion.yml +++ b/github/review/examples/dbt-pr-review.yml @@ -1,4 +1,4 @@ -# Example: drop-in dbt PR review for a dbt repo (e.g. altimate-ingestion). +# Example: drop-in dbt PR review for a dbt repo. # Copy to the target repo as `.github/workflows/dbt-pr-review.yml` and adjust the # dbt adapter + profiles/secrets to match the warehouse. # diff --git a/packages/opencode/test/altimate/review-action.test.ts b/packages/opencode/test/altimate/review-action.test.ts new file mode 100644 index 0000000000..36f0417224 --- /dev/null +++ b/packages/opencode/test/altimate/review-action.test.ts @@ -0,0 +1,24 @@ +import { describe, test, expect } from "bun:test" +import { promises as fs } from "node:fs" +import path from "node:path" +import YAML from "yaml" + +/** The GitHub Action's optional artifact preparation stays opt-in and injection-safe. */ +const actionPath = path.join(import.meta.dir, "../../../../github/review/action.yml") + +describe("action: dbt_prepare", () => { + test("is opt-in, and its script reads inputs only from env", async () => { + const action = YAML.parse(await fs.readFile(actionPath, "utf8")) + expect(action.inputs.dbt_prepare.default).toBe("none") + expect(action.inputs.dbt_command.default).toBe("dbt") + const steps: Array<{ name: string; if?: string; run?: string; env?: Record }> = action.runs.steps + const prep = steps.find((s) => s.name === "Prepare dbt artifacts")! + const review = steps.findIndex((s) => s.name === "Run dbt PR review") + expect(steps.indexOf(prep)).toBeLessThan(review) + expect(prep.if).toBe("inputs.dbt_prepare != 'none'") + expect(prep.run).not.toContain("${{") + expect(prep.env?.IN_DBT).toBe("${{ inputs.dbt_command }}") + // The base is compiled into target-base/, where the compiled-SQL resolver looks. + expect(prep.run).toContain('"$PROJECT/target-base"') + }) +}) diff --git a/packages/opencode/test/skill/release-v0.8.5-adversarial.test.ts b/packages/opencode/test/skill/release-v0.8.5-adversarial.test.ts index 00b1ee1724..77d91cbf04 100644 --- a/packages/opencode/test/skill/release-v0.8.5-adversarial.test.ts +++ b/packages/opencode/test/skill/release-v0.8.5-adversarial.test.ts @@ -238,7 +238,7 @@ describe("v0.8.5 adversarial - composite action", () => { const changelog = await fs.readFile(path.join(repoRoot, "CHANGELOG.md"), "utf8") expect(changelog).toContain(`## [${version}]`) - for (const relative of ["docs/docs/usage/dbt-pr-review.md", "github/review/examples/altimate-ingestion.yml"]) { + for (const relative of ["docs/docs/usage/dbt-pr-review.md", "github/review/examples/dbt-pr-review.yml"]) { const content = await fs.readFile(path.join(repoRoot, relative), "utf8") expect(content).toContain(`AltimateAI/altimate-code/github/review@v${version}`) expect(content).not.toContain("AltimateAI/altimate-code/github/review@v0.8.4") From d0dcecc8fabccd351d6b02d93701b0bdb5ba51e3 Mon Sep 17 00:00:00 2001 From: RaghwendraSingh Date: Fri, 13 Nov 2026 13:29:00 +0530 Subject: [PATCH 2/5] fix(review): column-set rules work in `dbt parse`-only CI; dedup windows split across lines are read whole MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found by replaying the PRs behind known escaped defects: - without compiled SQL, output columns come from the raw model's final SELECT (Jinja blanked; a Jinja expression in the list makes it unknown). `on_schema_change='fail'` with new columns now fires on the two real escapes it targets, in a parse-only CI setup - `row_number() over (…)` windows are read from the whole file, so an `ORDER BY` on its own line is seen: the "dedup has no ORDER BY" rule no longer fires falsely on multi-line windows, and the load-timestamp rule sees them. A window counts only when an added line falls inside it Co-Authored-By: Claude Opus 5.5 --- .../src/altimate/review/dbt-patterns.ts | 44 ++++++++++++---- .../src/altimate/review/orchestrate.ts | 46 +++++++++++----- .../review-incremental-semantics.test.ts | 52 +++++++++++++++++++ 3 files changed, 120 insertions(+), 22 deletions(-) diff --git a/packages/opencode/src/altimate/review/dbt-patterns.ts b/packages/opencode/src/altimate/review/dbt-patterns.ts index 91053d7515..f96eca2649 100644 --- a/packages/opencode/src/altimate/review/dbt-patterns.ts +++ b/packages/opencode/src/altimate/review/dbt-patterns.ts @@ -225,15 +225,42 @@ const detectNotIn: Detector = (c) => { }) } +/** + * `row_number() over (…)` windows in the model, read from the whole file so a + * window split across lines is seen whole. A window counts as part of the + * change when one of the PR's added lines falls inside it. + */ +function dedupWindows(c: Ctx): Array<{ spec: string; text: string }> { + const sql = stripComments(c.newSql) + const added = c.added.map((l) => stripComments(l).trim()).filter((l) => l.length >= 3) + const out: Array<{ spec: string; text: string }> = [] + for (const m of sql.matchAll(/row_number\s*\(\s*\)\s*over\s*\(/gi)) { + const open = m.index! + m[0].length - 1 + let depth = 0 + let close = -1 + for (let i = open; i < sql.length; i++) { + if (sql[i] === "(") depth++ + else if (sql[i] === ")" && --depth === 0) { + close = i + break + } + } + if (close < 0) continue + const text = sql.slice(m.index!, close + 1) + if (!added.some((l) => text.includes(l) || l.includes(text.replace(/\s+/g, " ")))) continue + out.push({ spec: sql.slice(open + 1, close).replace(/\s+/g, " ").trim(), text: text.replace(/\s+/g, " ") }) + } + return out +} + // 9. Dedup via ROW_NUMBER()/QUALIFY whose ORDER BY has no unique tiebreaker. const detectDedupTie: Detector = (c) => { - const hit = c.added.find((l) => /row_number\s*\(\s*\)\s*over\s*\(/i.test(stripComments(l))) - if (!hit) return null // Only flag dedup with NO ORDER BY — that is unambiguously non-deterministic. // A present ORDER BY (even single-column) is the developer's deterministic // choice and the dbt-recommended pattern; warning "it might not be unique" // on every dedup is noise (false positives on the common, correct case). - if (/order\s+by/i.test(stripComments(hit))) return null + const hit = dedupWindows(c).find((w) => !/order\s+by/i.test(w.spec)) + if (!hit) return null return makeFinding({ severity: "warning", category: "dedup", @@ -244,7 +271,7 @@ const detectDedupTie: Detector = (c) => { file: c.file.path, model: c.model, confidence: "high", - evidence: { tool: "dbt-patterns", result: { rule: "dedup_no_tiebreaker", line: hit.trim() } }, + evidence: { tool: "dbt-patterns", result: { rule: "dedup_no_tiebreaker", line: hit.text } }, ruleKey: "dedup:no-tiebreaker", }) } @@ -257,11 +284,8 @@ const detectDedupTie: Detector = (c) => { const LOAD_TS_RE = /^(_?(ingest(ion|ed)?|load(ed)?|etl|extract(ed)?|batch)(_at|_ts|_time|_timestamp|_date|_dt)|_?(ingestion|load|etl|batch)_timestamp|_loaded_at)$/i const detectDedupLoadTimestamp: Detector = (c) => { - for (const line of c.added) { - const s = stripComments(line) - const m = /row_number\s*\(\s*\)\s*over\s*\(([^)]*)\)/i.exec(s) - if (!m) continue - const order = /order\s+by\s+(.+)$/i.exec(m[1])?.[1] + for (const w of dedupWindows(c)) { + const order = /order\s+by\s+(.+)$/i.exec(w.spec)?.[1] if (!order) continue const keys = order.split(",").map((k) => k.trim().replace(/\s+(asc|desc)\b.*$/i, "").replace(/\s+nulls\s+(first|last)$/i, "")) if (keys.length !== 1) continue @@ -277,7 +301,7 @@ const detectDedupLoadTimestamp: Detector = (c) => { file: c.file.path, model: c.model, confidence: "high", - evidence: { tool: "dbt-patterns", result: { rule: "dedup_load_timestamp_only", line: line.trim() } }, + evidence: { tool: "dbt-patterns", result: { rule: "dedup_load_timestamp_only", line: w.text } }, ruleKey: "dedup:load-timestamp-only", }) } diff --git a/packages/opencode/src/altimate/review/orchestrate.ts b/packages/opencode/src/altimate/review/orchestrate.ts index a994dba02e..6608d49a00 100644 --- a/packages/opencode/src/altimate/review/orchestrate.ts +++ b/packages/opencode/src/altimate/review/orchestrate.ts @@ -661,19 +661,41 @@ async function columnDelta( dialect: string, ): Promise<{ added: string[]; removed: string[]; current: Set }> { const none = { added: [], removed: [], current: new Set() } - const { file, engineNewSql, engineOldSql } = ctx - if (!engineNewSql || file.status === "deleted") return none - const current = await outputColumns(runner, engineNewSql, dialect) - if (!current.size) return none - if (!hasOldSide(file)) return { added: [...current], removed: [], current } - if (!engineOldSql) return none - const before = await outputColumns(runner, engineOldSql, dialect) - if (!before.size) return none - return { - added: [...current].filter((c) => !before.has(c)), - removed: [...before].filter((c) => !current.has(c)), - current, + const { file, engineNewSql, engineOldSql, newSql, oldSql } = ctx + if (file.status === "deleted") return none + const withOld = hasOldSide(file) + const delta = (current: Set, before?: Set) => + !current.size || (withOld && !before?.size) + ? none + : { + added: [...current].filter((c) => !before?.has(c)), + removed: before ? [...before].filter((c) => !current.has(c)) : [], + current, + } + // Compiled SQL on both sides: column lineage plus the final SELECT's names. + if (engineNewSql && (!withOld || engineOldSql)) { + const current = await outputColumns(runner, engineNewSql, dialect) + const before = withOld ? await outputColumns(runner, engineOldSql!, dialect) : undefined + return delta(current, before) } + // No compiled SQL (CI that only runs `dbt parse`): the raw model's own SELECT + // list, when it names its columns. Columns produced inside a macro stay unknown. + if (!newSql) return none + const current = rawSelectList(newSql) + const before = withOld && oldSql ? rawSelectList(oldSql) : undefined + return delta(current, before) +} + +/** Output names of a raw (Jinja) model's final SELECT; empty when a Jinja expression is in the list. */ +function rawSelectList(sql: string): Set { + const plain = sql + .replace(/\{#[\s\S]*?#\}/g, " ") + .replace(/\{%[\s\S]*?%\}/g, " ") + .replace(/\{\{[\s\S]*?\}\}/g, " __jinja__ ") + const names = selectListNames(plain) + if (!names || names.includes("__jinja__")) return new Set() + // A Jinja expression inside an item (e.g. `{{ col }} as x`) leaves its alias intact. + return new Set(names) } const ticks = (xs: string[]) => xs.map((x) => `\`${x}\``).join(", ") diff --git a/packages/opencode/test/altimate/review-incremental-semantics.test.ts b/packages/opencode/test/altimate/review-incremental-semantics.test.ts index bbf424ceb6..5fd78deef1 100644 --- a/packages/opencode/test/altimate/review-incremental-semantics.test.ts +++ b/packages/opencode/test/altimate/review-incremental-semantics.test.ts @@ -299,3 +299,55 @@ describe("end to end through the orchestrator", () => { expect(env.verdict).not.toBe("REQUEST_CHANGES") }) }) + +describe("dedup windows split across lines", () => { + const sql = "select *,\n row_number() over (\n partition by job_id\n order by ingestion_timestamp desc\n ) as rn\nfrom t" + const rulesFor = (diffAdded: string) => + detectModelPatterns({ path: "models/m.sql", status: "modified", diff: diffAdded }, sql, DEFAULT_RUBRIC).map( + (f) => (f.evidence?.result as { rule?: string })?.rule, + ) + test("an ORDER BY on its own line is seen: no false 'no ORDER BY', and the load-timestamp rule fires", () => { + const r = rulesFor("+ row_number() over (\n") + expect(r).not.toContain("dedup_no_tiebreaker") + expect(r).toContain("dedup_load_timestamp_only") + }) + test("a window the PR doesn't touch is not reported", () => { + expect(rulesFor("+select 1 as unrelated\n")).not.toContain("dedup_load_timestamp_only") + }) +}) + +describe("column changes without compiled SQL (dbt parse-only CI)", () => { + test("on_schema_change='fail' sees columns added to the raw model's SELECT", async () => { + const cfg = "{{ config(materialized='incremental', unique_key='id', on_schema_change='fail') }}\n" + const oldSql = `${cfg}select id, amount from {{ ref('src') }}` + const newSql = `${cfg}select id, amount, region from {{ ref('src') }}` + const env = await runReview({ + changedFiles: [{ path: "models/marts/fct.sql", status: "modified", diff: `-${oldSql}\n+${newSql}\n` }], + config: { ...DEFAULT_REVIEW_CONFIG, reviewers: ["dbt_patterns"] }, + rubric: DEFAULT_RUBRIC, + mode: "comment", + runner: { + async impact() { + return { hasManifest: true, severity: "SAFE", directCount: 0, transitiveCount: 0, testCount: 0 } + }, + async grade() { + return {} + }, + async check() { + return { issues: [], ran: true } + }, + async equivalence() { + return { decided: false } + }, + async detectPii() { + return { columns: [] } + }, + async columnLineage() { + return [] + }, + } as ReviewRunner, + getContent: async (_f, side) => (side === "old" ? oldSql : newSql), + }) + expect(env.findings.find((f) => f.title.includes("on_schema_change"))?.body).toContain("adds `region`") + }) +}) From 27d00a8a466a78b8a15227d91d4e617e94d3bdc2 Mon Sep 17 00:00:00 2001 From: RaghwendraSingh Date: Fri, 13 Nov 2026 13:32:13 +0530 Subject: [PATCH 3/5] feat(review): data tests that can't do their job: self-comparisons, inherited warn/thresholds, loosened project thresholds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - a test added or changed in the PR that compares a model with itself (`dbt_utils.equality`/`equal_rowcount`/… against its own `ref`, or a `relationships` test to its own column) passes by construction (warning) - new tests that inherit `severity: warn` or an `error_if` threshold from `dbt_project.yml` (walked from the top through the project name and folders, deeper wins) are reported once per yml (suggestion) - a `dbt_project.yml` change that turns a folder's tests to `warn` or sets an `error_if` threshold is reported (warning) - reads dbt 1.8+ `arguments:` too; all findings are catalog origin (never block) Co-Authored-By: Claude Opus 5.5 --- docs/docs/usage/dbt-pr-review.md | 8 + .../src/altimate/review/orchestrate.ts | 11 + .../src/altimate/review/test-adequacy.ts | 214 ++++++++++++++++++ .../altimate/review-test-adequacy.test.ts | 83 +++++++ 4 files changed, 316 insertions(+) create mode 100644 packages/opencode/src/altimate/review/test-adequacy.ts create mode 100644 packages/opencode/test/altimate/review-test-adequacy.test.ts diff --git a/docs/docs/usage/dbt-pr-review.md b/docs/docs/usage/dbt-pr-review.md index c36e62aeda..d7f0731019 100644 --- a/docs/docs/usage/dbt-pr-review.md +++ b/docs/docs/usage/dbt-pr-review.md @@ -526,6 +526,14 @@ The deterministic rules group into four families: - **dbt unit tests.** A unit test whose `given` inputs are all `rows: []` is reported: it either never tests the model's logic, or expects rows the model has nothing to build from. +- **Data tests that can't do their job.** These are checked on tests added or + changed in the PR: + - a test that compares a model with itself (`dbt_utils.equality` or + `equal_rowcount` against its own `ref`, or a `relationships` test pointing + at its own column) passes by construction; + - new tests that inherit `severity: warn` or an `error_if` threshold from + `dbt_project.yml` report failures without failing the build; + - a `dbt_project.yml` change that loosens those thresholds for a whole folder. - **Incremental and snapshot semantics.** Config is read from the manifest (project, yml and inline, as dbt merges them) with the head's inline `{{ config() }}` on top. Formatting-only edits never count as a change. diff --git a/packages/opencode/src/altimate/review/orchestrate.ts b/packages/opencode/src/altimate/review/orchestrate.ts index 6608d49a00..051ea3325a 100644 --- a/packages/opencode/src/altimate/review/orchestrate.ts +++ b/packages/opencode/src/altimate/review/orchestrate.ts @@ -26,6 +26,7 @@ import { detectModelPatterns, detectSchemaYmlPatterns, splitDiff } from "./dbt-p import { listMirrorsModel, type ConsumerColumnList } from "./consumer-lists" import { detectUnitTestFixtures } from "./unit-tests" import { selectListNames } from "./select-list" +import { detectTestAdequacy, detectThresholdLoosening } from "./test-adequacy" import { incrementalFindings, snapshotFindings, snapshotYmlFindings } from "./incremental-semantics" import { type AiLaneStatus, type AiReviewInput, type AiReviewResult, safeReason } from "./ai-review" @@ -1753,6 +1754,9 @@ export async function runReview(input: OrchestrateInput): Promise f.kind === "schema_yml") + // The head's dbt_project.yml (test thresholds inherited by new tests), read once. + const projectYmlPath = `${input.layout?.root ? `${input.layout.root.replace(/\/$/, "")}/` : ""}dbt_project.yml` + const projectYml = schemaFiles.length ? await getContent?.(projectYmlPath, "new").catch(() => undefined) : undefined if (schemaFiles.length) { const schemaFindingSets = await Promise.all( schemaFiles.map(async (file) => { @@ -1766,6 +1770,7 @@ export async function runReview(input: OrchestrateInput): Promise f.kind === "project_config" && /(^|\/)dbt_project\.ya?ml$/.test(f.path))) { + if (file.status !== "modified") continue + const [oldContent, newContent] = await Promise.all([getContent?.(file.path, "old"), getContent?.(file.path, "new")]) + all.push(detectThresholdLoosening(file, oldContent, newContent)) + } // Architectural dedup: the regex `dbt-patterns`/`rule-catalog` layer is a // diff-aware, Jinja-aware, offline fallback. For the STRUCTURAL SQL checks diff --git a/packages/opencode/src/altimate/review/test-adequacy.ts b/packages/opencode/src/altimate/review/test-adequacy.ts new file mode 100644 index 0000000000..65fb2848cc --- /dev/null +++ b/packages/opencode/src/altimate/review/test-adequacy.ts @@ -0,0 +1,214 @@ +import YAML from "yaml" +import { makeFinding, type Finding } from "./finding" +import type { ChangedFile, DbtProjectLayout } from "./diff-filter" + +/** + * Data tests that cannot do their job, found from yml alone (any adapter): + * - a test that compares a model with itself (`dbt_utils.equality` / + * `equal_rowcount` against its own `ref`, or a `relationships` test pointing + * at its own column) passes by construction; + * - tests added in the PR that inherit `severity: warn` (or an `error_if` + * threshold) from `dbt_project.yml` report failures without failing the build; + * - a PR that loosens those thresholds in `dbt_project.yml` weakens every test + * under that path. + * Only tests added or changed in the PR are checked. + */ + +interface TestRef { + entity: string + column?: string + test: string + args: Record +} + +const SELF_COMPARE = new Set([ + "equality", + "equal_rowcount", + "cardinality_equality", + "fewer_rows_than", + "expect_table_row_count_to_equal_other_table", + "expect_table_aggregation_to_equal_other_table", +]) + +function parse(text?: string): any { + if (!text) return undefined + try { + return YAML.parse(text) + } catch { + return undefined + } +} + +function testsOf(doc: any): TestRef[] { + const out: TestRef[] = [] + const read = (list: unknown, entity: string, column?: string) => { + if (!Array.isArray(list)) return + for (const t of list) { + if (typeof t === "string") out.push({ entity, column, test: t, args: {} }) + else if (t && typeof t === "object") { + const [k] = Object.keys(t) + if (k) out.push({ entity, column, test: k, args: (t as any)[k] ?? {} }) + } + } + } + for (const group of ["models", "snapshots", "seeds"]) { + for (const e of Array.isArray(doc?.[group]) ? doc[group] : []) { + if (typeof e?.name !== "string") continue + read(e.data_tests ?? e.tests, e.name) + for (const c of Array.isArray(e.columns) ? e.columns : []) + if (typeof c?.name === "string") read(c.data_tests ?? c.tests, e.name, c.name) + } + } + return out +} + +const key = (t: TestRef) => `${t.entity}|${t.column ?? ""}|${t.test}|${JSON.stringify(t.args)}` +const arg = (t: TestRef, name: string) => t.args?.[name] ?? t.args?.arguments?.[name] +const shortName = (test: string) => test.split(".").pop()!.toLowerCase() +const refsEntity = (v: unknown, entity: string) => + typeof v === "string" && new RegExp(`ref\\(\\s*['"]${entity.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}['"]\\s*\\)`).test(v) + +function finding(file: ChangedFile, model: string | undefined, rule: string, severity: Finding["severity"], title: string, body: string, extra = ""): Finding { + return makeFinding({ + severity, + category: "test_coverage", + title, + body, + file: file.path, + model, + confidence: "high", + evidence: { tool: "test-adequacy", result: { rule } }, + ruleKey: `test_coverage:${rule}${extra}`, + }) +} + +/** `+severity` / `+error_if` / `+warn_if` (with or without `+`) on one config node. */ +function thresholds(node: any): { severity?: string; error_if?: string; warn_if?: string } { + if (!node || typeof node !== "object") return {} + const get = (k: string) => (node[`+${k}`] ?? node[k]) as unknown + const str = (v: unknown) => (typeof v === "string" ? v.trim().toLowerCase() : undefined) + return { severity: str(get("severity")), error_if: str(get("error_if")), warn_if: str(get("warn_if")) } +} + +/** + * Effective test thresholds for a yml under `segments` (folders below the model + * path), walking `data_tests:` (or `tests:`) from the top, through the project + * name, down the folders; deeper levels override. `path` is where the effective + * value was set. + */ +export function inheritedThresholds(projectYml: any, segments: string[]): { severity?: string; error_if?: string; path: string[] } { + const tree = projectYml?.data_tests ?? projectYml?.tests + if (!tree || typeof tree !== "object") return { path: [] } + const name = projectYml?.name + const chain = name && tree[name] && typeof tree[name] === "object" ? [String(name), ...segments] : segments + const levels: Array<[string[], any]> = [[[], tree]] + let node = tree + for (let i = 0; i < chain.length; i++) { + node = node?.[chain[i]] + if (!node || typeof node !== "object") break + levels.push([chain.slice(0, i + 1), node]) + } + const eff: { severity?: string; error_if?: string } = {} + let path: string[] = [] + for (const [at, n] of levels) { + const t = thresholds(n) + if (t.severity) eff.severity = t.severity + if (t.error_if) eff.error_if = t.error_if + if (t.severity || t.error_if) path = at.length ? at : ["(all tests)"] + } + return { ...eff, path } +} + +/** Folders of a yml below its model path (e.g. `models/marts/x.yml` → `["marts"]`). */ +function segmentsOf(file: string, layout?: DbtProjectLayout): string[] { + const root = layout?.root ? layout.root.replace(/\/$/, "") + "/" : "" + const rel = root && file.startsWith(root) ? file.slice(root.length) : file + const parts = rel.split("/").slice(0, -1) + const modelDirs = layout?.dirs.model ?? ["models"] + return modelDirs.includes(parts[0]) ? parts.slice(1) : parts +} + +export function detectTestAdequacy(input: { + file: ChangedFile + oldContent?: string + newContent?: string + projectYml?: string + layout?: DbtProjectLayout +}): Finding[] { + const { file } = input + if (file.status === "deleted") return [] + const newDoc = parse(input.newContent) + if (!newDoc) return [] + const before = new Set(testsOf(parse(input.oldContent)).map(key)) + const added = testsOf(newDoc).filter((t) => !before.has(key(t))) + const out: Finding[] = [] + + for (const t of added) { + const name = shortName(t.test) + const self = + (SELF_COMPARE.has(name) && ["compare_model", "to", "compare_to"].some((a) => refsEntity(arg(t, a), t.entity))) || + (name === "relationships" && refsEntity(arg(t, "to"), t.entity) && String(arg(t, "field") ?? "").toLowerCase() === (t.column ?? "").toLowerCase()) + if (self) + out.push( + finding(file, t.entity, "test-compares-model-with-itself", "warning", `${t.entity}: \`${t.test}\` compares the model with itself`, + `This test checks \`${t.entity}\`${t.column ? `.\`${t.column}\`` : ""} against \`ref('${t.entity}')\`, so it passes by construction. ` + + "Point it at the independent source of truth it is meant to reconcile against.", `:${key(t)}`), + ) + } + + const project = parse(input.projectYml) + if (project && added.length) { + const inherited = inheritedThresholds(project, segmentsOf(file.path, input.layout)) + const own = (t: TestRef) => { + const c = t.args?.config ?? {} + return [c.severity, c.error_if, c.warn_if, t.args?.severity, t.args?.error_if].some((v) => v !== undefined) + } + const loose = added.filter((t) => !own(t)) + const where = inherited.path.length ? ` (\`data_tests:\` → ${inherited.path.map((p) => `\`${p}\``).join(" → ")})` : "" + if (loose.length && inherited.severity === "warn") + out.push( + finding(file, undefined, "test-inherits-warn", "suggestion", `${loose.length} new test(s) in ${file.path} inherit \`severity: warn\``, + `They inherit \`severity: warn\` from \`dbt_project.yml\`${where}, so failures are reported but never fail \`dbt build\`. ` + + "If a test should gate the build, set `config: {severity: error}` on it."), + ) + else if (loose.length && inherited.error_if && inherited.error_if !== "!= 0") + out.push( + finding(file, undefined, "test-inherits-threshold", "suggestion", `${loose.length} new test(s) in ${file.path} inherit \`error_if: ${inherited.error_if}\``, + `They fail only when \`${inherited.error_if}\` rows fail, a threshold inherited from \`dbt_project.yml\`${where}. ` + + "Set `config: {error_if: '!= 0'}` on tests that must catch any bad row."), + ) + } + return out +} + +/** A `dbt_project.yml` change that loosens test thresholds for a whole path. */ +export function detectThresholdLoosening(file: ChangedFile, oldContent?: string, newContent?: string): Finding[] { + const oldDoc = parse(oldContent) + const newDoc = parse(newContent) + if (!oldDoc || !newDoc) return [] + const flat = (doc: any) => { + const out = new Map>() + const visit = (node: any, path: string[]) => { + if (!node || typeof node !== "object") return + out.set(path.join(" → ") || "(all tests)", thresholds(node)) + for (const [k, v] of Object.entries(node)) if (!k.startsWith("+") && v && typeof v === "object") visit(v, [...path, k]) + } + visit(doc?.data_tests ?? doc?.tests, []) + return out + } + const before = flat(oldDoc) + const out: Finding[] = [] + for (const [path, t] of flat(newDoc)) { + const prev = before.get(path) ?? {} + const toWarn = t.severity === "warn" && prev.severity !== "warn" + const newThreshold = t.error_if && t.error_if !== "!= 0" && t.error_if !== prev.error_if + if (!toWarn && !newThreshold) continue + out.push( + finding(file, undefined, "test-thresholds-loosened", "warning", `test thresholds loosened for \`${path}\``, + (toWarn ? "`severity: warn` now applies" : `\`error_if: ${t.error_if}\` now applies`) + + ` to every data test under \`${path}\` in \`dbt_project.yml\`, so those tests stop failing the build. ` + + "Prefer setting it on the specific tests that need it.", `:${path}`), + ) + } + return out +} diff --git a/packages/opencode/test/altimate/review-test-adequacy.test.ts b/packages/opencode/test/altimate/review-test-adequacy.test.ts new file mode 100644 index 0000000000..644e664d14 --- /dev/null +++ b/packages/opencode/test/altimate/review-test-adequacy.test.ts @@ -0,0 +1,83 @@ +import { describe, test, expect } from "bun:test" +import YAML from "yaml" +import { detectTestAdequacy, detectThresholdLoosening, inheritedThresholds } from "../../src/altimate/review/test-adequacy" + +/** Data tests that can't do their job, from yml alone (any adapter). */ + +const file = { path: "models/marts/_fct.yml", status: "modified" as const } +const rule = (fs: { evidence?: { result?: unknown } }[]) => fs.map((f) => (f.evidence?.result as { rule: string }).rule) + +describe("tests that compare a model with itself", () => { + const yml = (compare: string) => `models: + - name: fct_orders + data_tests: + - dbt_utils.equal_rowcount: + compare_model: ${compare} + columns: + - name: customer_id + data_tests: + - relationships: + to: ref('fct_orders') + field: customer_id +` + test("own-ref comparisons and self relationships pass by construction", () => { + const out = detectTestAdequacy({ file, newContent: yml("ref('fct_orders')") }) + expect(rule(out)).toEqual(["test-compares-model-with-itself", "test-compares-model-with-itself"]) + }) + + test("a comparison with another model is fine; unchanged tests are skipped", () => { + const other = yml("ref('stg_orders')").replace("to: ref('fct_orders')", "to: ref('dim_customers')") + expect(detectTestAdequacy({ file, newContent: other })).toEqual([]) + expect(detectTestAdequacy({ file, oldContent: yml("ref('fct_orders')"), newContent: yml("ref('fct_orders')") })).toEqual([]) + }) + + test("dbt 1.8+ `arguments:` form is read too", () => { + const y = `models:\n - name: m\n data_tests:\n - dbt_utils.equality:\n arguments:\n compare_model: ref("m")\n` + expect(rule(detectTestAdequacy({ file: { path: "models/_m.yml", status: "added" }, newContent: y }))).toEqual(["test-compares-model-with-itself"]) + }) +}) + +describe("thresholds inherited from dbt_project.yml", () => { + const project = `name: shop +data_tests: + shop: + marts: + +severity: warn + staging: + +error_if: ">100" +` + const newTests = (own = "") => `models: + - name: fct_orders + columns: + - name: id + data_tests: + - not_null${own} +` + test("the effective value walks project → folders, deeper wins", () => { + const doc = YAML.parse(project) + expect(inheritedThresholds(doc, ["marts"])).toEqual({ severity: "warn", path: ["shop", "marts"] }) + expect(inheritedThresholds(doc, ["staging", "x"])).toEqual({ error_if: ">100", path: ["shop", "staging"] }) + expect(inheritedThresholds(doc, ["core"])).toEqual({ path: [] }) + }) + + test("new tests under a warn folder are reported; a test with its own severity is not", () => { + const out = detectTestAdequacy({ file, newContent: newTests(), projectYml: project }) + expect(rule(out)).toEqual(["test-inherits-warn"]) + expect(out[0].body).toContain("`shop` → `marts`") + const own = newTests(":\n config:\n severity: error").replace("- not_null:", "- not_null:") + expect(detectTestAdequacy({ file, newContent: own, projectYml: project })).toEqual([]) + }) + + test("an inherited error_if threshold is reported", () => { + const out = detectTestAdequacy({ file: { path: "models/staging/_stg.yml", status: "modified" }, newContent: newTests(), projectYml: project }) + expect(rule(out)).toEqual(["test-inherits-threshold"]) + }) + + test("a dbt_project.yml change that loosens thresholds is reported", () => { + const before = "name: shop\ndata_tests:\n shop:\n marts:\n +severity: error\n" + const after = "name: shop\ndata_tests:\n shop:\n marts:\n +severity: warn\n" + const out = detectThresholdLoosening({ path: "dbt_project.yml", status: "modified" }, before, after) + expect(out.map((f) => f.title)).toEqual(["test thresholds loosened for `shop → marts`"]) + expect(detectThresholdLoosening({ path: "dbt_project.yml", status: "modified" }, after, after)).toEqual([]) + }) +}) From 7283ada8b7ecf422522fe7d4ec037145b4e361b3 Mon Sep 17 00:00:00 2001 From: RaghwendraSingh Date: Fri, 20 Nov 2026 10:28:50 +0530 Subject: [PATCH 4/5] fix(review): address review comments: conditional raw columns, dedup window touch, self-compare tests, `error_if` loosening, prepare-step failures - `orchestrate.ts`: a `{% if %}` / `{% for %}` tag inside a raw model's select list makes the list unknown, so conditional columns no longer read as added or removed in the parse-only column delta - `dbt-patterns.ts`: a dedup window counts as changed only when an added line falls inside its line span (head line numbers from the hunk headers), not when an added line's text happens to occur in it - `test-adequacy.ts`: drop `fewer_rows_than` (it fails against itself, it doesn't pass by construction); an aggregation comparison on the same model is flagged only when both expressions match - `test-adequacy.ts`: `error_if` is reported as loosened only when the new bound tolerates more failing rows; `> 0` / `!=0` / `>= 1` count as the strict default - `action.yml`: a failed `dbt docs generate` or base checkout warns instead of failing the step; a stale `target-base/` is removed before the base compile - tests for each Co-Authored-By: Claude Opus 5.5 --- github/review/action.yml | 7 ++- .../src/altimate/review/dbt-patterns.ts | 34 +++++++++++-- .../src/altimate/review/orchestrate.ts | 7 ++- .../src/altimate/review/test-adequacy.ts | 32 ++++++++++-- .../test/altimate/review-action.test.ts | 9 ++++ .../review-incremental-semantics.test.ts | 50 +++++++++++++++++++ .../altimate/review-test-adequacy.test.ts | 27 ++++++++++ 7 files changed, 155 insertions(+), 11 deletions(-) diff --git a/github/review/action.yml b/github/review/action.yml index f39818fc6b..02f6f7d7e9 100644 --- a/github/review/action.yml +++ b/github/review/action.yml @@ -207,7 +207,8 @@ runs: build "$PROJECT" "$PROJECT/target" if [[ "$IN_PREPARE" == "full" ]]; then echo "Generating catalog.json…" - ( cd "$PROJECT" && "${DBT[@]}" docs generate --target-path "$PROJECT/target" ${PROFILES[@]+"${PROFILES[@]}"} ) + ( cd "$PROJECT" && "${DBT[@]}" docs generate --target-path "$PROJECT/target" ${PROFILES[@]+"${PROFILES[@]}"} ) || + echo "::warning::dbt docs generate failed; continuing without catalog.json." fi # The base side: the PR's merge-base, compiled into target-base/. BASE="${IN_BASE:-$EVENT_BASE}" @@ -221,8 +222,10 @@ runs: fi if [[ -z "$MB" ]]; then echo "::warning::Could not find the merge-base with $BASE; skipping the base compile."; exit 0; fi WT="$RUNNER_TEMP/altimate-review-base" - git worktree add --detach "$WT" "$MB" >/dev/null trap 'git worktree remove --force "$WT" >/dev/null 2>&1 || true' EXIT + git worktree add --detach "$WT" "$MB" >/dev/null || { echo "::warning::Could not check out the base; skipping the base compile."; exit 0; } + # A stale target-base/ from an earlier run must not stand in for a failed compile. + rm -rf "$PROJECT/target-base" echo "Compiling the base ($BASE @ ${MB:0:12})…" build "$WT/$REL" "$PROJECT/target-base" || echo "::warning::The base compile failed; equivalence will be marked unverified." diff --git a/packages/opencode/src/altimate/review/dbt-patterns.ts b/packages/opencode/src/altimate/review/dbt-patterns.ts index f96eca2649..02ceb357f5 100644 --- a/packages/opencode/src/altimate/review/dbt-patterns.ts +++ b/packages/opencode/src/altimate/review/dbt-patterns.ts @@ -225,13 +225,38 @@ const detectNotIn: Detector = (c) => { }) } +/** Head-side line numbers of the added lines; undefined when the diff has no hunk headers. */ +function addedLineNumbers(diff: string | undefined): Set | undefined { + if (!diff || !/^@@ /m.test(diff)) return undefined + const out = new Set() + let right = 0 + let inHunk = false + for (const line of diff.split("\n")) { + const h = /^@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@/.exec(line) + if (h) { + right = Number(h[1]) + inHunk = true + continue + } + if (!inHunk || line.startsWith("\\")) continue + if (line.startsWith("+")) out.add(right++) + else if (line.startsWith(" ") || line === "") right++ + else if (!line.startsWith("-")) inHunk = false + } + return out +} + /** * `row_number() over (…)` windows in the model, read from the whole file so a * window split across lines is seen whole. A window counts as part of the - * change when one of the PR's added lines falls inside it. + * change when one of the PR's added lines falls inside its line span (by text + * only when the diff carries no hunk headers). */ function dedupWindows(c: Ctx): Array<{ spec: string; text: string }> { - const sql = stripComments(c.newSql) + // Comments blanked with newlines kept, so offsets map to head line numbers. + const sql = c.newSql.replace(/\/\*[\s\S]*?\*\/|--[^\n]*|\{#[\s\S]*?#\}/g, (m) => m.replace(/[^\n]/g, " ")) + const lineAt = (offset: number) => sql.slice(0, offset).split("\n").length + const addedLines = addedLineNumbers(c.file.diff) const added = c.added.map((l) => stripComments(l).trim()).filter((l) => l.length >= 3) const out: Array<{ spec: string; text: string }> = [] for (const m of sql.matchAll(/row_number\s*\(\s*\)\s*over\s*\(/gi)) { @@ -247,7 +272,10 @@ function dedupWindows(c: Ctx): Array<{ spec: string; text: string }> { } if (close < 0) continue const text = sql.slice(m.index!, close + 1) - if (!added.some((l) => text.includes(l) || l.includes(text.replace(/\s+/g, " ")))) continue + const touched = addedLines + ? [...addedLines].some((n) => n >= lineAt(m.index!) && n <= lineAt(close)) + : added.some((l) => text.includes(l) || l.includes(text.replace(/\s+/g, " "))) + if (!touched) continue out.push({ spec: sql.slice(open + 1, close).replace(/\s+/g, " ").trim(), text: text.replace(/\s+/g, " ") }) } return out diff --git a/packages/opencode/src/altimate/review/orchestrate.ts b/packages/opencode/src/altimate/review/orchestrate.ts index 051ea3325a..3102780020 100644 --- a/packages/opencode/src/altimate/review/orchestrate.ts +++ b/packages/opencode/src/altimate/review/orchestrate.ts @@ -687,11 +687,14 @@ async function columnDelta( return delta(current, before) } -/** Output names of a raw (Jinja) model's final SELECT; empty when a Jinja expression is in the list. */ +/** Output names of a raw (Jinja) model's final SELECT; empty when a Jinja + * expression or tag is in the list. */ function rawSelectList(sql: string): Set { const plain = sql .replace(/\{#[\s\S]*?#\}/g, " ") - .replace(/\{%[\s\S]*?%\}/g, " ") + // A tag in the list (`{% if %}` / `{% for %}`) makes its columns conditional: + // it becomes its own `__jinja__` item, which marks the list unknown. + .replace(/\{%[\s\S]*?%\}/g, " , __jinja__ , ") .replace(/\{\{[\s\S]*?\}\}/g, " __jinja__ ") const names = selectListNames(plain) if (!names || names.includes("__jinja__")) return new Set() diff --git a/packages/opencode/src/altimate/review/test-adequacy.ts b/packages/opencode/src/altimate/review/test-adequacy.ts index 65fb2848cc..7d2713719e 100644 --- a/packages/opencode/src/altimate/review/test-adequacy.ts +++ b/packages/opencode/src/altimate/review/test-adequacy.ts @@ -25,7 +25,6 @@ const SELF_COMPARE = new Set([ "equality", "equal_rowcount", "cardinality_equality", - "fewer_rows_than", "expect_table_row_count_to_equal_other_table", "expect_table_aggregation_to_equal_other_table", ]) @@ -82,6 +81,19 @@ function finding(file: ChangedFile, model: string | undefined, rule: string, sev }) } +/** + * Failing rows an `error_if` condition tolerates: `!= 0` / `> 0` → 0, `> 10` → 10, + * `>= 10` → 9. Undefined when the condition isn't a simple bound. + */ +function allowedFailures(cond: string | undefined): number | undefined { + const m = cond ? /^(!=|<>|>=|>)\s*(\d+)$/.exec(cond.trim()) : null + if (!m) return undefined + const n = Number(m[2]) + if (m[1] === ">=") return Math.max(0, n - 1) + if (m[1] === ">") return n + return n === 0 ? 0 : undefined +} + /** `+severity` / `+error_if` / `+warn_if` (with or without `+`) on one config node. */ function thresholds(node: any): { severity?: string; error_if?: string; warn_if?: string } { if (!node || typeof node !== "object") return {} @@ -145,8 +157,14 @@ export function detectTestAdequacy(input: { for (const t of added) { const name = shortName(t.test) + // Different expressions on the same model are a valid invariant (e.g. sum of + // line items equals the order total); only the same expression is circular. + const sameExpression = + name !== "expect_table_aggregation_to_equal_other_table" || + String(arg(t, "compare_expression") ?? arg(t, "expression") ?? "").replace(/\s+/g, "").toLowerCase() === + String(arg(t, "expression") ?? "").replace(/\s+/g, "").toLowerCase() const self = - (SELF_COMPARE.has(name) && ["compare_model", "to", "compare_to"].some((a) => refsEntity(arg(t, a), t.entity))) || + (SELF_COMPARE.has(name) && sameExpression && ["compare_model", "to", "compare_to"].some((a) => refsEntity(arg(t, a), t.entity))) || (name === "relationships" && refsEntity(arg(t, "to"), t.entity) && String(arg(t, "field") ?? "").toLowerCase() === (t.column ?? "").toLowerCase()) if (self) out.push( @@ -171,7 +189,7 @@ export function detectTestAdequacy(input: { `They inherit \`severity: warn\` from \`dbt_project.yml\`${where}, so failures are reported but never fail \`dbt build\`. ` + "If a test should gate the build, set `config: {severity: error}` on it."), ) - else if (loose.length && inherited.error_if && inherited.error_if !== "!= 0") + else if (loose.length && inherited.error_if && allowedFailures(inherited.error_if) !== 0) out.push( finding(file, undefined, "test-inherits-threshold", "suggestion", `${loose.length} new test(s) in ${file.path} inherit \`error_if: ${inherited.error_if}\``, `They fail only when \`${inherited.error_if}\` rows fail, a threshold inherited from \`dbt_project.yml\`${where}. ` + @@ -201,7 +219,13 @@ export function detectThresholdLoosening(file: ChangedFile, oldContent?: string, for (const [path, t] of flat(newDoc)) { const prev = before.get(path) ?? {} const toWarn = t.severity === "warn" && prev.severity !== "warn" - const newThreshold = t.error_if && t.error_if !== "!= 0" && t.error_if !== prev.error_if + // Only a threshold that tolerates more failing rows than before loosens; + // a changed condition that is not a simple bound is still reported. + const allowed = allowedFailures(t.error_if) + const newThreshold = + !!t.error_if && + t.error_if !== prev.error_if && + (allowed === undefined || allowed > (allowedFailures(prev.error_if) ?? 0)) if (!toWarn && !newThreshold) continue out.push( finding(file, undefined, "test-thresholds-loosened", "warning", `test thresholds loosened for \`${path}\``, diff --git a/packages/opencode/test/altimate/review-action.test.ts b/packages/opencode/test/altimate/review-action.test.ts index 36f0417224..fda2d21a26 100644 --- a/packages/opencode/test/altimate/review-action.test.ts +++ b/packages/opencode/test/altimate/review-action.test.ts @@ -21,4 +21,13 @@ describe("action: dbt_prepare", () => { // The base is compiled into target-base/, where the compiled-SQL resolver looks. expect(prep.run).toContain('"$PROJECT/target-base"') }) + + test("a failed docs generate or base checkout warns and lets the review run", async () => { + const action = YAML.parse(await fs.readFile(actionPath, "utf8")) + const prep = (action.runs.steps as Array<{ name: string; run?: string }>).find((s) => s.name === "Prepare dbt artifacts")! + const run = prep.run! + expect(run).toMatch(/docs generate[^\n]*\) \|\|\n\s*echo "::warning::/) + expect(run).toMatch(/git worktree add[^\n]*\|\| \{ echo "::warning::/) + expect(run.indexOf('rm -rf "$PROJECT/target-base"')).toBeLessThan(run.indexOf('build "$WT/$REL"')) + }) }) diff --git a/packages/opencode/test/altimate/review-incremental-semantics.test.ts b/packages/opencode/test/altimate/review-incremental-semantics.test.ts index 5fd78deef1..ffddb310b7 100644 --- a/packages/opencode/test/altimate/review-incremental-semantics.test.ts +++ b/packages/opencode/test/altimate/review-incremental-semantics.test.ts @@ -314,6 +314,19 @@ describe("dedup windows split across lines", () => { test("a window the PR doesn't touch is not reported", () => { expect(rulesFor("+select 1 as unrelated\n")).not.toContain("dedup_load_timestamp_only") }) + test("with hunk headers, an added line matching window text elsewhere doesn't touch the window", () => { + const noOrder = + "select\n tenant\n , row_number() over (partition by tenant, id) as rn\nfrom t\nunion all\nselect\n tenant\n , id\nfrom u" + const rules = (diff: string) => + detectModelPatterns({ path: "models/m.sql", status: "modified", diff }, noOrder, DEFAULT_RUBRIC).map( + (f) => (f.evidence?.result as { rule?: string })?.rule, + ) + // Line 8 (` , id`) in the second branch was added; the window on line 3 is old code. + expect(rules("@@ -7,1 +7,2 @@\n tenant\n+ , id\n")).not.toContain("dedup_no_tiebreaker") + expect(rules("@@ -3,1 +3,1 @@\n- , row_number() over () as rn\n+ , row_number() over (partition by tenant, id) as rn\n")).toContain( + "dedup_no_tiebreaker", + ) + }) }) describe("column changes without compiled SQL (dbt parse-only CI)", () => { @@ -350,4 +363,41 @@ describe("column changes without compiled SQL (dbt parse-only CI)", () => { }) expect(env.findings.find((f) => f.title.includes("on_schema_change"))?.body).toContain("adds `region`") }) + + test("columns inside a `{% if %}` block are unknown, not added", async () => { + const cfg = "{{ config(materialized='incremental', unique_key='id', on_schema_change='fail') }}\n" + const review = (oldSql: string, newSql: string) => + runReview({ + changedFiles: [{ path: "models/marts/fct.sql", status: "modified", diff: `-${oldSql}\n+${newSql}\n` }], + config: { ...DEFAULT_REVIEW_CONFIG, reviewers: ["dbt_patterns"] }, + rubric: DEFAULT_RUBRIC, + mode: "comment", + runner: { + async impact() { + return { hasManifest: true, severity: "SAFE", directCount: 0, transitiveCount: 0, testCount: 0 } + }, + async grade() { + return {} + }, + async check() { + return { issues: [], ran: true } + }, + async equivalence() { + return { decided: false } + }, + async detectPii() { + return { columns: [] } + }, + async columnLineage() { + return [] + }, + } as ReviewRunner, + getContent: async (_f, side) => (side === "old" ? oldSql : newSql), + }) + const conditional = await review( + `${cfg}select id, amount from {{ ref('src') }}`, + `${cfg}select id, amount {% if var('with_region', false) %}, region {% endif %} from {{ ref('src') }}`, + ) + expect(conditional.findings.some((f) => f.title.includes("on_schema_change"))).toBe(false) + }) }) diff --git a/packages/opencode/test/altimate/review-test-adequacy.test.ts b/packages/opencode/test/altimate/review-test-adequacy.test.ts index 644e664d14..49a4a7c9e7 100644 --- a/packages/opencode/test/altimate/review-test-adequacy.test.ts +++ b/packages/opencode/test/altimate/review-test-adequacy.test.ts @@ -31,6 +31,22 @@ describe("tests that compare a model with itself", () => { expect(detectTestAdequacy({ file, oldContent: yml("ref('fct_orders')"), newContent: yml("ref('fct_orders')") })).toEqual([]) }) + test("fewer_rows_than against itself, or an aggregation with a different expression, is not flagged", () => { + const y = `models: + - name: fct_orders + data_tests: + - dbt_utils.fewer_rows_than: + compare_model: ref('fct_orders') + - dbt_expectations.expect_table_aggregation_to_equal_other_table: + expression: sum(line_total) + compare_model: ref('fct_orders') + compare_expression: sum(order_total) +` + expect(detectTestAdequacy({ file, newContent: y })).toEqual([]) + const same = y.replace("sum(order_total)", "sum( line_total )") + expect(rule(detectTestAdequacy({ file, newContent: same }))).toEqual(["test-compares-model-with-itself"]) + }) + test("dbt 1.8+ `arguments:` form is read too", () => { const y = `models:\n - name: m\n data_tests:\n - dbt_utils.equality:\n arguments:\n compare_model: ref("m")\n` expect(rule(detectTestAdequacy({ file: { path: "models/_m.yml", status: "added" }, newContent: y }))).toEqual(["test-compares-model-with-itself"]) @@ -80,4 +96,15 @@ data_tests: expect(out.map((f) => f.title)).toEqual(["test thresholds loosened for `shop → marts`"]) expect(detectThresholdLoosening({ path: "dbt_project.yml", status: "modified" }, after, after)).toEqual([]) }) + + test("error_if: only a bound that tolerates more failures loosens; strict forms are the default", () => { + const at = (cond: string) => `name: shop\ndata_tests:\n shop:\n marts:\n +error_if: "${cond}"\n` + const loosened = (from: string, to: string) => + detectThresholdLoosening({ path: "dbt_project.yml", status: "modified" }, at(from), at(to)).length > 0 + expect(loosened(">100", ">10")).toBe(false) + expect(loosened(">10", ">100")).toBe(true) + expect(loosened("!= 0", "> 0")).toBe(false) + expect(loosened("!= 0", ">=1")).toBe(false) + expect(loosened("!= 0", ">5")).toBe(true) + }) }) From b50675c03af8cb76696b0161f5b3b2c675b8dc84 Mon Sep 17 00:00:00 2001 From: RaghwendraSingh Date: Fri, 20 Nov 2026 12:55:53 +0530 Subject: [PATCH 5/5] fix(review): address review feedback: dedup window text fallback, complete raw select lists, effective test thresholds, `cardinality_equality` field, prepare-step head handling - `dbt-patterns.ts`: without hunk headers, a dedup window counts as touched only by an added line that holds window syntax (`row_number`, `over`, `partition by`, `order by`), with whitespace normalised on both sides; a bare identifier that also occurs in the window no longer flags it - `select-list.ts` / `orchestrate.ts`: the parse-only column delta treats the raw select list as unknown when any item has no name of its own (e.g. `amount::numeric(18,2)`); comments updated for the compiled-on-one-side case - `test-adequacy.ts`: `severity` and `error_if` are resolved separately, so a local `warn_if` / `error_if` no longer hides an inherited `severity: warn` - `test-adequacy.ts`: threshold loosening compares effective values across the union of config paths in both versions, reports only the topmost loosened path, ignores redundant overrides and catches deleted stricter overrides - `test-adequacy.ts`: `cardinality_equality` against the same model is flagged only when `field` is the same column; seed/snapshot yml paths strip their resource dir before walking `data_tests:` - `action.yml`: `dbt_prepare` skips when `head` is not the checkout, rejects an empty `dbt_command`, warns instead of failing on a head compile error, builds the head into the `manifest_path` directory, and copies a runtime `profiles.yml` into the base worktree; docs drop the offline claim - tests for each Co-Authored-By: Claude Opus 5.5 --- docs/docs/usage/dbt-pr-review.md | 7 +- github/review/action.yml | 29 +++++- .../src/altimate/review/dbt-patterns.ts | 12 ++- .../src/altimate/review/orchestrate.ts | 13 ++- .../src/altimate/review/select-list.ts | 7 +- .../src/altimate/review/test-adequacy.ts | 99 ++++++++++++------- .../test/altimate/review-action.test.ts | 17 ++++ .../review-incremental-semantics.test.ts | 17 ++++ .../altimate/review-test-adequacy.test.ts | 46 +++++++++ 9 files changed, 194 insertions(+), 53 deletions(-) diff --git a/docs/docs/usage/dbt-pr-review.md b/docs/docs/usage/dbt-pr-review.md index d7f0731019..595d9c6d0d 100644 --- a/docs/docs/usage/dbt-pr-review.md +++ b/docs/docs/usage/dbt-pr-review.md @@ -221,10 +221,13 @@ The action can build these for you with `dbt_prepare` (off by default): - **`compile`:** runs `dbt compile` on the head, then compiles the PR's merge-base into `target-base/` in a temporary worktree. This enables equivalence and - column-breakage proofs. It usually works offline. + column-breakage proofs. dbt needs a profile it can load, and many adapters + connect to the warehouse while compiling. The head is compiled into the + directory of `manifest_path`. When `head` is set to a commit other than the + checkout, preparation is skipped. - **`full`:** also runs `dbt docs generate` for `catalog.json` (complete column types). That needs warehouse credentials in the profile. -- A base compile that fails is a warning: the review still runs, and the +- A head or base compile that fails is a warning: the review still runs, and the affected models are listed as not verified. To build them yourself, compile the **merge-base**, not the base branch's tip: diff --git a/github/review/action.yml b/github/review/action.yml index 02f6f7d7e9..9d21643045 100644 --- a/github/review/action.yml +++ b/github/review/action.yml @@ -32,11 +32,11 @@ inputs: required: false default: "true" dbt_prepare: - description: "Build the dbt artifacts the full review uses before reviewing. `none` (default): you build them in earlier steps. `compile`: `dbt compile` on the head, and on the PR's merge-base into `target-base/` (equivalence and lineage proofs). `full`: also `dbt docs generate` for `catalog.json` (complete column types). `compile` usually works offline; `full` needs warehouse credentials in the profile." + description: "Build the dbt artifacts the full review uses before reviewing. `none` (default): you build them in earlier steps. `compile`: `dbt compile` on the head, and on the PR's merge-base into `target-base/` (equivalence and lineage proofs). `full`: also `dbt docs generate` for `catalog.json` (complete column types). `compile` needs a profile dbt can load, and many adapters connect while compiling; `full` needs warehouse credentials. A failed compile is a warning and the review runs without that side's artifacts." required: false default: "none" dbt_command: - description: "How to invoke dbt for `dbt_prepare`, e.g. `dbt`, `uv run dbt`, `poetry run dbt`." + description: "How to invoke dbt for `dbt_prepare`, e.g. `dbt`, `uv run dbt`, `poetry run dbt`. Split on whitespace (no shell quoting)." required: false default: "dbt" dbt_profiles_dir: @@ -188,12 +188,21 @@ runs: IN_DBT: ${{ inputs.dbt_command }} IN_PROFILES: ${{ inputs.dbt_profiles_dir }} IN_PROJECT_DIR: ${{ inputs.project_dir }} + IN_MANIFEST: ${{ inputs.manifest_path }} IN_BASE: ${{ inputs.base }} + IN_HEAD: ${{ inputs.head }} EVENT_BASE: ${{ github.event.pull_request.base.ref }} run: | set -euo pipefail case "$IN_PREPARE" in compile|full) ;; *) echo "::error::dbt_prepare must be none, compile or full (got '$IN_PREPARE')"; exit 1 ;; esac read -ra DBT <<< "$IN_DBT" + if [[ ${#DBT[@]} -eq 0 ]]; then echo "::error::dbt_command is empty; set it to how dbt is invoked (e.g. dbt)."; exit 1; fi + # Artifacts are built from the checkout: a different `head` would be reviewed + # against another commit's compiled SQL, so skip preparing instead. + if [[ -n "${IN_HEAD:-}" && "$(git rev-parse --verify -q "$IN_HEAD^{commit}" || true)" != "$(git rev-parse HEAD)" ]]; then + echo "::warning::head ($IN_HEAD) is not the checked-out commit; skipping dbt_prepare. Check out the head to build its artifacts." + exit 0 + fi PROJECT="$(cd "${IN_PROJECT_DIR:-.}" && pwd)" REL="$(git -C "$PROJECT" rev-parse --show-prefix)" PROFILES=() @@ -203,11 +212,18 @@ runs: [[ -f packages.yml || -f dependencies.yml ]] && "${DBT[@]}" deps ${PROFILES[@]+"${PROFILES[@]}"} "${DBT[@]}" compile --target-path "$2" ${PROFILES[@]+"${PROFILES[@]}"} ) } + # The head goes where the review reads the manifest (`manifest_path`). + MANIFEST="${IN_MANIFEST:-target/manifest.json}" + [[ "$MANIFEST" == /* ]] || MANIFEST="$PROJECT/$MANIFEST" + TARGET="$(dirname "$MANIFEST")" echo "Compiling the head…" - build "$PROJECT" "$PROJECT/target" + if ! build "$PROJECT" "$TARGET"; then + echo "::warning::The head compile failed (dbt may need a warehouse connection to compile); the review runs without compiled artifacts." + exit 0 + fi if [[ "$IN_PREPARE" == "full" ]]; then echo "Generating catalog.json…" - ( cd "$PROJECT" && "${DBT[@]}" docs generate --target-path "$PROJECT/target" ${PROFILES[@]+"${PROFILES[@]}"} ) || + ( cd "$PROJECT" && "${DBT[@]}" docs generate --target-path "$TARGET" ${PROFILES[@]+"${PROFILES[@]}"} ) || echo "::warning::dbt docs generate failed; continuing without catalog.json." fi # The base side: the PR's merge-base, compiled into target-base/. @@ -226,6 +242,11 @@ runs: git worktree add --detach "$WT" "$MB" >/dev/null || { echo "::warning::Could not check out the base; skipping the base compile."; exit 0; } # A stale target-base/ from an earlier run must not stand in for a failed compile. rm -rf "$PROJECT/target-base" + # A profiles.yml generated into the project dir at runtime is untracked, so + # the base worktree lacks it. + if [[ ${#PROFILES[@]} -eq 0 && -f "$PROJECT/profiles.yml" && ! -f "$WT/$REL/profiles.yml" ]]; then + cp "$PROJECT/profiles.yml" "$WT/$REL/profiles.yml" + fi echo "Compiling the base ($BASE @ ${MB:0:12})…" build "$WT/$REL" "$PROJECT/target-base" || echo "::warning::The base compile failed; equivalence will be marked unverified." diff --git a/packages/opencode/src/altimate/review/dbt-patterns.ts b/packages/opencode/src/altimate/review/dbt-patterns.ts index 02ceb357f5..275d902f53 100644 --- a/packages/opencode/src/altimate/review/dbt-patterns.ts +++ b/packages/opencode/src/altimate/review/dbt-patterns.ts @@ -250,14 +250,20 @@ function addedLineNumbers(diff: string | undefined): Set | undefined { * `row_number() over (…)` windows in the model, read from the whole file so a * window split across lines is seen whole. A window counts as part of the * change when one of the PR's added lines falls inside its line span (by text - * only when the diff carries no hunk headers). + * only when the diff carries no hunk headers, and then only for added lines + * that themselves hold window syntax). */ function dedupWindows(c: Ctx): Array<{ spec: string; text: string }> { // Comments blanked with newlines kept, so offsets map to head line numbers. const sql = c.newSql.replace(/\/\*[\s\S]*?\*\/|--[^\n]*|\{#[\s\S]*?#\}/g, (m) => m.replace(/[^\n]/g, " ")) const lineAt = (offset: number) => sql.slice(0, offset).split("\n").length const addedLines = addedLineNumbers(c.file.diff) - const added = c.added.map((l) => stripComments(l).trim()).filter((l) => l.length >= 3) + // Text fallback (diff without hunk headers): only added lines that are part + // of a window spec can touch one; a bare identifier elsewhere never does. + const squash = (t: string) => t.replace(/\s+/g, " ").trim() + const added = c.added + .map((l) => squash(stripComments(l))) + .filter((l) => l.length >= 3 && /\b(row_number|over|partition\s+by|order\s+by)\b/i.test(l)) const out: Array<{ spec: string; text: string }> = [] for (const m of sql.matchAll(/row_number\s*\(\s*\)\s*over\s*\(/gi)) { const open = m.index! + m[0].length - 1 @@ -274,7 +280,7 @@ function dedupWindows(c: Ctx): Array<{ spec: string; text: string }> { const text = sql.slice(m.index!, close + 1) const touched = addedLines ? [...addedLines].some((n) => n >= lineAt(m.index!) && n <= lineAt(close)) - : added.some((l) => text.includes(l) || l.includes(text.replace(/\s+/g, " "))) + : added.some((l) => squash(text).includes(l) || l.includes(squash(text))) if (!touched) continue out.push({ spec: sql.slice(open + 1, close).replace(/\s+/g, " ").trim(), text: text.replace(/\s+/g, " ") }) } diff --git a/packages/opencode/src/altimate/review/orchestrate.ts b/packages/opencode/src/altimate/review/orchestrate.ts index 3102780020..d7cda43a61 100644 --- a/packages/opencode/src/altimate/review/orchestrate.ts +++ b/packages/opencode/src/altimate/review/orchestrate.ts @@ -679,8 +679,9 @@ async function columnDelta( const before = withOld ? await outputColumns(runner, engineOldSql!, dialect) : undefined return delta(current, before) } - // No compiled SQL (CI that only runs `dbt parse`): the raw model's own SELECT - // list, when it names its columns. Columns produced inside a macro stay unknown. + // Compiled SQL missing on either side (`dbt parse`-only CI, or a base that did + // not compile): the raw model's own SELECT list, when it names every column. + // Columns produced inside a macro stay unknown. if (!newSql) return none const current = rawSelectList(newSql) const before = withOld && oldSql ? rawSelectList(oldSql) : undefined @@ -688,7 +689,8 @@ async function columnDelta( } /** Output names of a raw (Jinja) model's final SELECT; empty when a Jinja - * expression or tag is in the list. */ + * expression or tag is in the list, or when any item has no name of its own + * (e.g. `amount::numeric(18,2)`): this list is the whole answer, not a supplement. */ function rawSelectList(sql: string): Set { const plain = sql .replace(/\{#[\s\S]*?#\}/g, " ") @@ -696,9 +698,10 @@ function rawSelectList(sql: string): Set { // it becomes its own `__jinja__` item, which marks the list unknown. .replace(/\{%[\s\S]*?%\}/g, " , __jinja__ , ") .replace(/\{\{[\s\S]*?\}\}/g, " __jinja__ ") - const names = selectListNames(plain) + const names = selectListNames(plain, { complete: true }) + // A Jinja expression inside an item (e.g. `{{ col }} as x`) leaves its alias + // intact; only a Jinja item standing alone is unknown. if (!names || names.includes("__jinja__")) return new Set() - // A Jinja expression inside an item (e.g. `{{ col }} as x`) leaves its alias intact. return new Set(names) } diff --git a/packages/opencode/src/altimate/review/select-list.ts b/packages/opencode/src/altimate/review/select-list.ts index 9536bcc1bd..a167dbebc1 100644 --- a/packages/opencode/src/altimate/review/select-list.ts +++ b/packages/opencode/src/altimate/review/select-list.ts @@ -63,8 +63,10 @@ const unquote = (s: string) => s.replace(/^["`\[]|["`\]]$/g, "") const SET_OP = /\b(union|intersect|except|minus)\b(\s+(all|distinct))?\s*(\bselect\b|\()/i /** Output names of the final top-level SELECT, or null when unknown (`*`, unparseable). - * A set operation (`UNION` …) takes its names from its first branch. */ -export function selectListNames(sql: string): string[] | null { + * A set operation (`UNION` …) takes its names from its first branch. Items it + * cannot name contribute nothing, unless `complete` is set: then any unnamed + * item makes the whole list unknown (null). */ +export function selectListNames(sql: string, opts: { complete?: boolean } = {}): string[] | null { const flat = flatten(sql) const selects = [...flat.matchAll(/\bselect\b/gi)] if (!selects.length) return null @@ -93,6 +95,7 @@ export function selectListNames(sql: string): string[] | null { if (m && before && (m[1] || /([A-Za-z0-9_$"`\]')]|\bend)$/i.test(before))) name = unquote(m[2]) } if (name && (IDENT.test(name) || /^[^"`]+$/.test(name)) && !/^(end|null|true|false)$/i.test(name)) names.push(name.toLowerCase()) + else if (opts.complete) return null } return names } diff --git a/packages/opencode/src/altimate/review/test-adequacy.ts b/packages/opencode/src/altimate/review/test-adequacy.ts index 7d2713719e..6d5baff0a9 100644 --- a/packages/opencode/src/altimate/review/test-adequacy.ts +++ b/packages/opencode/src/altimate/review/test-adequacy.ts @@ -131,13 +131,13 @@ export function inheritedThresholds(projectYml: any, segments: string[]): { seve return { ...eff, path } } -/** Folders of a yml below its model path (e.g. `models/marts/x.yml` → `["marts"]`). */ +/** Folders of a yml below its resource path (e.g. `models/marts/x.yml` → `["marts"]`, `seeds/x.yml` → `[]`). */ function segmentsOf(file: string, layout?: DbtProjectLayout): string[] { const root = layout?.root ? layout.root.replace(/\/$/, "") + "/" : "" const rel = root && file.startsWith(root) ? file.slice(root.length) : file const parts = rel.split("/").slice(0, -1) - const modelDirs = layout?.dirs.model ?? ["models"] - return modelDirs.includes(parts[0]) ? parts.slice(1) : parts + const resourceDirs = layout ? Object.values(layout.dirs).flat() : ["models", "seeds", "snapshots", "tests"] + return resourceDirs.includes(parts[0]) ? parts.slice(1) : parts } export function detectTestAdequacy(input: { @@ -163,8 +163,12 @@ export function detectTestAdequacy(input: { name !== "expect_table_aggregation_to_equal_other_table" || String(arg(t, "compare_expression") ?? arg(t, "expression") ?? "").replace(/\s+/g, "").toLowerCase() === String(arg(t, "expression") ?? "").replace(/\s+/g, "").toLowerCase() + // `cardinality_equality` compares this column with `field` on the other + // model; against itself it is circular only when the field is the same column. + const sameField = + name !== "cardinality_equality" || String(arg(t, "field") ?? "").toLowerCase() === (t.column ?? "").toLowerCase() const self = - (SELF_COMPARE.has(name) && sameExpression && ["compare_model", "to", "compare_to"].some((a) => refsEntity(arg(t, a), t.entity))) || + (SELF_COMPARE.has(name) && sameExpression && sameField && ["compare_model", "to", "compare_to"].some((a) => refsEntity(arg(t, a), t.entity))) || (name === "relationships" && refsEntity(arg(t, "to"), t.entity) && String(arg(t, "field") ?? "").toLowerCase() === (t.column ?? "").toLowerCase()) if (self) out.push( @@ -177,21 +181,23 @@ export function detectTestAdequacy(input: { const project = parse(input.projectYml) if (project && added.length) { const inherited = inheritedThresholds(project, segmentsOf(file.path, input.layout)) - const own = (t: TestRef) => { - const c = t.args?.config ?? {} - return [c.severity, c.error_if, c.warn_if, t.args?.severity, t.args?.error_if].some((v) => v !== undefined) - } - const loose = added.filter((t) => !own(t)) + // dbt resolves `severity` and `error_if` separately: a local `warn_if` or + // `error_if` still leaves `severity` inherited, and vice versa. + const local = (t: TestRef, k: "severity" | "error_if") => t.args?.config?.[k] ?? t.args?.[k] + const inheritingWarn = added.filter((t) => local(t, "severity") === undefined) + const inheritingThreshold = added.filter( + (t) => local(t, "error_if") === undefined && String(local(t, "severity") ?? inherited.severity ?? "error").toLowerCase() !== "warn", + ) const where = inherited.path.length ? ` (\`data_tests:\` → ${inherited.path.map((p) => `\`${p}\``).join(" → ")})` : "" - if (loose.length && inherited.severity === "warn") + if (inheritingWarn.length && inherited.severity === "warn") out.push( - finding(file, undefined, "test-inherits-warn", "suggestion", `${loose.length} new test(s) in ${file.path} inherit \`severity: warn\``, + finding(file, undefined, "test-inherits-warn", "suggestion", `${inheritingWarn.length} new test(s) in ${file.path} inherit \`severity: warn\``, `They inherit \`severity: warn\` from \`dbt_project.yml\`${where}, so failures are reported but never fail \`dbt build\`. ` + "If a test should gate the build, set `config: {severity: error}` on it."), ) - else if (loose.length && inherited.error_if && allowedFailures(inherited.error_if) !== 0) + else if (inheritingThreshold.length && inherited.error_if && allowedFailures(inherited.error_if) !== 0) out.push( - finding(file, undefined, "test-inherits-threshold", "suggestion", `${loose.length} new test(s) in ${file.path} inherit \`error_if: ${inherited.error_if}\``, + finding(file, undefined, "test-inherits-threshold", "suggestion", `${inheritingThreshold.length} new test(s) in ${file.path} inherit \`error_if: ${inherited.error_if}\``, `They fail only when \`${inherited.error_if}\` rows fail, a threshold inherited from \`dbt_project.yml\`${where}. ` + "Set `config: {error_if: '!= 0'}` on tests that must catch any bad row."), ) @@ -204,34 +210,53 @@ export function detectThresholdLoosening(file: ChangedFile, oldContent?: string, const oldDoc = parse(oldContent) const newDoc = parse(newContent) if (!oldDoc || !newDoc) return [] - const flat = (doc: any) => { - const out = new Map>() - const visit = (node: any, path: string[]) => { - if (!node || typeof node !== "object") return - out.set(path.join(" → ") || "(all tests)", thresholds(node)) - for (const [k, v] of Object.entries(node)) if (!k.startsWith("+") && v && typeof v === "object") visit(v, [...path, k]) + // Every config path in either version: a deleted override loosens its subtree too. + const paths = new Map() + const visit = (node: any, path: string[]) => { + if (!node || typeof node !== "object") return + paths.set(path.join("\u0000"), path) + for (const [k, v] of Object.entries(node)) if (!k.startsWith("+") && v && typeof v === "object") visit(v, [...path, k]) + } + visit(oldDoc?.data_tests ?? oldDoc?.tests, []) + visit(newDoc?.data_tests ?? newDoc?.tests, []) + // Effective values at a path, walking the tree from the top (no project-name prepend: + // the path already names it when the config does). + const effective = (doc: any, path: string[]) => { + const eff: { severity: string; error_if?: string } = { severity: "error" } + let node = doc?.data_tests ?? doc?.tests + for (let i = 0; i <= path.length && node && typeof node === "object"; i++) { + const t = thresholds(node) + if (t.severity) eff.severity = t.severity + if (t.error_if) eff.error_if = t.error_if + node = node[path[i]] } - visit(doc?.data_tests ?? doc?.tests, []) - return out + return eff } - const before = flat(oldDoc) - const out: Finding[] = [] - for (const [path, t] of flat(newDoc)) { - const prev = before.get(path) ?? {} - const toWarn = t.severity === "warn" && prev.severity !== "warn" - // Only a threshold that tolerates more failing rows than before loosens; - // a changed condition that is not a simple bound is still reported. - const allowed = allowedFailures(t.error_if) - const newThreshold = - !!t.error_if && - t.error_if !== prev.error_if && + const loosened = (path: string[]) => { + const prev = effective(oldDoc, path) + const next = effective(newDoc, path) + const toWarn = next.severity === "warn" && prev.severity !== "warn" + // Only a bound that tolerates more failing rows loosens; a changed condition + // that isn't a simple bound is still reported. + const allowed = allowedFailures(next.error_if) + const looserThreshold = + next.severity !== "warn" && + prev.severity !== "warn" && + next.error_if !== prev.error_if && + !!next.error_if && (allowed === undefined || allowed > (allowedFailures(prev.error_if) ?? 0)) - if (!toWarn && !newThreshold) continue + return toWarn ? { why: "`severity: warn` now applies" } : looserThreshold ? { why: `\`error_if: ${next.error_if}\` now applies` } : undefined + } + const out: Finding[] = [] + for (const path of paths.values()) { + const hit = loosened(path) + // Report only the topmost loosened path, not each folder below it. + if (!hit || (path.length && loosened(path.slice(0, -1)))) continue + const label = path.join(" → ") || "(all tests)" out.push( - finding(file, undefined, "test-thresholds-loosened", "warning", `test thresholds loosened for \`${path}\``, - (toWarn ? "`severity: warn` now applies" : `\`error_if: ${t.error_if}\` now applies`) + - ` to every data test under \`${path}\` in \`dbt_project.yml\`, so those tests stop failing the build. ` + - "Prefer setting it on the specific tests that need it.", `:${path}`), + finding(file, undefined, "test-thresholds-loosened", "warning", `test thresholds loosened for \`${label}\``, + `${hit.why} to every data test under \`${label}\` in \`dbt_project.yml\`, so those tests stop failing the build. ` + + "Prefer setting it on the specific tests that need it.", `:${label}`), ) } return out diff --git a/packages/opencode/test/altimate/review-action.test.ts b/packages/opencode/test/altimate/review-action.test.ts index fda2d21a26..4a8390cdda 100644 --- a/packages/opencode/test/altimate/review-action.test.ts +++ b/packages/opencode/test/altimate/review-action.test.ts @@ -30,4 +30,21 @@ describe("action: dbt_prepare", () => { expect(run).toMatch(/git worktree add[^\n]*\|\| \{ echo "::warning::/) expect(run.indexOf('rm -rf "$PROJECT/target-base"')).toBeLessThan(run.indexOf('build "$WT/$REL"')) }) + + test("a mismatched head, an empty dbt_command or a failed head compile never builds the wrong artifacts or fails the job", async () => { + const action = YAML.parse(await fs.readFile(actionPath, "utf8")) + const prep = (action.runs.steps as Array<{ name: string; run?: string; env?: Record }>).find( + (s) => s.name === "Prepare dbt artifacts", + )! + const run = prep.run! + expect(prep.env?.IN_HEAD).toBe("${{ inputs.head }}") + expect(prep.env?.IN_MANIFEST).toBe("${{ inputs.manifest_path }}") + expect(run).toMatch(/\$\{#DBT\[@\]\} -eq 0 \]\]; then echo "::error::/) + // The head check runs before anything is compiled. + expect(run.indexOf('"$IN_HEAD^{commit}"')).toBeLessThan(run.indexOf('build "$PROJECT"')) + expect(run).toMatch(/if ! build "\$PROJECT" "\$TARGET"; then\n\s*echo "::warning::/) + expect(run).toContain('TARGET="$(dirname "$MANIFEST")"') + expect(run).toContain('cp "$PROJECT/profiles.yml" "$WT/$REL/profiles.yml"') + expect(action.inputs.dbt_prepare.description).not.toContain("usually works offline") + }) }) diff --git a/packages/opencode/test/altimate/review-incremental-semantics.test.ts b/packages/opencode/test/altimate/review-incremental-semantics.test.ts index ffddb310b7..053e483fd9 100644 --- a/packages/opencode/test/altimate/review-incremental-semantics.test.ts +++ b/packages/opencode/test/altimate/review-incremental-semantics.test.ts @@ -327,6 +327,16 @@ describe("dedup windows split across lines", () => { "dedup_no_tiebreaker", ) }) + test("without hunk headers, a bare identifier that also appears in the window doesn't touch it", () => { + const window = "select customer_id, row_number() over (partition by customer_id) as rn\nfrom t\ngroup by\n customer_id" + const rules = (diff: string) => + detectModelPatterns({ path: "models/m.sql", status: "modified", diff }, window, DEFAULT_RUBRIC).map( + (f) => (f.evidence?.result as { rule?: string })?.rule, + ) + expect(rules("+ customer_id\n")).not.toContain("dedup_no_tiebreaker") + // An added line that is the window itself (whitespace differing) still counts. + expect(rules("+select customer_id, row_number() over (partition by customer_id) as rn\n")).toContain("dedup_no_tiebreaker") + }) }) describe("column changes without compiled SQL (dbt parse-only CI)", () => { @@ -399,5 +409,12 @@ describe("column changes without compiled SQL (dbt parse-only CI)", () => { `${cfg}select id, amount {% if var('with_region', false) %}, region {% endif %} from {{ ref('src') }}`, ) expect(conditional.findings.some((f) => f.title.includes("on_schema_change"))).toBe(false) + // An item with no name of its own (a cast) makes the list unknown, so a + // column turned into a cast is neither removed nor are others added. + const cast = await review( + `${cfg}select id, amount from {{ ref('src') }}`, + `${cfg}select id, amount::numeric(18,2), region from {{ ref('src') }}`, + ) + expect(cast.findings.some((f) => f.title.includes("on_schema_change"))).toBe(false) }) }) diff --git a/packages/opencode/test/altimate/review-test-adequacy.test.ts b/packages/opencode/test/altimate/review-test-adequacy.test.ts index 49a4a7c9e7..d759c20771 100644 --- a/packages/opencode/test/altimate/review-test-adequacy.test.ts +++ b/packages/opencode/test/altimate/review-test-adequacy.test.ts @@ -107,4 +107,50 @@ data_tests: expect(loosened("!= 0", ">=1")).toBe(false) expect(loosened("!= 0", ">5")).toBe(true) }) + + test("loosening is judged on effective values: a deleted stricter override is reported, a redundant warn is not", () => { + const yml = (body: string) => `name: shop\ndata_tests:\n +severity: warn\n shop:\n${body}` + const ch = (a: string, b: string) => detectThresholdLoosening({ path: "dbt_project.yml", status: "modified" }, a, b) + // Root warn, marts override to error deleted: every marts test now warns. + const deleted = ch(yml(" marts:\n +severity: error\n"), yml(" marts:\n +materialized: view\n")) + expect(deleted.map((f) => f.title)).toEqual(["test thresholds loosened for `shop → marts`"]) + // A redundant warn under an already-warn root changes nothing. + expect(ch(yml(" marts:\n +enabled: true\n"), yml(" marts:\n +severity: warn\n"))).toEqual([]) + // Loosening at the root is reported once, not again for each folder below it. + const root = ch("data_tests:\n shop:\n marts:\n x:\n +enabled: true\n", "data_tests:\n +severity: warn\n shop:\n marts:\n x:\n +enabled: true\n") + expect(root.map((f) => f.title)).toEqual(["test thresholds loosened for `(all tests)`"]) + }) + + test("a local warn_if or error_if leaves severity inherited", () => { + const withCfg = (cfg: string) => newTests(`:\n config:\n ${cfg}`) + expect(rule(detectTestAdequacy({ file, newContent: withCfg("warn_if: '>10'"), projectYml: project }))).toEqual(["test-inherits-warn"]) + expect(rule(detectTestAdequacy({ file, newContent: withCfg("error_if: '!= 0'"), projectYml: project }))).toEqual(["test-inherits-warn"]) + // Under an inherited threshold, a local severity: error still inherits error_if. + const stg = { path: "models/staging/_stg.yml", status: "modified" as const } + expect(rule(detectTestAdequacy({ file: stg, newContent: withCfg("severity: error"), projectYml: project }))).toEqual(["test-inherits-threshold"]) + expect(detectTestAdequacy({ file: stg, newContent: withCfg("error_if: '!= 0'"), projectYml: project })).toEqual([]) + }) + + test("a seed yml is resolved below the seed path, not as a `seeds` folder", () => { + const proj = "name: shop\ndata_tests:\n shop:\n +severity: warn\n seeds:\n +severity: error\n" + const seedYml = "seeds:\n - name: countries\n columns:\n - name: code\n data_tests:\n - not_null\n" + const out = detectTestAdequacy({ file: { path: "seeds/_seeds.yml", status: "added" }, newContent: seedYml, projectYml: proj }) + expect(rule(out)).toEqual(["test-inherits-warn"]) + }) +}) + +describe("cardinality_equality against the same model", () => { + const y = (field: string) => `models: + - name: m + columns: + - name: a + data_tests: + - dbt_utils.cardinality_equality: + to: ref('m') + field: ${field} +` + test("a different field is a real check; the same field is circular", () => { + expect(detectTestAdequacy({ file, newContent: y("b") })).toEqual([]) + expect(rule(detectTestAdequacy({ file, newContent: y("a") }))).toEqual(["test-compares-model-with-itself"]) + }) })