diff --git a/packages/opencode/src/altimate/native/dbt/graph.ts b/packages/opencode/src/altimate/native/dbt/graph.ts index 3edff2baf9..e4daffde98 100644 --- a/packages/opencode/src/altimate/native/dbt/graph.ts +++ b/packages/opencode/src/altimate/native/dbt/graph.ts @@ -12,6 +12,8 @@ export interface GraphNode { label: string | null type: string path: string | null + /** The project-relative schema.yml that describes the node (dbt's patch_path without its `package://`), when there is one. */ + yml?: string materialized: string | null /** database.schema.table, lower-cased. */ relation: string | null @@ -57,6 +59,7 @@ function nodeOf(id: string, node: any): GraphNode { label: type === "exposure" ? node?.label || node?.name || id : null, type, path: node?.original_file_path ?? null, + ...(typeof node?.patch_path === "string" ? { yml: node.patch_path.replace(/^[^:/]+:\/\//, "") } : {}), materialized: node?.config?.materialized ?? null, relation: relationOf(node), columns: Object.entries(node?.columns ?? {}).map(([key, col]) => String(col?.name ?? key)), diff --git a/packages/opencode/src/altimate/tools/cte-detect.ts b/packages/opencode/src/altimate/tools/cte-detect.ts index 816aa6405f..01eeb4e887 100644 --- a/packages/opencode/src/altimate/tools/cte-detect.ts +++ b/packages/opencode/src/altimate/tools/cte-detect.ts @@ -301,6 +301,286 @@ export function detectCtes(text: string): CteInfo[] { return ctes } +/** A CTE name or a reference as written: its text without quotes, and whether it was quoted. */ +type Written = { name: string; quoted: boolean } + +function written(raw: string): Written { + const quote = raw[0] + if ((quote === '"' || quote === "`") && raw.length > 1 && raw.endsWith(quote)) return { name: raw.slice(1, -1).replaceAll(quote + quote, quote), quoted: true } + if (quote === "[" && raw.endsWith("]")) return { name: raw.slice(1, -1).replaceAll("]]", "]"), quoted: true } + return { name: raw, quoted: false } +} + +/** + * Whether a reference names a CTE. Two bare names match in any letter case; two quoted ones only exactly. A quoted + * and a bare one match when the quoted text is the bare name folded to lower case (Postgres, DuckDB, Redshift) or + * to upper case (Snowflake), the warehouse being unknown here: so `"ORDERS"` names `orders`, and `"Foo"` does not + * name `foo`. + */ +function sameName(a: Written, b: Written): boolean { + if (!a.quoted && !b.quoted) return a.name.toLowerCase() === b.name.toLowerCase() + if (a.quoted && b.quoted) return a.name === b.name + const [quoted, bare] = a.quoted ? [a, b] : [b, a] + return quoted.name === bare.name.toLowerCase() || quoted.name === bare.name.toUpperCase() +} + +const BARE_WORD = /[\p{L}_][\p{L}\p{N}_$]*/uy +const BRACKETED = /\[(?:[^\]\n]|\]\]){1,256}\]/y + +/** + * Position just past the Jinja delimiter `{{`, `{%` or `{#` at `pos` closes: a comment at its first `#}`; an + * expression or statement at its `}}` or `%}` outside the quoted strings in it, an expression's braces balanced, + * as Jinja's lexer reads them. + */ +function jinjaEnd(text: string, pos: number): number { + const kind = text[pos + 1] + if (kind === "#") { + const close = text.indexOf("#}", pos + 2) + return close === -1 ? text.length : close + 2 + } + let at = pos + 2 + let depth = 0 + while (at < text.length) { + const char = text[at] + if (char === "'" || char === '"') { + at++ + while (at < text.length && text[at] !== char) at += text[at] === "\\" ? 2 : 1 + at++ + } else if (kind === "{" && char === "{") { + depth++ + at++ + } else if (kind === "{" && char === "}" && depth > 0) { + depth-- + at++ + } else if (char === (kind === "{" ? "}" : "%") && text[at + 1] === "}") return at + 2 + else at++ + } + return text.length +} + +/** The blocks whose body is a string for dbt (`{% set x %}…{% endset %}`, a call, a macro), not the model's SQL. */ +const CAPTURING = new Set(["set", "call", "macro"]) + +/** + * The text with its Jinja blanked to spaces, offsets kept: every `{{ }}`, `{% %}` and `{# #}`, and the body of a + * captured `set`, `call` or `macro` block. Jinja renders before the SQL is read, so a quote, comment mark or name + * inside it is not SQL, wherever it sits (in a string, a comment, a quoted name). The text inside `{% raw %}` is + * kept as written. + */ +function blankJinja(text: string): string { + const out: string[] = [] + let kept = 0 + // Blanked to spaces, newlines too: a Jinja comment ends no SQL line comment. Whitespace control (`{{-`, `-}}`) + // takes the whitespace beside the tag with it, newlines included, as rendering does. + const blank = (from: number, to: number) => { + if (text[from + 2] === "-") while (from > kept && /\s/.test(text[from - 1])) from-- + if (text[to - 3] === "-") while (to < text.length && /\s/.test(text[to])) to++ + out.push(text.substring(kept, from), " ".repeat(to - from)) + kept = to + return to + } + let pos = 0 + while (pos < text.length) { + const at = text.indexOf("{", pos) + if (at === -1 || at + 1 >= text.length) break + const kind = text[at + 1] + if (kind !== "{" && kind !== "%" && kind !== "#") { + pos = at + 1 + continue + } + const end = jinjaEnd(text, at) + const head = kind === "%" ? /^\{%[-+]?\s*(\w+)/.exec(text.substring(at, Math.min(end, at + 200))) : null + const name = head?.[1] + if (name === "raw") { + // The text up to {% endraw %} is output as written. + const close = /\{%[-+]?\s*endraw\b[^%]*%\}/g + close.lastIndex = end + const found = close.exec(text) + blank(at, end) + if (found) { + out.push(text.substring(kept, found.index)) + kept = found.index + blank(found.index, close.lastIndex) + pos = close.lastIndex + } else pos = text.length + continue + } + const inline = name === "set" && /^\{%[-+]?\s*set\s+[\w.,\s()]+=(?!=)/.test(text.substring(at, end)) + if (name && CAPTURING.has(name) && !inline) { + // The block's body, to its own end tag (blocks of one kind nest), is blanked with it. + let depth = 1 + let scan = end + while (depth > 0 && scan < text.length) { + const next = text.indexOf("{", scan) + if (next === -1 || next + 1 >= text.length) { + scan = text.length + break + } + if (text[next + 1] !== "%" && text[next + 1] !== "{" && text[next + 1] !== "#") { + scan = next + 1 + continue + } + let tagEnd = jinjaEnd(text, next) + const tag = text[next + 1] === "%" ? /^\{%[-+]?\s*(end)?(\w+)/.exec(text.substring(next, Math.min(tagEnd, next + 200))) : null + if (tag && !tag[1] && tag[2] === "raw") { + // Text inside {% raw %} holds no tags: an `{% endset %}` there does not end the block. + const close = /\{%[-+]?\s*endraw\b[^%]*%\}/g + close.lastIndex = tagEnd + tagEnd = close.exec(text) ? close.lastIndex : text.length + } + if (tag?.[2] === name) { + const opensAgain = !tag[1] && !(name === "set" && /^\{%[-+]?\s*set\s+[\w.,\s()]+=(?!=)/.test(text.substring(next, tagEnd))) + if (tag[1]) depth-- + else if (opensAgain) depth++ + } + scan = tagEnd + } + pos = blank(at, scan) + continue + } + pos = blank(at, end) + } + out.push(text.substring(kept)) + return out.join("") +} + +/** + * Position just past the SQL quoted text at `pos`: a single-quoted string, or a `"` or `` ` `` quoted name. A + * doubled quote stays inside it; in a single-quoted string so does a backslash-escaped quote (Snowflake, + * BigQuery, Databricks, MySQL). Where a backslash is a plain character (Postgres, DuckDB), a string ending in one + * runs on past its quote and pairs the quotes after it wrongly: names after it may be missed, or a name in a + * later string read, so such a model's reads may be off either way. + */ +function skipQuoted(text: string, pos: number): number { + const quote = text[pos] + let at = pos + 1 + while (at < text.length) { + const char = text[at] + if (char === "\\" && quote === "'") at += 2 + else if (char === quote && text[at + 1] === quote) at += 2 + else if (char === quote) return at + 1 + else at++ + } + return text.length +} + +/** + * The CTEs the WITH clauses between `start` and `end` define: the query's own, and every subquery's that starts + * with WITH. Each with where its name is written (`at`), where its definition ends (`from`) and where the query + * holding it ends (`to`): its name means it from `from` to `to`. + */ +function shadowsIn(sql: string, start: number, end: number): { written: Written; at: number; from: number; to: number }[] { + const out: { written: Written; at: number; from: number; to: number }[] = [] + const add = (from: number, to: number) => { + const text = sql.substring(from, to) + for (const one of detectCtes(text)) { + // In a WITH RECURSIVE clause a CTE's name means that CTE across the whole clause, its own body included. + const afterWith = skipTrivia(text, one.withClauseStart + 4, true) + const recursive = /^recursive\b/i.test(text.substring(afterWith, afterWith + 10)) + out.push({ written: written(stripColumnList(one.name)), at: from + one.nameStart, from: from + (recursive ? one.withClauseStart : one.queryEnd), to }) + } + } + if (!/\bwith\b/i.test(sql.substring(start, end))) return out + add(start, end) + let pos = start + while (pos < end) { + const char = sql[pos] + if (char === "'" || char === '"' || char === "`") { + pos = skipQuoted(sql, pos) + continue + } + const skipped = skipLiteral(sql, pos) + if (skipped !== pos) { + pos = skipped + continue + } + if (char === "(") { + const first = skipTrivia(sql, pos + 1, false) + if (/^with\b/i.test(sql.substring(first, first + 5))) { + const close = findMatchingClosingParen(sql, pos) + if (close > pos) add(pos + 1, close) + } + } + pos++ + } + return out +} + +/** + * For each CTE, the positions (in `ctes`) of the earlier CTEs of its own WITH clause that its body names + * directly: a name outside strings, comments and Jinja (blanked first: `{{ }}`, `{% %}`, `{# #}`, and the SQL a + * `{% set %}`, `{% call %}` or `{% macro %}` block captures, which is a string for dbt), not qualified by a dot + * (`schema . orders`) or inside a longer word, that names the CTE (see `sameName`). Once a CTE the body's own + * WITH or a subquery's WITH defines has ended, its name is that inner CTE there, not a read of the outer one. + * A column or alias spelled like an earlier CTE counts too, so a CTE may be listed as read when it is not; a CTE + * named only through Jinja is not seen, nor one after a nested block comment. A client wanting a CTE's own time subtracts the CTEs it reads directly or through another (the + * closure of `reads`), each once. + */ +export function cteReads(text: string, ctes: CteInfo[]): number[][] { + const sql = blankJinja(text) + // Per WITH clause, the CTEs met so far, by the first letters of their names folded: built up in source order, + // so each body is scanned once and a reference is compared with few names. + const seen = new Map>() + const bucket = (name: string) => name.toLowerCase() + return ctes.map((cte, position) => { + const earlier = seen.get(cte.withClauseStart) ?? new Map() + seen.set(cte.withClauseStart, earlier) + const reads = new Set() + // The CTEs a WITH inside the body defines (its own, or a subquery's): each shadows an outer CTE of its name + // once its definition has ended, to the end of the query that holds it. + const inner = new Map() + for (const one of earlier.size > 0 ? shadowsIn(sql, cte.queryStart, cte.queryEnd) : []) inner.set(bucket(one.written.name), [...(inner.get(bucket(one.written.name)) ?? []), one]) + let pos = cte.queryStart + // The last character of SQL read before `pos`, past whitespace and comments: a dot qualifies the next name. + let previous = "" + while (earlier.size > 0 && pos < cte.queryEnd) { + const char = sql[pos] + if (/\s/.test(char)) { + pos++ + continue + } + if (char === "'") { + pos = skipQuoted(sql, pos) + previous = "'" + continue + } + let end = -1 + if (char === '"' || char === "`") end = skipQuoted(sql, pos) + else if (char === "[") { + BRACKETED.lastIndex = pos + if (BRACKETED.test(sql)) end = BRACKETED.lastIndex + } else { + // A comment or a dollar-quoted string is skipped whole. + const skipped = skipLiteral(sql, pos) + if (skipped !== pos) { + if (char === "$") previous = "$" + pos = skipped + continue + } + BARE_WORD.lastIndex = pos + if (/[\p{L}_]/u.test(String.fromCodePoint(sql.codePointAt(pos) ?? 32)) && BARE_WORD.test(sql)) end = BARE_WORD.lastIndex + } + if (end === -1) { + previous = char + pos++ + continue + } + if (previous !== "." && !/[\p{L}\p{N}_$]/u.test(sql[pos - 1] ?? " ")) { + const ref = written(sql.substring(pos, end)) + const named = (earlier.get(bucket(ref.name)) ?? []).filter((one) => sameName(one.written, ref)) + // An inner CTE's own name where it is defined is no reference at all, and one in scope shadows the outer. + const shadowed = (inner.get(bucket(ref.name)) ?? []).some((one) => one.at === pos || (one.from < pos && pos < one.to && sameName(one.written, ref))) + if (!shadowed) for (const one of named) reads.add(one.position) + } + previous = sql[end - 1] + pos = end + } + const own = written(stripColumnList(cte.name)) + earlier.set(bucket(own.name), [...(earlier.get(bucket(own.name)) ?? []), { written: own, position }]) + return [...reads].sort((a, b) => a - b) + }) +} + /** * Offset from which the CTEs could not be read, or undefined when every CTE was read: the * comma after the last CTE read of a WITH clause that goes on, or the start of a clause where none was read diff --git a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts index 9b5a60af1d..283dbd409d 100644 --- a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts +++ b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts @@ -9,7 +9,7 @@ import { isQueryTimeoutError } from "@altimateai/drivers/types" import { executeQuery } from "../native/query-execution" import { dbtAdapterFor } from "./dbt-adapter-access" import { untilAbort } from "./dbt-cost-estimate" -import { buildCountQuery, detectCtes, extractRowCount, stripColumnList, unclosedJinjaBlock, unreadCteAfter, type CteInfo } from "./cte-detect" +import { buildCountQuery, cteReads, detectCtes, extractRowCount, stripColumnList, unclosedJinjaBlock, unreadCteAfter, type CteInfo } from "./cte-detect" /** Folders a model search never enters. */ const SKIPPED_DIRS = new Set(["target", "dbt_packages", "dbt_modules", "node_modules", "logs", "venv", "site-packages"]) @@ -252,7 +252,7 @@ function failure(title: string, output: string, extra: Record = export const DbtCteProfileTool = Tool.define("dbt_cte_profile", { description: - "List a dbt model's CTEs, or time one CTE with a row-count query. Without `cte` or `position` it returns the CTE names in source order (name, line, position) and runs nothing, so a read-only call works. With one of them it runs `SELECT COUNT(*)` over that CTE and the CTEs before it in the same WITH clause on the warehouse, and returns the row count and the query's milliseconds on the warehouse (`queryMs`): dbt's own timing of the query's execute step where dbt reports one (run_results.json), else the call's time less that of a trivial query through dbt, so a command-line dbt's start-up of a second or more per query is not counted; `totalMs` is the count query's whole call through dbt, start-up included, and `timing` says which measure `queryMs` is (dbt, baseline, or wall when neither was available); that run needs write permission (`--allow-write`) because it starts the dbt adapter and compiles the model. Each run recomputes the earlier CTEs, so a model of N CTEs costs N queries. Only the model's own top-level WITH clause is read: a WITH inside a CTE body is part of that CTE. The list stops at a CTE whose name Jinja builds, and `incomplete` is set in the metadata then; a CTE whose count query reads is_incremental() or this, in its own SQL, an earlier CTE or the model header, is not run, because the count runs outside the model. Macros the query calls are not opened: one that reads is_incremental() inside is not caught, so treat the count of such a model as a full-history count.", + "List a dbt model's CTEs, or time one CTE with a row-count query. Without `cte` or `position` it returns the CTE names in source order (name, line, position, and `reads`: the positions of the earlier CTEs of its WITH clause its body names, directly; a CTE's own time is its count's time less the own times of the CTEs it reads directly or through another, each once, not less the CTE before it) and runs nothing, so a read-only call works. With one of them it runs `SELECT COUNT(*)` over that CTE and the CTEs before it in the same WITH clause on the warehouse, and returns the row count and the query's milliseconds on the warehouse (`queryMs`): dbt's own timing of the query's execute step where dbt reports one (run_results.json), else the call's time less that of a trivial query through dbt, so a command-line dbt's start-up of a second or more per query is not counted; `totalMs` is the count query's whole call through dbt, start-up included, and `timing` says which measure `queryMs` is (dbt, baseline, or wall when neither was available); that run needs write permission (`--allow-write`) because it starts the dbt adapter and compiles the model. Each run recomputes the earlier CTEs, so a model of N CTEs costs N queries. Only the model's own top-level WITH clause is read: a WITH inside a CTE body is part of that CTE. The list stops at a CTE whose name Jinja builds, and `incomplete` is set in the metadata then; a CTE whose count query reads is_incremental() or this, in its own SQL, an earlier CTE or the model header, is not run, because the count runs outside the model. Macros the query calls are not opened: one that reads is_incremental() inside is not caught, so treat the count of such a model as a full-history count.", parameters: z.object({ model: z .string() @@ -285,17 +285,19 @@ export const DbtCteProfileTool = Tool.define("dbt_cte_profile", { for (; counted < offset; counted++) if (text[counted] === "\n") line++ return line } + const reads = cteReads(text, ctes) const list = ctes.map((c, position) => ({ name: stripColumnList(c.name), line: lineAt(c.nameStart), index: c.index, clause: c.withClauseStart, position, + reads: reads[position], })) const stop = unreadCteAfter(text, ctes) const note = stop === undefined ? "" : `\nThe list is incomplete: the CTEs from line ${text.substring(0, stop).split("\n").length} on could not be read (its name is built by a Jinja expression or loop).` const output = (list.length - ? [`${list.length} CTE${list.length === 1 ? "" : "s"} in ${file}:`, ...list.map((c) => `position ${c.position}: ${c.name} (line ${c.line})`)].join("\n") + ? [`${list.length} CTE${list.length === 1 ? "" : "s"} in ${file}:`, ...list.map((c) => `position ${c.position}: ${c.name} (line ${c.line}${c.reads.length ? `; reads ${c.reads.map((i) => list[i].name).join(", ")}` : ""})`)].join("\n") : `No CTEs in ${file}.`) + note return { title: `dbt_cte_profile: ${list.length} CTE${list.length === 1 ? "" : "s"} in ${args.model}`, diff --git a/packages/opencode/test/altimate/column-reach.test.ts b/packages/opencode/test/altimate/column-reach.test.ts index 97ae23aad4..dd3b136f0c 100644 --- a/packages/opencode/test/altimate/column-reach.test.ts +++ b/packages/opencode/test/altimate/column-reach.test.ts @@ -966,6 +966,20 @@ describe("columnReach on SQL the engine cannot parse", () => { }) describe("lineageGraph", () => { + test("a node carries the schema.yml that describes it, without dbt's package prefix", () => { + const graph = lineageGraph( + { + nodes: { + "model.shop.orders": { ...model("orders", [], "select 1", []), patch_path: "shop://models/marts/schema.yml" }, + "model.shop.bare": model("bare", ["model.shop.orders"], "select 1", []), + }, + }, + "model.shop.orders", + ) + expect(graph.nodes["model.shop.orders"].yml).toBe("models/marts/schema.yml") + expect(graph.nodes["model.shop.bare"].yml).toBeUndefined() + }) + test("a twelve-hop chain comes back whole to its source; upstream past 500 nodes is cut and marked", () => { const chain: Record = {} for (let i = 1; i <= 12; i++) diff --git a/packages/opencode/test/altimate/cte-detect.test.ts b/packages/opencode/test/altimate/cte-detect.test.ts index f4adbd2346..c267307f65 100644 --- a/packages/opencode/test/altimate/cte-detect.test.ts +++ b/packages/opencode/test/altimate/cte-detect.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "bun:test" import { buildCountQuery, + cteReads, detectCtes as detectCtesInText, extractRowCount, quoteSqlIdentifier, @@ -1639,3 +1640,139 @@ describe("extractRowCount", () => { expect(extractRowCount([{ _PROFILE_COUNT: "not-a-number" }])).toBe(0) }) }) + +describe("cteReads", () => { + const readsOf = (sql: string) => { + const ctes = detectCtesInText(sql) + const reads = cteReads(sql, ctes) + return Object.fromEntries(ctes.map((c, i) => [c.name, reads[i].map((j) => ctes[j].name)])) + } + + it("lists only the earlier CTEs a body names: siblings read nothing of each other", () => { + const sql = `with orders as (select * from {{ ref('stg_orders') }}), +order_items as (select order_id, sum(price) as total from {{ ref('stg_order_items') }} group by 1), +payments as (select order_id, sum(amount) as paid from {{ ref('stg_payments') }} group by order_id), +customer_orders as ( + select orders.customer_id, sum(order_items.total), sum(payments.paid) + from orders left join order_items on orders.order_id = order_items.order_id + left join payments on orders.order_id = payments.order_id group by 1 +), +ranked as (select *, rank() over (order by 1) from customer_orders) +select * from ranked` + expect(readsOf(sql)).toEqual({ + orders: [], + order_items: [], + payments: [], + customer_orders: ["orders", "order_items", "payments"], + ranked: ["customer_orders"], + }) + }) + + it("does not count a name in a string, a comment, after a dot, or inside a longer word", () => { + const sql = `with orders as (select 1 as id), +a as (select 'orders' as s, x.orders, my_orders, orders2 from t as x -- orders + /* orders */ {# orders #}), +b as (select * from ORDERS) +select * from b` + expect(readsOf(sql)).toEqual({ orders: [], a: [], b: ["orders"] }) + }) + + it("matches a name in any letter case and quoting, brackets included, and keeps to its own WITH clause", () => { + const sql = `with "Big" as (select 1), other as (select 2), +q as (select * from "Big"), r as (select * from "OTHER"), s as (select * from [Big]), t as (select * from \`other\`), u as (select * from big) +select * from (with inner_cte as (select * from q) select * from inner_cte)` + // A bare name matches a quoted one only as its lower- or upper-case form: \`big\` is not "Big". + expect(readsOf(sql)).toEqual({ '"Big"': [], other: [], q: ['"Big"'], r: ["other"], s: ['"Big"'], t: ["other"], u: [] }) + expect(readsOf('with "Foo" as (select 1), foo as (select 2), keep as (select * from foo) select * from keep').keep).toEqual(["foo"]) + const bracketed = "with [Big Spenders] as (select 1), [we]]ird] as (select 2), b as (select * from [Big Spenders] join [we]]ird] on 1 = 1) select * from b" + expect(readsOf(bracketed)).toEqual({ "[Big Spenders]": [], "[we]]ird]": [], b: ["[Big Spenders]", "[we]]ird]"] }) + }) + + it("does not count a name inside Jinja: an expression, a ref with either quotes, a captured set block", () => { + const sql = `with orders as (select 1 as id), +a as (select * from {{ ref("orders") }} join {{ orders }} on 1 = 1), +b as ({% set q %}select * from orders{% endset %} select 2 as id), +c as ({% if true %}select * from orders{% endif %}) +select * from c` + expect(readsOf(sql)).toEqual({ orders: [], a: [], b: [], c: ["orders"] }) + }) + + it("does not count a qualified name with spaces around the dot", () => { + expect(readsOf("with orders as (select 1), a as (select * from analytics . orders) select * from a")).toEqual({ orders: [], a: [] }) + }) + + it("steps over a Jinja string holding }}, and a backslash-escaped quote in a SQL string", () => { + const jinja = `with orders as (select 1), b as (select {{ "}}" | length }} as n, {{ "'}}'" }} as m from orders) select * from b` + expect(readsOf(jinja)).toEqual({ orders: [], b: ["orders"] }) + const nested = `with orders as (select 1), b as (select {{ {"x": {"y": 1}}["x"].get("orders", 0) }} as n, {{ {"a": {"b": 1}}[orders]["b"] }} as m) select * from b` + expect(readsOf(nested)).toEqual({ orders: [], b: [] }) + const escaped = "with orders as (select 1), b as (select 'it\\'s orders \\'text' as label, 'x''orders' as y) select * from b" + expect(readsOf(escaped)).toEqual({ orders: [], b: [] }) + const jinjaInString = "with orders as (select 1), b as (select '{{ var('orders') }}' as label, '{% if x %}orders{% endif %}' as l2), c as (select '{{ var('x') }}' as l3 from orders) select * from c" + expect(readsOf(jinjaInString)).toEqual({ orders: [], b: [], c: ["orders"] }) + const jinjaInQuotes = `with orders as (select 1), b as (select "{{ var("orders") }}" from raw.customers), c as (select 'prefix {# 'orders' #} suffix' as label) select * from c` + expect(readsOf(jinjaInQuotes)).toEqual({ orders: [], b: [], c: [] }) + }) + + it("does not count a name qualified across a comment, and reads a name that starts outside the BMP", () => { + const sql = "with orders as (select 1), \u{10400}orders as (select 2), a as (select * from analytics . /* t */ orders join analytics.-- x\n orders on 1 = 1), b as (select * from \u{10400}orders) select * from b" + expect(readsOf(sql)).toEqual({ orders: [], "\u{10400}orders": [], a: [], b: ["\u{10400}orders"] }) + }) + + it("reads Jinja before SQL: delimiters and quotes inside Jinja, wherever it sits, are not SQL", () => { + const wrap = (body: string) => `with orders as (select 1), b as (${body}) select * from b` + const bodies = [ + `select $$ {{ var('x', '$$ orders $$') | length }} $$ as label`, + `select 1 /* {{ var('x', '*/ orders /*') | length }} */ as n`, + `select '{% set label = "%} 'orders'" %}constant' as label`, + `select '\\{{ var('orders', 'n') }}' as label`, + `select [col{# ] orders #}] from raw.t`, + `select "{{ var("orders") }}" from raw.customers`, + `select 'prefix {# 'orders' #} suffix' as label`, + `{% call statement('x') %}select * from orders{% endcall %} select 1 as n`, + `select 2 -- {#\n#} from orders`, + `select 2 -- {{ var('flag', 0) -}}\n from orders`, + `{% set q %}{% raw %}{% endset %}{% endraw %}\nselect * from orders\n{% endset %}\nselect 2`, + ] + for (const body of bodies) expect(readsOf(wrap(body)).b).toEqual([]) + // Jinja output between real SQL keeps the SQL readable, and {% raw %} text is SQL as written. + expect(readsOf(wrap(`select {{ var('n') }} from orders`)).b).toEqual(["orders"]) + expect(readsOf(wrap(`select * from {% raw %}orders{% endraw %}`)).b).toEqual(["orders"]) + }) + + it("a name the body's own WITH defines again is that inner CTE, not a read of the outer one", () => { + // Inside its own definition, or in an inner CTE before it, the name is still the outer CTE. + expect(readsOf("with orders as (select 1), b as (with x as (select * from orders), orders as (select 2) select * from x) select * from b").b).toEqual(["orders"]) + expect(readsOf("with orders as (select 1), b as (with orders as (select * from orders) select * from orders) select * from b").b).toEqual(["orders"]) + // In WITH RECURSIVE the inner CTE's own body already means it. + expect(readsOf("with dates as (select 999 as n), result as (with recursive dates(n) as (select 1 union all select n + 1 from dates where n < 3) select * from dates) select * from result").result).toEqual([]) + expect(readsOf("with dates as (select 999 as n), result as (with dates2(n) as (select n from dates) select * from dates2) select * from result").result).toEqual(["dates"]) + for (const gap of [" /* generate calendar */ ", " -- calendar\n ", " ".repeat(60)]) + expect(readsOf(`with dates as (select 999 as n), result as (with${gap}recursive dates(n) as (select 1 union all select n + 1 from dates where n < 3) select * from dates) select * from result`).result).toEqual([]) + // A subquery's WITH shadows too, only inside that subquery. + expect(readsOf("with orders as (select 1), b as (select * from (with orders as (select 2) select * from orders) s) select * from b").b).toEqual([]) + expect(readsOf("with orders as (select 1), b as (select * from (with orders as (select 2) select * from orders) s join orders on 1 = 1) select * from b").b).toEqual(["orders"]) + const sql = "with orders as (select 1), b as (with orders as (select 2) select * from orders), c as (with x as (select 3) select * from x join orders on 1 = 1) select * from c" + expect(readsOf(sql)).toEqual({ orders: [], b: [], c: ["orders"] }) + }) + + it("a body with thousands of inner CTEs is read in near-linear time", () => { + const inner = Array.from({ length: 10000 }, (_, i) => `i${i} as (select ${i} as x)`).join(",\n") + const sql = `with orders as (select 1), b as (with ${inner} select * from i9999 join orders on 1 = 1) select * from b` + const started = performance.now() + expect(readsOf(sql).b).toEqual(["orders"]) + expect(performance.now() - started).toBeLessThan(3000) + }) + + it("lists the reads of a model with thousands of CTEs in linear time", () => { + const parts = Array.from({ length: 10000 }, (_, i) => `c${i} as (select ${i} as x${i > 0 && i % 2 === 0 ? ` from c${i - 1}` : ""})`) + const sql = `with ${parts.join(",\n")}\nselect * from c9999` + const started = performance.now() + const ctes = detectCtesInText(sql) + const reads = cteReads(sql, ctes) + expect(reads.length).toBe(10000) + expect(reads[2]).toEqual([1]) + expect(reads[3]).toEqual([]) + expect(performance.now() - started).toBeLessThan(3000) + }) +}) diff --git a/packages/opencode/test/altimate/dbt-cte-profile.test.ts b/packages/opencode/test/altimate/dbt-cte-profile.test.ts index 6e747dbc51..df28398dca 100644 --- a/packages/opencode/test/altimate/dbt-cte-profile.test.ts +++ b/packages/opencode/test/altimate/dbt-cte-profile.test.ts @@ -130,14 +130,14 @@ describe("dbt_cte_profile list", () => { expect(result.metadata.success).toBe(true) expect(result.metadata.file).toBe(path.join("models", "marts", "customers.sql")) expect(result.metadata.ctes).toEqual([ - { name: "orders", line: 4, index: 0, clause: expect.any(Number), position: 0 }, - { name: '"Big Spenders"', line: 8, index: 1, clause: expect.any(Number), position: 1 }, - { name: "final", line: 12, index: 2, clause: expect.any(Number), position: 2 }, + { name: "orders", line: 4, index: 0, clause: expect.any(Number), position: 0, reads: [] }, + { name: '"Big Spenders"', line: 8, index: 1, clause: expect.any(Number), position: 1, reads: [0] }, + { name: "final", line: 12, index: 2, clause: expect.any(Number), position: 2, reads: [1] }, ]) expect(result.output).toContain("3 CTEs") // The numbers shown are the ones `position` takes: it counts from 0. expect(result.output).toContain("position 0: orders (line 4)") - expect(result.output).toContain("position 2: final (line 12)") + expect(result.output).toContain('position 2: final (line 12; reads "Big Spenders")') expect(queries).toEqual([]) })