From 38ce91241b18f055d5c54870dfeea0f9de4ab77b Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Fri, 20 Nov 2026 21:29:30 +0530 Subject: [PATCH 01/11] fix: dbt_cte_profile lists the earlier CTEs each CTE reads A CTE's own time was its count's time less the CTE before it, so a sibling that follows a slower CTE it does not read showed 0 ms. The listing now gives each CTE's reads (positions of the earlier CTEs of its WITH clause its body names), so a client can subtract only what the count actually runs. Co-Authored-By: Claude Opus 5.5 --- .../opencode/src/altimate/tools/cte-detect.ts | 53 +++++++++++++++++++ .../src/altimate/tools/dbt-cte-profile.ts | 8 +-- .../opencode/test/altimate/cte-detect.test.ts | 45 ++++++++++++++++ .../test/altimate/dbt-cte-profile.test.ts | 8 +-- 4 files changed, 107 insertions(+), 7 deletions(-) diff --git a/packages/opencode/src/altimate/tools/cte-detect.ts b/packages/opencode/src/altimate/tools/cte-detect.ts index 816aa6405f..3898ab24dd 100644 --- a/packages/opencode/src/altimate/tools/cte-detect.ts +++ b/packages/opencode/src/altimate/tools/cte-detect.ts @@ -301,6 +301,59 @@ export function detectCtes(text: string): CteInfo[] { return ctes } +/** A CTE's name for matching a reference: a quoted one as written, a bare one in lower case (as most warehouses fold it). */ +function referenceKey(name: string): string { + const quote = name[0] + if ((quote === '"' || quote === "`") && name.length > 1 && name.endsWith(quote)) return name.slice(1, -1).replaceAll(quote + quote, quote) + if (quote === "[" && name.endsWith("]")) return name.slice(1, -1).replaceAll("]]", "]") + return name.toLowerCase() +} + +const BARE_WORD = /[\p{L}_][\p{L}\p{N}_$]*/uy + +/** + * For each CTE, the positions (in `ctes`) of the earlier CTEs of its own WITH clause that its body names: + * a name outside strings and comments, not after a dot, equal to the CTE's (letter case aside when bare). + * A column or alias spelled like an earlier CTE counts too, so a CTE may be listed as read when it is not; + * a CTE that only a Jinja expression names is not seen. + */ +export function cteReads(text: string, ctes: CteInfo[]): number[][] { + return ctes.map((cte) => { + const earlier = new Map() + ctes.forEach((other, i) => { + if (other.withClauseStart !== cte.withClauseStart || other.index >= cte.index) return + const key = referenceKey(stripColumnList(other.name)) + earlier.set(key, [...(earlier.get(key) ?? []), i]) + }) + const reads = new Set() + let pos = cte.queryStart + while (earlier.size > 0 && pos < cte.queryEnd) { + const char = text[pos] + const quoted = char === '"' || char === "`" + const skipped = skipLiteral(text, pos) + // A string or a comment is skipped whole; a quoted name is read. + if (!quoted && skipped !== pos) { + pos = skipped + continue + } + let end = pos + 1 + if (quoted) end = skipped + else { + BARE_WORD.lastIndex = pos + if (BARE_WORD.test(text)) end = BARE_WORD.lastIndex + else { + pos = end + continue + } + } + const before = text[pos - 1] ?? " " + if (before !== "." && !/[\p{L}\p{N}_$]/u.test(before)) for (const i of earlier.get(referenceKey(text.substring(pos, end))) ?? []) reads.add(i) + pos = end + } + return [...reads].sort((a, b) => a - b) + }) +} + /** * Offset from which the CTEs could not be read, or undefined when every CTE was read: the * comma after the last CTE read of a WITH clause that goes on, or the start of a clause where none was read diff --git a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts index 9b5a60af1d..fdd61f5efe 100644 --- a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts +++ b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts @@ -9,7 +9,7 @@ import { isQueryTimeoutError } from "@altimateai/drivers/types" import { executeQuery } from "../native/query-execution" import { dbtAdapterFor } from "./dbt-adapter-access" import { untilAbort } from "./dbt-cost-estimate" -import { buildCountQuery, detectCtes, extractRowCount, stripColumnList, unclosedJinjaBlock, unreadCteAfter, type CteInfo } from "./cte-detect" +import { buildCountQuery, cteReads, detectCtes, extractRowCount, stripColumnList, unclosedJinjaBlock, unreadCteAfter, type CteInfo } from "./cte-detect" /** Folders a model search never enters. */ const SKIPPED_DIRS = new Set(["target", "dbt_packages", "dbt_modules", "node_modules", "logs", "venv", "site-packages"]) @@ -252,7 +252,7 @@ function failure(title: string, output: string, extra: Record = export const DbtCteProfileTool = Tool.define("dbt_cte_profile", { description: - "List a dbt model's CTEs, or time one CTE with a row-count query. Without `cte` or `position` it returns the CTE names in source order (name, line, position) and runs nothing, so a read-only call works. With one of them it runs `SELECT COUNT(*)` over that CTE and the CTEs before it in the same WITH clause on the warehouse, and returns the row count and the query's milliseconds on the warehouse (`queryMs`): dbt's own timing of the query's execute step where dbt reports one (run_results.json), else the call's time less that of a trivial query through dbt, so a command-line dbt's start-up of a second or more per query is not counted; `totalMs` is the count query's whole call through dbt, start-up included, and `timing` says which measure `queryMs` is (dbt, baseline, or wall when neither was available); that run needs write permission (`--allow-write`) because it starts the dbt adapter and compiles the model. Each run recomputes the earlier CTEs, so a model of N CTEs costs N queries. Only the model's own top-level WITH clause is read: a WITH inside a CTE body is part of that CTE. The list stops at a CTE whose name Jinja builds, and `incomplete` is set in the metadata then; a CTE whose count query reads is_incremental() or this, in its own SQL, an earlier CTE or the model header, is not run, because the count runs outside the model. Macros the query calls are not opened: one that reads is_incremental() inside is not caught, so treat the count of such a model as a full-history count.", + "List a dbt model's CTEs, or time one CTE with a row-count query. Without `cte` or `position` it returns the CTE names in source order (name, line, position, and `reads`: the positions of the earlier CTEs of its WITH clause its body names, so a CTE's own time is its count's time less the CTEs it reads, not less the CTE before it) and runs nothing, so a read-only call works. With one of them it runs `SELECT COUNT(*)` over that CTE and the CTEs before it in the same WITH clause on the warehouse, and returns the row count and the query's milliseconds on the warehouse (`queryMs`): dbt's own timing of the query's execute step where dbt reports one (run_results.json), else the call's time less that of a trivial query through dbt, so a command-line dbt's start-up of a second or more per query is not counted; `totalMs` is the count query's whole call through dbt, start-up included, and `timing` says which measure `queryMs` is (dbt, baseline, or wall when neither was available); that run needs write permission (`--allow-write`) because it starts the dbt adapter and compiles the model. Each run recomputes the earlier CTEs, so a model of N CTEs costs N queries. Only the model's own top-level WITH clause is read: a WITH inside a CTE body is part of that CTE. The list stops at a CTE whose name Jinja builds, and `incomplete` is set in the metadata then; a CTE whose count query reads is_incremental() or this, in its own SQL, an earlier CTE or the model header, is not run, because the count runs outside the model. Macros the query calls are not opened: one that reads is_incremental() inside is not caught, so treat the count of such a model as a full-history count.", parameters: z.object({ model: z .string() @@ -285,17 +285,19 @@ export const DbtCteProfileTool = Tool.define("dbt_cte_profile", { for (; counted < offset; counted++) if (text[counted] === "\n") line++ return line } + const reads = cteReads(text, ctes) const list = ctes.map((c, position) => ({ name: stripColumnList(c.name), line: lineAt(c.nameStart), index: c.index, clause: c.withClauseStart, position, + reads: reads[position], })) const stop = unreadCteAfter(text, ctes) const note = stop === undefined ? "" : `\nThe list is incomplete: the CTEs from line ${text.substring(0, stop).split("\n").length} on could not be read (its name is built by a Jinja expression or loop).` const output = (list.length - ? [`${list.length} CTE${list.length === 1 ? "" : "s"} in ${file}:`, ...list.map((c) => `position ${c.position}: ${c.name} (line ${c.line})`)].join("\n") + ? [`${list.length} CTE${list.length === 1 ? "" : "s"} in ${file}:`, ...list.map((c) => `position ${c.position}: ${c.name} (line ${c.line}${c.reads.length ? `; reads ${c.reads.map((i) => list[i].name).join(", ")}` : ""})`)].join("\n") : `No CTEs in ${file}.`) + note return { title: `dbt_cte_profile: ${list.length} CTE${list.length === 1 ? "" : "s"} in ${args.model}`, diff --git a/packages/opencode/test/altimate/cte-detect.test.ts b/packages/opencode/test/altimate/cte-detect.test.ts index f4adbd2346..c0e799f72f 100644 --- a/packages/opencode/test/altimate/cte-detect.test.ts +++ b/packages/opencode/test/altimate/cte-detect.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "bun:test" import { buildCountQuery, + cteReads, detectCtes as detectCtesInText, extractRowCount, quoteSqlIdentifier, @@ -1639,3 +1640,47 @@ describe("extractRowCount", () => { expect(extractRowCount([{ _PROFILE_COUNT: "not-a-number" }])).toBe(0) }) }) + +describe("cteReads", () => { + const readsOf = (sql: string) => { + const ctes = detectCtesInText(sql) + const reads = cteReads(sql, ctes) + return Object.fromEntries(ctes.map((c, i) => [c.name, reads[i].map((j) => ctes[j].name)])) + } + + it("lists only the earlier CTEs a body names: siblings read nothing of each other", () => { + const sql = `with orders as (select * from {{ ref('stg_orders') }}), +order_items as (select order_id, sum(price) as total from {{ ref('stg_order_items') }} group by 1), +payments as (select order_id, sum(amount) as paid from {{ ref('stg_payments') }} group by order_id), +customer_orders as ( + select orders.customer_id, sum(order_items.total), sum(payments.paid) + from orders left join order_items on orders.order_id = order_items.order_id + left join payments on orders.order_id = payments.order_id group by 1 +), +ranked as (select *, rank() over (order by 1) from customer_orders) +select * from ranked` + expect(readsOf(sql)).toEqual({ + orders: [], + order_items: [], + payments: [], + customer_orders: ["orders", "order_items", "payments"], + ranked: ["customer_orders"], + }) + }) + + it("does not count a name in a string, a comment, after a dot, or inside a longer word", () => { + const sql = `with orders as (select 1 as id), +a as (select 'orders' as s, x.orders, my_orders, orders2 from t as x -- orders + /* orders */ {# orders #}), +b as (select * from ORDERS) +select * from b` + expect(readsOf(sql)).toEqual({ orders: [], a: [], b: ["orders"] }) + }) + + it("matches a quoted name as written, a bare one in any case, and keeps to its own WITH clause", () => { + const sql = `with "Big" as (select 1), big as (select 2), +q as (select * from "Big"), r as (select * from Big) +select * from (with inner_cte as (select * from q) select * from inner_cte)` + expect(readsOf(sql)).toEqual({ '"Big"': [], big: [], q: ['"Big"'], r: ["big"] }) + }) +}) diff --git a/packages/opencode/test/altimate/dbt-cte-profile.test.ts b/packages/opencode/test/altimate/dbt-cte-profile.test.ts index 6e747dbc51..df28398dca 100644 --- a/packages/opencode/test/altimate/dbt-cte-profile.test.ts +++ b/packages/opencode/test/altimate/dbt-cte-profile.test.ts @@ -130,14 +130,14 @@ describe("dbt_cte_profile list", () => { expect(result.metadata.success).toBe(true) expect(result.metadata.file).toBe(path.join("models", "marts", "customers.sql")) expect(result.metadata.ctes).toEqual([ - { name: "orders", line: 4, index: 0, clause: expect.any(Number), position: 0 }, - { name: '"Big Spenders"', line: 8, index: 1, clause: expect.any(Number), position: 1 }, - { name: "final", line: 12, index: 2, clause: expect.any(Number), position: 2 }, + { name: "orders", line: 4, index: 0, clause: expect.any(Number), position: 0, reads: [] }, + { name: '"Big Spenders"', line: 8, index: 1, clause: expect.any(Number), position: 1, reads: [0] }, + { name: "final", line: 12, index: 2, clause: expect.any(Number), position: 2, reads: [1] }, ]) expect(result.output).toContain("3 CTEs") // The numbers shown are the ones `position` takes: it counts from 0. expect(result.output).toContain("position 0: orders (line 4)") - expect(result.output).toContain("position 2: final (line 12)") + expect(result.output).toContain('position 2: final (line 12; reads "Big Spenders")') expect(queries).toEqual([]) }) From cb559bf9f5330d9519af004f9d0e6ae95c9f583b Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Fri, 20 Nov 2026 21:40:38 +0530 Subject: [PATCH 02/11] fix: CTE reads skip Jinja, match brackets and any letter case, scan once; graph nodes carry their schema.yml - cteReads skips {{ }} expressions and captured set/call/macro blocks, reads [bracketed] names whole, matches names case-insensitively across quoting (Snowflake folds to upper case), ignores a name after a spaced dot, and keeps a per-clause name map so a model with thousands of CTEs is scanned once. - The tool description says a client subtracts the closure of reads. - dbt_lineage's include_graph nodes carry yml (the node's patch_path), so a client of a large manifest can tell when a model's schema.yml changed. Co-Authored-By: Claude Opus 5.5 --- .../opencode/src/altimate/native/dbt/graph.ts | 3 + .../opencode/src/altimate/tools/cte-detect.ts | 80 ++++++++++++------- .../src/altimate/tools/dbt-cte-profile.ts | 2 +- .../test/altimate/column-reach.test.ts | 14 ++++ .../opencode/test/altimate/cte-detect.test.ts | 35 +++++++- 5 files changed, 101 insertions(+), 33 deletions(-) diff --git a/packages/opencode/src/altimate/native/dbt/graph.ts b/packages/opencode/src/altimate/native/dbt/graph.ts index 3edff2baf9..e4daffde98 100644 --- a/packages/opencode/src/altimate/native/dbt/graph.ts +++ b/packages/opencode/src/altimate/native/dbt/graph.ts @@ -12,6 +12,8 @@ export interface GraphNode { label: string | null type: string path: string | null + /** The project-relative schema.yml that describes the node (dbt's patch_path without its `package://`), when there is one. */ + yml?: string materialized: string | null /** database.schema.table, lower-cased. */ relation: string | null @@ -57,6 +59,7 @@ function nodeOf(id: string, node: any): GraphNode { label: type === "exposure" ? node?.label || node?.name || id : null, type, path: node?.original_file_path ?? null, + ...(typeof node?.patch_path === "string" ? { yml: node.patch_path.replace(/^[^:/]+:\/\//, "") } : {}), materialized: node?.config?.materialized ?? null, relation: relationOf(node), columns: Object.entries(node?.columns ?? {}).map(([key, col]) => String(col?.name ?? key)), diff --git a/packages/opencode/src/altimate/tools/cte-detect.ts b/packages/opencode/src/altimate/tools/cte-detect.ts index 3898ab24dd..fc71dddfdb 100644 --- a/packages/opencode/src/altimate/tools/cte-detect.ts +++ b/packages/opencode/src/altimate/tools/cte-detect.ts @@ -301,55 +301,79 @@ export function detectCtes(text: string): CteInfo[] { return ctes } -/** A CTE's name for matching a reference: a quoted one as written, a bare one in lower case (as most warehouses fold it). */ +/** + * A CTE's name for matching a reference, quotes removed and in lower case: warehouses fold a bare name + * (Snowflake to upper case, most others to lower), so `orders` and `"ORDERS"` may name the same CTE. + */ function referenceKey(name: string): string { const quote = name[0] - if ((quote === '"' || quote === "`") && name.length > 1 && name.endsWith(quote)) return name.slice(1, -1).replaceAll(quote + quote, quote) - if (quote === "[" && name.endsWith("]")) return name.slice(1, -1).replaceAll("]]", "]") + if ((quote === '"' || quote === "`") && name.length > 1 && name.endsWith(quote)) return name.slice(1, -1).replaceAll(quote + quote, quote).toLowerCase() + if (quote === "[" && name.endsWith("]")) return name.slice(1, -1).replaceAll("]]", "]").toLowerCase() return name.toLowerCase() } const BARE_WORD = /[\p{L}_][\p{L}\p{N}_$]*/uy +const BRACKETED = /\[(?:[^\]\n]|\]\]){1,256}\]/y + +/** Whether the last character before `pos` that is not whitespace is a dot: the name there is qualified (`schema . orders`). */ +function afterDot(text: string, pos: number): boolean { + let at = pos - 1 + while (at >= 0 && /\s/.test(text[at])) at-- + return text[at] === "." +} /** - * For each CTE, the positions (in `ctes`) of the earlier CTEs of its own WITH clause that its body names: - * a name outside strings and comments, not after a dot, equal to the CTE's (letter case aside when bare). - * A column or alias spelled like an earlier CTE counts too, so a CTE may be listed as read when it is not; - * a CTE that only a Jinja expression names is not seen. + * For each CTE, the positions (in `ctes`) of the earlier CTEs of its own WITH clause that its body names + * directly: a name outside strings, comments and Jinja (`{{ }}`, and `{% set %}`, `{% call %}` and `{% macro %}` + * blocks, whose SQL is a string for dbt), not after a dot or inside a longer word, equal to the CTE's in any + * letter case and quoting. A column or alias spelled like an earlier CTE counts too, so a CTE may be listed as + * read when it is not; a CTE named only through Jinja is not seen. A client wanting a CTE's own time subtracts + * the CTEs it reads directly or through another (the closure of `reads`), each once. */ export function cteReads(text: string, ctes: CteInfo[]): number[][] { - return ctes.map((cte) => { - const earlier = new Map() - ctes.forEach((other, i) => { - if (other.withClauseStart !== cte.withClauseStart || other.index >= cte.index) return - const key = referenceKey(stripColumnList(other.name)) - earlier.set(key, [...(earlier.get(key) ?? []), i]) - }) + // Per WITH clause, the CTEs met so far by name: built up in source order, so each body is scanned once. + const seen = new Map>() + return ctes.map((cte, position) => { + const earlier = seen.get(cte.withClauseStart) ?? new Map() + seen.set(cte.withClauseStart, earlier) const reads = new Set() let pos = cte.queryStart while (earlier.size > 0 && pos < cte.queryEnd) { const char = text[pos] - const quoted = char === '"' || char === "`" - const skipped = skipLiteral(text, pos) - // A string or a comment is skipped whole; a quoted name is read. - if (!quoted && skipped !== pos) { - pos = skipped + if (char === "{" && text[pos + 1] === "{") { + const close = text.indexOf("}}", pos + 2) + pos = close === -1 ? text.length : close + 2 continue } - let end = pos + 1 - if (quoted) end = skipped - else { - BARE_WORD.lastIndex = pos - if (BARE_WORD.test(text)) end = BARE_WORD.lastIndex - else { - pos = end + if (char === "{" && text[pos + 1] === "%") { + pos = skipJinjaBlock(text, pos) + continue + } + let end = -1 + if (char === '"' || char === "`") end = skipLiteral(text, pos) + else if (char === "[") { + BRACKETED.lastIndex = pos + if (BRACKETED.test(text)) end = BRACKETED.lastIndex + } else { + // A string or a comment is skipped whole. + const skipped = skipLiteral(text, pos) + if (skipped !== pos) { + pos = skipped continue } + BARE_WORD.lastIndex = pos + if (/[\p{L}_]/u.test(char) && BARE_WORD.test(text)) end = BARE_WORD.lastIndex + } + if (end === -1) { + pos++ + continue } - const before = text[pos - 1] ?? " " - if (before !== "." && !/[\p{L}\p{N}_$]/u.test(before)) for (const i of earlier.get(referenceKey(text.substring(pos, end))) ?? []) reads.add(i) + if (!/[\p{L}\p{N}_$]/u.test(text[pos - 1] ?? " ") && !afterDot(text, pos)) + for (const i of earlier.get(referenceKey(text.substring(pos, end))) ?? []) reads.add(i) pos = end } + const key = referenceKey(stripColumnList(cte.name)) + earlier.set(key, [...(earlier.get(key) ?? []), position]) return [...reads].sort((a, b) => a - b) }) } diff --git a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts index fdd61f5efe..283dbd409d 100644 --- a/packages/opencode/src/altimate/tools/dbt-cte-profile.ts +++ b/packages/opencode/src/altimate/tools/dbt-cte-profile.ts @@ -252,7 +252,7 @@ function failure(title: string, output: string, extra: Record = export const DbtCteProfileTool = Tool.define("dbt_cte_profile", { description: - "List a dbt model's CTEs, or time one CTE with a row-count query. Without `cte` or `position` it returns the CTE names in source order (name, line, position, and `reads`: the positions of the earlier CTEs of its WITH clause its body names, so a CTE's own time is its count's time less the CTEs it reads, not less the CTE before it) and runs nothing, so a read-only call works. With one of them it runs `SELECT COUNT(*)` over that CTE and the CTEs before it in the same WITH clause on the warehouse, and returns the row count and the query's milliseconds on the warehouse (`queryMs`): dbt's own timing of the query's execute step where dbt reports one (run_results.json), else the call's time less that of a trivial query through dbt, so a command-line dbt's start-up of a second or more per query is not counted; `totalMs` is the count query's whole call through dbt, start-up included, and `timing` says which measure `queryMs` is (dbt, baseline, or wall when neither was available); that run needs write permission (`--allow-write`) because it starts the dbt adapter and compiles the model. Each run recomputes the earlier CTEs, so a model of N CTEs costs N queries. Only the model's own top-level WITH clause is read: a WITH inside a CTE body is part of that CTE. The list stops at a CTE whose name Jinja builds, and `incomplete` is set in the metadata then; a CTE whose count query reads is_incremental() or this, in its own SQL, an earlier CTE or the model header, is not run, because the count runs outside the model. Macros the query calls are not opened: one that reads is_incremental() inside is not caught, so treat the count of such a model as a full-history count.", + "List a dbt model's CTEs, or time one CTE with a row-count query. Without `cte` or `position` it returns the CTE names in source order (name, line, position, and `reads`: the positions of the earlier CTEs of its WITH clause its body names, directly; a CTE's own time is its count's time less the own times of the CTEs it reads directly or through another, each once, not less the CTE before it) and runs nothing, so a read-only call works. With one of them it runs `SELECT COUNT(*)` over that CTE and the CTEs before it in the same WITH clause on the warehouse, and returns the row count and the query's milliseconds on the warehouse (`queryMs`): dbt's own timing of the query's execute step where dbt reports one (run_results.json), else the call's time less that of a trivial query through dbt, so a command-line dbt's start-up of a second or more per query is not counted; `totalMs` is the count query's whole call through dbt, start-up included, and `timing` says which measure `queryMs` is (dbt, baseline, or wall when neither was available); that run needs write permission (`--allow-write`) because it starts the dbt adapter and compiles the model. Each run recomputes the earlier CTEs, so a model of N CTEs costs N queries. Only the model's own top-level WITH clause is read: a WITH inside a CTE body is part of that CTE. The list stops at a CTE whose name Jinja builds, and `incomplete` is set in the metadata then; a CTE whose count query reads is_incremental() or this, in its own SQL, an earlier CTE or the model header, is not run, because the count runs outside the model. Macros the query calls are not opened: one that reads is_incremental() inside is not caught, so treat the count of such a model as a full-history count.", parameters: z.object({ model: z .string() diff --git a/packages/opencode/test/altimate/column-reach.test.ts b/packages/opencode/test/altimate/column-reach.test.ts index 97ae23aad4..dd3b136f0c 100644 --- a/packages/opencode/test/altimate/column-reach.test.ts +++ b/packages/opencode/test/altimate/column-reach.test.ts @@ -966,6 +966,20 @@ describe("columnReach on SQL the engine cannot parse", () => { }) describe("lineageGraph", () => { + test("a node carries the schema.yml that describes it, without dbt's package prefix", () => { + const graph = lineageGraph( + { + nodes: { + "model.shop.orders": { ...model("orders", [], "select 1", []), patch_path: "shop://models/marts/schema.yml" }, + "model.shop.bare": model("bare", ["model.shop.orders"], "select 1", []), + }, + }, + "model.shop.orders", + ) + expect(graph.nodes["model.shop.orders"].yml).toBe("models/marts/schema.yml") + expect(graph.nodes["model.shop.bare"].yml).toBeUndefined() + }) + test("a twelve-hop chain comes back whole to its source; upstream past 500 nodes is cut and marked", () => { const chain: Record = {} for (let i = 1; i <= 12; i++) diff --git a/packages/opencode/test/altimate/cte-detect.test.ts b/packages/opencode/test/altimate/cte-detect.test.ts index c0e799f72f..6182ee88e7 100644 --- a/packages/opencode/test/altimate/cte-detect.test.ts +++ b/packages/opencode/test/altimate/cte-detect.test.ts @@ -1677,10 +1677,37 @@ select * from b` expect(readsOf(sql)).toEqual({ orders: [], a: [], b: ["orders"] }) }) - it("matches a quoted name as written, a bare one in any case, and keeps to its own WITH clause", () => { - const sql = `with "Big" as (select 1), big as (select 2), -q as (select * from "Big"), r as (select * from Big) + it("matches a name in any letter case and quoting, brackets included, and keeps to its own WITH clause", () => { + const sql = `with "Big" as (select 1), other as (select 2), +q as (select * from big), r as (select * from "OTHER"), s as (select * from [Big]), t as (select * from \`other\`) select * from (with inner_cte as (select * from q) select * from inner_cte)` - expect(readsOf(sql)).toEqual({ '"Big"': [], big: [], q: ['"Big"'], r: ["big"] }) + expect(readsOf(sql)).toEqual({ '"Big"': [], other: [], q: ['"Big"'], r: ["other"], s: ['"Big"'], t: ["other"] }) + const bracketed = "with [Big Spenders] as (select 1), [we]]ird] as (select 2), b as (select * from [Big Spenders] join [WE]]IRD] on 1 = 1) select * from b" + expect(readsOf(bracketed)).toEqual({ "[Big Spenders]": [], "[we]]ird]": [], b: ["[Big Spenders]", "[we]]ird]"] }) + }) + + it("does not count a name inside Jinja: an expression, a ref with either quotes, a captured set block", () => { + const sql = `with orders as (select 1 as id), +a as (select * from {{ ref("orders") }} join {{ orders }} on 1 = 1), +b as ({% set q %}select * from orders{% endset %} select 2 as id), +c as ({% if true %}select * from orders{% endif %}) +select * from c` + expect(readsOf(sql)).toEqual({ orders: [], a: [], b: [], c: ["orders"] }) + }) + + it("does not count a qualified name with spaces around the dot", () => { + expect(readsOf("with orders as (select 1), a as (select * from analytics . orders) select * from a")).toEqual({ orders: [], a: [] }) + }) + + it("lists the reads of a model with thousands of CTEs in linear time", () => { + const parts = Array.from({ length: 10000 }, (_, i) => `c${i} as (select ${i} as x${i > 0 && i % 2 === 0 ? ` from c${i - 1}` : ""})`) + const sql = `with ${parts.join(",\n")}\nselect * from c9999` + const started = performance.now() + const ctes = detectCtesInText(sql) + const reads = cteReads(sql, ctes) + expect(reads.length).toBe(10000) + expect(reads[2]).toEqual([1]) + expect(reads[3]).toEqual([]) + expect(performance.now() - started).toBeLessThan(3000) }) }) From a8607466ded44db04d5c2d4e927388f09fd4fbf3 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Fri, 20 Nov 2026 21:48:44 +0530 Subject: [PATCH 03/11] fix: CTE reads step over Jinja strings holding }}, backslash-escaped quotes and comments after a dot; a name may start outside the BMP Co-Authored-By: Claude Opus 5.5 --- .../opencode/src/altimate/tools/cte-detect.ts | 66 +++++++++++++++---- .../opencode/test/altimate/cte-detect.test.ts | 12 ++++ 2 files changed, 64 insertions(+), 14 deletions(-) diff --git a/packages/opencode/src/altimate/tools/cte-detect.ts b/packages/opencode/src/altimate/tools/cte-detect.ts index fc71dddfdb..ce88e5c021 100644 --- a/packages/opencode/src/altimate/tools/cte-detect.ts +++ b/packages/opencode/src/altimate/tools/cte-detect.ts @@ -315,20 +315,45 @@ function referenceKey(name: string): string { const BARE_WORD = /[\p{L}_][\p{L}\p{N}_$]*/uy const BRACKETED = /\[(?:[^\]\n]|\]\]){1,256}\]/y -/** Whether the last character before `pos` that is not whitespace is a dot: the name there is qualified (`schema . orders`). */ -function afterDot(text: string, pos: number): boolean { - let at = pos - 1 - while (at >= 0 && /\s/.test(text[at])) at-- - return text[at] === "." +/** Position just past the `}}` that closes the Jinja expression at `pos`, stepping over quoted strings inside it. */ +function skipJinjaExpression(text: string, pos: number): number { + let at = pos + 2 + while (at < text.length) { + const char = text[at] + if (char === "'" || char === '"') { + at++ + while (at < text.length && text[at] !== char) at += text[at] === "\\" ? 2 : 1 + at++ + } else if (char === "}" && text[at + 1] === "}") return at + 2 + else at++ + } + return text.length +} + +/** + * Position just past the single-quoted string at `pos`. A doubled quote and a backslash-escaped one (Snowflake, + * BigQuery, Databricks, MySQL) stay inside it; where a backslash is a plain character (Postgres, DuckDB), a string + * ending in one runs on past its quote, so the names after it are missed and the CTE is listed as reading less. + */ +function skipSingleQuoted(text: string, pos: number): number { + let at = pos + 1 + while (at < text.length) { + if (text[at] === "\\") at += 2 + else if (text[at] === "'" && text[at + 1] === "'") at += 2 + else if (text[at] === "'") return at + 1 + else at++ + } + return text.length } /** * For each CTE, the positions (in `ctes`) of the earlier CTEs of its own WITH clause that its body names * directly: a name outside strings, comments and Jinja (`{{ }}`, and `{% set %}`, `{% call %}` and `{% macro %}` - * blocks, whose SQL is a string for dbt), not after a dot or inside a longer word, equal to the CTE's in any - * letter case and quoting. A column or alias spelled like an earlier CTE counts too, so a CTE may be listed as - * read when it is not; a CTE named only through Jinja is not seen. A client wanting a CTE's own time subtracts - * the CTEs it reads directly or through another (the closure of `reads`), each once. + * blocks, whose SQL is a string for dbt), not qualified by a dot (`schema . orders`) or inside a longer word, + * equal to the CTE's in any letter case and quoting. A column or alias spelled like an earlier CTE counts too, + * so a CTE may be listed as read when it is not; a CTE named only through Jinja is not seen, nor one after a + * nested block comment. A client wanting a CTE's own time subtracts the CTEs it reads directly or through + * another (the closure of `reads`), each once. */ export function cteReads(text: string, ctes: CteInfo[]): number[][] { // Per WITH clause, the CTEs met so far by name: built up in source order, so each body is scanned once. @@ -338,38 +363,51 @@ export function cteReads(text: string, ctes: CteInfo[]): number[][] { seen.set(cte.withClauseStart, earlier) const reads = new Set() let pos = cte.queryStart + // The last character of SQL read before `pos`, past whitespace, comments and Jinja: a dot qualifies the next name. + let previous = "" while (earlier.size > 0 && pos < cte.queryEnd) { const char = text[pos] + if (/\s/.test(char)) { + pos++ + continue + } if (char === "{" && text[pos + 1] === "{") { - const close = text.indexOf("}}", pos + 2) - pos = close === -1 ? text.length : close + 2 + pos = skipJinjaExpression(text, pos) continue } if (char === "{" && text[pos + 1] === "%") { pos = skipJinjaBlock(text, pos) continue } + if (char === "'") { + pos = skipSingleQuoted(text, pos) + previous = "'" + continue + } let end = -1 if (char === '"' || char === "`") end = skipLiteral(text, pos) else if (char === "[") { BRACKETED.lastIndex = pos if (BRACKETED.test(text)) end = BRACKETED.lastIndex } else { - // A string or a comment is skipped whole. + // A comment is skipped whole. const skipped = skipLiteral(text, pos) if (skipped !== pos) { + if (char === "$") previous = "$" pos = skipped continue } BARE_WORD.lastIndex = pos - if (/[\p{L}_]/u.test(char) && BARE_WORD.test(text)) end = BARE_WORD.lastIndex + if (/[\p{L}_]/u.test(String.fromCodePoint(text.codePointAt(pos) ?? 32)) && BARE_WORD.test(text)) end = BARE_WORD.lastIndex } if (end === -1) { + previous = char pos++ continue } - if (!/[\p{L}\p{N}_$]/u.test(text[pos - 1] ?? " ") && !afterDot(text, pos)) + if (previous !== "." && !/[\p{L}\p{N}_$]/u.test(text[pos - 1] ?? " ")) for (const i of earlier.get(referenceKey(text.substring(pos, end))) ?? []) reads.add(i) + previous = text[end - 1] pos = end } const key = referenceKey(stripColumnList(cte.name)) diff --git a/packages/opencode/test/altimate/cte-detect.test.ts b/packages/opencode/test/altimate/cte-detect.test.ts index 6182ee88e7..8a37a7142d 100644 --- a/packages/opencode/test/altimate/cte-detect.test.ts +++ b/packages/opencode/test/altimate/cte-detect.test.ts @@ -1699,6 +1699,18 @@ select * from c` expect(readsOf("with orders as (select 1), a as (select * from analytics . orders) select * from a")).toEqual({ orders: [], a: [] }) }) + it("steps over a Jinja string holding }}, and a backslash-escaped quote in a SQL string", () => { + const jinja = `with orders as (select 1), b as (select {{ "}}" | length }} as n, {{ "'}}'" }} as m from orders) select * from b` + expect(readsOf(jinja)).toEqual({ orders: [], b: ["orders"] }) + const escaped = "with orders as (select 1), b as (select 'it\\'s orders \\'text' as label, 'x''orders' as y) select * from b" + expect(readsOf(escaped)).toEqual({ orders: [], b: [] }) + }) + + it("does not count a name qualified across a comment, and reads a name that starts outside the BMP", () => { + const sql = "with orders as (select 1), \u{10400}orders as (select 2), a as (select * from analytics . /* t */ orders join analytics.-- x\n orders on 1 = 1), b as (select * from \u{10400}orders) select * from b" + expect(readsOf(sql)).toEqual({ orders: [], "\u{10400}orders": [], a: [], b: ["\u{10400}orders"] }) + }) + it("lists the reads of a model with thousands of CTEs in linear time", () => { const parts = Array.from({ length: 10000 }, (_, i) => `c${i} as (select ${i} as x${i > 0 && i % 2 === 0 ? ` from c${i - 1}` : ""})`) const sql = `with ${parts.join(",\n")}\nselect * from c9999` From 49b2fd04f410111e28235e998645548f52867276 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Fri, 20 Nov 2026 21:54:44 +0530 Subject: [PATCH 04/11] fix: a Jinja expression's braces are balanced before its closing }} Co-Authored-By: Claude Opus 5.5 --- packages/opencode/src/altimate/tools/cte-detect.ts | 12 +++++++++++- packages/opencode/test/altimate/cte-detect.test.ts | 2 ++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/opencode/src/altimate/tools/cte-detect.ts b/packages/opencode/src/altimate/tools/cte-detect.ts index ce88e5c021..452d8835d1 100644 --- a/packages/opencode/src/altimate/tools/cte-detect.ts +++ b/packages/opencode/src/altimate/tools/cte-detect.ts @@ -315,15 +315,25 @@ function referenceKey(name: string): string { const BARE_WORD = /[\p{L}_][\p{L}\p{N}_$]*/uy const BRACKETED = /\[(?:[^\]\n]|\]\]){1,256}\]/y -/** Position just past the `}}` that closes the Jinja expression at `pos`, stepping over quoted strings inside it. */ +/** + * Position just past the `}}` that closes the Jinja expression at `pos`: quoted strings inside it are stepped + * over and braces balanced, as Jinja's lexer does, so `{{ {"a": {"b": 1}}["a"] }}` ends at its last `}}`. + */ function skipJinjaExpression(text: string, pos: number): number { let at = pos + 2 + let depth = 0 while (at < text.length) { const char = text[at] if (char === "'" || char === '"') { at++ while (at < text.length && text[at] !== char) at += text[at] === "\\" ? 2 : 1 at++ + } else if (char === "{") { + depth++ + at++ + } else if (char === "}" && depth > 0) { + depth-- + at++ } else if (char === "}" && text[at + 1] === "}") return at + 2 else at++ } diff --git a/packages/opencode/test/altimate/cte-detect.test.ts b/packages/opencode/test/altimate/cte-detect.test.ts index 8a37a7142d..f00c34edfd 100644 --- a/packages/opencode/test/altimate/cte-detect.test.ts +++ b/packages/opencode/test/altimate/cte-detect.test.ts @@ -1702,6 +1702,8 @@ select * from c` it("steps over a Jinja string holding }}, and a backslash-escaped quote in a SQL string", () => { const jinja = `with orders as (select 1), b as (select {{ "}}" | length }} as n, {{ "'}}'" }} as m from orders) select * from b` expect(readsOf(jinja)).toEqual({ orders: [], b: ["orders"] }) + const nested = `with orders as (select 1), b as (select {{ {"x": {"y": 1}}["x"].get("orders", 0) }} as n, {{ {"a": {"b": 1}}[orders]["b"] }} as m) select * from b` + expect(readsOf(nested)).toEqual({ orders: [], b: [] }) const escaped = "with orders as (select 1), b as (select 'it\\'s orders \\'text' as label, 'x''orders' as y) select * from b" expect(readsOf(escaped)).toEqual({ orders: [], b: [] }) }) From 99a6dbf3787491118caaa9e0b47872b094d62e39 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Fri, 20 Nov 2026 22:07:14 +0530 Subject: [PATCH 05/11] fix: Jinja inside a SQL string stays inside it; the backslash limit is documented both ways Co-Authored-By: Claude Opus 5.5 --- packages/opencode/src/altimate/tools/cte-detect.ts | 12 +++++++++--- packages/opencode/test/altimate/cte-detect.test.ts | 2 ++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/packages/opencode/src/altimate/tools/cte-detect.ts b/packages/opencode/src/altimate/tools/cte-detect.ts index 452d8835d1..d19ab9e812 100644 --- a/packages/opencode/src/altimate/tools/cte-detect.ts +++ b/packages/opencode/src/altimate/tools/cte-detect.ts @@ -342,13 +342,19 @@ function skipJinjaExpression(text: string, pos: number): number { /** * Position just past the single-quoted string at `pos`. A doubled quote and a backslash-escaped one (Snowflake, - * BigQuery, Databricks, MySQL) stay inside it; where a backslash is a plain character (Postgres, DuckDB), a string - * ending in one runs on past its quote, so the names after it are missed and the CTE is listed as reading less. + * BigQuery, Databricks, MySQL) stay inside it, and so does Jinja (`'{{ var('x') }}'`, whose own quotes do not + * end the string). Where a backslash is a plain character (Postgres, DuckDB), a string ending in one runs on past + * its quote and pairs the quotes after it wrongly: names after it may be missed, or a name in a later string + * read, so such a model's reads may be off either way. */ function skipSingleQuoted(text: string, pos: number): number { let at = pos + 1 while (at < text.length) { - if (text[at] === "\\") at += 2 + if (text[at] === "{" && text[at + 1] === "{") at = skipJinjaExpression(text, at) + else if (text[at] === "{" && text[at + 1] === "%") { + const close = text.indexOf("%}", at + 2) + at = close === -1 ? text.length : close + 2 + } else if (text[at] === "\\") at += 2 else if (text[at] === "'" && text[at + 1] === "'") at += 2 else if (text[at] === "'") return at + 1 else at++ diff --git a/packages/opencode/test/altimate/cte-detect.test.ts b/packages/opencode/test/altimate/cte-detect.test.ts index f00c34edfd..948aa85bc6 100644 --- a/packages/opencode/test/altimate/cte-detect.test.ts +++ b/packages/opencode/test/altimate/cte-detect.test.ts @@ -1706,6 +1706,8 @@ select * from c` expect(readsOf(nested)).toEqual({ orders: [], b: [] }) const escaped = "with orders as (select 1), b as (select 'it\\'s orders \\'text' as label, 'x''orders' as y) select * from b" expect(readsOf(escaped)).toEqual({ orders: [], b: [] }) + const jinjaInString = "with orders as (select 1), b as (select '{{ var('orders') }}' as label, '{% if x %}orders{% endif %}' as l2), c as (select '{{ var('x') }}' as l3 from orders) select * from c" + expect(readsOf(jinjaInString)).toEqual({ orders: [], b: [], c: ["orders"] }) }) it("does not count a name qualified across a comment, and reads a name that starts outside the BMP", () => { From cec32c88bafb9cc55f437cd7461ecbf87c403b27 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Fri, 20 Nov 2026 22:13:49 +0530 Subject: [PATCH 06/11] fix: quoted names and strings step over Jinja comments and Jinja holding the same quote Co-Authored-By: Claude Opus 5.5 --- .../opencode/src/altimate/tools/cte-detect.ts | 31 ++++++++++--------- .../opencode/test/altimate/cte-detect.test.ts | 2 ++ 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/packages/opencode/src/altimate/tools/cte-detect.ts b/packages/opencode/src/altimate/tools/cte-detect.ts index d19ab9e812..18d30fc33c 100644 --- a/packages/opencode/src/altimate/tools/cte-detect.ts +++ b/packages/opencode/src/altimate/tools/cte-detect.ts @@ -341,22 +341,25 @@ function skipJinjaExpression(text: string, pos: number): number { } /** - * Position just past the single-quoted string at `pos`. A doubled quote and a backslash-escaped one (Snowflake, - * BigQuery, Databricks, MySQL) stay inside it, and so does Jinja (`'{{ var('x') }}'`, whose own quotes do not - * end the string). Where a backslash is a plain character (Postgres, DuckDB), a string ending in one runs on past - * its quote and pairs the quotes after it wrongly: names after it may be missed, or a name in a later string - * read, so such a model's reads may be off either way. + * Position just past the quoted text at `pos`: a single-quoted string, or a `"` or `` ` `` quoted name. A doubled + * quote stays inside it, and so does Jinja (`'{{ var('x') }}'`, `"{{ var("x") }}"`, `'{# 'x' #}'`), whose own + * quotes do not end it. In a single-quoted string a backslash-escaped quote stays inside too (Snowflake, BigQuery, + * Databricks, MySQL); where a backslash is a plain character (Postgres, DuckDB), a string ending in one runs on + * past its quote and pairs the quotes after it wrongly: names after it may be missed, or a name in a later + * string read, so such a model's reads may be off either way. */ -function skipSingleQuoted(text: string, pos: number): number { +function skipQuoted(text: string, pos: number): number { + const quote = text[pos] let at = pos + 1 while (at < text.length) { - if (text[at] === "{" && text[at + 1] === "{") at = skipJinjaExpression(text, at) - else if (text[at] === "{" && text[at + 1] === "%") { - const close = text.indexOf("%}", at + 2) + const char = text[at] + if (char === "{" && text[at + 1] === "{") at = skipJinjaExpression(text, at) + else if (char === "{" && (text[at + 1] === "%" || text[at + 1] === "#")) { + const close = text.indexOf(text[at + 1] + "}", at + 2) at = close === -1 ? text.length : close + 2 - } else if (text[at] === "\\") at += 2 - else if (text[at] === "'" && text[at + 1] === "'") at += 2 - else if (text[at] === "'") return at + 1 + } else if (char === "\\" && quote === "'") at += 2 + else if (char === quote && text[at + 1] === quote) at += 2 + else if (char === quote) return at + 1 else at++ } return text.length @@ -396,12 +399,12 @@ export function cteReads(text: string, ctes: CteInfo[]): number[][] { continue } if (char === "'") { - pos = skipSingleQuoted(text, pos) + pos = skipQuoted(text, pos) previous = "'" continue } let end = -1 - if (char === '"' || char === "`") end = skipLiteral(text, pos) + if (char === '"' || char === "`") end = skipQuoted(text, pos) else if (char === "[") { BRACKETED.lastIndex = pos if (BRACKETED.test(text)) end = BRACKETED.lastIndex diff --git a/packages/opencode/test/altimate/cte-detect.test.ts b/packages/opencode/test/altimate/cte-detect.test.ts index 948aa85bc6..88c3feaf77 100644 --- a/packages/opencode/test/altimate/cte-detect.test.ts +++ b/packages/opencode/test/altimate/cte-detect.test.ts @@ -1708,6 +1708,8 @@ select * from c` expect(readsOf(escaped)).toEqual({ orders: [], b: [] }) const jinjaInString = "with orders as (select 1), b as (select '{{ var('orders') }}' as label, '{% if x %}orders{% endif %}' as l2), c as (select '{{ var('x') }}' as l3 from orders) select * from c" expect(readsOf(jinjaInString)).toEqual({ orders: [], b: [], c: ["orders"] }) + const jinjaInQuotes = `with orders as (select 1), b as (select "{{ var("orders") }}" from raw.customers), c as (select 'prefix {# 'orders' #} suffix' as label) select * from c` + expect(readsOf(jinjaInQuotes)).toEqual({ orders: [], b: [], c: [] }) }) it("does not count a name qualified across a comment, and reads a name that starts outside the BMP", () => { From 8d856394d65697b9b9d02c5497301fe5b2b16c4d Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Fri, 20 Nov 2026 22:19:34 +0530 Subject: [PATCH 07/11] fix: CTE reads blank Jinja before reading the SQL, and a body's own WITH shadows an outer CTE of the same name Jinja renders before the SQL is read, so every {{ }}, {% %} and {# #} (and the body of a captured set/call/macro block) is blanked first, wherever it sits; the SQL scan then needs no Jinja awareness. Co-Authored-By: Claude Opus 5.5 --- .../opencode/src/altimate/tools/cte-detect.ts | 160 +++++++++++++----- .../opencode/test/altimate/cte-detect.test.ts | 23 +++ 2 files changed, 141 insertions(+), 42 deletions(-) diff --git a/packages/opencode/src/altimate/tools/cte-detect.ts b/packages/opencode/src/altimate/tools/cte-detect.ts index 18d30fc33c..9e52e7e9ef 100644 --- a/packages/opencode/src/altimate/tools/cte-detect.ts +++ b/packages/opencode/src/altimate/tools/cte-detect.ts @@ -316,10 +316,16 @@ const BARE_WORD = /[\p{L}_][\p{L}\p{N}_$]*/uy const BRACKETED = /\[(?:[^\]\n]|\]\]){1,256}\]/y /** - * Position just past the `}}` that closes the Jinja expression at `pos`: quoted strings inside it are stepped - * over and braces balanced, as Jinja's lexer does, so `{{ {"a": {"b": 1}}["a"] }}` ends at its last `}}`. + * Position just past the Jinja delimiter `{{`, `{%` or `{#` at `pos` closes: a comment at its first `#}`; an + * expression or statement at its `}}` or `%}` outside the quoted strings in it, an expression's braces balanced, + * as Jinja's lexer reads them. */ -function skipJinjaExpression(text: string, pos: number): number { +function jinjaEnd(text: string, pos: number): number { + const kind = text[pos + 1] + if (kind === "#") { + const close = text.indexOf("#}", pos + 2) + return close === -1 ? text.length : close + 2 + } let at = pos + 2 let depth = 0 while (at < text.length) { @@ -328,36 +334,108 @@ function skipJinjaExpression(text: string, pos: number): number { at++ while (at < text.length && text[at] !== char) at += text[at] === "\\" ? 2 : 1 at++ - } else if (char === "{") { + } else if (kind === "{" && char === "{") { depth++ at++ - } else if (char === "}" && depth > 0) { + } else if (kind === "{" && char === "}" && depth > 0) { depth-- at++ - } else if (char === "}" && text[at + 1] === "}") return at + 2 + } else if (char === (kind === "{" ? "}" : "%") && text[at + 1] === "}") return at + 2 else at++ } return text.length } +/** The blocks whose body is a string for dbt (`{% set x %}…{% endset %}`, a call, a macro), not the model's SQL. */ +const CAPTURING = new Set(["set", "call", "macro"]) + +/** + * The text with its Jinja blanked to spaces, offsets kept: every `{{ }}`, `{% %}` and `{# #}`, and the body of a + * captured `set`, `call` or `macro` block. Jinja renders before the SQL is read, so a quote, comment mark or name + * inside it is not SQL, wherever it sits (in a string, a comment, a quoted name). The text inside `{% raw %}` is + * kept as written. + */ +function blankJinja(text: string): string { + const out: string[] = [] + let kept = 0 + const blank = (from: number, to: number) => { + out.push(text.substring(kept, from), text.substring(from, to).replace(/[^\n]/g, " ")) + kept = to + } + let pos = 0 + while (pos < text.length) { + const at = text.indexOf("{", pos) + if (at === -1 || at + 1 >= text.length) break + const kind = text[at + 1] + if (kind !== "{" && kind !== "%" && kind !== "#") { + pos = at + 1 + continue + } + const end = jinjaEnd(text, at) + const head = kind === "%" ? /^\{%[-+]?\s*(\w+)/.exec(text.substring(at, Math.min(end, at + 200))) : null + const name = head?.[1] + if (name === "raw") { + // The text up to {% endraw %} is output as written. + const close = /\{%[-+]?\s*endraw\b[^%]*%\}/g + close.lastIndex = end + const found = close.exec(text) + blank(at, end) + if (found) { + out.push(text.substring(kept, found.index)) + kept = found.index + blank(found.index, close.lastIndex) + pos = close.lastIndex + } else pos = text.length + continue + } + const inline = name === "set" && /^\{%[-+]?\s*set\s+[\w.,\s()]+=(?!=)/.test(text.substring(at, end)) + if (name && CAPTURING.has(name) && !inline) { + // The block's body, to its own end tag (blocks of one kind nest), is blanked with it. + let depth = 1 + let scan = end + while (depth > 0 && scan < text.length) { + const next = text.indexOf("{", scan) + if (next === -1 || next + 1 >= text.length) { + scan = text.length + break + } + if (text[next + 1] !== "%" && text[next + 1] !== "{" && text[next + 1] !== "#") { + scan = next + 1 + continue + } + const tagEnd = jinjaEnd(text, next) + const tag = text[next + 1] === "%" ? /^\{%[-+]?\s*(end)?(\w+)/.exec(text.substring(next, Math.min(tagEnd, next + 200))) : null + if (tag?.[2] === name) { + const opensAgain = !tag[1] && !(name === "set" && /^\{%[-+]?\s*set\s+[\w.,\s()]+=(?!=)/.test(text.substring(next, tagEnd))) + if (tag[1]) depth-- + else if (opensAgain) depth++ + } + scan = tagEnd + } + blank(at, scan) + pos = scan + continue + } + blank(at, end) + pos = end + } + out.push(text.substring(kept)) + return out.join("") +} + /** - * Position just past the quoted text at `pos`: a single-quoted string, or a `"` or `` ` `` quoted name. A doubled - * quote stays inside it, and so does Jinja (`'{{ var('x') }}'`, `"{{ var("x") }}"`, `'{# 'x' #}'`), whose own - * quotes do not end it. In a single-quoted string a backslash-escaped quote stays inside too (Snowflake, BigQuery, - * Databricks, MySQL); where a backslash is a plain character (Postgres, DuckDB), a string ending in one runs on - * past its quote and pairs the quotes after it wrongly: names after it may be missed, or a name in a later - * string read, so such a model's reads may be off either way. + * Position just past the SQL quoted text at `pos`: a single-quoted string, or a `"` or `` ` `` quoted name. A + * doubled quote stays inside it; in a single-quoted string so does a backslash-escaped quote (Snowflake, + * BigQuery, Databricks, MySQL). Where a backslash is a plain character (Postgres, DuckDB), a string ending in one + * runs on past its quote and pairs the quotes after it wrongly: names after it may be missed, or a name in a + * later string read, so such a model's reads may be off either way. */ function skipQuoted(text: string, pos: number): number { const quote = text[pos] let at = pos + 1 while (at < text.length) { const char = text[at] - if (char === "{" && text[at + 1] === "{") at = skipJinjaExpression(text, at) - else if (char === "{" && (text[at + 1] === "%" || text[at + 1] === "#")) { - const close = text.indexOf(text[at + 1] + "}", at + 2) - at = close === -1 ? text.length : close + 2 - } else if (char === "\\" && quote === "'") at += 2 + if (char === "\\" && quote === "'") at += 2 else if (char === quote && text[at + 1] === quote) at += 2 else if (char === quote) return at + 1 else at++ @@ -367,66 +445,64 @@ function skipQuoted(text: string, pos: number): number { /** * For each CTE, the positions (in `ctes`) of the earlier CTEs of its own WITH clause that its body names - * directly: a name outside strings, comments and Jinja (`{{ }}`, and `{% set %}`, `{% call %}` and `{% macro %}` - * blocks, whose SQL is a string for dbt), not qualified by a dot (`schema . orders`) or inside a longer word, - * equal to the CTE's in any letter case and quoting. A column or alias spelled like an earlier CTE counts too, - * so a CTE may be listed as read when it is not; a CTE named only through Jinja is not seen, nor one after a - * nested block comment. A client wanting a CTE's own time subtracts the CTEs it reads directly or through - * another (the closure of `reads`), each once. + * directly: a name outside strings, comments and Jinja (blanked first: `{{ }}`, `{% %}`, `{# #}`, and the SQL a + * `{% set %}`, `{% call %}` or `{% macro %}` block captures, which is a string for dbt), not qualified by a dot + * (`schema . orders`) or inside a longer word, equal to the CTE's in any letter case and quoting. A name its own + * WITH defines again (`with orders as (…) select * from orders` inside the body) is that inner CTE, not a read. + * A column or alias spelled like an earlier CTE counts too, so a CTE may be listed as read when it is not; a CTE + * named only through Jinja is not seen, nor one shadowed in a deeper subquery, nor one after a nested block + * comment. A client wanting a CTE's own time subtracts the CTEs it reads directly or through another (the + * closure of `reads`), each once. */ export function cteReads(text: string, ctes: CteInfo[]): number[][] { + const sql = blankJinja(text) // Per WITH clause, the CTEs met so far by name: built up in source order, so each body is scanned once. const seen = new Map>() return ctes.map((cte, position) => { const earlier = seen.get(cte.withClauseStart) ?? new Map() seen.set(cte.withClauseStart, earlier) const reads = new Set() + // The names the body's own WITH defines shadow the outer CTEs of that name. + const body = sql.substring(cte.queryStart, cte.queryEnd) + const shadowed = new Set(earlier.size > 0 && /\bwith\b/i.test(body) ? detectCtes(body).map((inner) => referenceKey(stripColumnList(inner.name))) : []) let pos = cte.queryStart - // The last character of SQL read before `pos`, past whitespace, comments and Jinja: a dot qualifies the next name. + // The last character of SQL read before `pos`, past whitespace and comments: a dot qualifies the next name. let previous = "" while (earlier.size > 0 && pos < cte.queryEnd) { - const char = text[pos] + const char = sql[pos] if (/\s/.test(char)) { pos++ continue } - if (char === "{" && text[pos + 1] === "{") { - pos = skipJinjaExpression(text, pos) - continue - } - if (char === "{" && text[pos + 1] === "%") { - pos = skipJinjaBlock(text, pos) - continue - } if (char === "'") { - pos = skipQuoted(text, pos) + pos = skipQuoted(sql, pos) previous = "'" continue } let end = -1 - if (char === '"' || char === "`") end = skipQuoted(text, pos) + if (char === '"' || char === "`") end = skipQuoted(sql, pos) else if (char === "[") { BRACKETED.lastIndex = pos - if (BRACKETED.test(text)) end = BRACKETED.lastIndex + if (BRACKETED.test(sql)) end = BRACKETED.lastIndex } else { - // A comment is skipped whole. - const skipped = skipLiteral(text, pos) + // A comment or a dollar-quoted string is skipped whole. + const skipped = skipLiteral(sql, pos) if (skipped !== pos) { if (char === "$") previous = "$" pos = skipped continue } BARE_WORD.lastIndex = pos - if (/[\p{L}_]/u.test(String.fromCodePoint(text.codePointAt(pos) ?? 32)) && BARE_WORD.test(text)) end = BARE_WORD.lastIndex + if (/[\p{L}_]/u.test(String.fromCodePoint(sql.codePointAt(pos) ?? 32)) && BARE_WORD.test(sql)) end = BARE_WORD.lastIndex } if (end === -1) { previous = char pos++ continue } - if (previous !== "." && !/[\p{L}\p{N}_$]/u.test(text[pos - 1] ?? " ")) - for (const i of earlier.get(referenceKey(text.substring(pos, end))) ?? []) reads.add(i) - previous = text[end - 1] + const key = referenceKey(sql.substring(pos, end)) + if (previous !== "." && !/[\p{L}\p{N}_$]/u.test(sql[pos - 1] ?? " ") && !shadowed.has(key)) for (const i of earlier.get(key) ?? []) reads.add(i) + previous = sql[end - 1] pos = end } const key = referenceKey(stripColumnList(cte.name)) diff --git a/packages/opencode/test/altimate/cte-detect.test.ts b/packages/opencode/test/altimate/cte-detect.test.ts index 88c3feaf77..a198e3388d 100644 --- a/packages/opencode/test/altimate/cte-detect.test.ts +++ b/packages/opencode/test/altimate/cte-detect.test.ts @@ -1717,6 +1717,29 @@ select * from c` expect(readsOf(sql)).toEqual({ orders: [], "\u{10400}orders": [], a: [], b: ["\u{10400}orders"] }) }) + it("reads Jinja before SQL: delimiters and quotes inside Jinja, wherever it sits, are not SQL", () => { + const wrap = (body: string) => `with orders as (select 1), b as (${body}) select * from b` + const bodies = [ + `select $$ {{ var('x', '$$ orders $$') | length }} $$ as label`, + `select 1 /* {{ var('x', '*/ orders /*') | length }} */ as n`, + `select '{% set label = "%} 'orders'" %}constant' as label`, + `select '\\{{ var('orders', 'n') }}' as label`, + `select [col{# ] orders #}] from raw.t`, + `select "{{ var("orders") }}" from raw.customers`, + `select 'prefix {# 'orders' #} suffix' as label`, + `{% call statement('x') %}select * from orders{% endcall %} select 1 as n`, + ] + for (const body of bodies) expect(readsOf(wrap(body)).b).toEqual([]) + // Jinja output between real SQL keeps the SQL readable, and {% raw %} text is SQL as written. + expect(readsOf(wrap(`select {{ var('n') }} from orders`)).b).toEqual(["orders"]) + expect(readsOf(wrap(`select * from {% raw %}orders{% endraw %}`)).b).toEqual(["orders"]) + }) + + it("a name the body's own WITH defines again is that inner CTE, not a read of the outer one", () => { + const sql = "with orders as (select 1), b as (with orders as (select 2) select * from orders), c as (with x as (select 3) select * from x join orders on 1 = 1) select * from c" + expect(readsOf(sql)).toEqual({ orders: [], b: [], c: ["orders"] }) + }) + it("lists the reads of a model with thousands of CTEs in linear time", () => { const parts = Array.from({ length: 10000 }, (_, i) => `c${i} as (select ${i} as x${i > 0 && i % 2 === 0 ? ` from c${i - 1}` : ""})`) const sql = `with ${parts.join(",\n")}\nselect * from c9999` From 11305b6f311b429bd4769db6a36efb4a58b61df2 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Fri, 20 Nov 2026 22:26:24 +0530 Subject: [PATCH 08/11] fix: CTE reads: shadowing starts after the inner CTE's definition, quoted names keep their case, Jinja blanks newlines and follows whitespace control, raw text does not end a captured block Co-Authored-By: Claude Opus 5.5 --- .../opencode/src/altimate/tools/cte-detect.ts | 76 +++++++++++++------ .../opencode/test/altimate/cte-detect.test.ts | 14 +++- 2 files changed, 63 insertions(+), 27 deletions(-) diff --git a/packages/opencode/src/altimate/tools/cte-detect.ts b/packages/opencode/src/altimate/tools/cte-detect.ts index 9e52e7e9ef..51cf0e9692 100644 --- a/packages/opencode/src/altimate/tools/cte-detect.ts +++ b/packages/opencode/src/altimate/tools/cte-detect.ts @@ -301,15 +301,27 @@ export function detectCtes(text: string): CteInfo[] { return ctes } +/** A CTE name or a reference as written: its text without quotes, and whether it was quoted. */ +type Written = { name: string; quoted: boolean } + +function written(raw: string): Written { + const quote = raw[0] + if ((quote === '"' || quote === "`") && raw.length > 1 && raw.endsWith(quote)) return { name: raw.slice(1, -1).replaceAll(quote + quote, quote), quoted: true } + if (quote === "[" && raw.endsWith("]")) return { name: raw.slice(1, -1).replaceAll("]]", "]"), quoted: true } + return { name: raw, quoted: false } +} + /** - * A CTE's name for matching a reference, quotes removed and in lower case: warehouses fold a bare name - * (Snowflake to upper case, most others to lower), so `orders` and `"ORDERS"` may name the same CTE. + * Whether a reference names a CTE. Two bare names match in any letter case; two quoted ones only exactly. A quoted + * and a bare one match when the quoted text is the bare name folded to lower case (Postgres, DuckDB, Redshift) or + * to upper case (Snowflake), the warehouse being unknown here: so `"ORDERS"` names `orders`, and `"Foo"` does not + * name `foo`. */ -function referenceKey(name: string): string { - const quote = name[0] - if ((quote === '"' || quote === "`") && name.length > 1 && name.endsWith(quote)) return name.slice(1, -1).replaceAll(quote + quote, quote).toLowerCase() - if (quote === "[" && name.endsWith("]")) return name.slice(1, -1).replaceAll("]]", "]").toLowerCase() - return name.toLowerCase() +function sameName(a: Written, b: Written): boolean { + if (!a.quoted && !b.quoted) return a.name.toLowerCase() === b.name.toLowerCase() + if (a.quoted && b.quoted) return a.name === b.name + const [quoted, bare] = a.quoted ? [a, b] : [b, a] + return quoted.name === bare.name.toLowerCase() || quoted.name === bare.name.toUpperCase() } const BARE_WORD = /[\p{L}_][\p{L}\p{N}_$]*/uy @@ -358,9 +370,14 @@ const CAPTURING = new Set(["set", "call", "macro"]) function blankJinja(text: string): string { const out: string[] = [] let kept = 0 + // Blanked to spaces, newlines too: a Jinja comment ends no SQL line comment. Whitespace control (`{{-`, `-}}`) + // takes the whitespace beside the tag with it, newlines included, as rendering does. const blank = (from: number, to: number) => { - out.push(text.substring(kept, from), text.substring(from, to).replace(/[^\n]/g, " ")) + if (text[from + 2] === "-") while (from > kept && /\s/.test(text[from - 1])) from-- + if (text[to - 3] === "-") while (to < text.length && /\s/.test(text[to])) to++ + out.push(text.substring(kept, from), " ".repeat(to - from)) kept = to + return to } let pos = 0 while (pos < text.length) { @@ -403,8 +420,14 @@ function blankJinja(text: string): string { scan = next + 1 continue } - const tagEnd = jinjaEnd(text, next) + let tagEnd = jinjaEnd(text, next) const tag = text[next + 1] === "%" ? /^\{%[-+]?\s*(end)?(\w+)/.exec(text.substring(next, Math.min(tagEnd, next + 200))) : null + if (tag && !tag[1] && tag[2] === "raw") { + // Text inside {% raw %} holds no tags: an `{% endset %}` there does not end the block. + const close = /\{%[-+]?\s*endraw\b[^%]*%\}/g + close.lastIndex = tagEnd + tagEnd = close.exec(text) ? close.lastIndex : text.length + } if (tag?.[2] === name) { const opensAgain = !tag[1] && !(name === "set" && /^\{%[-+]?\s*set\s+[\w.,\s()]+=(?!=)/.test(text.substring(next, tagEnd))) if (tag[1]) depth-- @@ -412,12 +435,10 @@ function blankJinja(text: string): string { } scan = tagEnd } - blank(at, scan) - pos = scan + pos = blank(at, scan) continue } - blank(at, end) - pos = end + pos = blank(at, end) } out.push(text.substring(kept)) return out.join("") @@ -447,8 +468,8 @@ function skipQuoted(text: string, pos: number): number { * For each CTE, the positions (in `ctes`) of the earlier CTEs of its own WITH clause that its body names * directly: a name outside strings, comments and Jinja (blanked first: `{{ }}`, `{% %}`, `{# #}`, and the SQL a * `{% set %}`, `{% call %}` or `{% macro %}` block captures, which is a string for dbt), not qualified by a dot - * (`schema . orders`) or inside a longer word, equal to the CTE's in any letter case and quoting. A name its own - * WITH defines again (`with orders as (…) select * from orders` inside the body) is that inner CTE, not a read. + * (`schema . orders`) or inside a longer word, that names the CTE (see `sameName`). Once a CTE the body's own + * WITH defines has ended, its name is that inner CTE, not a read of the outer one. * A column or alias spelled like an earlier CTE counts too, so a CTE may be listed as read when it is not; a CTE * named only through Jinja is not seen, nor one shadowed in a deeper subquery, nor one after a nested block * comment. A client wanting a CTE's own time subtracts the CTEs it reads directly or through another (the @@ -456,15 +477,18 @@ function skipQuoted(text: string, pos: number): number { */ export function cteReads(text: string, ctes: CteInfo[]): number[][] { const sql = blankJinja(text) - // Per WITH clause, the CTEs met so far by name: built up in source order, so each body is scanned once. - const seen = new Map>() + // Per WITH clause, the CTEs met so far, by the first letters of their names folded: built up in source order, + // so each body is scanned once and a reference is compared with few names. + const seen = new Map>() + const bucket = (name: string) => name.toLowerCase() return ctes.map((cte, position) => { - const earlier = seen.get(cte.withClauseStart) ?? new Map() + const earlier = seen.get(cte.withClauseStart) ?? new Map() seen.set(cte.withClauseStart, earlier) const reads = new Set() - // The names the body's own WITH defines shadow the outer CTEs of that name. + // The CTEs the body's own WITH defines: each shadows an outer CTE of its name once its definition has ended + // (in the later inner CTEs and the body's main query), not inside its own definition or before it. const body = sql.substring(cte.queryStart, cte.queryEnd) - const shadowed = new Set(earlier.size > 0 && /\bwith\b/i.test(body) ? detectCtes(body).map((inner) => referenceKey(stripColumnList(inner.name))) : []) + const inner = earlier.size > 0 && /\bwith\b/i.test(body) ? detectCtes(body).map((one) => ({ written: written(stripColumnList(one.name)), at: cte.queryStart + one.nameStart, from: cte.queryStart + one.queryEnd })) : [] let pos = cte.queryStart // The last character of SQL read before `pos`, past whitespace and comments: a dot qualifies the next name. let previous = "" @@ -500,13 +524,17 @@ export function cteReads(text: string, ctes: CteInfo[]): number[][] { pos++ continue } - const key = referenceKey(sql.substring(pos, end)) - if (previous !== "." && !/[\p{L}\p{N}_$]/u.test(sql[pos - 1] ?? " ") && !shadowed.has(key)) for (const i of earlier.get(key) ?? []) reads.add(i) + if (previous !== "." && !/[\p{L}\p{N}_$]/u.test(sql[pos - 1] ?? " ")) { + const ref = written(sql.substring(pos, end)) + // An inner CTE's own name where it is defined is no reference at all. + const shadowed = inner.some((one) => one.at === pos || (one.from < pos && sameName(one.written, ref))) + if (!shadowed) for (const one of earlier.get(bucket(ref.name)) ?? []) if (sameName(one.written, ref)) reads.add(one.position) + } previous = sql[end - 1] pos = end } - const key = referenceKey(stripColumnList(cte.name)) - earlier.set(key, [...(earlier.get(key) ?? []), position]) + const own = written(stripColumnList(cte.name)) + earlier.set(bucket(own.name), [...(earlier.get(bucket(own.name)) ?? []), { written: own, position }]) return [...reads].sort((a, b) => a - b) }) } diff --git a/packages/opencode/test/altimate/cte-detect.test.ts b/packages/opencode/test/altimate/cte-detect.test.ts index a198e3388d..98632a3005 100644 --- a/packages/opencode/test/altimate/cte-detect.test.ts +++ b/packages/opencode/test/altimate/cte-detect.test.ts @@ -1679,10 +1679,12 @@ select * from b` it("matches a name in any letter case and quoting, brackets included, and keeps to its own WITH clause", () => { const sql = `with "Big" as (select 1), other as (select 2), -q as (select * from big), r as (select * from "OTHER"), s as (select * from [Big]), t as (select * from \`other\`) +q as (select * from "Big"), r as (select * from "OTHER"), s as (select * from [Big]), t as (select * from \`other\`), u as (select * from big) select * from (with inner_cte as (select * from q) select * from inner_cte)` - expect(readsOf(sql)).toEqual({ '"Big"': [], other: [], q: ['"Big"'], r: ["other"], s: ['"Big"'], t: ["other"] }) - const bracketed = "with [Big Spenders] as (select 1), [we]]ird] as (select 2), b as (select * from [Big Spenders] join [WE]]IRD] on 1 = 1) select * from b" + // A bare name matches a quoted one only as its lower- or upper-case form: \`big\` is not "Big". + expect(readsOf(sql)).toEqual({ '"Big"': [], other: [], q: ['"Big"'], r: ["other"], s: ['"Big"'], t: ["other"], u: [] }) + expect(readsOf('with "Foo" as (select 1), foo as (select 2), keep as (select * from foo) select * from keep').keep).toEqual(["foo"]) + const bracketed = "with [Big Spenders] as (select 1), [we]]ird] as (select 2), b as (select * from [Big Spenders] join [we]]ird] on 1 = 1) select * from b" expect(readsOf(bracketed)).toEqual({ "[Big Spenders]": [], "[we]]ird]": [], b: ["[Big Spenders]", "[we]]ird]"] }) }) @@ -1728,6 +1730,9 @@ select * from c` `select "{{ var("orders") }}" from raw.customers`, `select 'prefix {# 'orders' #} suffix' as label`, `{% call statement('x') %}select * from orders{% endcall %} select 1 as n`, + `select 2 -- {#\n#} from orders`, + `select 2 -- {{ var('flag', 0) -}}\n from orders`, + `{% set q %}{% raw %}{% endset %}{% endraw %}\nselect * from orders\n{% endset %}\nselect 2`, ] for (const body of bodies) expect(readsOf(wrap(body)).b).toEqual([]) // Jinja output between real SQL keeps the SQL readable, and {% raw %} text is SQL as written. @@ -1736,6 +1741,9 @@ select * from c` }) it("a name the body's own WITH defines again is that inner CTE, not a read of the outer one", () => { + // Inside its own definition, or in an inner CTE before it, the name is still the outer CTE. + expect(readsOf("with orders as (select 1), b as (with x as (select * from orders), orders as (select 2) select * from x) select * from b").b).toEqual(["orders"]) + expect(readsOf("with orders as (select 1), b as (with orders as (select * from orders) select * from orders) select * from b").b).toEqual(["orders"]) const sql = "with orders as (select 1), b as (with orders as (select 2) select * from orders), c as (with x as (select 3) select * from x join orders on 1 = 1) select * from c" expect(readsOf(sql)).toEqual({ orders: [], b: [], c: ["orders"] }) }) From 79304cfb11ca2d28e8ad3206b1d490b6906ac792 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Fri, 20 Nov 2026 22:48:14 +0530 Subject: [PATCH 09/11] fix: a subquery's WITH shadows an outer CTE of the same name inside that subquery Co-Authored-By: Claude Opus 5.5 --- .../opencode/src/altimate/tools/cte-detect.ts | 50 ++++++++++++++++--- .../opencode/test/altimate/cte-detect.test.ts | 3 ++ 2 files changed, 45 insertions(+), 8 deletions(-) diff --git a/packages/opencode/src/altimate/tools/cte-detect.ts b/packages/opencode/src/altimate/tools/cte-detect.ts index 51cf0e9692..11340ff2d8 100644 --- a/packages/opencode/src/altimate/tools/cte-detect.ts +++ b/packages/opencode/src/altimate/tools/cte-detect.ts @@ -464,15 +464,50 @@ function skipQuoted(text: string, pos: number): number { return text.length } +/** + * The CTEs the WITH clauses between `start` and `end` define: the query's own, and every subquery's that starts + * with WITH. Each with where its name is written (`at`), where its definition ends (`from`) and where the query + * holding it ends (`to`): its name means it from `from` to `to`. + */ +function shadowsIn(sql: string, start: number, end: number): { written: Written; at: number; from: number; to: number }[] { + const out: { written: Written; at: number; from: number; to: number }[] = [] + const add = (from: number, to: number) => { + for (const one of detectCtes(sql.substring(from, to))) out.push({ written: written(stripColumnList(one.name)), at: from + one.nameStart, from: from + one.queryEnd, to }) + } + if (!/\bwith\b/i.test(sql.substring(start, end))) return out + add(start, end) + let pos = start + while (pos < end) { + const char = sql[pos] + if (char === "'" || char === '"' || char === "`") { + pos = skipQuoted(sql, pos) + continue + } + const skipped = skipLiteral(sql, pos) + if (skipped !== pos) { + pos = skipped + continue + } + if (char === "(") { + const first = skipTrivia(sql, pos + 1, false) + if (/^with\b/i.test(sql.substring(first, first + 5))) { + const close = findMatchingClosingParen(sql, pos) + if (close > pos) add(pos + 1, close) + } + } + pos++ + } + return out +} + /** * For each CTE, the positions (in `ctes`) of the earlier CTEs of its own WITH clause that its body names * directly: a name outside strings, comments and Jinja (blanked first: `{{ }}`, `{% %}`, `{# #}`, and the SQL a * `{% set %}`, `{% call %}` or `{% macro %}` block captures, which is a string for dbt), not qualified by a dot * (`schema . orders`) or inside a longer word, that names the CTE (see `sameName`). Once a CTE the body's own - * WITH defines has ended, its name is that inner CTE, not a read of the outer one. + * WITH or a subquery's WITH defines has ended, its name is that inner CTE there, not a read of the outer one. * A column or alias spelled like an earlier CTE counts too, so a CTE may be listed as read when it is not; a CTE - * named only through Jinja is not seen, nor one shadowed in a deeper subquery, nor one after a nested block - * comment. A client wanting a CTE's own time subtracts the CTEs it reads directly or through another (the + * named only through Jinja is not seen, nor one after a nested block comment. A client wanting a CTE's own time subtracts the CTEs it reads directly or through another (the * closure of `reads`), each once. */ export function cteReads(text: string, ctes: CteInfo[]): number[][] { @@ -485,10 +520,9 @@ export function cteReads(text: string, ctes: CteInfo[]): number[][] { const earlier = seen.get(cte.withClauseStart) ?? new Map() seen.set(cte.withClauseStart, earlier) const reads = new Set() - // The CTEs the body's own WITH defines: each shadows an outer CTE of its name once its definition has ended - // (in the later inner CTEs and the body's main query), not inside its own definition or before it. - const body = sql.substring(cte.queryStart, cte.queryEnd) - const inner = earlier.size > 0 && /\bwith\b/i.test(body) ? detectCtes(body).map((one) => ({ written: written(stripColumnList(one.name)), at: cte.queryStart + one.nameStart, from: cte.queryStart + one.queryEnd })) : [] + // The CTEs a WITH inside the body defines (its own, or a subquery's): each shadows an outer CTE of its name + // once its definition has ended, to the end of the query that holds it. + const inner = earlier.size > 0 ? shadowsIn(sql, cte.queryStart, cte.queryEnd) : [] let pos = cte.queryStart // The last character of SQL read before `pos`, past whitespace and comments: a dot qualifies the next name. let previous = "" @@ -527,7 +561,7 @@ export function cteReads(text: string, ctes: CteInfo[]): number[][] { if (previous !== "." && !/[\p{L}\p{N}_$]/u.test(sql[pos - 1] ?? " ")) { const ref = written(sql.substring(pos, end)) // An inner CTE's own name where it is defined is no reference at all. - const shadowed = inner.some((one) => one.at === pos || (one.from < pos && sameName(one.written, ref))) + const shadowed = inner.some((one) => one.at === pos || (one.from < pos && pos < one.to && sameName(one.written, ref))) if (!shadowed) for (const one of earlier.get(bucket(ref.name)) ?? []) if (sameName(one.written, ref)) reads.add(one.position) } previous = sql[end - 1] diff --git a/packages/opencode/test/altimate/cte-detect.test.ts b/packages/opencode/test/altimate/cte-detect.test.ts index 98632a3005..093059ed18 100644 --- a/packages/opencode/test/altimate/cte-detect.test.ts +++ b/packages/opencode/test/altimate/cte-detect.test.ts @@ -1744,6 +1744,9 @@ select * from c` // Inside its own definition, or in an inner CTE before it, the name is still the outer CTE. expect(readsOf("with orders as (select 1), b as (with x as (select * from orders), orders as (select 2) select * from x) select * from b").b).toEqual(["orders"]) expect(readsOf("with orders as (select 1), b as (with orders as (select * from orders) select * from orders) select * from b").b).toEqual(["orders"]) + // A subquery's WITH shadows too, only inside that subquery. + expect(readsOf("with orders as (select 1), b as (select * from (with orders as (select 2) select * from orders) s) select * from b").b).toEqual([]) + expect(readsOf("with orders as (select 1), b as (select * from (with orders as (select 2) select * from orders) s join orders on 1 = 1) select * from b").b).toEqual(["orders"]) const sql = "with orders as (select 1), b as (with orders as (select 2) select * from orders), c as (with x as (select 3) select * from x join orders on 1 = 1) select * from c" expect(readsOf(sql)).toEqual({ orders: [], b: [], c: ["orders"] }) }) From f7e8c8d75e4f947108abe7328d9fc994932f387f Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Fri, 20 Nov 2026 22:52:21 +0530 Subject: [PATCH 10/11] fix: an inner WITH RECURSIVE CTE shadows from its clause's start; inner shadows are indexed by name Co-Authored-By: Claude Opus 5.5 --- .../opencode/src/altimate/tools/cte-detect.ts | 17 ++++++++++++----- .../opencode/test/altimate/cte-detect.test.ts | 11 +++++++++++ 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/packages/opencode/src/altimate/tools/cte-detect.ts b/packages/opencode/src/altimate/tools/cte-detect.ts index 11340ff2d8..0f35b44811 100644 --- a/packages/opencode/src/altimate/tools/cte-detect.ts +++ b/packages/opencode/src/altimate/tools/cte-detect.ts @@ -472,7 +472,12 @@ function skipQuoted(text: string, pos: number): number { function shadowsIn(sql: string, start: number, end: number): { written: Written; at: number; from: number; to: number }[] { const out: { written: Written; at: number; from: number; to: number }[] = [] const add = (from: number, to: number) => { - for (const one of detectCtes(sql.substring(from, to))) out.push({ written: written(stripColumnList(one.name)), at: from + one.nameStart, from: from + one.queryEnd, to }) + const text = sql.substring(from, to) + for (const one of detectCtes(text)) { + // In a WITH RECURSIVE clause a CTE's name means that CTE across the whole clause, its own body included. + const recursive = /^with\s+recursive\b/i.test(text.substring(one.withClauseStart, one.withClauseStart + 40)) + out.push({ written: written(stripColumnList(one.name)), at: from + one.nameStart, from: from + (recursive ? one.withClauseStart : one.queryEnd), to }) + } } if (!/\bwith\b/i.test(sql.substring(start, end))) return out add(start, end) @@ -522,7 +527,8 @@ export function cteReads(text: string, ctes: CteInfo[]): number[][] { const reads = new Set() // The CTEs a WITH inside the body defines (its own, or a subquery's): each shadows an outer CTE of its name // once its definition has ended, to the end of the query that holds it. - const inner = earlier.size > 0 ? shadowsIn(sql, cte.queryStart, cte.queryEnd) : [] + const inner = new Map() + for (const one of earlier.size > 0 ? shadowsIn(sql, cte.queryStart, cte.queryEnd) : []) inner.set(bucket(one.written.name), [...(inner.get(bucket(one.written.name)) ?? []), one]) let pos = cte.queryStart // The last character of SQL read before `pos`, past whitespace and comments: a dot qualifies the next name. let previous = "" @@ -560,9 +566,10 @@ export function cteReads(text: string, ctes: CteInfo[]): number[][] { } if (previous !== "." && !/[\p{L}\p{N}_$]/u.test(sql[pos - 1] ?? " ")) { const ref = written(sql.substring(pos, end)) - // An inner CTE's own name where it is defined is no reference at all. - const shadowed = inner.some((one) => one.at === pos || (one.from < pos && pos < one.to && sameName(one.written, ref))) - if (!shadowed) for (const one of earlier.get(bucket(ref.name)) ?? []) if (sameName(one.written, ref)) reads.add(one.position) + const named = (earlier.get(bucket(ref.name)) ?? []).filter((one) => sameName(one.written, ref)) + // An inner CTE's own name where it is defined is no reference at all, and one in scope shadows the outer. + const shadowed = (inner.get(bucket(ref.name)) ?? []).some((one) => one.at === pos || (one.from < pos && pos < one.to && sameName(one.written, ref))) + if (!shadowed) for (const one of named) reads.add(one.position) } previous = sql[end - 1] pos = end diff --git a/packages/opencode/test/altimate/cte-detect.test.ts b/packages/opencode/test/altimate/cte-detect.test.ts index 093059ed18..382df04488 100644 --- a/packages/opencode/test/altimate/cte-detect.test.ts +++ b/packages/opencode/test/altimate/cte-detect.test.ts @@ -1744,6 +1744,9 @@ select * from c` // Inside its own definition, or in an inner CTE before it, the name is still the outer CTE. expect(readsOf("with orders as (select 1), b as (with x as (select * from orders), orders as (select 2) select * from x) select * from b").b).toEqual(["orders"]) expect(readsOf("with orders as (select 1), b as (with orders as (select * from orders) select * from orders) select * from b").b).toEqual(["orders"]) + // In WITH RECURSIVE the inner CTE's own body already means it. + expect(readsOf("with dates as (select 999 as n), result as (with recursive dates(n) as (select 1 union all select n + 1 from dates where n < 3) select * from dates) select * from result").result).toEqual([]) + expect(readsOf("with dates as (select 999 as n), result as (with dates2(n) as (select n from dates) select * from dates2) select * from result").result).toEqual(["dates"]) // A subquery's WITH shadows too, only inside that subquery. expect(readsOf("with orders as (select 1), b as (select * from (with orders as (select 2) select * from orders) s) select * from b").b).toEqual([]) expect(readsOf("with orders as (select 1), b as (select * from (with orders as (select 2) select * from orders) s join orders on 1 = 1) select * from b").b).toEqual(["orders"]) @@ -1751,6 +1754,14 @@ select * from c` expect(readsOf(sql)).toEqual({ orders: [], b: [], c: ["orders"] }) }) + it("a body with thousands of inner CTEs is read in near-linear time", () => { + const inner = Array.from({ length: 10000 }, (_, i) => `i${i} as (select ${i} as x)`).join(",\n") + const sql = `with orders as (select 1), b as (with ${inner} select * from i9999 join orders on 1 = 1) select * from b` + const started = performance.now() + expect(readsOf(sql).b).toEqual(["orders"]) + expect(performance.now() - started).toBeLessThan(3000) + }) + it("lists the reads of a model with thousands of CTEs in linear time", () => { const parts = Array.from({ length: 10000 }, (_, i) => `c${i} as (select ${i} as x${i > 0 && i % 2 === 0 ? ` from c${i - 1}` : ""})`) const sql = `with ${parts.join(",\n")}\nselect * from c9999` From 26df3ff05b658a80bbb033b90709b80b56f09df2 Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Fri, 20 Nov 2026 23:04:54 +0530 Subject: [PATCH 11/11] fix: WITH RECURSIVE is recognised across comments and any whitespace Co-Authored-By: Claude Opus 5.5 --- packages/opencode/src/altimate/tools/cte-detect.ts | 3 ++- packages/opencode/test/altimate/cte-detect.test.ts | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/opencode/src/altimate/tools/cte-detect.ts b/packages/opencode/src/altimate/tools/cte-detect.ts index 0f35b44811..01eeb4e887 100644 --- a/packages/opencode/src/altimate/tools/cte-detect.ts +++ b/packages/opencode/src/altimate/tools/cte-detect.ts @@ -475,7 +475,8 @@ function shadowsIn(sql: string, start: number, end: number): { written: Written; const text = sql.substring(from, to) for (const one of detectCtes(text)) { // In a WITH RECURSIVE clause a CTE's name means that CTE across the whole clause, its own body included. - const recursive = /^with\s+recursive\b/i.test(text.substring(one.withClauseStart, one.withClauseStart + 40)) + const afterWith = skipTrivia(text, one.withClauseStart + 4, true) + const recursive = /^recursive\b/i.test(text.substring(afterWith, afterWith + 10)) out.push({ written: written(stripColumnList(one.name)), at: from + one.nameStart, from: from + (recursive ? one.withClauseStart : one.queryEnd), to }) } } diff --git a/packages/opencode/test/altimate/cte-detect.test.ts b/packages/opencode/test/altimate/cte-detect.test.ts index 382df04488..c267307f65 100644 --- a/packages/opencode/test/altimate/cte-detect.test.ts +++ b/packages/opencode/test/altimate/cte-detect.test.ts @@ -1747,6 +1747,8 @@ select * from c` // In WITH RECURSIVE the inner CTE's own body already means it. expect(readsOf("with dates as (select 999 as n), result as (with recursive dates(n) as (select 1 union all select n + 1 from dates where n < 3) select * from dates) select * from result").result).toEqual([]) expect(readsOf("with dates as (select 999 as n), result as (with dates2(n) as (select n from dates) select * from dates2) select * from result").result).toEqual(["dates"]) + for (const gap of [" /* generate calendar */ ", " -- calendar\n ", " ".repeat(60)]) + expect(readsOf(`with dates as (select 999 as n), result as (with${gap}recursive dates(n) as (select 1 union all select n + 1 from dates where n < 3) select * from dates) select * from result`).result).toEqual([]) // A subquery's WITH shadows too, only inside that subquery. expect(readsOf("with orders as (select 1), b as (select * from (with orders as (select 2) select * from orders) s) select * from b").b).toEqual([]) expect(readsOf("with orders as (select 1), b as (select * from (with orders as (select 2) select * from orders) s join orders on 1 = 1) select * from b").b).toEqual(["orders"])