From ef50c07edc33d789ba950461a2c223653d15b71e Mon Sep 17 00:00:00 2001 From: Harish Kumar Date: Fri, 20 Nov 2026 12:00:05 +0530 Subject: [PATCH 01/37] =?UTF-8?q?fix:=20hands-on=20findings=20=E2=80=94=20?= =?UTF-8?q?real=20reach=20in=20the=20task=20band,=20hardcoded=20refs,=20sh?= =?UTF-8?q?ell=20edits,=20older=20doc=20gaps,=20pane=20follow=20and=20fit,?= =?UTF-8?q?=20package=20repos,=20value=20changes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - The task's manifest copies sit in target/ itself: altimate-code reads a seed's header from the folder above the manifest's, so a copy one folder deeper traced nothing through a select * and the band said 0 columns. A reach of 0 beside readers that may pass the column on unnamed is "not traced", never "every downstream model was read". - Compile and validate flags a model, seed or source named by its table instead of ref() or source(). - A model a shell command rewrites gets the per-edit impact note. - Only columns an edit made undocumented are its gaps; older ones are said once, quietly. - The lineage pane follows the edited model as the edit starts; a graph wider than the pane opens as its ends, with its dashboards named. - A package repo is compiled and validated from the project below it that installs it (integration_tests); missing packages are a one-line note. - With no select item changed, the band names the columns whose values change and their reach, else says values may change. Co-Authored-By: Claude Opus 5.5 --- plugins/altimate-code/CHANGELOG.md | 8 + plugins/altimate-code/hooks/dbt-mods-core.ts | 68 +++++- plugins/altimate-code/hooks/dbt-mods.test.tsx | 129 +++++++++++ plugins/altimate-code/hooks/dbt-mods.tsx | 203 ++++++++++++++---- plugins/altimate-code/hooks/features/index.ts | 3 +- .../altimate-code/hooks/impact-report.test.ts | 50 ++++- plugins/altimate-code/hooks/impact-report.ts | 88 +++++++- plugins/altimate-code/hooks/lineage.test.tsx | 106 ++++++++- plugins/altimate-code/hooks/lineage.tsx | 119 +++++++++- .../altimate-code/hooks/task-impact.test.tsx | 19 +- plugins/altimate-code/hooks/task-impact.tsx | 48 ++++- plugins/altimate-code/types/index.d.ts | 17 +- 12 files changed, 777 insertions(+), 81 deletions(-) diff --git a/plugins/altimate-code/CHANGELOG.md b/plugins/altimate-code/CHANGELOG.md index 14d356a1..80328f90 100644 --- a/plugins/altimate-code/CHANGELOG.md +++ b/plugins/altimate-code/CHANGELOG.md @@ -12,6 +12,14 @@ All notable changes to this plugin will be documented in this file. - When the Query results or lineage pane sits above the prompt (a terminal under 110 columns, or not fullscreen), Claude Code gives it the same height whatever size it asks for, so the Min / Default / Max row, which did nothing there, is replaced by a line saying so. - `--defer --state …`, `--favor-state` and `--target …` that `/defer` and `/target` add to Claude's dbt commands are no longer invisible: Claude's result says what the command ran as, and a line above the prompt shows it until the next prompt. - **Resolve** on a thread the lagging comment list still shows after it was resolved elsewhere is done (the thread is hidden) instead of answering "not an open thread"; a reply to such a thread says it is no longer open and reads the list again. +- The end-of-task band no longer says a changed column "reaches 0 columns in 0 models" on a compiled project when its readers `select *` from it (jaffle_shop's `stg_payments.amount`, read by `orders` and `customers`). The task's copy of the manifest sat one folder below `target/`, and altimate-code reads a seed's CSV header from the folder above the manifest's, so nothing was traced through a `select *` from a seed. The copies now sit in `target/` itself (`.altimate-manifest-before..json`, `.altimate-task-manifest..json`), and the band and `/lineage-report` trace what the per-edit note traces (6 columns in 2 models there). A reach of 0 beside downstream models that may pass the column on without naming it (`select *`, `NATURAL JOIN`, columns named at run time) is now "not traced: N downstream models may read it without naming it (select *)", with a lower-bound footer instead of "✓ Every downstream model was read". +- When no output column changed but values did (a column computed in a Jinja `{% set %}` block the select reads, as in Tuva's `core__medical_claim`), the band says "values change in paid_amount → reaches N columns in M models", tracing the names the change rewrote that the manifest has as the model's columns. When none can be named it says "values may change downstream; the model-level count stands", never a count of 0. +- Compile and validate flags a model, seed, snapshot or source named by its table instead of `ref()` or `source()` (`main.stg_orders`, `"db"."main"."stg_orders"`, `shop.customers`): "hardcoded main.stg_orders: use {{ ref('stg_orders') }} — lineage can't follow it", in the band, the dbt check pane and Claude's note, which asks Claude to put the `ref()` back unless the person asked for the table. +- Compile and validate warns only about the columns an edit left undocumented (new or renamed ones). Columns that were undocumented before the edit (the last check returned them, or the SQL from before the edit named them) are said once a conversation, quietly ("2 undocumented before this edit"), and do not make the check a warning. +- A package repo tested from a project below it (Tuva: `integration_tests/` installs the package with `local: ../`): when the package's own packages are not installed, compile and validate runs dbt from that project (selecting the model by package and name), says "ran from integration_tests", and the doctor counts the packages installed there. A project whose packages are not installed and that nothing installs gets a one-line "not checked: dbt's packages are not installed…" note instead of a failure. +- With `show_edit_impact` on, a model Claude changes through a shell command (`python3 …`, `sed -i`, `cat > …`) gets the same per-edit impact note as an Edit: the model files under the project's model folders modified since the command started are found after it ends, and their change from the last commit is traced. The note goes to Claude's context too. +- The lineage pane moves to the model Claude is editing as the edit starts (before its impact lookup and the edit itself), and to the model a writing shell command names as it starts, instead of after the edit lands. +- At its default size the lineage pane draws a graph wider than the pane as its ends only (which fit, dashboards included), saying so; `v` shows the whole graph. Dashboards (exposures) downstream that the drawing leaves out are named under it. ## [1.8.3] — 2026-11-19 diff --git a/plugins/altimate-code/hooks/dbt-mods-core.ts b/plugins/altimate-code/hooks/dbt-mods-core.ts index 996c1a90..153b979b 100644 --- a/plugins/altimate-code/hooks/dbt-mods-core.ts +++ b/plugins/altimate-code/hooks/dbt-mods-core.ts @@ -3,7 +3,7 @@ // each reading is tested on dbt's real output. import type { DbtFailure, DoctorCheck, ModelContext, QueryResult } from '../types' import { isDbtSubcommand, parseDbtCommand, type DbtInvocation } from './dbt-command' -import { withoutComment } from './lineage-graph' +import { selectItems, withoutComment } from './lineage-graph' const ANSI = /\u001b\[[0-9;]*m/g @@ -814,6 +814,72 @@ export function columnDrift(documented: string[], returned: string[], quoted: st } } +/** SQL with its comments gone and its string literals emptied, so neither is read as code. */ +function sqlCode(sql: string): string { + return sql.replace(/'(?:[^']|'')*'|--[^\n]*|\/\*[\s\S]*?\*\//g, one => (one.startsWith("'") ? "''" : ' ')) +} + +// An identifier part: double-quoted, backquoted, bracketed or bare. +const PART = String.raw`(?:"[^"]+"|` + '`[^`]+`' + String.raw`|\[[^\]]+\]|[A-Za-z_][\w$]*)` +const RELATION_AFTER = new RegExp(String.raw`\b(?:from|join)\s+(${PART}(?:\s*\.\s*${PART}){0,2})`, 'gi') +const unquote = (part: string) => part.trim().replace(/^["`\[]|["`\]]$/g, '').toLowerCase() + +/** + * Relations a model's SQL names directly after FROM or JOIN that are a dbt model, seed, snapshot or source of the + * manifest: each should be a `ref()` or `source()`, or dbt (and column lineage) cannot follow it. Jinja, comments, + * string literals and the model's own CTEs are left out; the model itself is never flagged. + */ +export function hardcodedRelations(sql: string, manifest: any, selfId?: string | null): { relation: string; use: string }[] { + const text = sqlCode(sql.replace(/\{\{[\s\S]*?\}\}|\{%[\s\S]*?%\}|\{#[\s\S]*?#\}/g, ' __jinja__ ')) + const ctes = new Set([...text.matchAll(/\b([A-Za-z_]\w*)\s+as\s*(?:not\s+)?(?:materialized\s+)?\(/gi)].map(one => one[1]!.toLowerCase())) + // Every relation the manifest knows, by its full, schema-qualified and bare spellings. + const known = new Map() + const add = (id: string, node: any, use: string) => { + if (id === selfId) return + const parts = typeof node?.relation_name === 'string' ? node.relation_name.split('.').map(unquote) : [node?.database, node?.schema, node?.alias ?? node?.identifier ?? node?.name].filter(Boolean).map((one: string) => String(one).toLowerCase()) + if (!parts.length) return + for (let i = 0; i < parts.length; i++) known.set(parts.slice(i).join('.'), { relation: parts.slice(i).join('.'), use }) + } + for (const [id, node] of Object.entries(manifest?.nodes ?? {})) { + if (['model', 'seed', 'snapshot'].includes(node?.resource_type) && typeof node?.name === 'string') add(id, node, `{{ ref('${node.name}') }}`) + } + for (const [id, node] of Object.entries(manifest?.sources ?? {})) { + if (typeof node?.source_name === 'string' && typeof node?.name === 'string') add(id, node, `{{ source('${node.source_name}', '${node.name}') }}`) + } + const found = new Map() + for (const match of text.matchAll(RELATION_AFTER)) { + const written = match[1]! + const parts = written.split(/\s*\.\s*/).map(unquote) + if (parts.includes('__jinja__') || (parts.length === 1 && ctes.has(parts[0]!))) continue + const hit = known.get(parts.join('.')) + if (hit && !found.has(hit.use)) found.set(hit.use, { relation: written.replace(/\s+/g, ''), use: hit.use }) + } + return [...found.values()] +} + +/** + * Which undocumented columns this edit made, and which were undocumented already. A column is the edit's when the SQL + * after the edit names it in a select list and the SQL before did not (a new or renamed column), and the last check + * of the model did not return it. Any other (named before, or not named by the edit at all: a `select *`, a macro, a + * package's passthrough columns) was there before. With neither the SQL from before nor an earlier check, or no + * change seen between the two texts, every column counts as the edit's. + */ +export function undocumentedByEdit(undocumented: string[], before: string | null, after: string | null, earlier: string[] | null): { fresh: string[]; old: string[] } { + if (before === null && earlier === null) return { fresh: undocumented, old: [] } + // The file reads the same as before the edit: what the edit changed is not seen, so nothing is set aside. + if (earlier === null && before !== null && after !== null && before === after) return { fresh: undocumented, old: [] } + const was = before === null ? null : selectItems(sqlCode(before)) + const now = after === null ? null : selectItems(sqlCode(after)) + const returned = new Set((earlier ?? []).map(one => one.toLowerCase())) + const old = (name: string) => { + const key = name.toLowerCase() + if (returned.has(key)) return true + if (!was || !now) return false + return was.has(key) || !now.has(key) + } + return { fresh: undocumented.filter(one => !old(one)), old: undocumented.filter(old) } +} + // What the history may keep: the plugin store takes 4 MiB of JSON in all, the history and bookmarks among // other keys, so a cell, a query and the whole history are bounded, in UTF-8 bytes as the store counts them. const SAVED_CELL = 200 diff --git a/plugins/altimate-code/hooks/dbt-mods.test.tsx b/plugins/altimate-code/hooks/dbt-mods.test.tsx index c5ba3339..84712a51 100644 --- a/plugins/altimate-code/hooks/dbt-mods.test.tsx +++ b/plugins/altimate-code/hooks/dbt-mods.test.tsx @@ -2,6 +2,8 @@ import { expect, mock, test as kitTest } from 'claude-code/testing' import { columnDrift, + hardcodedRelations, + undocumentedByEdit, columnWidths, dbtCommand, dbtCommands, @@ -681,6 +683,133 @@ test('an edit is compiled and run against the warehouse; column drift from schem await band.unmount() }) +kitTest('an undocumented column is the edit\'s only when the edit names it and the SQL before did not', () => { + const before = 'select id, amount / 100 as amount from {{ ref("raw") }}' + // A rewritten expression keeps its column: not a new gap. A column from a macro or `*` is not the edit's either. + expect(undocumentedByEdit(['amount', 'x_passthrough'], before, 'select id, amount from {{ ref("raw") }}', null)).toEqual({ fresh: [], old: ['amount', 'x_passthrough'] }) + // A new or renamed column is. + expect(undocumentedByEdit(['amount_cents'], before, 'select id, amount as amount_cents from {{ ref("raw") }}', null)).toEqual({ fresh: ['amount_cents'], old: [] }) + // An earlier check that returned it: there before. + expect(undocumentedByEdit(['amount_cents'], null, null, ['id', 'amount_cents'])).toEqual({ fresh: [], old: ['amount_cents'] }) + // A check after another (no SQL from before), of a column the last check did not return: new. + expect(undocumentedByEdit(['amount_cents'], null, null, ['id'])).toEqual({ fresh: ['amount_cents'], old: [] }) + // Nothing to compare with: every column is the edit's, as before. + expect(undocumentedByEdit(['amount'], null, null, null)).toEqual({ fresh: ['amount'], old: [] }) +}) + +test('only the columns an edit left undocumented are its gaps; ones undocumented before it are said once, quietly', async ($, on) => { + let release!: () => void + const hold = new Promise(resolve => (release = resolve)) + const show: Options['show'] = { rows: [{ order_id: 1, amount: 10, status: 'placed', amount_cents: 1000 }] } + const { clock, files } = await start($, on, { show, holdEdit: hold, files: { 'models/orders.sql': 'select order_id, amount, status from {{ ref("stg_orders") }}' } }) + const call = $.tool.call({ tool: 'Edit', file_path: ORDERS, old_string: 'status from', new_string: 'status, amount * 100 as amount_cents from' }) + await clock.settle() + files['models/orders.sql'] = 'select order_id, amount, status, amount * 100 as amount_cents from {{ ref("stg_orders") }}' + release() + const result: any = await call + await clock.settle() + const note = result.context?.find((one: string) => one.startsWith('Altimate compile-and-validate')) + expect(note).toContain('- returned but not documented in schema.yml: amount_cents\n') + expect(note).toContain("- already undocumented before this edit (not this edit's doing; nothing to fix for it): status") + const band = await $.ui.mount({ ...BAND, surface: 'terminal' }) + expect(await band.find({ type: 'Text', text: /undocumented amount_cents · 1 undocumented before this edit/ })).toBeDefined() + await band.unmount() + const pane = await $.ui.mount({ ...CHECK, surface: 'terminal' }) + expect(await pane.find({ type: 'Text', text: ' not documented, from before this edit' })).toBeDefined() + await pane.unmount() + + // The next edit adds no column: nothing is the edit's gap, and the older ones are not said again. + show.rows = [{ order_id: 1, amount: 10, status: 'placed' }] + files['models/orders.sql'] = 'select order_id, amount, status from {{ ref("stg_orders") }}' + const again: any = await $.tool.call({ tool: 'Edit', file_path: ORDERS, old_string: 'amount, status', new_string: 'amount, status' }) + await clock.settle() + const second = again.context?.find((one: string) => one.startsWith('Altimate compile-and-validate')) + expect(second).not.toContain('not documented') + expect(second).not.toContain('already undocumented') + const after = await $.ui.mount({ ...BAND, surface: 'terminal' }) + expect(await after.find({ type: 'Text', text: /no new gaps with schema\.yml/ })).toBeDefined() + expect(await after.find({ type: 'Text', text: /undocumented before this edit/ })).toBeUndefined() + await after.unmount() +}) + +test('a package tested from integration_tests is compiled and validated from there, where its packages are installed', async ($, on) => { + const { clock, runs, cwds, toasts } = await start($, on, { + pathDbtPython: '/usr/bin/python3', + show: { rows: [{ order_id: 1, amount: 10 }], node: 'orders' }, + files: { + 'packages.yml': 'packages:\n - package: dbt-labs/dbt_utils\n version: 1.0.0\n', + 'integration_tests/dbt_project.yml': "name: 'shop_tests'\n", + 'integration_tests/packages.yml': 'packages:\n - local: ../\n - package: dbt-labs/dbt_utils\n version: 1.0.0\n', + 'integration_tests/dbt_packages/shop/dbt_project.yml': "name: 'shop'\n", + // The package's own packages installed too: its models are still run where its tests enable them. + 'dbt_packages/dbt_utils/dbt_project.yml': "name: 'dbt_utils'\n", + }, + }) + const result: any = await $.tool.call({ tool: 'Edit', file_path: ORDERS, old_string: 'select *', new_string: 'select *' }) + await clock.settle() + const at = runs.findIndex(argv => argv[1] === 'show') + // `path:` would read from integration_tests: the package and the model's name select it there. + expect(runs[at]).toEqual(['dbt', 'show', '--select', 'package:shop,orders', '--indirect-selection', 'empty', '--limit', '1', '--output', 'json', '--log-format', 'json']) + expect(cwds[at]).toBe(`${ROOT}/integration_tests`) + const note = result.context?.find((one: string) => one.startsWith('Altimate compile-and-validate')) + expect(note).toContain('- compiled and ran from integration_tests (the project that installs this package); the warehouse returned 2 columns.') + expect(toasts).toContain('✓ orders · compiled and validated against the warehouse') + const band = await $.ui.mount({ ...BAND, surface: 'terminal' }) + expect(await band.find({ type: 'Text', text: /match schema\.yml · ran from integration_tests/ })).toBeDefined() + await band.unmount() +}) + +test('a project whose packages are not installed gets a one-line note, not a failure, when nothing installs it elsewhere', async ($, on) => { + const { clock, toasts } = await start($, on, { + show: { error: 'dbt expects 1 package(s) based on packages specified in packages.yml, but found only 0 package(s) installed in dbt_packages. Run "dbt deps"' }, + files: { 'packages.yml': 'packages:\n - package: dbt-labs/dbt_utils\n version: 1.0.0\n' }, + }) + const result: any = await $.tool.call({ tool: 'Edit', file_path: ORDERS, old_string: 'select *', new_string: 'select *' }) + await clock.settle() + const note = result.context?.find((one: string) => one.startsWith('Altimate compile-and-validate')) + expect(note).not.toContain('FAILED') + expect(note).toContain("- not checked: dbt's packages are not installed in this project, so dbt could not compile it (run `dbt deps` there, or run dbt from the project that installs this package).") + expect(toasts.some(one => one.startsWith('✖'))).toBe(false) + const band = await $.ui.mount({ ...BAND, surface: 'terminal' }) + expect(await band.find({ type: 'Text', text: /^◦ Compile & validate / })).toBeDefined() + await band.unmount() +}) + +kitTest('a model, seed or source named by its table instead of ref() or source() is found, in any spelling; ref(), CTEs, comments and strings are not', () => { + const manifest = { + nodes: { + 'model.jaffle_shop.stg_orders': { resource_type: 'model', name: 'stg_orders', relation_name: '"jaffle_shop"."main"."stg_orders"' }, + 'model.jaffle_shop.orders': { resource_type: 'model', name: 'orders', relation_name: '"jaffle_shop"."main"."orders"' }, + 'seed.jaffle_shop.raw_payments': { resource_type: 'seed', name: 'raw_payments', relation_name: '"jaffle_shop"."main"."raw_payments"' }, + }, + sources: { 'source.jaffle_shop.shop.customers': { source_name: 'shop', name: 'customers', relation_name: '"raw"."shop"."customers"' } }, + } + const self = 'model.jaffle_shop.orders' + expect(hardcodedRelations("with orders as (select * from {{ ref('stg_orders') }}) select * from orders", manifest, self)).toEqual([]) + expect(hardcodedRelations('with orders as (select * from main.stg_orders) select * from orders', manifest, self)).toEqual([{ relation: 'main.stg_orders', use: "{{ ref('stg_orders') }}" }]) + expect(hardcodedRelations('select * from "jaffle_shop"."main"."stg_orders" o join raw_payments p using (order_id)', manifest, self)).toEqual([ + { relation: '"jaffle_shop"."main"."stg_orders"', use: "{{ ref('stg_orders') }}" }, + { relation: 'raw_payments', use: "{{ ref('raw_payments') }}" }, + ]) + expect(hardcodedRelations('select * from shop.customers', manifest, self)).toEqual([{ relation: 'shop.customers', use: "{{ source('shop', 'customers') }}" }]) + // Not code: a comment, a string; and the model itself. + expect(hardcodedRelations("-- from main.stg_orders\nselect 'from main.stg_orders' as note from {{ ref('stg_orders') }} /* join raw_payments */", manifest, self)).toEqual([]) + expect(hardcodedRelations('select * from main.orders', manifest, self)).toEqual([]) +}) + +test('an edit that swaps a ref() for the table it builds is flagged: lineage cannot follow it', async ($, on) => { + const { clock, toasts } = await start($, on, { files: { 'models/orders.sql': 'select order_id, amount from stg_orders' } }) + const result: any = await $.tool.call({ tool: 'Edit', file_path: ORDERS, old_string: 'from stg_orders', new_string: 'from stg_orders' }) + await clock.settle() + const note = result.context?.find((one: string) => one.startsWith('Altimate compile-and-validate')) + expect(note).toContain("- hardcoded stg_orders: use {{ ref('stg_orders') }}. dbt does not know this model reads it") + expect(note).toContain('Put the ref() or source() back unless the user asked for the table by name, and tell the user.') + expect(toasts).toContain('⚠ orders · compiled and validated against the warehouse') + const band = await $.ui.mount({ ...BAND, surface: 'terminal' }) + expect(await band.find({ type: 'Text', text: /hardcoded stg_orders: use \{\{ ref\('stg_orders'\) \}\} — lineage can't follow it/ })).toBeDefined() + await band.unmount() +}) + test('a dbt command that only rewrites the manifest starts no check; a change to the model\'s schema.yml checks it again', async ($, on) => { const { clock, runs, mtimes, setBash } = await start($, on, { show: { rows: [{ order_id: 1, order_total: 10 }] } }) await $.tool.call({ tool: 'Edit', file_path: ORDERS, old_string: 'as amount', new_string: 'as order_total' }) diff --git a/plugins/altimate-code/hooks/dbt-mods.tsx b/plugins/altimate-code/hooks/dbt-mods.tsx index c838bd8e..de60f6fa 100644 --- a/plugins/altimate-code/hooks/dbt-mods.tsx +++ b/plugins/altimate-code/hooks/dbt-mods.tsx @@ -11,6 +11,8 @@ import { loadPaneSizesOnce, noteTerminal, paneTerminalColumns, paneRequest, size import { cellText, columnDrift, + hardcodedRelations, + undocumentedByEdit, columnWidths, dbtCommands, type DbtCommand, @@ -102,6 +104,10 @@ let doctorNoteDue: string | null = null const queues = new Map>() // The newest check per file: an older one of the same file landing late does not overwrite it. const checkSeq = new Map() +// The columns each model's last check got back, by project and file, less the gaps an edit made: a column there is not a new gap. +const lastReturned = new Map() +// Models whose columns undocumented from before an edit were said once in this conversation: not said again. +const oldGapsSaid = new Set() let seq = 0 // Bumped when a conversation ends (/clear): work started in the last one lands nowhere in this one. let generation = 0 @@ -360,6 +366,73 @@ async function projectName($: EngineInterface, root: string): Promise ''))?.[1] ?? null } +/** What a project's packages.yml (or a dependencies.yml with packages) asks for, and whether anything is installed; null when it declares none. */ +async function packagesOf($: EngineInterface, root: string): Promise<{ file: string; installPath: string; installed: boolean } | null> { + // A dependencies.yml that declares only `projects:` has no packages to install. + const file = (await $.fs.exists(`${root}/packages.yml`).catch(() => false)) + ? 'packages.yml' + : /^packages:/m.test(await $.fs.read(`${root}/dependencies.yml`).catch(() => '')) + ? 'dependencies.yml' + : null + if (!file) return null + // `packages-install-path` in dbt_project.yml, `dbt_packages` when absent. + const configured = /^packages-install-path:\s*['"]?([^'"#\s]+)/m.exec(await $.fs.read(`${root}/dbt_project.yml`).catch(() => ''))?.[1] + // A Jinja value is dbt's to render: dbt's default stands in. + const installPath = configured && !configured.includes('{{') ? configured : 'dbt_packages' + const installed = (await $.fs.list(installPath.startsWith('/') ? installPath : `${root}/${installPath}`).catch(() => [])).length > 0 + return { file, installPath, installed } +} + +// The project below a package's folder that installs it, by package folder: found once a session. +const runners = new Map>() + +/** + * A dbt project in a folder below `root`, a level or two down (a package's `integration_tests`), whose + * packages.yml or dependencies.yml installs `root` itself by a local path (`local: ../`). That is where dbt is run + * for a package repo: its packages are installed there, not in the package's own folder. + */ +function runnerOf($: EngineInterface, root: string): Promise { + const known = runners.get(root) + if (known) return known + const finding = (async () => { + let level = [root] + for (let depth = 0; depth < 2; depth++) { + const next: string[] = [] + for (const dir of level) { + for (const entry of await $.fs.list(dir).catch(() => [])) { + if (entry.kind !== 'dir' || entry.name.startsWith('.') || ['node_modules', 'target', 'dbt_packages', 'dbt_modules', 'logs', 'venv'].includes(entry.name)) continue + const sub = `${dir}/${entry.name}` + next.push(sub) + if (!(await $.fs.exists(`${sub}/dbt_project.yml`).catch(() => false))) continue + const declared = `${await $.fs.read(`${sub}/packages.yml`).catch(() => '')}\n${await $.fs.read(`${sub}/dependencies.yml`).catch(() => '')}` + for (const one of declared.matchAll(/^\s*-?\s*local:\s*['"]?([^'"#\n]+?)['"]?\s*(?:#.*)?$/gm)) { + const path = one[1]!.trim() + if (normalize(path.startsWith('/') ? path : `${sub}/${path}`) === root) return sub + } + } + } + level = next.slice(0, 200) + } + return null + })() + runners.set(root, finding) + return finding +} + +/** + * Where the plugin runs dbt for a model of `root`: the project below it that installs it (a package tested from + * integration_tests, whose vars enable its models and whose packages are installed there), else the project itself. + * The project itself is kept when only its own packages are installed. + */ +async function runPlaceOf($: EngineInterface, root: string): Promise { + const runner = await runnerOf($, root) + if (!runner) return root + const theirs = await packagesOf($, runner) + if (!theirs || theirs.installed) return runner + const own = await packagesOf($, root) + return own && own.installed ? root : runner +} + /** The project's artifact folder (`target-path` in dbt_project.yml): absolute, or under the project. */ async function targetOf($: EngineInterface, root: string): Promise { const path = targetPath(await $.fs.read(`${root}/dbt_project.yml`).catch(() => '')) @@ -396,7 +469,7 @@ async function pythonFor($: EngineInterface, root: string): Promise { * manifest over 4 MiB (more than $.fs.read takes) is read for one file's node only, `rel`, through the * project's Python, as the lineage half reads one through altimate-code. */ -async function readManifest($: EngineInterface, root: string, rel?: string, dir?: string): Promise<{ manifest: any; mtimeMs: number } | null> { +async function readManifest($: EngineInterface, root: string, rel?: string, dir?: string, pkg?: string | null): Promise<{ manifest: any; mtimeMs: number } | null> { const path = `${dir ?? (await targetOf($, root))}/manifest.json` const stat = await $.fs.stat(path).catch(() => null) if (!stat || stat.kind !== 'file') return null @@ -413,7 +486,7 @@ async function readManifest($: EngineInterface, root: string, rel?: string, dir? if (cached) return { manifest: await cached, mtimeMs: stat.mtimeMs } const slicing = (async () => { const python = await pythonFor($, root) - const project = (await projectName($, root)) ?? '' + const project = (pkg !== undefined ? pkg : await projectName($, root)) ?? '' const run = await $.process.run([python, '-c', MANIFEST_SLICE, path, rel, project], { cwd: root, timeoutMs: 60_000 }) if (run.exitCode !== 0 || run.isStdoutTruncated) throw new Error('manifest slice failed') return JSON.parse(run.stdout) @@ -448,7 +521,7 @@ function addContext(result: T, note: string | const schemaOf = (node: any): string | null => (typeof node?.patch_path === 'string' ? node.patch_path.replace(/^[^:]*:\/\//, '') : null) /** Compiles the model and runs it against the warehouse (one row), then holds its columns to schema.yml. */ -async function compileAndValidate($: EngineInterface, root: string, file: string, mineGeneration = generation): Promise<{ view: CheckView; note: string; current: () => boolean; newest: () => boolean }> { +async function compileAndValidate($: EngineInterface, root: string, file: string, mineGeneration = generation, sqlBefore: string | null = null): Promise<{ view: CheckView; note: string; current: () => boolean; newest: () => boolean }> { const rel = relative(root, file) // A check from a conversation that has ended takes no place in the order: it can neither land nor push a // current check out. Within this conversation, `current` is the newest check of its file (an older one of the @@ -462,11 +535,16 @@ async function compileAndValidate($: EngineInterface, root: string, file: string const newest = () => seq === mine && generation === mineGeneration // Changes made from here on are newer than this check: the next recheck sees them. const started = await $.clock.now() + // Where dbt runs: the project, or for a package repo whose own packages are not installed, the project below it + // that installs it (integration_tests), where its models are this package's. + const place = await runPlaceOf($, root) + const viaRunner = place !== root + const pkg = await projectName($, root) // The model's own name, from the manifest when it knows the file (shown to the person; dbt selects by path). - const before = await readManifest($, root, rel) - const known = before ? nodeForFile(before.manifest, rel, await projectName($, root)) : null + const before = await readManifest($, place, rel, undefined, pkg) + const known = before ? nodeForFile(before.manifest, rel, pkg) : null const model = typeof known?.node?.name === 'string' ? known.node.name : basename(file).replace(/\.sql$/i, '') - const blank = { model, root, file: rel, errors: [], missing: [], undocumented: [], columns: null, returned: [], documented: [], reason: null, ran: false, schema: schemaOf(known?.node), target: await targetOf($, root) } + const blank = { model, root, file: rel, errors: [], missing: [], undocumented: [], columns: null, returned: [], documented: [], reason: null, ran: false, schema: schemaOf(known?.node), target: await targetOf($, root), ...(viaRunner && { ranFrom: relative(root, place) }) } // Not run at all: a reason the check could not start, said as "not checked". const notChecked = async (reason: string) => { const view: CheckView = { ...blank, status: 'partial', reason, ms: 0, at: started } @@ -475,7 +553,7 @@ async function compileAndValidate($: EngineInterface, root: string, file: string } // dbt splits a selection on spaces (a union of the pieces): a path with whitespace would select, and run, other models. if (/\s/.test(rel)) return notChecked("the model's path has a space, which dbt's selector would split") - if (unrunnable.has(root)) return notChecked(NO_DBT) + if (unrunnable.has(place)) return notChecked(NO_DBT) await setCheck($, { ...blank, status: 'running', ms: 0, at: started }, newest) // The file itself, by path (a name can be a version's or another package's), and only it: dbt's eager // indirect selection would also run its tests, and show a test's failure as the model's. @@ -483,17 +561,19 @@ async function compileAndValidate($: EngineInterface, root: string, file: string // package (`a,b`) selects only the project's model. dbt prints a preview only for a node whose name is in the // selection string, so the model's name joins the intersection: a versioned model whose file has another // name (`defined_in`) is still previewed, and the path keeps it to that one version. - const pkg = await projectName($, root) - const selectorFor = (name: string | null) => [`path:${rel}`, ...(pkg ? [`package:${pkg}`] : []), ...(name ? [name] : [])].join(',') - const scratch = await scratchOf($, root) - const run1 = (args: string[], read?: () => Promise): Promise => runDbt($, root, args, DBT_TIMEOUT_MS, current, true, read).catch(error => ({ exitCode: -1, stdout: '', stderr: String(error) })) + // From the project that installs the package, `path:` reads from that project's folder: the package and the model's + // name select it instead (the file's name when the manifest does not know it). + const selectorFor = (name: string | null) => + viaRunner ? [...(pkg ? [`package:${pkg}`] : []), name ?? basename(file).replace(/\.sql$/i, '')].join(',') : [`path:${rel}`, ...(pkg ? [`package:${pkg}`] : []), ...(name ? [name] : [])].join(',') + const scratch = await scratchOf($, place) + const run1 = (args: string[], read?: () => Promise): Promise => runDbt($, place, args, DBT_TIMEOUT_MS, current, true, read).catch(error => ({ exitCode: -1, stdout: '', stderr: String(error) })) // The manifest the show wrote is read while the plugin's dbt queue is still held, so no other run of it replaces it first. - const show = (select: string) => run1(['show', '--select', select, '--indirect-selection', 'empty', '--limit', '1', '--output', 'json', '--log-format', 'json'], () => readManifest($, root, rel, scratch)) + const show = (select: string) => run1(['show', '--select', select, '--indirect-selection', 'empty', '--limit', '1', '--output', 'json', '--log-format', 'json'], () => readManifest($, place, rel, scratch, pkg)) // A newer edit replaced this check before its turn: nothing ran, nothing to report. const replaced = () => notChecked('a newer check replaced this one').then(one => ({ ...one, note: '' })) const knownName = typeof known?.node?.name === 'string' ? known.node.name : null let name = knownName - if (!name) { + if (!name && !viaRunner) { // No manifest that lists the file: dbt runs the query whatever the selection and prints a preview only // for a name it carries, so a second show would run it again. Ask dbt for the name first: `ls` runs no query. const listing = await run1(['ls', '--select', selectorFor(null), '--resource-type', 'model', '--indirect-selection', 'empty', '--output', 'json', '--output-keys', 'name resource_type', '--log-format', 'text']) @@ -508,7 +588,7 @@ async function compileAndValidate($: EngineInterface, root: string, file: string if (cannotRun(run)) return notChecked(NO_DBT) // dbt show re-parsed the project: the manifest it wrote (in the plugin's own folder) holds the model and schema.yml as they are. let after = run.artifact ?? null - let node = after ? nodeForFile(after.manifest, rel, await projectName($, root)) : null + let node = after ? nodeForFile(after.manifest, rel, pkg) : null let named = typeof node?.node?.name === 'string' ? node.node.name : null let preview = run.exitCode === 0 ? showPreview(run.stdout, node?.id ?? known?.id) : null // No preview, and the fresh manifest names the model with a name the selector did not carry (an old name that @@ -519,13 +599,23 @@ async function compileAndValidate($: EngineInterface, root: string, file: string if (again === SKIPPED) return replaced() run = again after = run.artifact ?? null - node = after ? nodeForFile(after.manifest, rel, await projectName($, root)) : null + node = after ? nodeForFile(after.manifest, rel, pkg) : null named = typeof node?.node?.name === 'string' ? node.node.name : null preview = run.exitCode === 0 ? showPreview(run.stdout, node?.id ?? known?.id) : null } const ms = (await $.clock.now()) - started const errors = run.exitCode === 0 ? [] : logErrors(run.stdout) if (run.exitCode !== 0 && errors.length === 0) errors.push(dbtTail(run, 6) || `dbt exited ${run.exitCode}`) + // dbt stopped before the model because the project's packages are not installed: nothing about the model is + // known, so it is a one-line note, not the model's failure. + if (run.exitCode !== 0 && errors.some(one => DEPS_MISSING.test(one))) { + const where = relative(root, place) || 'this project' + const reason = `dbt's packages are not installed in ${where}, so dbt could not compile it (run \`dbt deps\` there, or run dbt from the project that installs this package)` + const view: CheckView = { ...blank, status: 'partial', reason, ms, at: started } + await setCheck($, view, newest) + if (current()) checkEnded(`${root}/${rel}`, mine, null) + return { view, note: checkNote(view), current, newest } + } // dbt exited 0 but showed nothing. A model the manifest still lists is a failure; a file it does not // list (a model just disabled, not a model at all) was not checked. const nothingShown = run.exitCode === 0 && !preview @@ -550,8 +640,17 @@ async function compileAndValidate($: EngineInterface, root: string, file: string : documented.length === 0 ? 'schema.yml documents no columns for this model' : null - const status: CheckView['status'] = errors.length ? 'error' : reason ? 'partial' : drift.missing.length || drift.undocumented.length ? 'warn' : 'ok' - const view: CheckView = { ...blank, model: named ?? model, schema: node ? schemaOf(node.node) : blank.schema, status, reason, ran: !errors.length && !nothingShown, errors: errors.map(one => one.slice(0, 1500)), missing: drift.missing, undocumented: drift.undocumented, columns: returned ? returned.length : null, returned: returned ?? [], documented, ms, at: started } + // Only the columns this edit left undocumented are its gaps; ones undocumented before it are said once, quietly. + const sqlNow = await $.fs.read(file).catch(() => null) + const gaps = undocumentedByEdit(drift.undocumented, sqlBefore, sqlBefore === null ? null : sqlNow, lastReturned.get(fileKey) ?? null) + // A model, seed, snapshot or source named by its table instead of ref() or source(): dbt and lineage cannot follow it. + const hardcoded = sqlNow !== null && !errors.length ? hardcodedRelations(sqlNow, after?.manifest ?? before?.manifest, node?.id ?? known?.id ?? null) : [] + // A gap this session's edit made stays the edit's on the next check, until schema.yml or the model closes it. + if (returned && !stale) lastReturned.set(fileKey, returned.filter(one => !gaps.fresh.includes(one))) + const said = oldGapsSaid.has(fileKey) + if (gaps.old.length && !stale && current()) oldGapsSaid.add(fileKey) + const status: CheckView['status'] = errors.length ? 'error' : hardcoded.length ? 'warn' : reason ? 'partial' : drift.missing.length || gaps.fresh.length ? 'warn' : 'ok' + const view: CheckView = { ...blank, model: named ?? model, schema: node ? schemaOf(node.node) : blank.schema, status, reason, ran: !errors.length && !nothingShown, errors: errors.map(one => one.slice(0, 1500)), missing: drift.missing, undocumented: gaps.fresh, ...(gaps.old.length ? { undocumentedBefore: gaps.old, ...(said && { oldGapsSaid: true }) } : {}), ...(hardcoded.length ? { hardcoded } : {}), columns: returned ? returned.length : null, returned: returned ?? [], documented, ms, at: started } await setCheck($, view, newest) if (current()) checkEnded(`${root}/${rel}`, mine, view.ran && after ? { root, file: `${root}/${rel}`, manifest: `${scratch}/manifest.json`, mtimeMs: after.mtimeMs } : null) // An edit of a schema file or macro that came while this ran checks it again now. @@ -662,6 +761,9 @@ function editConcerns(check: CheckView, edited: string): boolean { const NO_DBT = 'dbt could not be run here' +// dbt's own words when packages.yml lists packages that are not installed. +const DEPS_MISSING = /expects \d+ package\(s\)[^\n]*but found only \d+ package\(s\) installed|Run "?dbt deps"?/i + const STATUS_ICON: Record = { running: '…', ok: '✓', partial: '◦', warn: '⚠', error: '✖' } // An error that says nothing about the model's own SQL: a parent relation not built yet, a profile or a connection. @@ -682,11 +784,14 @@ function checkNote(view: CheckView): string { ].join('\n') } if (!view.ran) return [head, `- not checked: ${view.reason}.`].join('\n') - const lines = [head, `- compiled and ran${view.columns !== null ? `; the warehouse returned ${view.columns} columns` : ''}.`] + const lines = [head, `- compiled and ran${view.ranFrom ? ` from ${view.ranFrom} (the project that installs this package)` : ''}${view.columns !== null ? `; the warehouse returned ${view.columns} columns` : ''}.`] + for (const one of view.hardcoded ?? []) lines.push(`- hardcoded ${one.relation}: use ${one.use}. dbt does not know this model reads it, so it is left out of the DAG and of column lineage, and it reads whatever that table holds in this target.`) if (view.reason) lines.push(`- its columns were not compared with schema.yml: ${view.reason}.`) if (view.missing.length) lines.push(`- documented in schema.yml but not returned: ${view.missing.join(', ')}`) if (view.undocumented.length) lines.push(`- returned but not documented in schema.yml: ${view.undocumented.join(', ')}`) - if (view.status === 'warn') lines.push('Update schema.yml or the model so they agree, or tell the user why they differ.') + if (view.undocumentedBefore?.length && !view.oldGapsSaid) lines.push(`- already undocumented before this edit (not this edit's doing; nothing to fix for it): ${view.undocumentedBefore.join(', ')}`) + if (view.hardcoded?.length) lines.push('Put the ref() or source() back unless the user asked for the table by name, and tell the user.') + if (view.status === 'warn' && (view.missing.length || view.undocumented.length)) lines.push('Update schema.yml or the model so they agree, or tell the user why they differ.') return lines.join('\n') } @@ -1333,21 +1438,16 @@ async function runDoctor($: EngineInterface, root: string, connection = false): checks.push({ id: 'adapter', label: 'Adapter', status: 'ok', detail: `${type}${adapter ? ` ${adapter.version}` : ''}`, fix: null }) } - // Packages: packages.yml with nothing installed (at packages-install-path) fails every parse. - // A dependencies.yml that declares only `projects:` has no packages to install. - const packagesFile = (await $.fs.exists(`${root}/packages.yml`).catch(() => false)) - ? 'packages.yml' - : /^packages:/m.test(await $.fs.read(`${root}/dependencies.yml`).catch(() => '')) - ? 'dependencies.yml' - : null - if (packagesFile) { - // `packages-install-path` in dbt_project.yml, `dbt_packages` when absent. - const configured = /^packages-install-path:\s*['"]?([^'"#\s]+)/m.exec(await $.fs.read(`${root}/dbt_project.yml`).catch(() => ''))?.[1] - // A Jinja value is dbt's to render: dbt's default stands in. - const installPath = configured && !configured.includes('{{') ? configured : 'dbt_packages' - const installed = installPath.startsWith('/') ? installPath : `${root}/${installPath}` - const has = (await $.fs.list(installed).catch(() => [])).length > 0 - checks.push({ id: 'packages', label: 'Packages', status: has ? 'ok' : 'fail', detail: has ? `${installPath} installed` : `${packagesFile} is not installed`, fix: has ? null : 'dbt deps' }) + // Packages: packages.yml with nothing installed (at packages-install-path) fails every parse. A package repo + // whose own packages are not installed is fine when the project below it that installs it (integration_tests) + // has them: dbt is run from there. + const packages = await packagesOf($, root) + if (packages) { + const runner = packages.installed ? null : await runnerOf($, root) + const theirs = runner ? await packagesOf($, runner) : null + const has = packages.installed || (!!runner && (!theirs || theirs.installed)) + const detail = packages.installed ? `${packages.installPath} installed` : has ? `installed in ${relative(root, runner!)}, where dbt runs for this package` : `${packages.file} is not installed` + checks.push({ id: 'packages', label: 'Packages', status: has ? 'ok' : 'fail', detail, fix: has ? null : 'dbt deps' }) } // Manifest freshness. @@ -1428,6 +1528,8 @@ async function openDoctor($: EngineInterface) { export async function startDbtMods($: EngineInterface, isInteractive: boolean) { const asked = generation contextGiven.clear() + oldGapsSaid.clear() + lastReturned.clear() pendingNotes.splice(0) doctorNoteDue = null unrunnable.clear() @@ -1495,10 +1597,13 @@ export async function dbtModsBand($: EngineInterface, ui: { Box: any; Button: an : check.status === 'error' ? (check.errors[0] ?? 'failed').split('\n').filter(Boolean).slice(-2).join(' ').slice(0, 160) : [ + ...(check.hardcoded ?? []).map(one => `hardcoded ${one.relation}: use ${one.use} — lineage can't follow it`), check.columns !== null ? `${check.columns} columns` : 'no row returned', check.missing.length ? `missing ${check.missing.join(', ')}` : null, check.undocumented.length ? `undocumented ${check.undocumented.join(', ')}` : null, - check.status === 'ok' && check.columns !== null ? 'match schema.yml' : null, + check.status === 'ok' && check.columns !== null ? (check.undocumentedBefore?.length ? 'no new gaps with schema.yml' : 'match schema.yml') : null, + check.undocumentedBefore?.length && !check.oldGapsSaid ? `${check.undocumentedBefore.length} undocumented before this edit` : null, + check.ranFrom ? `ran from ${check.ranFrom}` : null, `${(check.ms / 1000).toFixed(1)}s`, ] .filter(Boolean) @@ -1719,8 +1824,11 @@ function pendingLines(agentId: string | null): string[] { return withPending({ context: [] as string[] }, agentId).context } +/** Compile and validate reads a model's text from before an edit, to tell the columns the edit left undocumented from older gaps. */ +export const DBT_MODS_EDIT_READS_BEFORE = true + /** After a model Claude edited: compile it and run it against the warehouse, then check its columns. */ -export async function dbtModsEditAfter($: EngineInterface, e: { tool: string; agentId?: string }, result: ToolCallResult, file: { path: string }): Promise { +export async function dbtModsEditAfter($: EngineInterface, e: { tool: string; agentId?: string }, result: ToolCallResult, file: { path: string; before?: string | null }): Promise { if (!enabled || !EDIT_TOOLS.has(e.tool)) return [] const path = normalize(file.path) // The conversation this edit belongs to, taken before the edit ran, so a /clear during it is seen. @@ -1732,7 +1840,7 @@ export async function dbtModsEditAfter($: EngineInterface, e: { tool: string; ag const folders = root ? modelPaths(await $.fs.read(`${root}/dbt_project.yml`).catch(() => '')) : [] const inModels = !!root && folders.some(folder => (folder === '' ? !/^(target|dbt_packages|dbt_modules|logs)\//.test(relative(root, path)) : relative(root, path).startsWith(`${folder}/`))) if (root && inModels) { - const check = compileAndValidate($, root, path, asked) + const check = compileAndValidate($, root, path, asked, file.before ?? null) const landed = await within($, CHECK_HOLD_MS, check) // Landed, and still the newest check of this file in this conversation. if (landed && landed.current()) { @@ -1789,6 +1897,8 @@ export async function endDbtMods($: EngineInterface) { parsed = null forcedWhileBusy = null contextGiven.clear() + oldGapsSaid.clear() + lastReturned.clear() pendingNotes.splice(0) doctorNoteDue = null await update($, checkAtom, () => null) @@ -1954,12 +2064,13 @@ export function registerDbtMods(on: On, context: { options: PluginOptions }) { running: 'Compiling and running it against the warehouse…', ok: 'Compiles, runs, and its columns match schema.yml', partial: check.ran ? `Compiles and runs; columns not compared: ${check.reason}` : `Not checked: ${check.reason}`, - warn: 'Compiles and runs, but its columns differ from schema.yml', + warn: check.hardcoded?.length ? `Compiles and runs, but names ${check.hardcoded.map(one => one.relation).join(', ')} by table: lineage can't follow it` : 'Compiles and runs, but its columns differ from schema.yml', error: 'dbt could not compile or run it', }[check.status] // The exact names the core compared: a quoted column is not matched without case here. const missing = new Set(check.missing) const extra = new Set(check.undocumented) + const before = new Set(check.undocumentedBefore ?? []) return ( @@ -1969,6 +2080,18 @@ export function registerDbtMods(on: On, context: { options: PluginOptions }) { {verdict} + {check.hardcoded?.length ? ( + + Named by table, not ref() or source() + {check.hardcoded.map(one => ( + + {one.relation} + use + {one.use} + + ))} + + ) : null} {check.errors.length ? ( dbt said @@ -1983,9 +2106,9 @@ export function registerDbtMods(on: On, context: { options: PluginOptions }) { {check.columns === null ? The query returned no row, so its columns could not be read. : null} {check.returned.map(name => ( - {check.status === 'partial' ? · : {extra.has(name) ? '+ ' : '✓ '}} + {check.status === 'partial' ? · : before.has(name) ? · : {extra.has(name) ? '+ ' : '✓ '}} {name} - {extra.has(name) ? returned, not documented : null} + {extra.has(name) ? returned, not documented : before.has(name) ? not documented, from before this edit : null} ))} {check.documented.filter(name => missing.has(name)).map(name => ( diff --git a/plugins/altimate-code/hooks/features/index.ts b/plugins/altimate-code/hooks/features/index.ts index 28f1b4bb..38de5bc7 100644 --- a/plugins/altimate-code/hooks/features/index.ts +++ b/plugins/altimate-code/hooks/features/index.ts @@ -18,6 +18,7 @@ import { dbtModsBashAfter, dbtModsCallStart, dbtModsEditAfter, + DBT_MODS_EDIT_READS_BEFORE as dbtModsEditReadsBefore, dbtModsReadOrEditAfter, endDbtMods, newTurnDbtMods, @@ -399,7 +400,7 @@ export async function featuresBashAfter($: EngineInterface, e: BashCall, result: export const EDIT_AFTER_TIMEOUT_MS = dbtModsCheckHoldMs + 5_000 /** Whether any feature's `editAfter` needs the file's text from before the edit. */ -export const EDIT_READS_BEFORE: boolean = runsEditReadsBefore || deferEditReadsBefore || toModelEditReadsBefore || commentsEditReadsBefore || costEditReadsBefore || healthEditReadsBefore || sqlGraphEditReadsBefore || docsEditReadsBefore || cteProfileEditReadsBefore +export const EDIT_READS_BEFORE: boolean = dbtModsEditReadsBefore || runsEditReadsBefore || deferEditReadsBefore || toModelEditReadsBefore || commentsEditReadsBefore || costEditReadsBefore || healthEditReadsBefore || sqlGraphEditReadsBefore || docsEditReadsBefore || cteProfileEditReadsBefore /** The context lines every feature adds after an edit landed. */ export async function featuresEditAfter($: EngineInterface, e: FileCall, result: ToolCallResult, file: EditedFile): Promise { diff --git a/plugins/altimate-code/hooks/impact-report.test.ts b/plugins/altimate-code/hooks/impact-report.test.ts index ce12439f..73d0c6fa 100644 --- a/plugins/altimate-code/hooks/impact-report.test.ts +++ b/plugins/altimate-code/hooks/impact-report.test.ts @@ -23,6 +23,7 @@ import { reportHtml, sectionOf, summaryLines, + valueCandidates, summaryOf, upstreamWords, WINDOWS_EXPLORER, @@ -154,7 +155,7 @@ describe('the summary', () => { test('a model with no changed output column shows its model-level count alone', () => { const change = { ...CHANGE, columns: [] } const lines = summaryLines(summaryOf([change], [{ model: 'fct_orders', column: null, down: null, up: null, modelLevel: 6, failed: false }], null)) - expect(lines).toEqual(['◆ Lineage after this task · 1 model changed', 'fct_orders · 6 downstream models at model level', ' no output column changed: the model-level count stands']) + expect(lines).toEqual(['◆ Lineage after this task · 1 model changed', 'fct_orders · 6 downstream models at model level', ' values may change downstream; the model-level count stands']) }) }) @@ -595,3 +596,50 @@ test('a reach left unknown by a model read only in part keeps the footer a lower expect(lines).toContain('⚠ Lower bound: 1 model could not be read') expect(lines).not.toContain('✓ Every downstream model was read') }) + +// altimate-code's real answer for jaffle_shop's stg_payments.amount (cents change), asked of a manifest copy one +// folder below target/: the seed's CSV header could not be found, so both readers, which select *, were read in +// full and the value was not traced through them. +const STAR_ANSWER = {"version": 1, "focus": {"model": "model.jaffle_shop.stg_payments", "column": "amount"}, "direction": "downstream", "inputs": {"sql": "compiled_code", "catalog": null, "engine": "altimate-core 0.9.0", "workarounds": ["relations keyed database.schema.name", "ephemeral parents expanded", "parent columns = documented + own lineage targets + SELECT list", "aggregate FILTER clause dropped before analysis", "WITH nested inside a CTE reported unresolved", "reads with zero traced columns reported unresolved"]}, "counts": {"columns": 0, "models": 0, "hops": 0, "contracts": 0, "ends": 0, "may_carry_models": 2, "not_traced_models": 0, "model_level": 2}, "nodes": [{"id": "model.jaffle_shop.stg_payments", "name": "stg_payments", "folder": "staging", "materialized": "view", "contract": false, "tests": 3, "exposures": []}], "columns": [{"model": 0, "column": "amount", "hop": 0, "end": false, "how": null, "how_from": null}], "edges": [], "also_reads": [], "unresolved": [], "may_carry": [{"id": "model.jaffle_shop.customers", "name": "customers", "reason": "star"}, {"id": "model.jaffle_shop.orders", "name": "orders", "reason": "star"}], "flags": [], "consumers": {"declared": 0, "exposures": []}, "truncated": false, "next": null} as any + +test('a reach of 0 beside readers that select * is not traced, and the footer never says every model was read', () => { + expect(unknownReach(STAR_ANSWER)).toBe('not traced: 2 downstream models may read it without naming it (select *)') + const change = { model: 'stg_payments', file: 'models/staging/stg_payments.sql', root: '/r', columns: [{ name: 'amount', before: 'amount / 100', after: 'amount' }], ended: '' } as any + const traces = [{ model: 'stg_payments', column: 'amount', down: STAR_ANSWER, up: null, modelLevel: 2, failed: false, unknown: unknownReach(STAR_ANSWER)! }] as any + const lines = summaryLines(summaryOf([change], traces, null)) + expect(lines).toEqual([ + '◆ Lineage after this task · 1 model changed', + 'stg_payments · 2 downstream models at model level', + ' amount not traced: 2 downstream models may read it without naming it (select *)', + '⚠ Lower bound: 2 models may read it without naming it', + ]) +}) + +test('a change no select item shows (a value computed in a Jinja block) names the columns whose values change, and their reach', () => { + // Tuva's core__medical_claim lists its columns in a `{% set %}` block the select reads. + const before = "{%- set cols -%}\n med.claim_id\n , med.paid_amount\n , med.allowed_amount\n{%- endset -%}\nselect {{ cols }} from {{ ref('x') }} as med\nwhere med.claim_id is not null" + const after = before.replace(' , med.paid_amount', ' , case when med.paid_amount < 0 then 0 else med.paid_amount end as paid_amount') + expect(changedColumns(before, after)).toEqual([]) + expect(valueCandidates(before, after)).toEqual(['paid_amount']) + // A filter names no column whose value changes. + expect(valueCandidates(before, before.replace('is not null', 'is not null and med.paid_amount > 0'))).toEqual([]) + const answer = { + version: 1, focus: { model: 'model.t.core__medical_claim', column: 'paid_amount' }, direction: 'downstream', + nodes: [{ id: 'model.t.core__medical_claim', name: 'core__medical_claim' }, { id: 'model.t.fact_claims', name: 'fact_claims' }], + columns: [{ model: 0, column: 'paid_amount', hop: 0 }, { model: 1, column: 'paid_amount', hop: 1, end: true }], + counts: { columns: 1, models: 1, ends: 1, hops: 1, may_carry_models: 3, model_level: 97 }, unresolved: [], may_carry: [], + } as any + const change = { model: 'core__medical_claim', file: 'models/core/final/core__medical_claim.sql', root: '/r', columns: [], values: ['paid_amount'], ended: '' } as any + const traces = [ + { model: 'core__medical_claim', column: null, down: null, up: null, modelLevel: 97, failed: false }, + { model: 'core__medical_claim', column: 'paid_amount', down: answer, up: null, modelLevel: 97, failed: false, value: true }, + ] as any + expect(summaryLines(summaryOf([change], traces, null))).toEqual([ + '◆ Lineage after this task · 1 model changed', + 'core__medical_claim · 97 downstream models at model level', + ' values change in paid_amount → reaches 1 column in 1 model (1 end column) · 3 more may change', + '✓ Every downstream model was read', + ]) + // None the manifest has: the values may still change, said without a count of 0. + expect(summaryLines(summaryOf([change], traces.slice(0, 1), null))).toContain(' values may change downstream; the model-level count stands') +}) diff --git a/plugins/altimate-code/hooks/impact-report.ts b/plugins/altimate-code/hooks/impact-report.ts index 65bc8848..5b221da1 100644 --- a/plugins/altimate-code/hooks/impact-report.ts +++ b/plugins/altimate-code/hooks/impact-report.ts @@ -38,6 +38,11 @@ export type ModelChange = { file: string root: string columns: { name: string; before: string | null; after: string | null }[] + /** + * With no output column changed: names on the lines the change rewrote that may be the model's columns, whose + * values may change (an expression in a Jinja block the select list reads). The manifest says which are columns. + */ + values?: string[] /** The file's fingerprint as the task ended, to tell whether it changed since. */ ended?: string } @@ -80,6 +85,45 @@ export function changedColumns(original: string, current: string): ModelChange[' .map(name => ({ name, before: lastExpr(before, name), after: lastExpr(after, name) })) } +// Words on a changed line that are SQL or Jinja, not a column. +const NOT_COLUMNS = new Set( + ('select from where and or not null is in as on case when then else end cast coalesce nullif greatest least ' + + 'join left right inner outer full cross group by order having limit distinct union all with over partition ' + + 'sum count avg min max abs round floor ceil true false between like ilike exists any interval date ' + + 'set endset if endif for endfor macro endmacro ref source config var this dbt type_string type_numeric ' + + 'type_int type_float type_timestamp type_boolean quote_column int integer numeric decimal varchar text float double boolean').split(' '), +) +// A changed line that filters or joins rather than computes a value. +const NOT_A_VALUE = /^\s*(?:where|and|or|on|from|join|left|right|inner|full|cross|group|order|having|limit|qualify|union|\)|\{%-?\s*(?:if|endif|for|endfor|else))\b/i + +/** + * When no output column changed: names on the lines the change rewrote or added (not filters or joins) that may + * be the model's columns, the ones named most often first. Whether each is a column is the manifest's to say. + */ +export function valueCandidates(original: string, current: string, max = MAX_TASK_COLUMNS * 2): string[] { + const lines = (text: string) => text.split('\n').map(line => line.trim()).filter(Boolean) + const was = lines(original) + const now = lines(current) + const left = new Map() + for (const line of was) left.set(line, (left.get(line) ?? 0) + 1) + const added = now.filter(line => { + const n = left.get(line) ?? 0 + if (n > 0) left.set(line, n - 1) + return n === 0 + }) + const counts = new Map() + for (const line of added) { + if (NOT_A_VALUE.test(line)) continue + const code = line.replace(/--.*$/, '').replace(/'(?:[^']|'')*'/g, ' ') + for (const match of code.matchAll(/(? b[1] - a[1]).slice(0, max).map(([name]) => name) +} + /** The column answer in an impact_analysis result's metadata, or null when it carries none. */ export function answerOf(metadata: unknown): ColumnAnswer | null { const m = metadata as any @@ -221,6 +265,8 @@ export type ColumnTrace = { unchecked?: boolean /** Why the reach is unknown though the column was found: the answer reached nothing, and models it reads were not read. */ unknown?: string + /** A column whose values the change alters though no select item changed (ModelChange.values). */ + value?: boolean consumers?: ReportConsumers } @@ -236,7 +282,14 @@ export function unknownReach(answer: ColumnAnswer | null): string | null { // whichever names more. const partly = Array.isArray(answer.may_carry) ? answer.may_carry.filter(one => PARTLY_READ.has(String((one as any)?.reason ?? ''))).length : 0 const unread = Math.max(r.unresolved, answer.unresolved?.length ?? 0, partly) - if (r.columns > 0 || r.models > 0 || unread === 0) return null + if (r.columns > 0 || r.models > 0) return null + if (unread === 0) { + // Read in full, but in a way that may pass the value on without naming it (`select *`): a 0 would be a guess. + const unseen = unseenOf(answer) + if (unseen.length === 0) return null + const how = [...new Set(unseen.map(one => UNSEEN[String((one as any)?.reason)]))].join(', ') + return `not traced: ${count(unseen.length, 'downstream model')} may read it without naming it (${how})` + } const missing = (answer.unresolved ?? []).filter(one => /no compiled sql/i.test(String(one?.reason ?? ''))).length if (missing > 0 && missing >= unread) return `not traced: ${count(missing, 'downstream model')} ${missing === 1 ? 'has' : 'have'} no compiled SQL (run \`dbt compile\`)` if (missing > 0) return `not traced: ${count(unread, 'downstream model')} could not be read (${missing} with no compiled SQL: run \`dbt compile\`)` @@ -244,6 +297,10 @@ export function unknownReach(answer: ColumnAnswer | null): string | null { return `not traced: ${count(unread, 'downstream model')} could not be read${why ? ` (${why})` : ''}` } +/** The may-carry reasons for a model read in full that may still pass the value on without naming it. */ +const UNSEEN: Record = { star: 'select *', natural_join: 'NATURAL JOIN', dynamic_name: 'columns named at run time', row_reference: 'a whole row as a value' } +const unseenOf = (answer: ColumnAnswer) => (Array.isArray(answer.may_carry) ? answer.may_carry.filter(one => String((one as any)?.reason ?? '') in UNSEEN) : []) + /** The may-carry reasons for a model whose SQL was not read in full: its reach from the column is not known. */ const PARTLY_READ = new Set(['unresolved', 'read_in_part', 'read_columns_unknown', 'unclear_sql', 'capped']) @@ -253,6 +310,7 @@ const ADDED_NOTE = 'not in the manifest from before the task (new in it?): its r /** The band's view of a finished task. */ export function summaryOf(changes: ModelChange[], traces: ColumnTrace[], report: string | null): TaskSummaryView { const unresolved = new Set() + const unseen = new Set() let truncated = false const models = changes.map(change => { const mine = traces.filter(one => one.model === change.model) @@ -264,13 +322,16 @@ export function summaryOf(changes: ModelChange[], traces: ColumnTrace[], report: for (const entry of one.down?.unresolved ?? []) unresolved.add(one.down!.nodes[entry.model]?.id ?? one.down!.nodes[entry.model]?.name ?? String(entry.model)) // A reach left unknown by models read only in part: those count as unread too, so the footer never says all were read. if (one.unknown) for (const entry of Array.isArray(one.down?.may_carry) ? one.down!.may_carry : []) if (PARTLY_READ.has(String((entry as any)?.reason ?? ''))) unresolved.add(String((entry as any)?.id ?? (entry as any)?.name)) + // Read in full but passing the value on unnamed: the footer says so, never that the reach is settled. + if (one.unknown && one.down) for (const entry of unseenOf(one.down)) unseen.add(String((entry as any)?.id ?? (entry as any)?.name)) if (reach?.truncated) truncated = true const note = one.unknown ?? (one.unchecked ? ADDED_NOTE : undefined) - return { name: one.column!, reach, failed: one.failed, ...(note && { note }) } + return { name: one.column!, reach, failed: one.failed, ...(note && { note }), ...(one.value && { value: true }) } }) return { name: change.model, modelLevel, columns } }) - return { state: 'ready', models, unresolved: unresolved.size, truncated, report, openFailed: false } + for (const id of unresolved) unseen.delete(id) + return { state: 'ready', models, unresolved: unresolved.size, ...(unseen.size ? { unseen: unseen.size } : {}), truncated, report, openFailed: false } } /** The band's lines, as text: what the summary says, for the log and for tests. */ @@ -279,22 +340,29 @@ export function summaryLines(view: TaskSummaryView): string[] { const lines = [`◆ Lineage after this task · ${plural(view.models.length, 'model')} changed`] for (const model of view.models) { lines.push(`${model.name}${model.modelLevel !== null ? ` · ${plural(model.modelLevel, 'downstream model')} at model level` : ''}`) - if (model.columns.length === 0) lines.push(' no output column changed: the model-level count stands') + // No output column changed, and none whose values change could be named: the values may still change. + if (model.columns.length === 0) lines.push(model.modelLevel === 0 ? ' no model reads it downstream' : ' values may change downstream; the model-level count stands') for (const col of model.columns) { + // A column named by its values changing, not by its select item: said so. + const name = col.value ? `values change in ${col.name} →` : `${col.name} ` if (view.state === 'tracing' && !col.reach) lines.push(` ${col.name} tracing…`) - else if (col.note) lines.push(` ${col.name} ${col.note}`) - else if (!col.reach) lines.push(` ${col.name} ${col.failed ? 'could not be traced' : `no column answer: update altimate-code to ${MIN_ALTIMATE_CODE} or later`}`) + else if (col.note) lines.push(` ${name} ${col.note}`) + else if (!col.reach) lines.push(` ${name} ${col.failed ? 'could not be traced' : `no column answer: update altimate-code to ${MIN_ALTIMATE_CODE} or later`}`) else { const r = col.reach // What the value reaches, and how many more it may reach: nothing is reported unaffected. const parts = [`reaches ${plural(r.columns, 'column')} in ${plural(r.models, 'model')} (${plural(r.ends, 'end column')})`] parts.push(`${r.mayCarry} more may change`) - lines.push(` ${col.name} ${parts.join(' · ')}`) + lines.push(` ${name} ${parts.join(' · ')}`) } } } - if (view.unresolved > 0 || view.truncated) { - const why = [view.unresolved ? `${plural(view.unresolved, 'model')} could not be read` : '', view.truncated ? 'the trace hit its limit' : ''].filter(Boolean).join('; ') + if (view.unresolved > 0 || view.truncated || (view.unseen ?? 0) > 0) { + const why = [ + view.unresolved ? `${plural(view.unresolved, 'model')} could not be read` : '', + view.unseen ? `${plural(view.unseen, 'model')} may read it without naming it` : '', + view.truncated ? 'the trace hit its limit' : '', + ].filter(Boolean).join('; ') lines.push(`⚠ Lower bound: ${why}`) } else if (view.state === 'ready' && view.models.some(one => one.columns.some(col => col.reach))) { lines.push('✓ Every downstream model was read') @@ -596,6 +664,8 @@ export function reportData(changes: ModelChange[], traces: ColumnTrace[], task: const focuses: ReportFocus[] = [] for (const change of changes) { if (change.columns.length === 0) focuses.push(focusOf(change, null, traces.find(one => one.model === change.model && one.column === null))) + // Columns whose values the change alters, found by name in the manifest: each its own section. + for (const trace of traces) if (trace.value && trace.model === change.model && trace.column) focuses.push(focusOf(change, { name: trace.column, before: null, after: null }, trace)) for (const col of change.columns) focuses.push(focusOf(change, col, traces.find(one => one.model === change.model && one.column === col.name))) } const names = changes.map(one => one.model) diff --git a/plugins/altimate-code/hooks/lineage.test.tsx b/plugins/altimate-code/hooks/lineage.test.tsx index b83ea663..913060f1 100644 --- a/plugins/altimate-code/hooks/lineage.test.tsx +++ b/plugins/altimate-code/hooks/lineage.test.tsx @@ -62,7 +62,7 @@ const REACH = { }, } -type Options = { mcp?: boolean; warehouse?: boolean; structured?: boolean; size?: number; mtime?: () => number; reach?: Record; capped?: boolean; lineageText?: string; toolResult?: (e: { tool: string }) => unknown; failColumn?: string; holdFirstManifestRead?: Promise; holdReach?: { column: string; until: Promise }; modelLineage?: Record; cliTooLarge?: boolean; cliCutOff?: (tool?: string) => boolean; cliCutOffExit?: number; cliStdout?: string; cliFailure?: boolean; sqlFiles?: Record; holdFirstSqlRead?: Promise; holdManifestStat?: { at: number; until: Promise }; noRootProject?: boolean; mcpHang?: string; narrow?: string; fileMtime?: (path: string) => number | undefined; manifest?: unknown; cliLineage?: Record; persisted?: { readable: boolean }; holdCliLineage?: Promise } +type Options = { fileText?: (path: string) => string | undefined; shell?: (argv: readonly string[]) => { exitCode: number; stdout: string } | undefined; mcp?: boolean; warehouse?: boolean; structured?: boolean; size?: number; mtime?: () => number; reach?: Record; capped?: boolean; lineageText?: string; toolResult?: (e: { tool: string }) => unknown; failColumn?: string; holdFirstManifestRead?: Promise; holdReach?: { column: string; until: Promise }; modelLineage?: Record; cliTooLarge?: boolean; cliCutOff?: (tool?: string) => boolean; cliCutOffExit?: number; cliStdout?: string; cliFailure?: boolean; sqlFiles?: Record; holdFirstSqlRead?: Promise; holdManifestStat?: { at: number; until: Promise }; noRootProject?: boolean; mcpHang?: string; narrow?: string; fileMtime?: (path: string) => number | undefined; manifest?: unknown; cliLineage?: Record; persisted?: { readable: boolean }; holdCliLineage?: Promise } /** * A dbt project at /repo, the plugin's MCP server (or the CLI when it is not connected), and the @@ -155,6 +155,8 @@ async function start($: any, on: any, options: Options = {}) { if (!options.persisted?.readable) throw new Error('EACCES') return { value: JSON.stringify({ ...reach, report: 'Impact' }) } } + const given = options.fileText?.(e.path) + if (given !== undefined) return { value: given } return { value: e.path === MANIFEST_PATH ? JSON.stringify(options.manifest ?? MANIFEST) : 'select 1' } }) on('tool.list', () => ({ @@ -203,6 +205,8 @@ async function start($: any, on: any, options: Options = {}) { on('process.run', async (_: unknown, e: { argv: readonly string[] }) => { const inert = nothingToReport(e.argv) if (inert) return { value: inert } + const shell = options.shell?.(e.argv) + if (shell) return { value: { ...shell, stderr: '' } } const [, command, , tool] = e.argv // A slow CLI dbt_lineage: it answers once the test lets it. if (command === 'tool' && tool === 'dbt_lineage' && options.holdCliLineage) await options.holdCliLineage @@ -275,6 +279,9 @@ test('reading a model points the pane at it: the manifest graph, then its column expect(logs.some(line => line.startsWith('lineage-pane · target/manifest.json · ') && line.endsWith('offline'))).toBe(true) const ui = await $.ui.mount({ ...PANE, surface: 'terminal' }) + // Wider than the pane, it opens as its ends: v picks the whole graph. + await ui.press({ key: 'lineage-view' }) + await clock.settle() expect(await ui.find({ type: 'Text', text: /lineage · fct_orders/ })).toBeDefined() expect(await ui.find({ type: 'Text', text: /GRAPH · manifest/ })).toBeDefined() expect(await ui.find({ type: 'Button', text: /int_orders/ })).toBeDefined() @@ -437,8 +444,8 @@ test('[c] picks a column and shows its reach; [d] unfolds downstream; a model in await clock.settle() const ui = await $.ui.mount({ ...PANE, surface: 'terminal' }) - // The text lists: graph → ends only → folders → list. - for (let i = 0; i < 3; i++) await ui.press({ key: 'lineage-view' }) + // The text lists: the whole graph (it opened as its ends, being wider than the pane) → ends only → folders → list. + for (let i = 0; i < 4; i++) await ui.press({ key: 'lineage-view' }) await clock.settle() expect(await ui.find({ type: 'Text', text: /2 more models/ })).toBeDefined() await ui.press({ key: 'lineage-down' }) @@ -468,6 +475,10 @@ test('the graph view draws boxes a press follows; [v] cycles the views and [l] t const ui = await $.ui.mount({ ...PANE, surface: 'terminal' }) expect(await ui.find({ type: 'Text', text: /┏━+┓/ })).toBeDefined() + // Wider than the pane, it opens as its ends: v picks the whole graph, and then cycles the views. + await ui.press({ key: 'lineage-view' }) + await clock.settle() + expect(await ui.find({ type: 'Text', text: /GRAPH · manifest/ })).toBeDefined() for (const title of [/ENDS ONLY · manifest/, /FOLDERS · manifest/, 'UPSTREAM · manifest', /GRAPH · manifest/]) { await ui.press({ key: 'lineage-view' }) await clock.settle() @@ -658,7 +669,8 @@ test('a graph too big for Claude Code to draw is shown as folders, saying why, n const { clock } = await start($, on, { manifest: { ...MANIFEST, nodes } }) await $.command.run({ command: 'lineage', args: 'fct_orders', origin: { kind: 'composer' }, presentation: { isFullscreen: true, columns: 200 } }) await clock.settle() - const ui = await $.ui.mount({ ...PANE, surface: 'terminal' }) + // A pane wide enough for the small graph below, so it is drawn whole rather than as its ends. + const ui = await $.ui.mount({ ...PANE, props: { ...PANE.props, bodyColumns: 200 }, surface: 'terminal' }) expect(await ui.find({ type: 'Text', text: /^FOLDERS · manifest/ })).toBeDefined() expect(await ui.find({ type: 'Text', text: /the whole graph is more than Claude Code can draw in a pane, so it is shown as folders/ })).toBeDefined() // A small graph (upstream of a report, away from the 600) is still drawn whole. @@ -817,6 +829,9 @@ test('a manifest over 4 MiB is read through altimate-code: the graph comes from await clock.settle() expect(calls).toEqual(['tool:Read', 'cli:dbt_lineage:graph']) const ui = await $.ui.mount({ ...PANE, surface: 'terminal' }) + // Wider than the pane, it opens as its ends: v picks the whole graph. + await ui.press({ key: 'lineage-view' }) + await clock.settle() expect(await ui.find({ type: 'Text', text: /GRAPH · altimate-code/ })).toBeDefined() expect(await ui.find({ type: 'Text', text: /5 direct · 1 transitive · 1 test ·/ })).toBeDefined() expect(await ui.find({ type: 'Text', text: /← int_orders\.revenue\s+Transformation/ })).toBeDefined() @@ -1241,6 +1256,9 @@ test('the graph is one canvas with every level and whole names; h and l pan it, await $.tool.call({ tool: 'Read', file_path: `${ROOT}/models/${name(3)}.sql` }) await clock.settle() const ui = await $.ui.mount({ ...PANE, props: { ...PANE.props, bodyColumns: 60 }, surface: 'terminal' }) + // Wider than the pane, it opens as its ends; v shows the whole graph. + await ui.press({ key: 'lineage-view' }) + await clock.settle() const range = async () => /columns (\d+)–(\d+) of (\d+)/.exec((await ui.find({ type: 'Text', text: /columns \d+–\d+ of \d+/ }))!.text!)!.slice(1).map(Number) const [start1, , total] = await range() // Centred on the model, whose name is drawn whole; never a fold or a cut. @@ -1278,6 +1296,8 @@ test('every parent is drawn, no "+N more", each followable after panning to it', await $.tool.call({ tool: 'Read', file_path: `${ROOT}/models/${long('focus')}.sql` }) await clock.settle() const ui = await $.ui.mount({ ...PANE, props: { ...PANE.props, bodyColumns: 120 }, surface: 'terminal' }) + await ui.press({ key: 'lineage-view' }) + await clock.settle() expect(await ui.find({ type: 'Button', text: /\+\d+ more/ })).toBeUndefined() // The parents sit left of the model: pan there. for (let i = 0; i < 10; i++) await ui.press({ key: 'lineage-pan-left' }) @@ -1751,6 +1771,9 @@ test("a press on a graph box asks for warehouse lineage; the pane following Clau await clock.settle() expect(calls).not.toContain('mcp:table_lineage') const ui = await $.ui.mount({ ...PANE, surface: 'terminal' }) + // Wider than the pane, it opens as its ends: v picks the whole graph. + await ui.press({ key: 'lineage-view' }) + await clock.settle() const box = await ui.find({ type: 'Button', text: /int_orders/ }) await ui.press({ key: box!.key! }) await clock.settle() @@ -1908,3 +1931,78 @@ optionTest('show_edit_impact also shows the per-edit impact note to the person', expect(debug).toContain(NOTE) expect(logs).toContain(NOTE) }) + +optionTest('a model a shell command rewrites gets the per-edit impact note too, to Claude and the person', { show_edit_impact: true }, async ($, on) => { + const before = 'select sum(amount) as revenue from orders' + let text = before + let changed = 1 + const { clock, logs, debug } = await start($, on, { + fileText: path => (path === FILE ? text : undefined), + fileMtime: path => (path === FILE ? changed : undefined), + // find lists the files changed in the last minute; git has the model's text from the last commit. + shell: argv => (argv[0] === 'find' ? { exitCode: 0, stdout: changed > 1 ? `${FILE}\n` : '' } : argv[0] === 'git' && argv[3] === 'rev-parse' ? { exitCode: 0, stdout: `${ROOT}\n` } : argv[0] === 'git' && argv[3] === 'show' ? { exitCode: 0, stdout: before } : undefined), + toolResult: e => { + // The command rewrites the model, as `python3 -c` or `sed -i` does. + if (e.tool === 'Bash') { + text = 'select sum(amount) - sum(refunds) as revenue from orders' + changed = Number.MAX_SAFE_INTEGER + } + return undefined + }, + }) + const result: any = await $.tool.call({ tool: 'Bash', command: `python3 -c "import pathlib; p = pathlib.Path('models/fct_orders.sql'); p.write_text(p.read_text().replace('sum(amount)', 'sum(amount) - sum(refunds)'))"` }) + await clock.settle() + expect(debug).toContain(NOTE) + expect(logs).toContain(NOTE) + expect(result.context?.some((one: string) => one.startsWith("Altimate lineage for 'fct_orders.revenue'") || one.startsWith('Altimate lineage for fct_orders.revenue'))).toBe(true) + // A command that reads a model changes nothing: no note. + const read: any = await $.tool.call({ tool: 'Bash', command: 'cat models/fct_orders.sql' }) + await clock.settle() + expect((read.context ?? []).some((one: string) => one.startsWith('Altimate lineage for'))).toBe(false) +}) + +test('the pane moves to the model being edited as the edit starts, before the edit lands', async ($, on) => { + let release!: () => void + const until = new Promise(resolve => (release = resolve)) + let ran = false + const { clock } = await start($, on, { + holdReach: { column: 'revenue', until }, + toolResult: e => { + if (e.tool === 'Edit') ran = true + return undefined + }, + }) + const pane = await $.ui.mount({ ...PANE, surface: 'terminal' }) + expect(await pane.find({ type: 'Text', text: /fct_orders/ })).toBeUndefined() + const edit = $.tool.call(EDIT) + // The edit is held while its impact lookup runs: the pane already shows the model being edited. + for (let i = 0; i < 10; i++) await clock.advance(100) + expect(ran).toBe(false) + expect(await pane.find({ type: 'Text', text: /fct_orders/ })).toBeDefined() + release() + await edit + await clock.settle() + await pane.unmount() +}) + +test('a graph wider than the pane opens as its ends, which fit, with its dashboards named; v shows the whole graph', async ($, on) => { + const { clock } = await start($, on) + await $.tool.call({ tool: 'Read', file_path: FILE }) + await clock.settle() + const ui = await $.ui.mount({ ...PANE, props: { ...PANE.props, bodyColumns: 50 }, surface: 'terminal' }) + expect(await ui.find({ type: 'Text', text: /^ENDS ONLY/ })).toBeDefined() + expect(await ui.find({ type: 'Text', text: /wider than the pane: ends only · v for the whole graph/ })).toBeDefined() + // No window onto a wider canvas: nothing runs off the right. + expect(await ui.find({ type: 'Text', text: /columns \d+–\d+ of \d+/ })).toBeUndefined() + // The dashboard is in view: drawn as a box, or named under the drawing. + const drawn = await ui.find({ type: 'Button', text: /Exec Revenue/ }) + const named = await ui.find({ type: 'Text', text: /dashboard.* downstream.*Exec Revenue/ }) + expect(drawn ?? named).toBeDefined() + await ui.press({ key: 'lineage-view' }) + await clock.settle() + expect(await ui.find({ type: 'Text', text: /^GRAPH/ })).toBeDefined() + expect(await ui.find({ type: 'Text', text: /columns \d+–\d+ of \d+/ })).toBeDefined() + // Whatever the window shows, the dashboards stay named. + expect((await ui.find({ type: 'Button', text: /Exec Revenue/ })) ?? (await ui.find({ type: 'Text', text: /Exec Revenue/ }))).toBeDefined() + await ui.unmount() +}) diff --git a/plugins/altimate-code/hooks/lineage.tsx b/plugins/altimate-code/hooks/lineage.tsx index c755e032..cf6577e9 100644 --- a/plugins/altimate-code/hooks/lineage.tsx +++ b/plugins/altimate-code/hooks/lineage.tsx @@ -23,6 +23,7 @@ import { countText, describeGraph, downstreamArt, + downstreamOf, findModel, findModelOrColumn, graphFromSql, @@ -76,6 +77,8 @@ const warehouseAtom = atom({ plugin: 'altimate-code', key: 'lineageWarehouse' } const expandedAtom = atom({ plugin: 'altimate-code', key: 'lineageExpanded' } as const, { up: false, down: false }) // How the pane draws the graph: boxes and arrows, only its ends, folders, or the text lists. const viewAtom = atom({ plugin: 'altimate-code', key: 'lineageView' } as const, 'graph' as ViewKind | 'list') +// Whether the person picked the view (v): until then a graph wider than the pane is shown as its ends, which fit. +const viewChosenAtom = atom({ plugin: 'altimate-code', key: 'isLineageViewChosen' } as const, false) // A model the pane should scroll down to once its graph is drawn: the canvas can be taller than the pane. let revealFor: string | null = null // The dbt models Claude's last answer names, in the order it names them. @@ -867,7 +870,7 @@ function cut(text: string, max: number): string { * columns, which downstream models and dashboards read that column. Logged, then added to * Claude's context with the edit's result. */ -async function impactFirst($: EngineInterface, e: any, file: string, deadline: number, progress: { project: boolean; failed?: boolean }): Promise { +async function impactFirst($: EngineInterface, e: any, file: string, deadline: number, progress: { project: boolean; failed?: boolean }, changedAt?: number | null): Promise { const root = await projectRoot($, file, true) if (!root) return null projectSeen = true @@ -890,8 +893,9 @@ async function impactFirst($: EngineInterface, e: any, file: string, deadline: n if (!graph || !id) return null const node = graph.nodes[id]! // The file already changed after the last compile: the manifest describes an older version of it. - const fileStat = await $.fs.stat(file).catch(() => null) - const stale = !!fileStat && fileStat.mtimeMs > stat.mtimeMs + // A file a shell command just changed is judged by its time before the command (unknown: not said). + const fileStat = changedAt === undefined ? await $.fs.stat(file).catch(() => null) : null + const stale = changedAt === undefined ? !!fileStat && fileStat.mtimeMs > stat.mtimeMs : changedAt !== null && changedAt > stat.mtimeMs const counts = impactOf(graph, id) const downstream = counts.direct + counts.transitive if (downstream === 0 && counts.exposures.length === 0) return null @@ -1150,6 +1154,47 @@ function drawRow(ui: Ui, $: EngineInterface, graph: LineageGraph, segments: Segm ) } +// A shell command that may write the SQL files it names (`sed -i`, a redirect, `tee`, a script). +const WRITES = /\bsed\b[^|;&]*\s-i|\bperl\b[^|;&]*\s-\w*i|>>?\s*['"]?[^\s'"|;&]*\.sql\b|\btee\b|\b(?:python3?|node|ruby|bun|deno)\b|\b(?:mv|cp|patch|truncate)\b|\bgit\s+(?:apply|checkout|restore)\b/ +// The most model files one shell command's note is about. +const MAX_BASH_MODELS = 3 +// A file's time can trail the clock read as the command started by this much. +const MTIME_SLACK_MS = 1_000 + +/** A shell command that may write, as it started: where it runs and when, to find the model files it changed after. */ +type BashStart = { command: string; startedAt: number } + +/** The project's model files a shell command changed: modified since it started, with their text from the last commit. */ +async function bashChanged($: EngineInterface, start: BashStart): Promise<{ root: string; files: { file: string; before: string }[] } | null> { + // From the session's folder, as the task summary reads it: Claude's shell folder can be slow to read, and this holds the command's result. + const base = await $.session.root().catch(() => '.') + const root = await projectRoot($, commandDir(start.command, base), false).catch(() => null) + if (!root) return null + const models = projectLayout(await $.fs.read(`${root}/dbt_project.yml`).catch(() => '')).models.map(one => one.replace(/^\.\//, '').replace(/\/+$/, '')).filter(Boolean) + const dirs = models.map(one => (one === '.' ? root : `${root}/${one}`)) + if (!dirs.length) return null + // Files touched in the last minute, then each one's own time against the command's start. + const listed = await $.process.run(['find', ...dirs, '-name', '*.sql', '-type', 'f', '-mmin', '-1'], { timeoutMs: 10_000 }).catch(() => null) + const top = (await $.process.run(['git', '-C', root, 'rev-parse', '--show-toplevel'], { timeoutMs: 5_000 }).catch(() => null))?.stdout.trim() ?? '' + const files: { file: string; before: string }[] = [] + for (const file of (listed?.stdout ?? '').split('\n').map(one => one.trim()).filter(Boolean)) { + if (files.length >= MAX_BASH_MODELS) break + if (/[\\/](?:target|dbt_packages|dbt_modules)[\\/]/.test(file.slice(root.length)) || !isModelPath(root, models, file)) continue + const stat = await $.fs.stat(file).catch(() => null) + if (!stat || stat.mtimeMs < start.startedAt) continue + // Its text before: the last commit's (an untracked file had none). + const head = top && file.startsWith(`${top}/`) ? await $.process.run(['git', '-C', top, 'show', `HEAD:${file.slice(top.length + 1)}`], { timeoutMs: 5_000 }).catch(() => null) : null + files.push({ file, before: head?.exitCode === 0 ? head.stdout : '' }) + } + return { root, files } +} + +/** Whether a file is under one of the project's model folders. */ +function isModelPath(root: string, models: string[], file: string): boolean { + const rel = file.slice(root.length + 1) + return file.startsWith(`${root}/`) && /\.sql$/i.test(rel) && models.some(one => one === '.' || rel.startsWith(`${one}/`)) +} + /** The lineage hooks whose events no other part of the plugin hooks. */ export function registerLineage(on: On, options: Record = {}) { showEditImpact = options.show_edit_impact === true || options.show_edit_impact === 'true' @@ -1266,6 +1311,19 @@ export function registerLineage(on: On, options: Record = {}) { let lookup: Promise | null = null let finished = false const progress: { project: boolean; failed?: boolean } = { project: false } + // The pane moves to the model being edited as the edit starts, not once it has landed. + if (isEdit) later($, async () => void (await follow($, { file: file!, auto: true }))) + // A shell command that may write (`python3 …`, `sed -i`): its start, so a model it rewrites gets the same note as + // an Edit. Nothing is read or run before it: a slow lookup never holds the command or the gates after this hook. + const bash: BashStart | null = e.tool === 'Bash' && 'command' in e && !(e as { agentId?: string }).agentId && WRITES.test(String(e.command)) ? { command: String(e.command), startedAt: (await $.clock.now()) - MTIME_SLACK_MS } : null + // The model file it names: the pane moves to it as the command starts, in the background. + if (bash && /\.sql\b/i.test(bash.command)) { + later($, async () => { + const base = (await $.session.cwd().catch(() => null)) ?? (await $.session.root().catch(() => '.')) + const first = sqlFilesIn(bash.command, commandDir(bash.command, base))[0] + if (first && (await $.fs.exists(first).catch(() => false))) await follow($, { file: first, auto: true }) + }) + } if (isEdit) { // The columns stop half a second before the hold ends, leaving time to write the note. lookup = impactFirst($, e, file!, (await $.clock.now()) + HOLD_TOTAL_MS - 500, progress) @@ -1288,6 +1346,27 @@ export function registerLineage(on: On, options: Record = {}) { // Any tool may have changed the shown model's file (Edit, a shell command): one stat says so. later($, () => recheckStale($)) if (result.deny || result.isError) return result + // The model files the shell command changed: the pane follows the first, and each gets the impact note an Edit gets. + const bashNotes: ImpactNote[] = [] + if (bash) { + const changed = await within($, 3_000, bashChanged($, bash).catch(() => null)) + if (changed?.files.length) { + clearSqlGraphs() + later($, async () => void (await follow($, { file: changed.files[0]!.file, auto: true }))) + const deadline = (await $.clock.now()) + HOLD_TOTAL_MS - 500 + const notes = await within( + $, + HOLD_TOTAL_MS, + Promise.all(changed.files.map(async one => impactFirst($, { tool: 'Edit', file_path: one.file, old_string: one.before, new_string: await $.fs.read(one.file).catch(() => '') }, one.file, deadline, { project: false }, null).catch(() => null))), + ) + for (const note of notes ?? []) { + if (!note) continue + bashNotes.push(note) + $.ui.log(note.toast, { to: 'debug' }) + if (showEditImpact) $.ui.log(note.toast) + } + } + } // After the edit lands: a .sql file or dbt_project.yml changed, so the SQL-file graphs are stale. if (file && EDIT_TOOLS.has(e.tool) && /(\.sql|\.py|dbt_project\.yml)$/i.test(file)) clearSqlGraphs() // Projects are found by their dbt_project.yml alone. @@ -1342,6 +1421,14 @@ export function registerLineage(on: On, options: Record = {}) { return { ...result, context: [...(result.context ?? []), note.context] } as typeof result } } + const fresh = bashNotes.filter(one => !contextGiven.has(one.key)) + if (fresh.length) { + for (const one of fresh) { + contextGiven.add(one.key) + $.ui.log(`impact-first · added to Claude's context: lineage for ${one.view.column ? `${one.view.model}.${one.view.column}` : one.view.model} (changed by a shell command)`, { to: 'debug' }) + } + return { ...result, context: [...(result.context ?? []), ...fresh.map(one => one.context)] } as typeof result + } if (lookupFailed) { const name = (file!.split(/[\\/]/).pop() ?? file!).replace(/\.sql$/i, '') $.ui.log(`impact-first · added to Claude's context: the lineage lookup failed for ${name}`) @@ -1408,6 +1495,7 @@ export function registerLineage(on: On, options: Record = {}) { const stale = await read($, staleAtom) const loading = await read($, loadingAtom) const chosenView = await read($, viewAtom) + const viewChosen = await read($, viewChosenAtom) const pan = await read($, panAtom) const answer = await read($, answerAtom) const node = focus && graph ? graph.nodes[focus.model] : null @@ -1499,6 +1587,18 @@ export function registerLineage(on: On, options: Record = {}) { ;({ drawn, room, maxPan, from, shownRows } = draw()) } } + // The whole graph is wider than the pane and the person has not picked a view: its ends, which fit, with the + // dashboards at the end, rather than a window with edges running off the right. + const fitted = !viewChosen && view === 'graph' && !!drawn && drawn.canvas > room + const wide = fitted ? drawn!.canvas : 0 + if (fitted) { + view = 'ends' + ;({ drawn, room, maxPan, from, shownRows } = draw()) + } + // Dashboards downstream that the drawing leaves out (past the pane, or folded): named under it. + const drawnIds = new Set(shownRows.flatMap(row => row.map(one => one.node).filter(Boolean))) + const dashboards = view === 'list' ? [] : downstreamOf(graph, focus.model).rows.filter(row => row.node.type === 'exposure') + const unseenDashboards = dashboards.filter(row => !drawnIds.has(row.node.id)) // From what is shown: a stored pan past the edge (a narrower pane, a right-clamped start) is not the start. const panBy = (step: number) => update($, panAtom, () => Math.min(maxPan, Math.max(0, from + step))) // Once a new model's graph is drawn, scroll the pane to it: the canvas can be taller than the pane. @@ -1578,6 +1678,7 @@ export function registerLineage(on: On, options: Record = {}) { {graph.upTruncated || graph.truncated ? ` · ${[graph.upTruncated && 'upstream', graph.truncated && 'downstream'].filter(Boolean).join(' and ')} cut at 500 nodes` : ''} {tooBig && ⚠ the whole graph is more than Claude Code can draw in a pane, so it is shown as folders · v: ends only, list} + {fitted && the whole graph is {wide} columns wide, wider than the pane: ends only · v for the whole graph} {view === 'graph' && ( {drawn.canvas > room &&