diff --git a/plugins/altimate-code/CHANGELOG.md b/plugins/altimate-code/CHANGELOG.md index e4a9ea5..14d356a 100644 --- a/plugins/altimate-code/CHANGELOG.md +++ b/plugins/altimate-code/CHANGELOG.md @@ -2,6 +2,17 @@ All notable changes to this plugin will be documented in this file. +## [Unreleased] + +### Fixed +- After `/clear`, the new conversation loads the team comments again (the band, and the threads Claude gets when it reads a commented model) instead of waiting for `/comments` or the next poll, and project health keeps its baseline, the changes not yet checked and the issues found (`/health` no longer says the session has no project; a check the `/clear` cut short runs again). The status line keeps the integrations and altimate-code parts; health draws its own again. +- The dbt runs the plugin starts by itself (compile and validate's `dbt show` and `dbt ls`, `/dbt-show` and Re-run, and the project health check's parse) write their artifacts to `target/.altimate-plugin/` instead of `target/`, so they no longer replace the project's `manifest.json` (with every model's compiled SQL) or `run_results.json`. The health check runs its CLI call in its own process for that, with the project's `catalog.json` copied beside its manifest. The doctor's **Run dbt parse** still refreshes the project's own manifest. +- After Claude edits a model's SQL that dbt has not compiled since, its documentation coverage says it is from before the edit, and once the edit's compile-and-validate check has run the model, `/propagate` shows the coverage of the model as it is now. +- The end-of-task summary for a model the task created says its columns are new with an unknown reach, not "update altimate-code". +- When the Query results or lineage pane sits above the prompt (a terminal under 110 columns, or not fullscreen), Claude Code gives it the same height whatever size it asks for, so the Min / Default / Max row, which did nothing there, is replaced by a line saying so. +- `--defer --state …`, `--favor-state` and `--target …` that `/defer` and `/target` add to Claude's dbt commands are no longer invisible: Claude's result says what the command ran as, and a line above the prompt shows it until the next prompt. +- **Resolve** on a thread the lagging comment list still shows after it was resolved elsewhere is done (the thread is hidden) instead of answering "not an open thread"; a reply to such a thread says it is no longer open and reads the list again. + ## [1.8.3] — 2026-11-19 ### Fixed diff --git a/plugins/altimate-code/hooks/altimate-cli.ts b/plugins/altimate-code/hooks/altimate-cli.ts index 07bf895..1c64640 100644 --- a/plugins/altimate-code/hooks/altimate-cli.ts +++ b/plugins/altimate-code/hooks/altimate-cli.ts @@ -18,11 +18,11 @@ export const NOT_FOUND = /ENOENT|not found|no such file/i export const MISSING_BINARY = new RegExp(`${NOT_FOUND.source}|(cannot|could not|failed to) (start|spawn|run)`, 'i') /** Runs the altimate CLI under the first name that is installed; 90 s unless the caller says otherwise. */ -export async function runAltimate($: EngineInterface, args: string[], timeoutMs = 90_000) { +export async function runAltimate($: EngineInterface, args: string[], timeoutMs = 90_000, env?: Record) { let lastError: unknown = null for (const bin of CLI_NAMES) { try { - return await $.process.run([bin, ...args], { timeoutMs }) + return await $.process.run([bin, ...args], { timeoutMs, ...(env && { env }) }) } catch (error) { if (!MISSING_BINARY.test(String(error))) throw error lastError = error @@ -191,6 +191,8 @@ export type RunToolOptions = { * item 4), as the skill does before it adds `--allow-write`. */ write?: boolean + /** Variables set for the CLI process (CLI route only, so a call that sets them never goes over MCP). */ + env?: Record } /** @@ -201,7 +203,7 @@ export type RunToolOptions = { */ export async function runTool($: EngineInterface, name: string, args: Record, options: RunToolOptions = {}) { const { until, write = false } = options - const cliOnly = options.cliOnly === true || write + const cliOnly = options.cliOnly === true || write || options.env !== undefined // With a deadline, nothing starts once it has passed, and the CLI is stopped at it. const left = async () => { const ms = until === undefined ? TOOL_TIMEOUT_MS : until - (await $.clock.now()) @@ -241,7 +243,7 @@ export async function runTool($: EngineInterface, name: string, args: Record }; idLagsUntil?: Promise } = {}): () => Promise { + let cleared = false + let held = false + const written = new Set() + const name = (e: Address) => `${e.plugin}\u0000${e.key}\u0000${e.id ?? ''}` + // A $-call's hook answers `{ value }`: here the value is the read, `{ value, version }`. The version stays, so a write still lands. + on('state.get', async (_: unknown, e: Address, next: (e: Address) => Promise<{ value?: { value: unknown; version: number } }>) => { + const stored = await next(e) + return cleared && !written.has(name(e)) && stored.value ? { ...stored, value: { ...stored.value, value: undefined } } : stored + }) + on('state.set', async (_: unknown, e: Address, next: (e: Address) => Promise) => { + const asked = clears + // The new conversation's first write of that value waits, so a test can act while it is under way. + if (cleared && !held && options.holdWriteOf?.key === e.key) { + held = true + await options.holdWriteOf.until + } + const done = await next(e) + // Read as written only once it is, and only in the conversation it was written in. + if (cleared && asked === clears) written.add(name(e)) + return done + }) + let clears = 0 + // The engine's id read can lag the /clear: until `idLagsUntil` settles it still names the session that ended. + let lagging = !!options.idLagsUntil + options.idLagsUntil?.then(() => (lagging = false)) + on('session.id', () => ({ value: `session-${lagging ? Math.max(0, clears - 1) : clears}` })) + return async () => { + await $.session.end({ reason: 'clear', sessionId: `session-${clears}` }) + clears++ + cleared = true + written.clear() + } +} diff --git a/plugins/altimate-code/hooks/comments.test.tsx b/plugins/altimate-code/hooks/comments.test.tsx index 87f197d..e84a5d2 100644 --- a/plugins/altimate-code/hooks/comments.test.tsx +++ b/plugins/altimate-code/hooks/comments.test.tsx @@ -1,5 +1,6 @@ import { expect, mock, test } from 'claude-code/testing' +import { playsClear } from './clear-state.testkit' import { PHASE_TIMEOUT_MS } from './tool-chain' import { FIRST_READ_WAIT_MS, contextBlock, groupByModel, placeThread, projectName, readOptions, sameText, relativePath, threadsOnFile, toThread } from './features/comments' @@ -795,14 +796,98 @@ test('a folder without a project of its own is looked up again from another proj expect(cli.filter(c => c.args.action === 'list').map(c => c.args.project)).toEqual(['jaffle_shop', 'other_shop', 'jaffle_shop', 'other_shop']) }) -test('a post to a thread a teammate already resolved says nothing was sent and does not lock the other threads', async ($, on) => { - const { clock, cli } = await start($, on, { failPost: true, failPostCode: 'unknown_thread' }) +test('resolving a thread a teammate already resolved, which the lagging list still shows, is done: it leaves the pane and the list is read again', async ($, on) => { + const second = { ...RAW, group_id: 12, file_path: 'models/customers.sql' } + const { clock, cli } = await start($, on, { failPost: true, failPostCode: 'unknown_thread', threads: [RAW, second] }) const ui = await $.ui.mount({ ...PANE, surface: 'terminal' }) await clock.settle() await ui.press({ key: '7:11-resolve' }) for (let i = 0; i < 3; i++) await clock.settle() - await ui.press({ key: '7:11-resolve' }) + expect(await ui.find({ type: 'Text', text: /Already resolved: the list was behind/ })).toBeDefined() + expect(await ui.find({ type: 'Button', key: '7:11-resolve' })).toBeUndefined() + expect(await ui.find({ type: 'Text', text: /Thread 11 is not an open thread/ })).toBeUndefined() + expect(cli.filter(c => c.args.action === 'list')).toHaveLength(2) + // The other thread is not locked. + await ui.press({ key: '7:12-resolve' }) for (let i = 0; i < 3; i++) await clock.settle() expect(cli.filter(c => c.args.action === 'resolve')).toHaveLength(2) await ui.unmount() }) + +test('a reply to a thread that is no longer open says so plainly and reads the list again', async ($, on) => { + const { clock, cli } = await start($, on, { failPost: true, failPostCode: 'unknown_thread' }) + const ui = await $.ui.mount({ ...PANE, surface: 'terminal' }) + await clock.settle() + await ui.input({ key: '7:11-reply', text: 'Yes' }) + for (let i = 0; i < 3; i++) await clock.settle() + expect(cli.filter(c => c.args.action === 'reply')).toHaveLength(1) + expect(await ui.find({ type: 'Text', text: /That thread is no longer open \(it was resolved, or the list was behind\): nothing was sent/ })).toBeDefined() + expect(cli.filter(c => c.args.action === 'list')).toHaveLength(2) + await ui.unmount() +}) + +test('after /clear the new conversation loads the threads again: the pane lists them and a read of the model gets them', async ($, on) => { + const clear = playsClear($, on) + const { clock, cli } = await start($, on) + await clear() + await clock.settle() + expect(cli.filter(c => c.args.action === 'list')).toHaveLength(2) + const ui = await $.ui.mount({ ...PANE, surface: 'terminal' }) + expect(await ui.find({ type: 'Text', text: 'models/orders.sql' })).toBeDefined() + await ui.unmount() + const result: any = await $.tool.call({ tool: 'Read', file_path: FILE }) + expect(result.context?.[0]).toContain('Grace Hopper: No, gross.') +}) + +test('after /clear the first read of a model waits for the new conversation\'s list, as a session\'s first read does', async ($, on) => { + const clear = playsClear($, on) + let release!: () => void + const held = new Promise(resolve => (release = resolve)) + const { clock } = await start($, on, { holdListCall: { n: 1, until: held } }) + await clear() + const reading = $.tool.call({ tool: 'Read', file_path: FILE }) + await clock.advance(10) + release() + await clock.advance(10) + const result: any = await reading + expect(result.context?.[0]).toContain('line 3, model, on “sum(amount)”') +}) + +test('a read right after /clear, before the new conversation\'s comments are loaded, waits for them even when the last conversation spent its first-read wait', async ($, on) => { + let newId!: () => void + const clear = playsClear($, on, { idLagsUntil: new Promise(resolve => (newId = resolve)) }) + let first!: () => void + let second!: () => void + const holdLists = { 0: new Promise(resolve => (first = resolve)), 1: new Promise(resolve => (second = resolve)) } + const { clock } = await start($, on, { settle: false, holdLists }) + // The last conversation's first read comes before its list: it spends its wait. + const early = $.tool.call({ tool: 'Read', file_path: FILE }) + await clock.advance(FIRST_READ_WAIT_MS + 10) + await early + first() + await clock.settle() + await clear() + // No settling: the read comes before the session id watcher has seen the new id. + const reading = $.tool.call({ tool: 'Read', file_path: FILE }) + await clock.advance(10) + second() + await clock.advance(10) + const result: any = await reading + newId() + expect(result.context?.[0]).toContain('Grace Hopper: No, gross.') +}) + +test('a read of another project right after /clear, before the new conversation\'s comments are loaded, keeps that project', async ($, on) => { + let newId!: () => void + const clear = playsClear($, on, { idLagsUntil: new Promise(resolve => (newId = resolve)) }) + const other = '/other' + const { clock, cli } = await start($, on, { otherProject: other, files: { [FILE]: SQL, [`${other}/models/orders.sql`]: SQL } }) + await clear() + await $.tool.call({ tool: 'Read', file_path: `${other}/models/orders.sql` }) + newId() + await clock.settle() + const ui = await $.ui.mount({ ...PANE, surface: 'terminal' }) + expect(await ui.find({ type: 'Text', text: /Team comments · other_shop/ })).toBeDefined() + await ui.unmount() + expect(cli.filter(c => c.args.action === 'list').at(-1)?.args.project).toBe('other_shop') +}) diff --git a/plugins/altimate-code/hooks/dbt-mods-core.ts b/plugins/altimate-code/hooks/dbt-mods-core.ts index 301bc0a..996c1a9 100644 --- a/plugins/altimate-code/hooks/dbt-mods-core.ts +++ b/plugins/altimate-code/hooks/dbt-mods-core.ts @@ -250,8 +250,9 @@ const CONTEXT_FLAGS: Record = { '--target': '--target', '-t': '- const SELECT_FLAGS: Record = { '--select': '--select', '-s': '--select', '--models': '--select', '-m': '--select', '--exclude': '--exclude', '--selector': '--selector' } // Flags that change how dbt reports or how fast it runs, never what it runs. const DISPLAY_FLAGS = new Set(['--log-format', '--log-format-file', '--log-level', '--log-level-file', '--log-path', '--quiet', '-q', '--debug', '-d', '--no-print', '--print', '--use-colors', '--no-use-colors', '--printer-width', '--threads', '--no-version-check', '--version-check', '--write-json', '--no-write-json', '--send-anonymous-usage-stats', '--no-send-anonymous-usage-stats', '--log-cache-events', '--record-timing-info']) -// Flags a re-run of Claude's dbt show replaces: the pane reads JSON, and keeps the limit itself. -const OUTPUT_FLAGS = new Set(['--output', '-o', '--log-format', '--limit', '--log-level']) +// Flags a re-run of Claude's dbt show replaces: the pane reads JSON, and keeps the limit itself. Its artifact +// folder is the plugin's own (SCRATCH_TARGET), so a folder Claude named is dropped too. +const OUTPUT_FLAGS = new Set(['--output', '-o', '--log-format', '--limit', '--log-level', '--target-path']) const BARE_OUTPUT_FLAGS = new Set(['--quiet', '-q']) /** The last value given to `--flag value` or `--flag=value` in an argv; null when none, or when it holds an expansion the shell would make. */ @@ -472,6 +473,16 @@ export function targetPath(yml: string): string { return value && !value.includes('{{') ? value : 'target' } +/** + * The folder, inside the project's artifact folder, where the dbt runs the plugin starts by itself (a check's + * `dbt show`, a health check's parse) write their artifacts. dbt rewrites manifest.json on every run with the + * compiled SQL of only the nodes that run compiled: in the project's own folder that would drop every other + * model's compiled SQL (and replace run_results.json), so those runs go here and are read from here. + */ +export const SCRATCH_TARGET = '.altimate-plugin' +/** The health check's own folder inside SCRATCH_TARGET: its parse runs outside the plugin's dbt queue, so it never writes the manifest a check is reading. */ +export const HEALTH_SCRATCH = 'health' + /** * The folders a project's files live in, from dbt_project.yml: its model, seed, snapshot, macro, test * and analysis paths (dbt's defaults when a key is absent). `''` is the project folder itself. diff --git a/plugins/altimate-code/hooks/dbt-mods.test.tsx b/plugins/altimate-code/hooks/dbt-mods.test.tsx index 2c9f9c5..c5ba333 100644 --- a/plugins/altimate-code/hooks/dbt-mods.test.tsx +++ b/plugins/altimate-code/hooks/dbt-mods.test.tsx @@ -135,7 +135,7 @@ const DOCTOR = { ...RESULTS, requestId: 'altimate-doctor', props: { ...RESULTS.p type Options = { /** What dbt show answers: rows, or a failure's error lines. */ - show?: { rows?: Record[]; error?: string; node?: string | null; onlyName?: string; mustInclude?: string; raw?: { exitCode: number; stdout: string; stderr?: string }; writes?: string; invocation?: string } + show?: { rows?: Record[]; error?: string; node?: string | null; onlyName?: string; mustInclude?: string; raw?: { exitCode: number; stdout: string; stderr?: string }; writes?: string; claudeWrites?: string; invocation?: string } /** The node names `dbt ls` prints; the default is none. */ listed?: (string | { name: string; resource_type: string })[] /** What stops dbt from running: its spawn failing, or the shell finding no such command. Asked at each run. */ @@ -223,6 +223,9 @@ type Options = { async function start($: any, on: any, options: Options = {}) { const clock = mock.clock(on, { now: options.now ?? 1_000 }) const runs: string[][] = [] + const envs: Record[] = [] + // dbt processes and manifest reads, in the order they happened. + const order: string[] = [] const cwds: string[] = [] const toasts: string[] = [] const prompts: string[] = [] @@ -326,12 +329,13 @@ async function start($: any, on: any, options: Options = {}) { if (!exists(e.path)) throw new Error('ENOENT') const isDir = !(rel in files) const mtimeMs = rel === 'target/run_results.json' ? (options.runResultsAt ?? 5_000) : (mtimes[rel] ?? 500) - const size = rel === 'target/manifest.json' && options.manifestSize ? options.manifestSize : rel === 'target/run_results.json' && options.runResultsSize ? options.runResultsSize : (files[rel]?.length ?? 0) + const size = /(^|\/)manifest\.json$/.test(rel) && options.manifestSize ? options.manifestSize : rel === 'target/run_results.json' && options.runResultsSize ? options.runResultsSize : (files[rel]?.length ?? 0) return { value: { kind: isDir ? 'dir' : 'file', size, mtimeMs, isLink: false } } }) on('fs.read', (_: unknown, e: { path: string }) => { if (options.pathDbtPython && e.path === '/x/bin/dbt') return { value: `#!${options.pathDbtPython}\nimport sys\n` } const rel = e.path.slice(ROOT.length + 1) + if (rel.endsWith('manifest.json')) order.push(`read ${rel}`) if (!(rel in files)) throw new Error('ENOENT') if (rel === 'target/run_results.json' && (options.runResultsSize ?? 0) > 4 * 1024 * 1024) throw new Error('file is larger than 4 MiB') return { value: files[rel] } @@ -349,8 +353,13 @@ async function start($: any, on: any, options: Options = {}) { }) let held = false let shows = 0 - on('process.run', async (_: unknown, e: { argv: readonly string[]; init?: { cwd?: string; stdin?: string } }) => { + on('process.run', async (_: unknown, e: { argv: readonly string[]; init?: { cwd?: string; stdin?: string; env?: Record } }) => { runs.push([...e.argv]) + envs.push(e.init?.env ?? {}) + if (e.argv[0] === DBT) order.push(`dbt ${e.argv[1]}`) + // Where this dbt writes its artifacts: DBT_TARGET_PATH when set, else the project's target folder. + const given = e.init?.env?.DBT_TARGET_PATH + const out = given ? (given.startsWith(`${ROOT}/`) ? given.slice(ROOT.length + 1) : given) : (options.target ?? 'target') if (e.argv[2] === PROFILE_REDACT) { redactSent.push({ argv: [...e.argv], stdin: e.init?.stdin }) await options.holdRedact @@ -395,11 +404,14 @@ async function start($: any, on: any, options: Options = {}) { } if (verb === 'show') { const show = options.show ?? { rows: [{ order_id: 1, amount: 10 }] } - // A dbt show writes run_results.json, whoever's invocation is in it by the time the process exits. - if (show.writes) files['target/run_results.json'] = JSON.stringify({ ...RUN_RESULTS, metadata: { invocation_id: show.writes, generated_at: 'x' } }) - if (options.manifestAfterShow) { - files['target/manifest.json'] = JSON.stringify(options.manifestAfterShow) - mtimes['target/manifest.json'] = (mtimes['target/manifest.json'] ?? 1_000) + 1 + // A dbt show writes run_results.json to its own artifact folder; a run of Claude's that ends meanwhile writes the project's. + if (show.writes) files[`${out}/run_results.json`] = JSON.stringify({ ...RUN_RESULTS, metadata: { invocation_id: show.writes, generated_at: 'x' } }) + if (show.claudeWrites) files[`${options.target ?? 'target'}/run_results.json`] = JSON.stringify({ ...RUN_RESULTS, metadata: { invocation_id: show.claudeWrites, generated_at: 'x' } }) + // dbt parses the project and writes the manifest to its artifact folder, every time. + const parsed = options.manifestAfterShow ? JSON.stringify(options.manifestAfterShow) : files[`${options.target ?? 'target'}/manifest.json`] + if (parsed) { + files[`${out}/manifest.json`] = parsed + mtimes[`${out}/manifest.json`] = (mtimes[`${out}/manifest.json`] ?? 1_000) + 1 } // dbt selects a model by its name: the one the test's manifest gives the file, if it gave one. const asked = e.argv[e.argv.indexOf('--select') + 1] @@ -434,7 +446,7 @@ async function start($: any, on: any, options: Options = {}) { }) await $.session.start({ cwd: ROOT, surface: 'terminal', isInteractive: options.interactive ?? false }) await clock.settle() - return { redactSent, clock, runs, cwds, toasts, prompts, store, registered, storeGets, mtimes, opened, sizes, files, setBash: (next: typeof bash) => void (bash = next) } + return { redactSent, clock, runs, envs, order, cwds, toasts, prompts, store, registered, storeGets, mtimes, opened, sizes, files, setBash: (next: typeof bash) => void (bash = next) } } // ─── What the helpers read from dbt ───────────────────────────────────────────────────────────── @@ -2682,10 +2694,27 @@ test('a docked pane reports the transcript\'s width, which never sets the next d await clock.settle() expect(sizes.at(-1)).toMatchObject({ columns: 90 }) const inline = await $.ui.mount({ ...RESULTS, surface: 'terminal', props: { ...RESULTS.props, placement: 'inline' }, viewport: { columns: 100, rows: 50, isFullscreen: false } }) - await inline.press({ key: 'results-size-max' }) + await inline.unmount() + // The inline drawing gave the terminal's width: the next dock is sized from it. + const again = await $.ui.mount({ ...RESULTS, surface: 'terminal', viewport: { columns: 55, rows: 50, isFullscreen: true } }) + await again.press({ key: 'results-size-max' }) await clock.settle() expect(sizes.at(-1)).toMatchObject({ columns: 45, rows: 60 }) + await again.unmount() +}) + +test('seated inline above the prompt, where Claude Code gives the pane the same height whatever it asks for, the pane has no Min / Default / Max that would do nothing, and says why', async ($, on) => { + const { clock } = await start($, on) + await $.command.run({ command: 'dbt-show', args: 'orders' }) + await clock.settle() + const inline = await $.ui.mount({ ...RESULTS, surface: 'terminal', props: { ...RESULTS.props, placement: 'inline' }, viewport: { columns: 100, rows: 42, isFullscreen: false } }) + for (const size of ['min', 'default', 'max']) expect(await inline.find({ type: 'Button', key: `results-size-${size}` })).toBeUndefined() + expect(await inline.find({ type: 'Text', text: /Above the prompt the pane is as tall as Claude Code lets it be/ })).toBeDefined() await inline.unmount() + // Docked, the three are there. + const docked = await $.ui.mount({ ...RESULTS, surface: 'terminal' }) + expect(await docked.find({ type: 'Button', key: 'results-size-max' })).toBeDefined() + await docked.unmount() }) test('Max asks for the width Default asks for, Min for less, and the first open asks as Default does', async ($, on) => { @@ -3308,36 +3337,34 @@ test('a run_results.json over 4 MiB that is the one seen before the command is n await band.unmount() }) -for (const [what, show] of [ - ['one that changed across the plugin\'s process', { writes: 'plugin-show' }], -] as const) { - test(`${what}: the artifact the plugin's own dbt show wrote while Claude's command ran is not that command's result`, async ($, on) => { - let ctx: any - ctx = await start($, on, { - runResultsAt: 9_000_000, - show: { rows: [{ order_id: 1 }], ...show }, - // The plugin's check finishes while Claude's command is running. - duringBash: async () => { - await editOrders($) - await ctx.clock.settle() - }, - }) - ctx.setBash({ stdout: CUSTOMERS_FAILED, isError: true }) - await $.tool.call({ tool: 'Bash', command: 'dbt run -s customers' }) - await ctx.clock.settle() - const band = await $.ui.mount({ ...BAND, surface: 'terminal' }) - expect(await band.find({ type: 'Text', text: /customers/ })).toBeDefined() - expect(await band.find({ type: 'Text', text: /not_null_orders_amount/ })).toBeUndefined() - await band.unmount() +test('the plugin\'s own dbt show, run while Claude\'s command runs, writes its run results to its own folder: the project\'s are left to Claude\'s command', async ($, on) => { + let ctx: any + ctx = await start($, on, { + runResultsAt: 9_000_000, + show: { rows: [{ order_id: 1 }], writes: 'plugin-show' }, + // The plugin's check finishes while Claude's command is running. + duringBash: async () => { + await editOrders($) + await ctx.clock.settle() + }, }) -} + const projectResults = ctx.files['target/run_results.json'] + ctx.setBash({ stdout: CUSTOMERS_FAILED, isError: true }) + await $.tool.call({ tool: 'Bash', command: 'dbt run -s customers' }) + await ctx.clock.settle() + expect(ctx.files['target/run_results.json']).toBe(projectResults) + expect(ctx.files['target/.altimate-plugin/run_results.json']).toContain('plugin-show') + const band = await $.ui.mount({ ...BAND, surface: 'terminal' }) + expect(await band.find({ type: 'Text', text: /customers/ })).toBeDefined() + await band.unmount() +}) test('Claude\'s run that wrote run_results.json after the plugin\'s own dbt show, before the plugin read it, is still recorded', async ($, on) => { let ctx: any // The plugin's dbt printed its own id; by the time the process exited Claude's run had written the file. ctx = await start($, on, { runResultsAt: 9_000_000, - show: { rows: [{ order_id: 1 }], writes: 'claude-run', invocation: 'plugin-show' }, + show: { rows: [{ order_id: 1 }], claudeWrites: 'claude-run', invocation: 'plugin-show' }, duringBash: async () => { await editOrders($) await ctx.clock.settle() @@ -3888,3 +3915,75 @@ kitTest('dbt debug\'s error is scrubbed whole before it is cut, so a value that clearRedactions() } }) + +test('the plugin\'s own dbt runs write their artifacts to its own folder: the project\'s manifest, with every model\'s compiled SQL, is left as it was', async ($, on) => { + const { clock, runs, envs, files, mtimes } = await start($, on, { manifestAfterShow: { ...MANIFEST, metadata: { invocation_id: 'plugin-show' } } }) + const kept = files['target/manifest.json'] + const keptAt = mtimes['target/manifest.json'] + // The check of an edit, then /dbt-show. + const result: any = await $.tool.call({ tool: 'Edit', file_path: `${ROOT}/models/orders.sql`, old_string: 'a', new_string: 'b' }) + await clock.settle() + await $.command.run({ command: 'dbt-show', args: 'orders' }) + await clock.settle() + const shows = runs.map((argv, i) => ({ argv, env: envs[i]! })).filter(one => one.argv[0] === DBT && one.argv.includes('show')) + expect(shows).toHaveLength(2) + for (const one of shows) expect(one.env.DBT_TARGET_PATH).toBe(`${ROOT}/target/.altimate-plugin`) + expect(files['target/manifest.json']).toBe(kept) + expect(mtimes['target/manifest.json']).toBe(keptAt) + // The check reads the manifest its own dbt show wrote: the model as it is now. + expect(files['target/.altimate-plugin/manifest.json']).toContain('plugin-show') + expect(result.context?.find((one: string) => one.startsWith('Altimate compile-and-validate'))).toBeDefined() +}) + +test('the doctor\'s Run dbt parse, which the person presses to refresh a stale manifest, writes the project\'s own manifest', async ($, on) => { + const { clock, runs, envs } = await start($, on, { interactive: true, now: 200_000, mtimes: { 'models/orders.sql': 5_000 } }) + const band = await $.ui.mount({ ...BAND, surface: 'terminal' }) + await band.press({ key: 'dbt-doctor-parse' }) + await clock.settle() + await band.unmount() + const parse = runs.findIndex(argv => argv[0] === DBT && argv[1] === 'parse') + expect(parse).toBeGreaterThanOrEqual(0) + expect(envs[parse]!.DBT_TARGET_PATH).toBeUndefined() +}) + +test('Re-run of Claude\'s dbt show drops the artifact folder Claude named: the replay writes to the plugin\'s own', async ($, on) => { + expect(replayArgv(['dbt', 'show', '-s', 'orders', '--target-path', 'target'], 5)).toEqual(['dbt', 'show', '-s', 'orders', '--limit', '5', '--output', 'json', '--log-format', 'json']) + expect(replayArgv(['dbt', 'show', '--target-path=target', '-s', 'orders'], 5)).toEqual(['dbt', 'show', '-s', 'orders', '--limit', '5', '--output', 'json', '--log-format', 'json']) + const { clock, runs, envs, setBash, files } = await start($, on) + const kept = files['target/manifest.json'] + setBash({ stdout: showLog([{ order_id: 1 }], 'orders') }) + await $.tool.call({ tool: 'Bash', command: 'dbt show -s orders --target-path target' }) + await clock.settle() + const ui = await $.ui.mount({ ...RESULTS, surface: 'terminal' }) + await ui.press({ key: 'results-rerun' }) + await clock.settle() + await ui.unmount() + const replay = runs.findIndex(argv => argv[0] === DBT && argv[1] === 'show') + expect(replay).toBeGreaterThanOrEqual(0) + expect(runs[replay]).not.toContain('--target-path') + expect(envs[replay]!.DBT_TARGET_PATH).toBe(`${ROOT}/target/.altimate-plugin`) + expect(files['target/manifest.json']).toBe(kept) +}) + +test('a check reads the manifest its dbt show wrote before the plugin\'s next queued dbt run can write over it', async ($, on) => { + let release!: () => void + let readable!: () => void + // The check's read of its manifest is slow: the queued run would start meanwhile if the queue were not held. + const statGate = { path: 'target/.altimate-plugin/manifest.json', until: new Promise(resolve => (readable = resolve)) } + const { clock, order } = await start($, on, { holdShow: new Promise(resolve => (release = resolve)), statGate }) + const edit = $.tool.call({ tool: 'Edit', file_path: `${ROOT}/models/orders.sql`, old_string: 'a', new_string: 'b' }) + await clock.advance(10) + // A /dbt-show queued behind the check's own show. + const shown = $.command.run({ command: 'dbt-show', args: 'orders' }) + await clock.advance(10) + release() + for (let i = 0; i < 5; i++) await clock.advance(10) + readable() + await Promise.all([edit, shown]) + await clock.settle() + const shows = order.map((one, i) => (one === 'dbt show' ? i : -1)).filter(i => i >= 0) + expect(shows.length).toBeGreaterThanOrEqual(2) + const read = order.findIndex((one, i) => i > shows[0]! && one === 'read target/.altimate-plugin/manifest.json') + expect(read).toBeGreaterThan(shows[0]!) + expect(read).toBeLessThan(shows[1]!) +}) diff --git a/plugins/altimate-code/hooks/dbt-mods.tsx b/plugins/altimate-code/hooks/dbt-mods.tsx index 4076785..c838bd8 100644 --- a/plugins/altimate-code/hooks/dbt-mods.tsx +++ b/plugins/altimate-code/hooks/dbt-mods.tsx @@ -46,6 +46,7 @@ import { dbtVerdict, RUN_VERBS, targetPath, + SCRATCH_TARGET, showPreview, stripAnsi, textTable, @@ -186,6 +187,10 @@ async function dbtBinary($: EngineInterface, root: string): Promise { return 'dbt' } +// A run's process result, with what `read` gave of its artifacts while the queue was held (null when that failed). +type Artifact = { manifest: any; mtimeMs: number } | null +type ProcessDone = { exitCode: number; stdout: string; stderr: string; isStdoutTruncated?: boolean; artifact?: Artifact } + // What a run that was no longer wanted when its turn came answers: it never started. const SKIPPED = { exitCode: -2, stdout: '', stderr: 'skipped: a newer check replaced it' } @@ -193,7 +198,7 @@ const SKIPPED = { exitCode: -2, stdout: '', stderr: 'skipped: a newer check repl * Runs a command in a project, after any dbt this plugin already started there. `wanted` is asked * when its turn comes: a check a newer edit replaced runs no warehouse query. */ -function runQueued($: EngineInterface, root: string, argv: () => Promise, options: { cwd?: string; timeoutMs?: number; wanted?: () => boolean } = {}) { +function runQueued($: EngineInterface, root: string, argv: () => Promise, options: { cwd?: string; timeoutMs?: number; wanted?: () => boolean; scratch?: boolean; read?: () => Promise } = {}): Promise { const before = queues.get(root) ?? Promise.resolve() const run = before .catch(() => {}) @@ -203,16 +208,22 @@ function runQueued($: EngineInterface, root: string, argv: () => Promise word === 'show' || word === 'ls') + // own only when it changed across the process, and when dbt printed its own id, only that one is. A run in + // the plugin's own folder leaves the project's artifact alone, unless the command names a folder itself. + const ownFolder = options.scratch && !command.some(word => word === '--target-path' || word.startsWith('--target-path=')) + const tracked = !ownFolder && command.some(word => word === 'show' || word === 'ls') const before = tracked ? await seenInvocation($, root).catch(() => null) : null - const done = await $.process.run(command, { cwd: options.cwd ?? root, timeoutMs: options.timeoutMs ?? DBT_TIMEOUT_MS }) + // A run the plugin starts by itself writes its artifacts to its own folder (a --target-path the command + // names still wins): the project's manifest keeps every model's compiled SQL. + const env = options.scratch ? { DBT_TARGET_PATH: await scratchOf($, root) } : undefined + const done = await $.process.run(command, { cwd: options.cwd ?? root, timeoutMs: options.timeoutMs ?? DBT_TIMEOUT_MS, ...(env && { env }) }) if (tracked) { const after = await seenInvocation($, root).catch(() => null) const own = printedInvocation(done.stdout) ?? (after !== null && after !== before ? after : null) if (own) ownInvocation.set(root, own) } - return done + // What the run wrote is read before the next queued run can write over it. + return options.read ? { ...done, artifact: await options.read().catch(() => null) } : done }) queues.set(root, run.catch(() => {})) return run @@ -251,8 +262,8 @@ function cannotRun(run: { exitCode: number; stdout: string; stderr: string }): b } /** Runs the project's dbt in the project, after any dbt this plugin already started there. */ -function runDbt($: EngineInterface, root: string, args: string[], timeoutMs = DBT_TIMEOUT_MS, wanted?: () => boolean) { - return runQueued($, root, async () => [await dbtBinary($, root), ...args], { timeoutMs, wanted }).then( +function runDbt($: EngineInterface, root: string, args: string[], timeoutMs = DBT_TIMEOUT_MS, wanted?: () => boolean, scratch = false, read?: () => Promise) { + return runQueued($, root, async () => [await dbtBinary($, root), ...args], { timeoutMs, wanted, scratch, read }).then( done => { if (done !== SKIPPED) { if (cannotRun(done)) unrunnable.add(root) @@ -355,6 +366,11 @@ async function targetOf($: EngineInterface, root: string): Promise { return normalize(path.startsWith('/') ? path : `${root}/${path}`) } +/** Where the plugin's own dbt runs in the project write their artifacts (SCRATCH_TARGET): never the project's own. */ +async function scratchOf($: EngineInterface, root: string): Promise { + return `${await targetOf($, root)}/${SCRATCH_TARGET}` +} + /** The interpreter dbt's own launcher names on its first line (`#!/x/bin/python`); undefined for a native binary or an `env` launcher. */ async function dbtShebang($: EngineInterface, dbt: string): Promise { const launcher = dbt === 'dbt' ? await $.process.run(['which', 'dbt'], { timeoutMs: 5_000 }).then(one => one.stdout.trim()).catch(() => '') : dbt @@ -380,8 +396,8 @@ async function pythonFor($: EngineInterface, root: string): Promise { * manifest over 4 MiB (more than $.fs.read takes) is read for one file's node only, `rel`, through the * project's Python, as the lineage half reads one through altimate-code. */ -async function readManifest($: EngineInterface, root: string, rel?: string): Promise<{ manifest: any; mtimeMs: number } | null> { - const path = `${await targetOf($, root)}/manifest.json` +async function readManifest($: EngineInterface, root: string, rel?: string, dir?: string): Promise<{ manifest: any; mtimeMs: number } | null> { + const path = `${dir ?? (await targetOf($, root))}/manifest.json` const stat = await $.fs.stat(path).catch(() => null) if (!stat || stat.kind !== 'file') return null try { @@ -469,8 +485,10 @@ async function compileAndValidate($: EngineInterface, root: string, file: string // name (`defined_in`) is still previewed, and the path keeps it to that one version. const pkg = await projectName($, root) const selectorFor = (name: string | null) => [`path:${rel}`, ...(pkg ? [`package:${pkg}`] : []), ...(name ? [name] : [])].join(',') - const run1 = (args: string[]) => runDbt($, root, args, DBT_TIMEOUT_MS, current).catch(error => ({ exitCode: -1, stdout: '', stderr: String(error) })) - const show = (select: string) => run1(['show', '--select', select, '--indirect-selection', 'empty', '--limit', '1', '--output', 'json', '--log-format', 'json']) + const scratch = await scratchOf($, root) + const run1 = (args: string[], read?: () => Promise): Promise => runDbt($, root, args, DBT_TIMEOUT_MS, current, true, read).catch(error => ({ exitCode: -1, stdout: '', stderr: String(error) })) + // The manifest the show wrote is read while the plugin's dbt queue is still held, so no other run of it replaces it first. + const show = (select: string) => run1(['show', '--select', select, '--indirect-selection', 'empty', '--limit', '1', '--output', 'json', '--log-format', 'json'], () => readManifest($, root, rel, scratch)) // A newer edit replaced this check before its turn: nothing ran, nothing to report. const replaced = () => notChecked('a newer check replaced this one').then(one => ({ ...one, note: '' })) const knownName = typeof known?.node?.name === 'string' ? known.node.name : null @@ -488,8 +506,8 @@ async function compileAndValidate($: EngineInterface, root: string, file: string let run = await show(selector) if (run === SKIPPED) return replaced() if (cannotRun(run)) return notChecked(NO_DBT) - // dbt show re-parsed the project: the manifest now holds the model and schema.yml as they are. - let after = await readManifest($, root, rel) + // dbt show re-parsed the project: the manifest it wrote (in the plugin's own folder) holds the model and schema.yml as they are. + let after = run.artifact ?? null let node = after ? nodeForFile(after.manifest, rel, await projectName($, root)) : null let named = typeof node?.node?.name === 'string' ? node.node.name : null let preview = run.exitCode === 0 ? showPreview(run.stdout, node?.id ?? known?.id) : null @@ -500,7 +518,7 @@ async function compileAndValidate($: EngineInterface, root: string, file: string const again = await show(selectorFor(named)) if (again === SKIPPED) return replaced() run = again - after = await readManifest($, root, rel) + after = run.artifact ?? null node = after ? nodeForFile(after.manifest, rel, await projectName($, root)) : null named = typeof node?.node?.name === 'string' ? node.node.name : null preview = run.exitCode === 0 ? showPreview(run.stdout, node?.id ?? known?.id) : null @@ -512,7 +530,7 @@ async function compileAndValidate($: EngineInterface, root: string, file: string // list (a model just disabled, not a model at all) was not checked. const nothingShown = run.exitCode === 0 && !preview // A manifest that is there but could not be read (too big to slice, no interpreter) says nothing about this file. - const manifestUnread = !after && (await $.fs.stat(`${await targetOf($, root)}/manifest.json`).catch(() => null))?.kind === 'file' + const manifestUnread = !after && (await $.fs.stat(`${scratch}/manifest.json`).catch(() => null))?.kind === 'file' if (nothingShown && node) errors.push(`dbt show selected nothing for ${rel} (the model ${named ?? model}), so nothing was compiled or run. ${scrubDbt(stripAnsi(run.stdout)).split('\n').map(line => line.trim()).filter(Boolean).slice(-1)[0] ?? ''}`.trim()) const documentedColumns = node ? Object.values(node.node.columns ?? {}) : [] const documented = documentedColumns.map(column => String(column.name)) @@ -535,6 +553,7 @@ async function compileAndValidate($: EngineInterface, root: string, file: string const status: CheckView['status'] = errors.length ? 'error' : reason ? 'partial' : drift.missing.length || drift.undocumented.length ? 'warn' : 'ok' const view: CheckView = { ...blank, model: named ?? model, schema: node ? schemaOf(node.node) : blank.schema, status, reason, ran: !errors.length && !nothingShown, errors: errors.map(one => one.slice(0, 1500)), missing: drift.missing, undocumented: drift.undocumented, columns: returned ? returned.length : null, returned: returned ?? [], documented, ms, at: started } await setCheck($, view, newest) + if (current()) checkEnded(`${root}/${rel}`, mine, view.ran && after ? { root, file: `${root}/${rel}`, manifest: `${scratch}/manifest.json`, mtimeMs: after.mtimeMs } : null) // An edit of a schema file or macro that came while this ran checks it again now. if (forcedWhileBusy && current() && !rechecking) { const waiting = forcedWhileBusy @@ -546,6 +565,43 @@ async function compileAndValidate($: EngineInterface, root: string, file: string return { view, note: checkNote(view), current, newest } } +/** A check that ran its model: the model's file, and the manifest its dbt show wrote (in the plugin's own folder) with its time. */ +export type CheckLanded = { root: string; file: string; manifest: string; mtimeMs: number } +type Waiter = { after: number; resolve: (landed: CheckLanded | null) => void } +// Who waits for a check of a file, by the file's normalized path, with the last check started when they asked. +const landedWaiters = new Map() + +/** + * Resolves when the newest check of `file` that starts after this call ends: with its manifest when it compiled + * and ran the model (that manifest holds the model's SQL as it is now, where the project's own may hold it as it + * was at the last dbt compile), else null. A check already running when this is called is from before the edit, + * and does not answer it. Never rejects; `cancel` stops the wait (the caller's own deadline). + */ +export function nextCheckOf(file: string): { landed: Promise; cancel: () => void } { + const key = normalize(file) + let waiter!: Waiter + const landed = new Promise(resolve => (waiter = { after: seq, resolve })) + landedWaiters.set(key, [...(landedWaiters.get(key) ?? []), waiter]) + const cancel = () => { + const left = (landedWaiters.get(key) ?? []).filter(one => one !== waiter) + if (left.length) landedWaiters.set(key, left) + else landedWaiters.delete(key) + waiter.resolve(null) + } + return { landed, cancel } +} + +/** The check numbered `mine` of `file` ended: the waiters that asked before it started get its answer. */ +function checkEnded(file: string, mine: number, landed: CheckLanded | null) { + const key = normalize(file) + const waiting = landedWaiters.get(key) ?? [] + const answered = waiting.filter(one => one.after < mine) + const left = waiting.filter(one => one.after >= mine) + if (left.length) landedWaiters.set(key, left) + else landedWaiters.delete(key) + for (const one of answered) one.resolve(landed) +} + async function setCheck($: EngineInterface, view: CheckView, current: () => boolean) { if (!current()) return await update($, checkAtom, () => view) @@ -803,7 +859,7 @@ async function runShowWith($: EngineInterface, base: Omit runDbt($, root, args), asked) + return runShowWith($, { id: queryId(root, select, inline, []), title: titleOf(select, inline), root, select, inline, options: [], argv: null, cwd: null, limit, by: 'you' }, () => runDbt($, root, args, DBT_TIMEOUT_MS, undefined, true), asked) } /** @@ -821,7 +877,7 @@ async function rerun($: EngineInterface, query: QueryResult, asked = generation) const executable = written && (await $.fs.exists(written).catch(() => false)) ? written : await dbtBinary($, query.root) // The profile the replayed command reads is the one its own flags name, from the folder it runs in. await ensureRedactions($, query.root, { dir: cwd, profilesDir: profilesDirOf(query.argv), profile: flagOf(query.argv, '--profile') }) - return runShowWith($, query, () => runQueued($, query.root, async () => [executable, ...args], { cwd }), asked) + return runShowWith($, query, () => runQueued($, query.root, async () => [executable, ...args], { cwd, scratch: true }), asked) } // ─── 12 · Model context, before Claude guesses ─────────────────────────────────────────────────────── @@ -1725,6 +1781,9 @@ export async function dbtModsReadOrEditAfter($: EngineInterface, e: { tool: stri */ export async function endDbtMods($: EngineInterface) { generation++ + // A check of the conversation that ended answers nobody in the next. + for (const waiting of landedWaiters.values()) for (const one of waiting) one.resolve(null) + landedWaiters.clear() forgetRedactions() slices.clear() parsed = null @@ -1815,7 +1874,7 @@ export function registerDbtMods(on: On, context: { options: PluginOptions }) { const recent = history.filter(one => !one.bookmarked && one.id !== result?.id).slice(0, HISTORY_SHOWN) // Clear history is offered while any query that is not bookmarked is kept, the one shown included. const clearable = history.some(one => !one.bookmarked) - const sizes = await sizeControls($, { Box, Button, Text }, { id: RESULTS_PANE, title: RESULTS_TITLE, spec: RESULTS_SIZES, keyPrefix: 'results', terminal: paneTerminalColumns(e) }, work => later($, work)) + const sizes = await sizeControls($, { Box, Button, Text }, { id: RESULTS_PANE, title: RESULTS_TITLE, spec: RESULTS_SIZES, keyPrefix: 'results', terminal: paneTerminalColumns(e), placement: (e.props as { placement?: string } | undefined)?.placement }, work => later($, work)) const table = (() => { if (!result) return Run /dbt-show <model>, or ask Claude to `dbt show` a model. diff --git a/plugins/altimate-code/hooks/features/comments.tsx b/plugins/altimate-code/hooks/features/comments.tsx index 105436e..b0a016f 100644 --- a/plugins/altimate-code/hooks/features/comments.tsx +++ b/plugins/altimate-code/hooks/features/comments.tsx @@ -38,6 +38,8 @@ const MODEL_FILE = /\.(sql|ya?ml|py|md|csv)$/i const PARTIAL_NOTE = 'Not every open thread is listed: the service sent only part of the shares, or the list was cut for size.' const UNCONFIRMED_NOTE = 'An earlier post is not confirmed yet: press Refresh first.' const LOCKED_NOTE = ' The thread stays locked until the list shows the post or a few minutes have passed (the list can lag): press Refresh later.' +const NOT_OPEN_NOTE = 'That thread is no longer open (it was resolved, or the list was behind): nothing was sent. The list is read again.' +const ALREADY_RESOLVED_NOTE = 'Already resolved: the list was behind. It is hidden now.' const REFUSALS = new Set(['read_only', 'missing_text', 'missing_ids', 'unknown_thread', 'incomplete_list']) const MIN_POLL_SECONDS = 30 const MAX_POLL_SECONDS = 86_400 @@ -192,6 +194,12 @@ export function register(on: On, context: FeatureContext) { }) } +/** What a conversation keeps of its own, reset when one starts (a session's start, or a /clear's end). */ +function newConversation() { + firstWaitSpent = false + pendingFor = null +} + /** After the session started: the command, a first load, and the poll. */ export async function sessionStart($: EngineInterface, e: Args<'session.start'>) { await $.command.register({ name: COMMAND, description: 'Open comment threads from your team’s shared dbt docs; reply or resolve' }) @@ -199,8 +207,7 @@ export async function sessionStart($: EngineInterface, e: Args<'session.start'>) poll = null loadAgain = false generation++ - firstWaitSpent = false - pendingFor = null + newConversation() uncertain = null resolvedAt.clear() // Work still running when an earlier session ended can never finish, so it must not block this one. @@ -215,6 +222,20 @@ export async function sessionStart($: EngineInterface, e: Args<'session.start'>) await attach($, cwd) } +/** + * After a /clear: the new conversation's `$.state` is empty, so the project is found and its threads are + * loaded again now (sessionEnd has reset the first-read wait, so its first read of a model waits for them). The poll and a + * post's lock carry on (sessionEnd keeps them: they describe the shared docs, not the conversation). + */ +export async function clearStart($: EngineInterface) { + // A read in the new conversation that came first has found its project already: that one stays. + if ((await get($)).root) return + searched.clear() + within.clear() + const cwd = (await $.session.cwd().catch(() => null)) ?? (await $.session.root().catch(() => null)) + if (cwd) await attach($, cwd) +} + /** Also called for a subagent's turns, which carry `e.agentId`. */ export async function turnStart($: EngineInterface, e: Args<'turn.start'>) {} @@ -224,7 +245,12 @@ export async function turnComplete($: EngineInterface, e: Args<'turn.complete'>, export async function sessionEnd($: EngineInterface, e: Args<'session.end'>) { // /clear ends a session without starting another: the pane keeps its threads and drafts, so the poll goes on // and a post still running or unconfirmed keeps its lock (it describes the shared docs, not the conversation). - if (e.reason === 'clear') return + if (e.reason === 'clear') { + // The next conversation starts with empty state: its first read of a model waits for its list, as a session's + // first read does, even when it comes before clearStart. + newConversation() + return + } poll?.cancel() poll = null generation++ @@ -530,14 +556,22 @@ async function post($: EngineInterface, thread: CommentsThread, args: Record (mine !== generation ? current : { ...current, posting: null, note })) + // A thread that is no longer open: the list is read again, so it shows what the service holds now. + if (gone) { + release() + await load($) + } return } // Any other failure, a timeout included, may have landed: posting stays locked until a list of this @@ -563,7 +597,7 @@ async function post($: EngineInterface, thread: CommentsThread, args: Record one.shareId !== thread.shareId || one.groupId !== thread.groupId) : current.threads, }, diff --git a/plugins/altimate-code/hooks/features/defer.test.ts b/plugins/altimate-code/hooks/features/defer.test.ts index e48bab2..86f1375 100644 --- a/plugins/altimate-code/hooks/features/defer.test.ts +++ b/plugins/altimate-code/hooks/features/defer.test.ts @@ -1,6 +1,7 @@ import { expect, mock, test } from 'claude-code/testing' -import { effective, rewriteDeferCommand, runsHere, runsInProject, statusText } from './defer' +import { PHASE_TIMEOUT_MS } from '../tool-chain' +import { effective, rewriteDefer, rewriteDeferCommand, runsHere, runsInProject, statusText } from './defer' const on = { stateDir: '/work/prod-state' } @@ -224,6 +225,9 @@ test('the override decides, and the status text shows target and defer', () => { const ROOT = '/repo' /** The session's folder, answered by the fake host. */ let cwd = ROOT +/** Held while set: the project's defer setting read, and the Bash command itself. */ +let holdRead: Promise | null = null +let holdBash: Promise | null = null /** A dbt project at /repo whose defer setting the CLI reports as `answer`; Bash commands are recorded. */ function start($: any, on: any, answer: Record, fail: { times: number } = { times: 0 }, projects: string[] = [ROOT]) { @@ -235,11 +239,14 @@ function start($: any, on: any, answer: Record, fail: { times: on('session.start', (_: unknown, e: { cwd: string }) => ({ cwd: e.cwd })) on('session.root', () => ({ value: ROOT })) cwd = ROOT + holdRead = null + holdBash = null on('session.cwd', () => ({ value: cwd })) on('fs.exists', (_: unknown, e: { path: string }) => ({ value: projects.some(dir => e.path === `${dir}/dbt_project.yml`) })) - on('process.run', (_: unknown, e: { readonly argv: readonly string[] }) => { + on('process.run', async (_: unknown, e: { readonly argv: readonly string[] }) => { if (!e.argv.includes('dbt_defer_state')) return { value: { exitCode: 0, stdout: '{}', stderr: '' } } reads.push(e.argv.join(' ')) + await holdRead if (fail.times-- > 0) return { value: { exitCode: 1, stdout: '', stderr: 'not ready' } } return { value: { exitCode: 0, stdout: JSON.stringify({ status: 'success', output: '', metadata: { success: true, ...answer } }), stderr: '' }, @@ -249,8 +256,9 @@ function start($: any, on: any, answer: Record, fail: { times: statuses.push(e.text) return { value: undefined } }) - on('tool.call', (_: unknown, e: any) => { + on('tool.call', async (_: unknown, e: any) => { bash.push(e.command) + await holdBash return { result: { ok: true } } }) return { bash, statuses, reads, clock } @@ -411,3 +419,66 @@ test('a flag that takes no value before --project-dir leaves it a real option, a expect(rewriteDeferCommand('DBT_FAVOR_STATE=false env dbt run', options)).toBe('DBT_FAVOR_STATE=false env dbt run --defer --state /work/prod-state') expect(rewriteDeferCommand('echo hi; dbt run', options)).toBe('echo hi; dbt run --defer --state /work/prod-state --favor-state') }) + +test('the flags the plugin adds are told: rewriteDefer names each invocation\'s, the state folder relative to the project', () => { + const { command, added } = rewriteDefer('dbt build -s x && dbt test', { stateDir: `${ROOT}/prod`, favorState: true, target: 'dev' }, ROOT) + expect(command).toBe(`dbt build --defer --state ${ROOT}/prod --favor-state --target dev -s x && dbt test --defer --state ${ROOT}/prod --favor-state --target dev`) + expect(added).toEqual(['dbt build --defer --state prod --favor-state --target dev', 'dbt test --defer --state prod --favor-state --target dev']) + expect(rewriteDefer('ls', { stateDir: `${ROOT}/prod` }, ROOT).added).toEqual([]) +}) + +const BAND = { + plugin: 'altimate-code', + component: 'AbovePrompt', + props: { hasSurvey: false, isCollapsed: false, isWorking: false, maxRows: 10, bodyColumns: 120, scroll: { offset: 0, bodyRows: 10 }, view: {} }, +} as const + +test('Claude is told in the result, and the person in the band until the next prompt, the flags its dbt command ran with', async ($, on) => { + start($, on, DEFER) + on('ui.render', ($$: any, e: any) => { + const { Box } = $$.ui.resolve(e) + return h(Box, { key: 'engine' }) as any + }) + on('turn.start', (_: unknown, e: { turnId: string }) => ({ turnId: e.turnId })) + await $.session.start({ cwd: ROOT, surface: 'terminal', isInteractive: true }) + const ran: any = await $.tool.call({ tool: 'Bash', command: 'dbt build -s x' }) + expect(ran.context).toContain('◆ dbt · this command ran as dbt build --defer --state /work/prod-state --favor-state: /defer and /target added those flags (the command shown is as written), so its results are from that target and state') + const band = await $.ui.mount({ ...BAND, surface: 'terminal' }) + expect(await band.find({ type: 'Text', text: 'dbt build --defer --state /work/prod-state --favor-state' })).toBeDefined() + await band.unmount() + // A command the plugin left as written says nothing. + const own: any = await $.tool.call({ tool: 'Bash', command: 'dbt build --state ./mine' }) + expect((own.context ?? []).some((line: string) => line.startsWith('◆ dbt · this command ran as'))).toBe(false) + await $.turn.start({ turnId: 't2', prompt: 'next' } as any) + const after = await $.ui.mount({ ...BAND, surface: 'terminal' }) + expect(await after.find({ type: 'Text', text: /ran as/ })).toBeUndefined() + await after.unmount() +}) + +test('each call gets the flags added to it: a concurrent call that wrote the same flags itself takes none of them', async ($, on) => { + start($, on, DEFER) + await $.session.start({ cwd: ROOT, surface: 'terminal', isInteractive: true }) + // Claude's own command already names the flags the rewrite would add, so it is not rewritten. + const own: any = await $.tool.call({ tool: 'Bash', command: 'dbt build --defer --state /work/prod-state --favor-state -s x', tool_use_id: 'own' }) + expect((own.context ?? []).some((line: string) => line.startsWith('◆ dbt · this command ran as'))).toBe(false) + const ran: any = await $.tool.call({ tool: 'Bash', command: 'dbt build -s x', tool_use_id: 'rewritten' }) + expect(ran.context).toContain('◆ dbt · this command ran as dbt build --defer --state /work/prod-state --favor-state: /defer and /target added those flags (the command shown is as written), so its results are from that target and state') +}) + +test('a rewrite the chain gave up on, which finishes while the command runs as written, tells Claude nothing about flags it never added', async ($, on) => { + const seen = start($, on, DEFER) + await $.session.start({ cwd: ROOT, surface: 'terminal', isInteractive: true }) + let releaseRead!: () => void + let releaseBash!: () => void + holdRead = new Promise(resolve => (releaseRead = resolve)) + holdBash = new Promise(resolve => (releaseBash = resolve)) + const call = $.tool.call({ tool: 'Bash', command: 'dbt build -s x', tool_use_id: 'late' }) + // Past the phase deadline the command runs as written; the settings read then finishes while it runs. + await seen.clock.advance(PHASE_TIMEOUT_MS + 100) + releaseRead() + await seen.clock.advance(10) + releaseBash() + const ran: any = await call + expect(seen.bash.at(-1)).toBe('dbt build -s x') + expect((ran.context ?? []).some((line: string) => line.startsWith('◆ dbt · this command ran as'))).toBe(false) +}) diff --git a/plugins/altimate-code/hooks/features/defer.tsx b/plugins/altimate-code/hooks/features/defer.tsx index f913afb..888485b 100644 --- a/plugins/altimate-code/hooks/features/defer.tsx +++ b/plugins/altimate-code/hooks/features/defer.tsx @@ -16,6 +16,11 @@ import type { DeferState } from '../../types' import type { BandUi, FeatureContext } from './index' const deferAtom = atom({ plugin: 'altimate-code', key: 'defer' } as const, null as DeferState | null) +// What /defer and /target added to Claude's last dbt command, for the band, until the next prompt. +const ranAtom = atom({ plugin: 'altimate-code', key: 'deferRan' } as const, null as string | null) +// The flags added to each rewritten call, by the call's tool_use_id (the command as it ran when there is none), for its result. +const addedTo = new Map() +const callKey = (e: BashCall) => (typeof e.tool_use_id === 'string' && e.tool_use_id ? e.tool_use_id : `command:${e.command}`) // The commands that take `--defer` in every dbt version this reads state for (older dbt refuses the option on // `ls`, `parse`, `seed`, `run-operation` and others, and `retry` reads the previous run's results from `--state`). @@ -45,8 +50,17 @@ export type RewriteOptions = { * when nothing needs to change. An invocation with a word the parser cannot read (`$FLAGS`, `~/x`, a glob) is left as written. */ export function rewriteDeferCommand(command: string, options: RewriteOptions): string { + return rewriteDefer(command, options).command +} + +/** + * rewriteDeferCommand's command, with the flags it added to each dbt invocation, first invocation first, as the + * person reads them: the state folder named relative to `base` when it is inside it. + */ +export function rewriteDefer(command: string, options: RewriteOptions, base?: string): { command: string; added: string[] } { const { stateDir, favorState = false, target } = options - if (!stateDir && !target) return command + const added: string[] = [] + if (!stateDir && !target) return { command, added } let out = command // DBT_STATE or DBT_TARGET exported in the line is the user's own choice: none of its dbt commands gets the flag // (a line with an `export` and the variable anywhere in it counts, whatever the quoting). @@ -68,8 +82,17 @@ export function rewriteDeferCommand(command: string, options: RewriteOptions): s if (target && !NO_TARGET.has(one.subcommand) && one.target === null && !exportedTarget && !/\bDBT_TARGET=/.test(ownText)) flags.push('--target', quote(target)) if (flags.length === 0) continue out = `${out.slice(0, one.subcommandEnd)} ${flags.join(' ')}${out.slice(one.subcommandEnd)}` + const shown = flags.map(flag => (stateDir && flag === quote(stateDir) ? quote(relativeTo(base, stateDir)) : flag)) + added.unshift(`dbt ${one.subcommand} ${shown.join(' ')}`) } - return out + return { command: out, added } +} + +/** `path` relative to `base` when it is inside it, else as it is. */ +function relativeTo(base: string | undefined, path: string): string { + if (!base) return path + const root = base.replace(/[\\/]+$/, '') + return path.startsWith(`${root}/`) ? path.slice(root.length + 1) : path } type DeferAnswer = { @@ -411,13 +434,18 @@ export async function sessionStart($: EngineInterface, e: Args<'session.start'>) // A new session starts from the project's own settings, not from the last one's overrides. generation++ loading = null + addedTo.clear() await update($, deferAtom, () => null) + await update($, ranAtom, () => null) // Clearing the line is cosmetic: an engine that cannot draw it does not stop the session. await setStatusPart($, 'defer', null).catch(() => {}) } /** Also called for a subagent's turns, which carry `e.agentId`. */ -export async function turnStart($: EngineInterface, e: Args<'turn.start'>) {} +export async function turnStart($: EngineInterface, e: Args<'turn.start'>) { + // A new prompt: the band's line was about the last one's commands. A subagent's turn is part of the prompt's. + if (!(e as { agentId?: string }).agentId && (await read($, ranAtom))) await update($, ranAtom, () => null) +} /** After the turn's answer; `out` is what the turn ended with. Also called for a subagent's turns (`e.agentId`). */ export async function turnComplete($: EngineInterface, e: Args<'turn.complete'>, out: EventResult<'turn.complete'>) {} @@ -426,7 +454,16 @@ export async function sessionEnd($: EngineInterface, e: Args<'session.end'>) {} /** This feature's line in the band, under the lineage line; null when there is nothing to say. */ export async function bandLine($: EngineInterface, ui: BandUi): Promise { - return null + const ran = await read($, ranAtom) + if (!ran) return null + const { Text } = ui + return ( + + ↪ ran as + {ran} + · /defer, /target + + ) } /** The command to run in place of `e.command`; null keeps it. */ @@ -439,8 +476,12 @@ export async function bashRewrite($: EngineInterface, e: BashCall): Promise null)) ?? (await $.session.root().catch(() => null)) if (!root) return null if (!(await runsHere($, e.command, root, state.projectDir))) return null - const command = rewriteDeferCommand(e.command, effective(state)) - return command === e.command ? null : command + const { command, added } = rewriteDefer(e.command, effective(state), state.projectDir) + if (command === e.command) return null + // Kept for the call's after phase, which tells Claude what ran (a refused call lets go in bashDropped); at most a few are held. + if (addedTo.size >= 20) addedTo.delete(addedTo.keys().next().value!) + addedTo.set(callKey({ ...e, command }), { command, added }) + return command } /** @@ -451,7 +492,25 @@ export async function bashGate($: EngineInterface, e: BashCall): Promise<{ deny: return null } -export async function bashAfter($: EngineInterface, e: BashCall, result: ToolCallResult): Promise {} +/** + * The flags the plugin added to a dbt command: the transcript shows the command as Claude wrote it, so Claude is + * told in the result, and the person in the band, what really ran. + */ +export async function bashAfter($: EngineInterface, e: BashCall, result: ToolCallResult): Promise { + const kept = addedTo.get(callKey(e)) + if (!kept) return + addedTo.delete(callKey(e)) + // A rewrite the chain gave up on (its deadline passed) never ran: the command that ran says which did. + if (kept.command !== e.command) return + const text = kept.added.join('; ') + await update($, ranAtom, () => text) + return [`◆ dbt · this command ran as ${text}: /defer and /target added those flags (the command shown is as written), so its results are from that target and state`] +} + +/** A call a gate refused never reaches bashAfter: its flags are let go. */ +export async function bashDropped($: EngineInterface, e: BashCall): Promise { + addedTo.delete(callKey(e)) +} /** `file.before` holds the text from before the edit only when `editReadsBefore` is true. */ export async function editAfter($: EngineInterface, e: FileCall, result: ToolCallResult, file: EditedFile): Promise {} diff --git a/plugins/altimate-code/hooks/features/docs.test.tsx b/plugins/altimate-code/hooks/features/docs.test.tsx index a393dd4..0ea317f 100644 --- a/plugins/altimate-code/hooks/features/docs.test.tsx +++ b/plugins/altimate-code/hooks/features/docs.test.tsx @@ -7,6 +7,7 @@ const test = (name: string, body: Parameters[1]) => kitTest(name import { PHASE_TIMEOUT_MS } from '../tool-chain' import { ROUND_MS, applyPrompt, candidatesFrom, changedDescriptions, columnDescriptions, modelPaths, sentGroups } from './docs' import { nothingToReport } from '../inert-features' +import { playsClear } from '../clear-state.testkit' const ROOT = '/repo' const MANIFEST_PATH = `${ROOT}/target/manifest.json` @@ -200,7 +201,36 @@ const DOWNSTREAM_AFTER = `models: description: "The customer's key, as the shop issues it" ` -type Options = { refuse?: boolean; downstream?: string; reach?: Record; coverage?: Record | null; coverageError?: string; noManifest?: boolean; stale?: boolean; newFile?: boolean; manifest?: Record } +type Options = { + refuse?: boolean + downstream?: string + reach?: Record + coverage?: Record | null + coverageError?: string + noManifest?: boolean + stale?: boolean + newFile?: boolean + manifest?: Record + /** A model file written after the project's manifest: dbt has not run since the edit. */ + editNewer?: boolean + /** The edit's check (the dbt helpers): what its dbt show previews, the manifest it writes, and the coverage read from that manifest. */ + check?: { + rows: Record[] + node: string + manifest: Record + coverage: Record + /** The coverage read from the check's manifest, by how many dbt shows had finished when it was read. */ + coverageAfter?: (shows: number) => Record + /** The first dbt show waits for this. */ + holdFirst?: Promise + /** The coverage read from the project's manifest for the second edit waits for this. */ + holdSecondEdit?: Promise + /** The coverage read from the project's manifest for the first edit waits for this. */ + holdFirstEdit?: Promise + /** The check's manifest is rewritten (by another plugin run) while its coverage is read. */ + rewrittenWhileRead?: boolean + } +} /** A dbt project at /repo with the plugin's MCP server connected, and the session beneath the plugin. */ async function start($: any, on: any, options: Options = {}) { @@ -223,7 +253,11 @@ async function start($: any, on: any, options: Options = {}) { on('ui.open', () => ({ value: {} })) on('ui.log', () => ({ value: undefined })) on('ui.toast', () => ({ value: undefined })) - on('ui.status', () => ({ value: undefined })) + const statuses: (string | undefined)[] = [] + on('ui.status', (_: unknown, e: { text?: string }) => { + statuses.push(e.text) + return { value: undefined } + }) on('ui.render', ($$: any, e: any) => { const { Box } = $$.ui.resolve(e) return @@ -234,10 +268,12 @@ async function start($: any, on: any, options: Options = {}) { return { text: e.text } }) on('fs.exists', (_: unknown, e: { path: string }) => ({ value: e.path === `${ROOT}/dbt_project.yml` || ((e.path === SCHEMA && !options.newFile) || e.path === DOWNSTREAM) || (!options.noManifest && e.path === MANIFEST_PATH) })) - on('fs.stat', () => ({ value: { kind: 'file', size: 100, mtimeMs: 1, isLink: false } })) + let checkManifestAt = 1 + on('fs.stat', (_: unknown, e: { path: string }) => ({ value: { kind: 'file', size: 100, mtimeMs: options.editNewer && e.path.endsWith('.sql') ? 2 : e.path.includes('.altimate-plugin') ? checkManifestAt : 1, isLink: false } })) on('fs.read', (_: unknown, e: { path: string }) => { if (e.path === `${ROOT}/dbt_project.yml`) return { value: "name: 'shop'\n" } if (e.path === MANIFEST_PATH && options.manifest) return { value: JSON.stringify({ nodes: options.manifest }) } + if (e.path === `${ROOT}/target/.altimate-plugin/manifest.json` && options.check) return { value: JSON.stringify({ nodes: options.check.manifest }) } // The edit's text from before is read first, then the file as the edit left it. // Each edit reads the file three times: before it, after it, and when the traces are back. if (e.path === SCHEMA) { @@ -259,8 +295,22 @@ async function start($: any, on: any, options: Options = {}) { if (e.tool === 'impact_analysis') return { value: { content: [{ type: 'text', text: JSON.stringify({ ...(options.reach ?? REACH), report: 'Impact' }) }], isError: false } } return { value: { content: [], isError: true } } }) + let shows = 0 + let plainReads = 0 + let firstReads = 0 + const previewOf = () => ({ data: { is_inline: false, node_name: options.check!.node, output_format: 'json', preview: JSON.stringify(options.check!.rows), unique_id: `model.shop.${options.check!.node}` }, info: { code: 'Q041', level: 'info', msg: '…' } }) on('process.run', (_: unknown, e: { argv: readonly string[] }) => { const [, command, , tool] = e.argv + // The edit's check: dbt show previews the model as the edit left it. + if (e.argv[0] === 'dbt' && command === 'show' && options.check) { + if (shows === 0 && options.check.holdFirst) { + shows++ + return options.check.holdFirst.then(() => ({ value: { exitCode: 0, stdout: JSON.stringify(previewOf()), stderr: '' } })) + } + shows++ + const preview = { data: { is_inline: false, node_name: options.check.node, output_format: 'json', preview: JSON.stringify(options.check.rows), unique_id: `model.shop.${options.check.node}` }, info: { code: 'Q041', level: 'info', msg: '…' } } + return { value: { exitCode: 0, stdout: JSON.stringify(preview), stderr: '' } } + } if (command !== 'tool' || tool === 'docs_share_comments') return { value: { exitCode: 1, stdout: '', stderr: 'not this test' } } // The project health check asks for its own tool at session start: not a call this feature makes. if (tool === 'dbt_health') return { value: nothingToReport(e.argv)! } @@ -268,7 +318,18 @@ async function start($: any, on: any, options: Options = {}) { toolArgs.push(String(e.argv[5])) if (tool === 'dbt_docs_coverage') { if (options.coverageError) return { value: { exitCode: 1, stdout: JSON.stringify({ status: 'error', output: options.coverageError, metadata: { success: false } }), stderr: '' } } - const body = options.coverage === null ? { status: 'error', output: 'Unknown tool: dbt_docs_coverage', metadata: {} } : { status: 'ok', output: 'coverage', metadata: options.coverage } + const fromCheck = options.check && String(e.argv[5]).includes('.altimate-plugin') + if (!fromCheck && options.check?.holdFirstEdit && ++firstReads === 1) { + const answer = { value: { exitCode: 0, stdout: JSON.stringify({ status: 'ok', output: 'coverage', metadata: options.coverage }), stderr: '' } } + return options.check.holdFirstEdit.then(() => answer) + } + if (!fromCheck && options.check?.holdSecondEdit && ++plainReads === 2) { + const answer = { value: { exitCode: 0, stdout: JSON.stringify({ status: 'ok', output: 'coverage', metadata: options.coverage }), stderr: '' } } + return options.check.holdSecondEdit.then(() => answer) + } + if (fromCheck && options.check!.rewrittenWhileRead) checkManifestAt++ + const checked = fromCheck ? (options.check!.coverageAfter?.(shows) ?? options.check!.coverage) : null + const body = options.coverage === null ? { status: 'error', output: 'Unknown tool: dbt_docs_coverage', metadata: {} } : { status: 'ok', output: 'coverage', metadata: checked ?? options.coverage } return { value: { exitCode: options.coverage === null ? 1 : 0, stdout: JSON.stringify(body), stderr: '' } } } return { value: { exitCode: 0, stdout: JSON.stringify({ status: 'ok', output: '', metadata: options.reach ?? REACH }), stderr: '' } } @@ -281,6 +342,7 @@ async function start($: any, on: any, options: Options = {}) { calls, toolArgs, submitted, + statuses, /** The next schema edit removes the description instead of writing it. */ clear: () => { clearing = true @@ -622,3 +684,96 @@ test('a plain description with a blank line in it is read to its last paragraph, ['orders.status', 'Plain'], ]) }) + +const COVERAGE_PANE = { plugin: 'altimate-code', component: 'Pane', requestId: 'altimate-docs', props: { title: 'Altimate docs', isFocused: false, bodyColumns: 120, placement: 'dock', scroll: { offset: 0, bodyRows: 30 }, view: {} } } as const + +kitTest('a SQL edit dbt has not compiled yet: the coverage says it is from before the edit, and the edit\'s check replaces it with the model as it is now', { options: { dbt_mods: true } }, async ($, on) => { + const node = (columns: string[]) => ({ 'model.shop.orders': { resource_type: 'model', name: 'orders', package_name: 'shop', original_file_path: 'models/marts/orders.sql', columns: Object.fromEntries(columns.map(name => [name, { name, description: 'd' }])) } }) + // The project's manifest is from before the rename: every column it knows is documented. + const before = { success: true, model: 'orders', has_description: true, output_columns: ['order_id', 'payment_count'], documented: ['order_id', 'payment_count'], missing: [], percent: 100 } + // The check's manifest has the model as the edit left it: payment_count is now payments, which nothing documents. + const after = { success: true, model: 'orders', has_description: true, output_columns: ['order_id', 'payments'], documented: ['order_id'], missing: ['payments'], percent: 50 } + const { clock } = await start($, on, { editNewer: true, manifest: node(['order_id', 'payment_count']), coverage: before, check: { rows: [{ order_id: 1, payments: 2 }], node: 'orders', manifest: node(['order_id', 'payment_count']), coverage: after } }) + const result: any = await $.tool.call({ tool: 'Edit', file_path: `${ROOT}/models/marts/orders.sql`, old_string: 'payment_count', new_string: 'payments' }) + const line = result.context?.find((one: string) => one.startsWith('◆ docs')) + expect(line).toContain('as of the last dbt compile, before this edit') + await clock.settle() + const ui = await $.ui.mount({ ...COVERAGE_PANE, surface: 'terminal' }) + expect(await ui.find({ type: 'Text', text: /orders: 1 of 2 columns documented \(50%\) · no description: payments/ })).toBeDefined() + expect(await ui.find({ type: 'Text', text: /2 of 2 columns documented/ })).toBeUndefined() + await ui.unmount() +}) + +kitTest('the coverage of an edit is read only from its own check: a check still running from an earlier edit does not answer it', { options: { dbt_mods: true } }, async ($, on) => { + const node = { 'model.shop.orders': { resource_type: 'model', name: 'orders', package_name: 'shop', original_file_path: 'models/marts/orders.sql', columns: {} } } + const before = { success: true, model: 'orders', has_description: true, output_columns: ['order_id'], documented: ['order_id'], missing: [], percent: 100 } + let release!: () => void + const holdFirst = new Promise(resolve => (release = resolve)) + let releaseB!: () => void + const holdSecondEdit = new Promise(resolve => (releaseB = resolve)) + // The coverage read from the check's manifest says which dbt show it followed. + const coverageAfter = (shows: number) => ({ success: true, model: 'orders', has_description: true, output_columns: ['order_id', `col_${shows}`], documented: ['order_id'], missing: [`col_${shows}`], percent: 50 }) + const { clock } = await start($, on, { editNewer: true, manifest: node, coverage: before, check: { rows: [{ order_id: 1 }], node: 'orders', manifest: node, coverage: before, coverageAfter, holdFirst, holdSecondEdit } }) + const file = `${ROOT}/models/marts/orders.sql` + // Edit A's check is held; edit B comes while it runs, and A's check ends before B's own check has started. + const a = $.tool.call({ tool: 'Edit', file_path: file, old_string: 'a', new_string: 'b' }) + await clock.advance(10) + const b = $.tool.call({ tool: 'Edit', file_path: file, old_string: 'b', new_string: 'c' }) + await clock.advance(10) + release() + await a + await clock.advance(10) + releaseB() + await b + await clock.settle() + const ui = await $.ui.mount({ ...COVERAGE_PANE, surface: 'terminal' }) + // B's coverage comes from the second show (B's own check), never the first (A's). + expect(await ui.find({ type: 'Text', text: /no description: col_2/ })).toBeDefined() + expect(await ui.find({ type: 'Text', text: /col_1/ })).toBeUndefined() + await ui.unmount() +}) + +kitTest('a check manifest another plugin run rewrote while its coverage was read is not used: the coverage stays marked as from before the edit', { options: { dbt_mods: true } }, async ($, on) => { + const node = { 'model.shop.orders': { resource_type: 'model', name: 'orders', package_name: 'shop', original_file_path: 'models/marts/orders.sql', columns: {} } } + const before = { success: true, model: 'orders', has_description: true, output_columns: ['order_id'], documented: ['order_id'], missing: [], percent: 100 } + const after = { success: true, model: 'orders', has_description: true, output_columns: ['order_id', 'payments'], documented: ['order_id'], missing: ['payments'], percent: 50 } + const { clock } = await start($, on, { editNewer: true, manifest: node, coverage: before, check: { rows: [{ order_id: 1 }], node: 'orders', manifest: node, coverage: after, rewrittenWhileRead: true } }) + await $.tool.call({ tool: 'Edit', file_path: `${ROOT}/models/marts/orders.sql`, old_string: 'a', new_string: 'b' }) + await clock.settle() + const ui = await $.ui.mount({ ...COVERAGE_PANE, surface: 'terminal' }) + expect(await ui.find({ type: 'Text', text: /from the manifest of before the last edit/ })).toBeDefined() + expect(await ui.find({ type: 'Text', text: /no description: payments/ })).toBeUndefined() + await ui.unmount() +}) + +kitTest('an edit\'s first coverage read that comes back late does not replace the coverage a later edit\'s check already gave', { options: { dbt_mods: true } }, async ($, on) => { + const node = { 'model.shop.orders': { resource_type: 'model', name: 'orders', package_name: 'shop', original_file_path: 'models/marts/orders.sql', columns: {} } } + const before = { success: true, model: 'orders', has_description: true, output_columns: ['order_id'], documented: ['order_id'], missing: [], percent: 100 } + const after = { success: true, model: 'orders', has_description: true, output_columns: ['order_id', 'payments'], documented: ['order_id'], missing: ['payments'], percent: 50 } + let releaseA!: () => void + const holdFirstEdit = new Promise(resolve => (releaseA = resolve)) + const { clock } = await start($, on, { editNewer: true, manifest: node, coverage: before, check: { rows: [{ order_id: 1 }], node: 'orders', manifest: node, coverage: after, holdFirstEdit } }) + const file = `${ROOT}/models/marts/orders.sql` + const a = $.tool.call({ tool: 'Edit', file_path: file, old_string: 'a', new_string: 'b' }) + await clock.advance(10) + await $.tool.call({ tool: 'Edit', file_path: file, old_string: 'b', new_string: 'c' }) + await clock.settle() + releaseA() + await a + await clock.settle() + const ui = await $.ui.mount({ ...COVERAGE_PANE, surface: 'terminal' }) + expect(await ui.find({ type: 'Text', text: /orders: 1 of 2 columns documented \(50%\) · no description: payments/ })).toBeDefined() + expect(await ui.find({ type: 'Text', text: /1 of 1 columns documented/ })).toBeUndefined() + await ui.unmount() +}) + +test('/clear drops the "docs N to copy" status part with the offers it counted, which the new conversation does not have', async ($, on) => { + const clear = playsClear($, on) + const { clock, statuses } = await start($, on) + await $.tool.call(EDIT) + await clock.settle() + expect(statuses.at(-1)).toContain('docs 2 to copy') + await clear() + await clock.settle() + expect(statuses.at(-1) ?? '').not.toContain('docs') +}) diff --git a/plugins/altimate-code/hooks/features/docs.tsx b/plugins/altimate-code/hooks/features/docs.tsx index 4d7c841..8f32461 100644 --- a/plugins/altimate-code/hooks/features/docs.tsx +++ b/plugins/altimate-code/hooks/features/docs.tsx @@ -9,7 +9,8 @@ import { atom, read, update } from 'claude-code' import type { Args, EngineInterface, EventResult, On, RenderElement, ToolCallResult } from 'claude-code' import type { DocsCandidate, DocsCoverageView, DocsGroup, DocsState } from '../../types' -import { later, runTool } from '../altimate-cli' +import { later, runTool, within } from '../altimate-cli' +import { nextCheckOf, type CheckLanded } from '../dbt-mods' import { setStatusPart } from '../status-line' import type { BashCall, ContextLines, EditedFile, FileCall } from '../tool-chain' import type { BandUi, FeatureContext } from './index' @@ -350,11 +351,11 @@ type ManifestModel = { resource_type?: string; package_name?: string; name?: str * versioned models its latest version stands for them. A model the manifest does not list, or a manifest * that cannot be read, gets the id dbt would give it, which finds nothing rather than another package's model. */ -async function modelRef($: EngineInterface, project: Project, model: string, byFile = false): Promise { +async function modelRef($: EngineInterface, project: Project, model: string, byFile = false, manifest = project.manifest): Promise { if (!project.package) return model const built = `model.${project.package}.${model}` try { - const nodes = (JSON.parse((await $.fs.read(project.manifest)) ?? '') as { nodes?: Record }).nodes ?? {} + const nodes = (JSON.parse((await $.fs.read(manifest)) ?? '') as { nodes?: Record }).nodes ?? {} const own = Object.entries(nodes).filter(([, node]) => node?.resource_type === 'model' && node.package_name === project.package) const stem = (node: ManifestModel) => (node.original_file_path ?? '').replace(/\.sql$/i, '').split('/').pop() if (byFile) { @@ -482,21 +483,17 @@ async function schemaEdited($: EngineInterface, file: EditedFile, created: boole return lines } -async function sqlEdited($: EngineInterface, file: EditedFile): Promise { - const project = await projectOf($, file.path) - if (!project) return [] - const model = normal(file.path).split('/').pop()!.replace(/\.sql$/i, '') - if (!(await $.fs.exists(project.manifest).catch(() => false))) return [] - const until = (await $.clock.now()) + ROUND_MS - const result = await runTool($, 'dbt_docs_coverage', { model: await modelRef($, project, model, true), manifest_path: project.manifest }, { until, cliOnly: true }).catch(() => null) - if (!result) return [] +/** The model's documentation coverage as `manifest` has it; a line for Claude when the tool is missing; null when it could not be read. */ +async function coverageOf($: EngineInterface, project: Project, model: string, manifest: string, until: number): Promise<{ view: DocsCoverageView; star: boolean } | { line: string } | null> { + const result = await runTool($, 'dbt_docs_coverage', { model: await modelRef($, project, model, true, manifest), manifest_path: manifest }, { until, cliOnly: true }).catch(() => null) + if (!result) return null const meta = result.metadata if (!result.ok || !meta || meta.success !== true) { if (MISSING_TOOL.test(result.text) && !saidOldCoverage) { saidOldCoverage = true - return ['◆ docs · this altimate-code has no dbt_docs_coverage: npm i -g @altimateai/altimate-code'] + return { line: '◆ docs · this altimate-code has no dbt_docs_coverage: npm i -g @altimateai/altimate-code' } } - return [] + return null } const view: DocsCoverageView = { model: typeof meta.model === 'string' && meta.model ? meta.model : model, @@ -505,17 +502,73 @@ async function sqlEdited($: EngineInterface, file: EditedFile): Promise { + // Taken before the first wait: an edit that comes later, or a session's end, makes this one's coverage old. + const mine = ++coverageSeq + const project = await projectOf($, file.path) + if (!project) return [] + const model = normal(file.path).split('/').pop()!.replace(/\.sql$/i, '') + const manifest = await $.fs.stat(project.manifest).catch(() => null) + if (!manifest || manifest.kind !== 'file') return [] + // dbt wrote the manifest before this edit: its compiled SQL is the model as it was, not as it is now. + const edited = await $.fs.stat(file.path).catch(() => null) + const stale = !!edited && manifest.mtimeMs < edited.mtimeMs + if (mine !== coverageSeq) return [] + const asked = stale ? { file: normal(file.path), model, project } : null + waiting = asked + // Asked for before the check starts (the dbt helpers run after this feature in the edit phase), so it is not missed. + if (asked) { + const { landed, cancel } = nextCheckOf(file.path) + later($, () => coverageAfterCheck($, asked, landed, cancel)) + } + const read = await coverageOf($, project, model, project.manifest, (await $.clock.now()) + ROUND_MS) + if (!read) return [] + if ('line' in read) return [read.line] + const view = { ...read.view, stale } + // A later edit (or its check's coverage) has set the coverage shown since this read began: it is not replaced. + if (mine === coverageSeq && waiting === asked) { + await update($, docsAtom, prev => ({ ...(prev ?? empty()), coverage: view })) + await showStatus($) } - await update($, docsAtom, prev => ({ ...(prev ?? empty()), coverage: view })) - await showStatus($) const gaps = [...(view.hasDescription ? [] : ['the model itself']), ...view.missing] // A select list that is not plain column names (a `*`, an unaliased expression, a UNION) is counted by the manifest's columns only. - const star = meta.source === 'manifest_columns' ? " · the model's select list is not plain column names (a `*`, an unaliased expression or a UNION), so only the columns the manifest lists are counted" : '' - if (!gaps.length && !star) return [] + const star = read.star ? " · the model's select list is not plain column names (a `*`, an unaliased expression or a UNION), so only the columns the manifest lists are counted" : '' + if (!gaps.length && !star && !stale) return [] // Names come from the project and this line goes to Claude, so each is a quoted string. const shown = gaps.slice(0, MAX_NAMES).map(one => JSON.stringify(one)).join(', ') + (gaps.length > MAX_NAMES ? `, +${gaps.length - MAX_NAMES} more` : '') - // The manifest's compiled SQL is from the last dbt compile, not from this edit. - return [`◆ docs · ${JSON.stringify(view.model)}: ${view.documented} of ${view.total} columns documented (${view.percent}%)${gaps.length ? ` · no description for: ${shown}` : ''}${star} · as of the last dbt compile`] + // The manifest's compiled SQL is from the last dbt compile: from before this edit when dbt has not run since. + const asOf = stale ? ' · as of the last dbt compile, before this edit: columns this edit added, renamed or dropped are not counted yet' : ' · as of the last dbt compile' + return [`◆ docs · ${JSON.stringify(view.model)}: ${view.documented} of ${view.total} columns documented (${view.percent}%)${gaps.length ? ` · no description for: ${shown}` : ''}${star}${asOf}`] +} + +/** Once the edit's check ran its model, the coverage shown from before that edit is read again from the check's manifest. */ +async function coverageAfterCheck($: EngineInterface, asked: { file: string; model: string; project: Project }, check: Promise, cancel: () => void) { + const landed = await within($, CHECK_WAIT_MS, check) + // A wait that ran out leaves nothing behind. + if (!landed) return cancel() + if (waiting !== asked) return + const read = await coverageOf($, asked.project, asked.model, landed.manifest, (await $.clock.now()) + RECHECK_MS) + // The plugin's next dbt run rewrites that manifest: an answer read from another run's is not this edit's. + const now = await $.fs.stat(landed.manifest).catch(() => null) + if (now?.mtimeMs !== landed.mtimeMs) return + // A newer edit asks for its own coverage: this one is no longer shown. + if (waiting !== asked || !read || 'line' in read) return + waiting = null + await update($, docsAtom, prev => ({ ...(prev ?? empty()), coverage: read.view })) } async function showStatus($: EngineInterface) { @@ -559,6 +612,7 @@ export function register(on: On, context: FeatureContext) { {coverage.model}: {coverage.documented} of {coverage.total} columns documented ({coverage.percent}%) {coverage.missing.length ? ` · no description: ${coverage.missing.slice(0, MAX_NAMES).join(', ')}` : ''} + {coverage.stale ? ' · from the manifest of before the last edit (the edit\'s check reads it again)' : ''} ) : null} {groups.length === 0 ? No description to copy: when Claude writes a column description, the downstream columns that carry the same value are listed here. : null} @@ -653,6 +707,7 @@ export function register(on: On, context: FeatureContext) { /** After the session started: the place to register commands. */ export async function sessionStart($: EngineInterface, e: Args<'session.start'>) { + waiting = null saidOldReach = false saidOldCoverage = false applied.clear() @@ -669,7 +724,11 @@ export async function turnStart($: EngineInterface, e: Args<'turn.start'>) {} /** After the turn's answer; `out` is what the turn ended with. Also called for a subagent's turns (`e.agentId`). */ export async function turnComplete($: EngineInterface, e: Args<'turn.complete'>, out: EventResult<'turn.complete'>) {} -export async function sessionEnd($: EngineInterface, e: Args<'session.end'>) {} +export async function sessionEnd($: EngineInterface, e: Args<'session.end'>) { + // A coverage read still running belongs to the conversation that ended (a /clear has no session start). + waiting = null + coverageSeq++ +} /** This feature's line in the band, under the lineage line; null when there is nothing to say. */ export async function bandLine($: EngineInterface, ui: BandUi): Promise { diff --git a/plugins/altimate-code/hooks/features/health.tsx b/plugins/altimate-code/hooks/features/health.tsx index 58340d9..4b9629d 100644 --- a/plugins/altimate-code/hooks/features/health.tsx +++ b/plugins/altimate-code/hooks/features/health.tsx @@ -16,6 +16,7 @@ import type { HealthInsight, HealthState } from '../../types' import { NOT_FOUND, later, runTool, within } from '../altimate-cli' import { featureUsed } from '../usage' import { setStatusPart } from '../status-line' +import { HEALTH_SCRATCH, SCRATCH_TARGET, targetPath } from '../dbt-mods-core' import type { BashCall, ContextLines, EditedFile, FileCall } from '../tool-chain' import type { BandUi, FeatureContext } from './index' @@ -52,6 +53,26 @@ let enabled = false let eligible = false /** The problem told to the user last, so the same one is not told at every turn. */ let toldProblem: string | null = null +/** The files the running check was asked about: a check a /clear cuts short has them asked for again. */ +let inFlight: string[] = [] +/** What the conversation a /clear ended knew (its baseline, the changes not yet checked, the issues): the next one keeps it. */ +let carried: HealthState | null = null +/** A /clear whose conversation has not had clearStart yet. */ +let clearPending = false +/** clearStart's restore while it runs. */ +let restoring: Promise | null = null +/** The last health CLI call queued, settled either way: the next one starts after it. */ +let cliQueue: Promise = Promise.resolve() + +/** Runs `job` after every one queued before it; its place is taken at once, before anything is awaited. */ +function oneAtATime(job: () => Promise): Promise { + const run = cliQueue.then(job, job) + cliQueue = run.then( + () => {}, + () => {}, + ) + return run +} /** Which model each file belonged to in earlier checks: a deleted schema file or model no longer shows in the latest answer. */ let known: HealthInsight[] = [] /** Whether the status line carries the health part now. */ @@ -473,6 +494,29 @@ async function isModelFile($: EngineInterface, file: string): Promise { return project !== null && !(ROOT_FILES.has(path.split('/').pop() ?? '') && norm(project) === dir) } +/** + * What the check's CLI process is given so its parse never rewrites the project's manifest (which would drop + * every model's compiled SQL): dbt's artifact folder is the plugin's own (SCRATCH_TARGET), with the project's + * catalog copied in (DataPilot reads it beside the manifest), and the CLI runs the tool in its own process, since + * a running tool daemon keeps the environment it started with. + */ +async function checkEnv($: EngineInterface): Promise | undefined> { + const root = await $.session.root().catch(() => '') + const project = root ? await projectAbove($, root) : null + if (!project) return undefined + const path = targetPath(await $.fs.read(`${project}/dbt_project.yml`).catch(() => '')) + const target = path.startsWith('/') ? path : `${project}/${path}` + const scratch = `${target}/${SCRATCH_TARGET}/${HEALTH_SCRATCH}` + // Best effort: without the copy the check reads no catalog, as with a project that has none; an earlier copy is + // removed first, so a catalog the project no longer has, or one that failed to copy, is never read. + await $.process.run(['rm', '-f', `${scratch}/catalog.json`], { timeoutMs: 10_000 }).catch(() => null) + if (await $.fs.exists(`${target}/catalog.json`).catch(() => false)) { + await $.process.run(['mkdir', '-p', scratch], { timeoutMs: 10_000 }).catch(() => null) + await $.process.run(['cp', '-p', `${target}/catalog.json`, `${scratch}/catalog.json`], { timeoutMs: 60_000 }).catch(() => null) + } + return { DBT_TARGET_PATH: scratch, ALTIMATE_TOOL_DAEMON: '0' } +} + /** Asks altimate-code's dbt_health for the project's insights. */ async function check($: EngineInterface): Promise { const badConfig = configPathProblem(options.healthConfigPath) @@ -480,7 +524,12 @@ async function check($: EngineInterface): Promise { let result: Awaited> try { const until = (await $.clock.now()) + CHECK_MS - result = await runTool($, 'dbt_health', toolArgs(options.healthConfigPath), { write: true, until }) + // One CLI call at a time, its folder's preparation included, across a /clear or a new session too: a call whose + // loop was cut short still runs, and the next one would share its artifact folder, so it takes its turn after it. + result = await oneAtATime(async () => { + const env = await checkEnv($) + return runTool($, 'dbt_health', toolArgs(options.healthConfigPath), { write: true, until, ...(env && { env }) }) + }) } catch (error) { // Only a binary that is not there: one that is there and refused to start keeps its own words. if (NOT_FOUND.test(String(error))) return { problem: NO_CLI } @@ -531,6 +580,7 @@ async function loop($: EngineInterface, mine: number) { if (!needsBaseline && before.pending.length === 0) return // The baseline run leaves `pending` alone: edits made while it ran are checked by the next run. const batch = needsBaseline ? [] : before.pending + inFlight = batch await update($, healthAtom, prev => ({ ...(prev ?? fresh()), pending: needsBaseline ? (prev?.pending ?? []) : (prev?.pending ?? []).filter(f => !batch.includes(f)), running: true })) const run = await check($) if (mine !== generation) return @@ -629,6 +679,7 @@ export function register(on: On, context: FeatureContext) { options = context.options on('command.run', { command: COMMAND }, async $ => { featureUsed($, 'health', 'opened') + await clearStart($) const state = await read($, healthAtom) if (!state || !enabled) return { text: 'Project health checks dbt projects: this session has none open yet.' } // Asking again retries a first check that failed, and checks changes not yet checked. @@ -648,6 +699,10 @@ export async function sessionStart($: EngineInterface, e: Args<'session.start'>) generation += 1 running = null enabled = false + carried = null + clearPending = false + restoring = null + inFlight = [] toldProblem = null known = [] shellBefore.clear() @@ -660,25 +715,97 @@ export async function sessionStart($: EngineInterface, e: Args<'session.start'>) // The baseline takes a while (the dbt adapter starts): nothing waits for it. A session with nobody at // the prompt checks only when the push gate needs it. eligible = e.isInteractive || options.healthBlocksPush === true || options.healthBlocksPush === 'true' + await enable($) +} + +/** Turns the checks on when the session's folder is in a dbt project, and starts the baseline. */ +async function enable($: EngineInterface) { const root = await $.session.root().catch(() => '') if (!eligible || !root || !(await projectAbove($, root))) return enabled = true later($, () => kick($)) } +/** + * After a /clear: the new conversation's `$.state` is empty. The baseline, the changes not yet checked and + * the issues of the conversation that ended are the project's, so they are kept (a change made before the + * /clear still holds a push); with none kept, the baseline is taken as at a session's start. + */ +export async function clearStart($: EngineInterface) { + // Once per /clear, whichever comes first: the session id's change, or a hook of the new conversation. A caller + // that comes while it runs waits for it, so a push is never decided on the empty state. + // Another /clear may come while a restore runs: once it ends, the one that /clear asked for runs too. + for (;;) { + if (restoring) { + await restoring + continue + } + if (!clearPending) return + clearPending = false + const run = restore($) + restoring = run + try { + await run + } finally { + if (restoring === run) restoring = null + } + } +} + +async function restore($: EngineInterface) { + // The snapshot is let go only once it is written: a /clear while the write is under way carries it on. + const kept = carried + generation += 1 + running = null + if (!kept) { + await update($, healthAtom, () => fresh()) + await enable($) + return + } + enabled = true + // Changes the new conversation recorded before this ran are added to the kept ones, not replaced by them. + await update($, healthAtom, prev => (prev ? mergeKept(kept, prev) : { ...kept, running: false })) + if (carried === kept) carried = null + drawn = true + await drawStatus($, kept.issues) + if (await wantsCheck($)) later($, () => kick($)) +} + +/** The state a /clear carries, with the changes a conversation recorded on top of it. */ +function mergeKept(kept: HealthState, recorded: HealthState): HealthState { + return { + ...kept, + baseline: kept.baseline ?? recorded.baseline, + running: false, + pending: [...new Set([...kept.pending, ...recorded.pending])], + touched: [...new Set([...kept.touched, ...recorded.touched])], + // A new change asks for a check again after a failed one, as markChanged does. + ...(recorded.pending.length ? { problem: null } : {}), + } +} + /** Also called for a subagent's turns, which carry `e.agentId`. */ export async function turnStart($: EngineInterface, e: Args<'turn.start'>) {} /** After the turn's answer: when model files changed, check again. */ export async function turnComplete($: EngineInterface, e: Args<'turn.complete'>, out: EventResult<'turn.complete'>) { if ((e as { agentId?: string }).agentId) return + await clearStart($) if (await wantsCheck($)) later($, () => kick($)) } export async function sessionEnd($: EngineInterface, e: Args<'session.end'>) { + // A /clear: what this conversation knows is handed to the next one. A check it cuts short is asked for again there. + const state = e.reason === 'clear' && enabled ? await read($, healthAtom).catch(() => null) : null + const now = state ? { ...state, pending: state.running ? [...new Set([...inFlight, ...state.pending])] : state.pending, running: false } : null + // A second /clear before the first was restored: what that one carried and what this conversation recorded are both kept. + // A fresh object either way, so a restore still writing the earlier snapshot does not let this one go. + carried = e.reason !== 'clear' ? null : carried ? (now ? mergeKept(carried, now) : { ...carried }) : now + clearPending = e.reason === 'clear' generation += 1 running = null - enabled = false + // Kept on through a /clear that carries the state: a change made before clearStart restores it is still recorded, and merged. + if (!carried) enabled = false } /** This feature's line in the band, under the lineage line; null when there is nothing to say. */ @@ -714,7 +841,8 @@ export async function bashGate($: EngineInterface, e: BashCall): Promise<{ deny: return null } // The whole decision, folder lookups included, is bounded inside the chain's deadline: past it the gate is given up on and the push runs. - const decided = await within($, GATE_WAIT_MS, decide($, e)) + // Right after a /clear the kept issues may not be back yet: the push waits for them, inside the same deadline. + const decided = await within($, GATE_WAIT_MS, clearStart($).then(() => decide($, e))) const verdict = decided === null ? hold('the check of the changes could not be read in time.') : decided === 'run' ? null : decided featureUsed($, 'health_push_gate', verdict ? 'held' : 'allowed') return verdict diff --git a/plugins/altimate-code/hooks/features/index.ts b/plugins/altimate-code/hooks/features/index.ts index 89652ec..28f1b4b 100644 --- a/plugins/altimate-code/hooks/features/index.ts +++ b/plugins/altimate-code/hooks/features/index.ts @@ -9,6 +9,8 @@ // A function that throws, at once or as a rejection, is logged and skipped: one feature's failure never stops another or the call. A `register` failure is silent (`$` is not available at load). import type { Args, EngineInterface, EventResult, On, PluginOptions, RenderElement, ToolCallInput, ToolCallResult } from 'claude-code' +import { later } from '../altimate-cli' +import { keepStatusParts, restoreStatusParts } from '../status-line' import type { BashCall, EditedFile, FileCall } from '../tool-chain' import { CHECK_HOLD_MS as dbtModsCheckHoldMs, @@ -47,6 +49,7 @@ import { bashRewrite as deferBashRewrite, bashGate as deferBashGate, bashAfter as deferBashAfter, + bashDropped as deferBashDropped, editAfter as deferEditAfter, readOrEditAfter as deferReadOrEditAfter, editReadsBefore as deferEditReadsBefore, @@ -68,6 +71,7 @@ import { import { register as commentsRegister, sessionStart as commentsSessionStart, + clearStart as commentsClearStart, turnStart as commentsTurnStart, turnComplete as commentsTurnComplete, sessionEnd as commentsSessionEnd, @@ -96,6 +100,7 @@ import { import { register as healthRegister, sessionStart as healthSessionStart, + clearStart as healthClearStart, turnStart as healthTurnStart, turnComplete as healthTurnComplete, sessionEnd as healthSessionEnd, @@ -228,6 +233,37 @@ export async function featuresSessionStart($: EngineInterface, e: Args<'session. await attempt($, 'dbtMods', 'sessionStart', () => sessionStartDbtMods($, e)) } +/** + * A conversation that a /clear started: no `session.start` comes for it, and its `$.state` starts empty, so the + * features whose state describes the project rather than the conversation fill it again. + */ +export async function featuresClearStart($: EngineInterface) { + await attempt($, 'statusLine', 'clearStart', () => restoreStatusParts($)) + await attempt($, 'comments', 'clearStart', () => commentsClearStart($)) + await attempt($, 'health', 'clearStart', () => healthClearStart($)) +} + +// How often, and how many times, the session's id is read for the one a /clear starts. +const CLEAR_POLL_MS = 200 +const CLEAR_POLLS = 50 + +/** + * After a /clear: the conversation that follows runs under a new session id, and its state is that id's. Once + * the id has changed, featuresClearStart fills it; an engine that cannot say its id gets nothing. + */ +async function whenCleared($: EngineInterface, ended: string) { + for (let i = 0; i < CLEAR_POLLS; i++) { + let id: string + try { + id = await $.session.id() + } catch { + return + } + if (id !== ended) return featuresClearStart($) + await $.clock.sleep(CLEAR_POLL_MS) + } +} + export async function featuresTurnStart($: EngineInterface, e: Args<'turn.start'>) { await attempt($, 'runs', 'turnStart', () => runsTurnStart($, e)) await attempt($, 'defer', 'turnStart', () => deferTurnStart($, e)) @@ -254,6 +290,11 @@ export async function featuresTurnComplete($: EngineInterface, e: Args<'turn.com } export async function featuresSessionEnd($: EngineInterface, e: Args<'session.end'>) { + // Taken before the features end: a /clear goes on under another id once they have. + if (e.reason === 'clear' && typeof e.sessionId === 'string') { + keepStatusParts() + later($, () => whenCleared($, e.sessionId)) + } await attempt($, 'runs', 'sessionEnd', () => runsSessionEnd($, e)) await attempt($, 'defer', 'sessionEnd', () => deferSessionEnd($, e)) await attempt($, 'toModel', 'sessionEnd', () => toModelSessionEnd($, e)) @@ -332,6 +373,7 @@ export async function featuresCallStart($: EngineInterface, e: ToolCallInput): P /** A Bash call that a gate refused, or whose result is a refusal, never reaches bashAfter: the features that kept it let go. */ export async function featuresBashDropped($: EngineInterface, e: BashCall): Promise { await attempt($, 'runs', 'bashDropped', () => runsBashDropped($, e)) + await attempt($, 'defer', 'bashDropped', () => deferBashDropped($, e)) } /** The context lines every feature adds after a Bash call ran. */ diff --git a/plugins/altimate-code/hooks/health.test.tsx b/plugins/altimate-code/hooks/health.test.tsx index 0453e8f..e2cb22c 100644 --- a/plugins/altimate-code/hooks/health.test.tsx +++ b/plugins/altimate-code/hooks/health.test.tsx @@ -1,5 +1,6 @@ import { expect, mock, test as engineTest } from 'claude-code/testing' +import { playsClear } from './clear-state.testkit' import { PHASE_TIMEOUT_MS } from './tool-chain' import { GATE_WAIT_MS, bandText, canonical, configPathProblem, filesWrittenBy, isGitPush, modelsOfFiles, newIssues, pairKey, pushRefusal, reportText, simpleCommands, toolArgs } from './features/health' @@ -70,9 +71,10 @@ type Options = { async function start($: any, on: any, options: Options, session: { isInteractive?: boolean } = {}) { const clock = mock.clock(on) - const calls: { tool: string; args: Record; argv: readonly string[] }[] = [] + const calls: { tool: string; args: Record; argv: readonly string[]; env?: Record }[] = [] const toasts: string[] = [] const statuses: string[] = [] + const concurrent = { now: 0, most: 0 } /** The usage events sent: feature and outcome. */ const usage: string[][] = [] if (options.env) { @@ -137,8 +139,11 @@ async function start($: any, on: any, options: Options, session: { isInteractive } const args = JSON.parse(String(e.argv[e.argv.indexOf('--args') + 1] ?? '{}')) const index = run++ - calls.push({ tool, args, argv: e.argv }) + calls.push({ tool, args, argv: e.argv, env: (e as { init?: { env?: Record } }).init?.env }) + // How many tool calls run at once: a held one counts until it is released. + concurrent.most = Math.max(concurrent.most, ++concurrent.now) if (options.hold?.at === index) await options.hold.until + concurrent.now-- const answer = options.runs[Math.min(index, options.runs.length - 1)]! if ('missing' in answer) return { deny: 'spawn ENOENT' } if ('refused' in answer) return { deny: 'failed to spawn: operation not permitted' } @@ -149,7 +154,7 @@ async function start($: any, on: any, options: Options, session: { isInteractive }) await $.session.start({ cwd: ROOT, surface: 'terminal', isInteractive: session.isInteractive ?? true }) await clock.settle() - return { clock, calls, toasts, statuses, usage } + return { clock, calls, toasts, statuses, usage, concurrent } } async function endTurn($: any, clock: any, id: string) { @@ -178,6 +183,195 @@ test('a baseline run at session start; a changed model with a new insight shows await band.unmount() }) +test('after /clear the new conversation keeps the baseline and the issues: the band, /health and the push gate still have them', async ($, on) => { + const clear = playsClear($, on) + const { clock, calls, statuses } = await start($, on, { runs: [[OLD_STG, OLD_FCT], [OLD_STG, OLD_FCT, NEW_STG]] }) + await $.tool.call(edit(STG)) + await endTurn($, clock, 't1') + expect(calls).toHaveLength(2) + await clear() + await clock.settle() + expect(statuses.at(-1)).toContain('1 health issue') + // No second baseline: the one from the session's start still stands. + expect(calls).toHaveLength(2) + const band = await $.ui.mount({ ...BAND, surface: 'terminal' }) + expect(await band.find({ type: 'Text', text: '1 new health issue · stg_orders: hard coded references · /health' })).toBeDefined() + await band.unmount() + const shown: any = await $.command.run({ command: 'health' } as any) + expect(shown.text).toContain('1 new health issue · stg_orders: hard coded references') +}) + +test('a /clear while a check runs has the next conversation check those changes again', async ($, on) => { + const clear = playsClear($, on) + let release!: () => void + const { clock, calls } = await start($, on, { runs: [[OLD_STG], [OLD_STG], [OLD_STG, NEW_STG]], hold: { at: 1, until: new Promise(resolve => (release = resolve)) } }) + await $.tool.call(edit(STG)) + await endTurn($, clock, 't1') + expect(calls).toHaveLength(2) + await clear() + release() + await clock.settle() + expect(calls).toHaveLength(3) + const shown: any = await $.command.run({ command: 'health' } as any) + expect(shown.text).toContain('1 new health issue · stg_orders') +}) + +test('after /clear with no baseline yet (the first check failed), /health in the new conversation takes it', async ($, on) => { + const clear = playsClear($, on) + const { clock, calls } = await start($, on, { runs: [{ fail: 'boom' }, [OLD_STG]] }) + expect(calls).toHaveLength(1) + await clear() + await clock.settle() + const shown: any = await $.command.run({ command: 'health' } as any) + expect(shown.text).toBe('No new health issues in the models changed this session.') + expect(calls).toHaveLength(2) +}) + +test('the check\'s parse writes to the plugin\'s own artifact folder, in a CLI process of its own: the project\'s manifest keeps its compiled SQL', async ($, on) => { + const { calls } = await start($, on, { runs: [[OLD_STG]] }) + expect(calls).toHaveLength(1) + expect(calls[0]!.env).toEqual({ DBT_TARGET_PATH: `${ROOT}/target/.altimate-plugin/health`, ALTIMATE_TOOL_DAEMON: '0' }) +}) + +test('a change made right after /clear, before the kept state is put back, is merged into it and checked', async ($, on) => { + const clear = playsClear($, on) + const { clock, calls } = await start($, on, { runs: [[OLD_STG, OLD_FCT], [OLD_STG, OLD_FCT, NEW_FCT]] }) + await clear() + // The new conversation's first edit lands before the kept state is restored. + await $.tool.call(edit(FCT)) + await clock.settle() + await endTurn($, clock, 't1') + expect(calls).toHaveLength(2) + const shown: any = await $.command.run({ command: 'health' } as any) + expect(shown.text).toContain('1 new health issue · fct_orders: model fanout') +}) + +test('a git push right after /clear, before the kept issues are back, is still held by them', { options: { healthBlocksPush: true } }, async ($, on) => { + const clear = playsClear($, on) + const { clock, calls } = await start($, on, { runs: [[OLD_STG, OLD_FCT], [OLD_STG, OLD_FCT, NEW_STG]] }) + await $.tool.call(edit(STG)) + await endTurn($, clock, 't1') + await clear() + // No settling: the push is the new conversation's first call. + const denied: any = await $.tool.call(command('git push')) + expect(denied?.deny).toContain('1 new project health issue') + expect(calls).toHaveLength(2) +}) + +test('a git push that comes while the /clear restore is running waits for it and is held', { options: { healthBlocksPush: true } }, async ($, on) => { + let release!: () => void + const clear = playsClear($, on, { holdWriteOf: { key: 'health', until: new Promise(resolve => (release = resolve)) } }) + const { clock } = await start($, on, { runs: [[OLD_STG, OLD_FCT], [OLD_STG, OLD_FCT, NEW_STG]] }) + await $.tool.call(edit(STG)) + await endTurn($, clock, 't1') + await clear() + // /health starts the restore, whose write of the kept state is held; the push and a second /health come meanwhile. + const asked = $.command.run({ command: 'health' } as any) + const pushed = $.tool.call(command('git push')) + const again = $.command.run({ command: 'health' } as any) + await clock.advance(10) + release() + const [first, denied, second]: any[] = await Promise.all([asked, pushed, again]) + expect(denied?.deny).toContain('1 new project health issue') + for (const shown of [first, second]) expect(shown.text).toContain('1 new health issue · stg_orders') +}) + +test('a git push whose /clear restore does not finish within the gate\'s wait is held, not let through', { options: { healthBlocksPush: true } }, async ($, on) => { + const clear = playsClear($, on, { holdWriteOf: { key: 'health', until: new Promise(() => {}) } }) + const { clock } = await start($, on, { runs: [[OLD_STG, OLD_FCT], [OLD_STG, OLD_FCT, NEW_STG]] }) + await $.tool.call(edit(STG)) + await endTurn($, clock, 't1') + await clear() + const pushed = $.tool.call(command('git push')) + await clock.advance(GATE_WAIT_MS + 10) + const denied: any = await pushed + expect(denied?.deny).toContain('git push is held') +}) + +test('a /clear that comes while the previous one\'s restore is still writing keeps the state, and the push stays held', { options: { healthBlocksPush: true } }, async ($, on) => { + let release!: () => void + const clear = playsClear($, on, { holdWriteOf: { key: 'health', until: new Promise(resolve => (release = resolve)) } }) + const { clock, calls } = await start($, on, { runs: [[OLD_STG, OLD_FCT], [OLD_STG, OLD_FCT, NEW_STG]] }) + await $.tool.call(edit(STG)) + await endTurn($, clock, 't1') + await clear() + const asked = $.command.run({ command: 'health' } as any) + await clock.advance(10) + await clear() + release() + await asked + await clock.settle() + expect(calls).toHaveLength(2) + const denied: any = await $.tool.call(command('git push')) + expect(denied?.deny).toContain('1 new project health issue') +}) + +test('a git push after a second /clear that came while the first restore was writing waits for both restores and is held', { options: { healthBlocksPush: true } }, async ($, on) => { + let release!: () => void + const clear = playsClear($, on, { holdWriteOf: { key: 'health', until: new Promise(resolve => (release = resolve)) } }) + const { clock } = await start($, on, { runs: [[OLD_STG, OLD_FCT], [OLD_STG, OLD_FCT, NEW_STG]] }) + await $.tool.call(edit(STG)) + await endTurn($, clock, 't1') + await clear() + const asked = $.command.run({ command: 'health' } as any) + await clock.advance(10) + await clear() + const pushed = $.tool.call(command('git push')) + const again = $.command.run({ command: 'health' } as any) + await clock.advance(10) + release() + const [denied, shown]: any[] = await Promise.all([pushed, again]) + await asked + expect(denied?.deny).toContain('1 new project health issue') + expect(shown.text).toContain('1 new health issue · stg_orders') +}) + +test('the next conversation\'s check waits for a CLI call the /clear cut short, so the two never share the artifact folder', async ($, on) => { + const clear = playsClear($, on) + let release!: () => void + const { clock, calls } = await start($, on, { runs: [[OLD_STG], [OLD_STG], [OLD_STG, NEW_STG]], hold: { at: 1, until: new Promise(resolve => (release = resolve)) } }) + await $.tool.call(edit(STG)) + await endTurn($, clock, 't1') + expect(calls).toHaveLength(2) + await clear() + await clock.settle() + // The cut-short call is still running: the restored conversation's check has not started. + expect(calls).toHaveLength(2) + release() + await clock.settle() + expect(calls).toHaveLength(3) +}) + +test('two /clears while a cut-short CLI call runs still start their checks one after another', async ($, on) => { + const clear = playsClear($, on) + let release!: () => void + const { clock, calls, concurrent } = await start($, on, { runs: [[OLD_STG], [OLD_STG], [OLD_STG, NEW_STG]], hold: { at: 1, until: new Promise(resolve => (release = resolve)) } }) + await $.tool.call(edit(STG)) + await endTurn($, clock, 't1') + await clear() + await clock.settle() + await $.tool.call(edit(STG)) + await clear() + await clock.settle() + release() + await clock.settle() + expect(calls.length).toBeGreaterThanOrEqual(3) + expect(concurrent.most).toBe(1) +}) + +test('a second /clear before the first one\'s state was restored keeps what the first carried', async ($, on) => { + const clear = playsClear($, on) + const { clock, calls } = await start($, on, { runs: [[OLD_STG, OLD_FCT], [OLD_STG, OLD_FCT, NEW_STG]] }) + await $.tool.call(edit(STG)) + await endTurn($, clock, 't1') + await clear() + await clear() + await clock.settle() + expect(calls).toHaveLength(2) + const shown: any = await $.command.run({ command: 'health' } as any) + expect(shown.text).toContain('1 new health issue · stg_orders: hard coded references') +}) + test('two changed models give "2 new health issues" with the first model named', async ($, on) => { const { clock } = await start($, on, { runs: [[OLD_STG], [OLD_STG, NEW_STG, NEW_FCT]] }) await $.tool.call(edit(STG)) diff --git a/plugins/altimate-code/hooks/lineage.tsx b/plugins/altimate-code/hooks/lineage.tsx index 66c7e9b..c755e03 100644 --- a/plugins/altimate-code/hooks/lineage.tsx +++ b/plugins/altimate-code/hooks/lineage.tsx @@ -1541,7 +1541,7 @@ export function registerLineage(on: On, options: Record = {}) { ) : null const relationNames = new Map(Object.values(graph.nodes).filter(one => one.relation).map(one => [one.relation!.toLowerCase(), one])) - const sizes = await sizeControls($, { Box, Button, Text }, { id: PANE, title: PANE_TITLE, spec: PANE_SIZES, keyPrefix: 'lineage', terminal: paneTerminalColumns(e) }, work => later($, work)) + const sizes = await sizeControls($, { Box, Button, Text }, { id: PANE, title: PANE_TITLE, spec: PANE_SIZES, keyPrefix: 'lineage', terminal: paneTerminalColumns(e), placement: (e.props as { placement?: string } | undefined)?.placement }, work => later($, work)) return ( diff --git a/plugins/altimate-code/hooks/pane-size.tsx b/plugins/altimate-code/hooks/pane-size.tsx index d8b2387..fc1ab43 100644 --- a/plugins/altimate-code/hooks/pane-size.tsx +++ b/plugins/altimate-code/hooks/pane-size.tsx @@ -12,6 +12,7 @@ export type PaneSizes = Record export const SIZE_ORDER: PaneSize[] = ['min', 'default', 'max'] const SIZE_LABEL: Record = { min: 'Min', default: 'Default', max: 'Max' } const SIZES_KEY = 'pane-sizes' +export const INLINE_SIZE_NOTE = 'Above the prompt the pane is as tall as Claude Code lets it be; Min, Default and Max size it when it docks beside a fullscreen chat.' const sizeAtom = atom({ plugin: 'altimate-code', key: 'paneSizes' } as const, {} as Record) @@ -91,10 +92,21 @@ async function choose($: EngineInterface, id: string, title: string, spec: PaneS await $.ui.open({ id, title, ...paneSizes(spec)[size], closeOnEscape: true }) } -/** The Min / Default / Max row of a pane, the pane's current size marked. */ -export async function sizeControls($: EngineInterface, ui: { Box: any; Button: any; Text: any }, pane: { id: string; title: string; spec: PaneSpec; keyPrefix: string; terminal?: number }, later: (work: () => Promise) => void) { +/** + * The Min / Default / Max row of a pane, the pane's current size marked. Seated inline above the prompt (a + * terminal under 110 columns, or not fullscreen), the pane is as tall as the layout spares whatever it asks + * for, so the three would look the same: a line says so in their place. + */ +export async function sizeControls($: EngineInterface, ui: { Box: any; Button: any; Text: any }, pane: { id: string; title: string; spec: PaneSpec; keyPrefix: string; terminal?: number; placement?: string }, later: (work: () => Promise) => void) { const { Box, Button, Text } = ui noteTerminal(pane.terminal) + if (pane.placement === 'inline') { + return ( + + {INLINE_SIZE_NOTE} + + ) + } const current = (await read($, sizeAtom))[pane.id] ?? 'default' return ( diff --git a/plugins/altimate-code/hooks/status-line.ts b/plugins/altimate-code/hooks/status-line.ts index b9d1e4e..92eaced 100644 --- a/plugins/altimate-code/hooks/status-line.ts +++ b/plugins/altimate-code/hooks/status-line.ts @@ -22,15 +22,45 @@ export function statusLineOf(current: Record): string | undefine return shown.join(' · ') } +// The parts as last drawn: a /clear empties the session's state, and the line is the process's, so they are put back. +let drawnParts: Record = {} +// The parts drawn when a /clear ended the conversation, until the next one has them back. +let beforeClear: Record | null = null + /** Sets one part (null or empty removes it) and draws the status line again. */ export async function setStatusPart($: EngineInterface, id: StatusPartId, text: string | null) { + // A part its owner removes after a /clear is not brought back with the parts kept from before it. + if (beforeClear && !text) delete beforeClear[id] await update($, parts, prev => { const next = { ...prev } if (text) next[id] = text else delete next[id] return next }) - $.ui.status(statusLineOf(await read($, parts))) + drawnParts = await read($, parts) + $.ui.status(statusLineOf(drawnParts)) +} + +// The parts that describe the machine rather than the conversation: they hold after a /clear. A feature's part +// comes from its session state, which a /clear empties; the feature draws it again when it has it back. +const KEPT_ACROSS_CLEAR: readonly StatusPartId[] = ['integrations', 'altimateCode'] + +/** At a /clear: keeps the machine's parts drawn now, which a part set before restoreStatusParts must not replace. */ +export function keepStatusParts() { + const kept = Object.fromEntries(KEPT_ACROSS_CLEAR.filter(id => drawnParts[id]).map(id => [id, drawnParts[id]!])) + beforeClear = { ...(beforeClear ?? {}), ...kept } +} + +/** After a /clear: the parts drawn before it, under any the new conversation has set already. */ +export async function restoreStatusParts($: EngineInterface) { + const kept = beforeClear + if (!kept) return + // Read when the write applies, so a part removed while it waits stays removed; let go only once written, and only + // if no later /clear kept a snapshot of its own meanwhile. + await update($, parts, prev => ({ ...kept, ...prev })) + if (beforeClear === kept) beforeClear = null + drawnParts = await read($, parts) + $.ui.status(statusLineOf(drawnParts)) } /** @@ -38,6 +68,8 @@ export async function setStatusPart($: EngineInterface, id: StatusPartId, text: * set is not drawn again before its owner has set it in this one. */ export async function clearStatusParts($: EngineInterface) { + drawnParts = {} + beforeClear = null await update($, parts, () => ({})) $.ui.status(undefined) } diff --git a/plugins/altimate-code/hooks/task-impact.test.tsx b/plugins/altimate-code/hooks/task-impact.test.tsx index dbfd76f..777c98e 100644 --- a/plugins/altimate-code/hooks/task-impact.test.tsx +++ b/plugins/altimate-code/hooks/task-impact.test.tsx @@ -952,3 +952,17 @@ test('a column the SQL from before the task did not name but the manifest has (a expect(line).toContain('amount reaches 2 columns in 2 models') expect(line).not.toMatch(/not in the manifest from before the task/) }) + +test('a model the task created, which the manifest from before it lacks, has its columns called new with an unknown reach, not an altimate-code to update', async ($, on) => { + const NEW_MODEL = `${ROOT}/models/marts/region_revenue.sql` + const { turn, logs } = await start($, on, { + answer: args => (args.model === 'region_revenue' ? { status: 'ok', output: 'Model "region_revenue" not found in manifest. Available models: fct_orders, int_orders', metadata: { success: false, truncated: false } } : undefined), + }) + await turn('t1', async () => { + await $.tool.call({ tool: 'Write', file_path: NEW_MODEL, content: 'select\n region,\n sum(revenue) as revenue\nfrom {{ ref("fct_orders") }}\ngroup by 1\n' }) + }) + const line = logs.find(one => one.startsWith('◆ Lineage after this task'))! + expect(line).toContain('region_revenue') + expect(line).toContain('region not in the manifest from before the task (new in it?): its reach is unknown, not zero') + expect(line).not.toContain('update altimate-code') +}) diff --git a/plugins/altimate-code/hooks/task-impact.tsx b/plugins/altimate-code/hooks/task-impact.tsx index facfb88..d45dcc1 100644 --- a/plugins/altimate-code/hooks/task-impact.tsx +++ b/plugins/altimate-code/hooks/task-impact.tsx @@ -227,11 +227,14 @@ async function traceDown($: EngineInterface, done: Finished): Promise one.hop === 0) ? answer : null - const lacks = typeof (meta as any)?.column_answer_error === 'string' || (!!answer && !down) + // The manifest from before the task has no such model (one the task created): impact_analysis says so with + // success false, and no column answer to give. + const noModel = (meta as any)?.success === false + const lacks = noModel || typeof (meta as any)?.column_answer_error === 'string' || (!!answer && !down) // Only a column the SQL from before the task did not name, which the manifest lacks too, is called new. const unchecked = col.before === null && !!meta && lacks // An altimate-code from before the column answer gives neither the answer nor its error. - const stale = !answer && !!meta && (meta as any).column_answer === undefined && (meta as any).column_answer_error === undefined + const stale = !noModel && !answer && !!meta && (meta as any).column_answer === undefined && (meta as any).column_answer_error === undefined return { model: change.model, column: col.name, down, up: null, modelLevel: modelLevelOf(meta), failed: !down && !stale, stale, unchecked, ...unknownOf(down), consumers: consumersOf(meta) } }) } diff --git a/plugins/altimate-code/types/index.d.ts b/plugins/altimate-code/types/index.d.ts index 207b47a..9a547aa 100644 --- a/plugins/altimate-code/types/index.d.ts +++ b/plugins/altimate-code/types/index.d.ts @@ -592,6 +592,8 @@ export interface DocsCoverageView { /** Output columns without a description. */ missing: string[] hasDescription: boolean + /** Read from a manifest dbt wrote before the model's last edit; the edit's check reads it again. */ + stale: boolean } export interface DocsState { @@ -695,6 +697,8 @@ declare module 'claude-code' { runs: Shaped /** hooks/features/: DeferState. */ defer: DeferState | null + /** hooks/features/: the flags /defer and /target added to Claude's last dbt command, until the next prompt; null with none. */ + deferRan: string | null /** hooks/features/: ToModelState. */ toModel: ToModelState | null /** hooks/features/: CommentsState. */