diff --git a/.gitignore b/.gitignore index 89b72ee7..5db4b3bb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ __pycache__/ .context/ +node_modules/ diff --git a/plugins/altimate-code/hooks/features/cost.test.tsx b/plugins/altimate-code/hooks/features/cost.test.tsx new file mode 100644 index 00000000..38f7d808 --- /dev/null +++ b/plugins/altimate-code/hooks/features/cost.test.tsx @@ -0,0 +1,1303 @@ +import { expect, mock, test } from 'claude-code/testing' + +import { formatBytes, inlineSql } from './cost' + +const ROOT = '/repo' +const GIB = 1024 ** 3 + +type Options = { + /** What dbt_cost_estimate answers for a model or SQL, by model name or the SQL text. */ + answers?: Record | 'unknown-tool'> + /** The adapter's answer for everything else. */ + fallback?: Record + /** Answers given in turn for a model, before `answers` applies again. */ + sequence?: Record[]> + /** The CLI answers only once this resolves. */ + hold?: Promise + /** With `hold`: only the estimate of this model or SQL waits. */ + holdFor?: string + /** Per model: that model's estimate answers only once its promise resolves. */ + holds?: Record> + /** The session's directory; changed mid-test to stand for a /cd. */ + cwd?: string + /** Drawing the status line throws. */ + statusFails?: boolean + /** The Bash tool runs this command only once this resolves. */ + holdBash?: { command: string; until: Promise } +} + +/** The session beneath the plugin, a CLI whose dbt_cost_estimate is scripted, and a Bash tool that records what ran. */ +async function start($: any, on: any, options: Options = {}) { + const clock = mock.clock(on) + const estimates: string[] = [] + /** Estimates asked for, before any hold: one that has started and not yet answered is in here. */ + const started: string[] = [] + const ran: string[] = [] + const statuses: (string | undefined)[] = [] + on('command.register', (_: unknown, e: { name: string }) => ({ value: { command: e.name } })) + on('turn.start', (_: unknown, e: { turnId: string }) => ({ turnId: e.turnId })) + on('session.end', () => ({ sessionId: 'test' })) + on('turn.complete', (_: unknown, e: { answer: string }) => ({ text: e.answer })) + on('session.start', (_: unknown, e: { cwd: string }) => ({ cwd: e.cwd })) + on('session.root', () => ({ value: ROOT })) + on('session.cwd', () => ({ value: options.cwd ?? ROOT })) + on('ui.log', () => ({ value: undefined })) + on('ui.status', (_: unknown, e: { text?: string }) => { + if (options.statusFails) throw new Error('no status line') + statuses.push(e.text) + return { value: undefined } + }) + on('ui.render', ($$: any, e: any) => { + const { Box } = $$.ui.resolve(e) + return + }) + on('fs.exists', () => ({ value: false })) + on('process.run', async (_: unknown, e: { argv: readonly string[] }) => { + const [, command, , tool] = e.argv + if (command !== 'tool') return { value: { exitCode: 1, stdout: '', stderr: 'not this test' } } + // Another feature's tool (runs asks dbt_run_results) has nothing to report here, and is not an estimate. + if (tool !== 'dbt_cost_estimate') return { value: { exitCode: 0, stdout: JSON.stringify({ status: 'ok', output: '', metadata: { found: false } }), stderr: '' } } + // The adapter starts the project's dbt: the call must carry write permission. + expect(e.argv).toContain('--allow-write') + const args = JSON.parse(String(e.argv[e.argv.indexOf('--args') + 1] ?? '{}')) + const key = String(args.model ?? args.sql) + started.push(key) + if (options.holds?.[key]) await options.holds[key] + if (options.hold && (options.holdFor === undefined || options.holdFor === key)) await options.hold + estimates.push(key) + const answer = options.sequence?.[key]?.shift() ?? options.answers?.[key] ?? options.fallback + if (answer === 'unknown-tool') { + return { value: { exitCode: 1, stdout: JSON.stringify({ status: 'error', output: `Unknown tool: ${tool}. Run \`altimate-code tool list\` to see the available tools.`, metadata: {} }), stderr: '' } } + } + return { value: { exitCode: 0, stdout: JSON.stringify({ status: 'ok', output: '', metadata: answer ?? { success: false, error: 'no answer scripted' } }), stderr: '' } } + }) + // The Bash tool itself, beneath the plugin. + on('tool.call', async (_: unknown, e: { tool: string; command?: string }) => { + if (e.tool === 'Bash' && options.holdBash && options.holdBash.command === e.command) await options.holdBash.until + if (e.tool === 'Bash') ran.push(String(e.command)) + return { result: { text: 'ok', isError: false } } + }) + await $.session.start({ cwd: ROOT, surface: 'terminal', isInteractive: true }) + await clock.settle() + return { clock, estimates, ran, statuses, started } +} + +const bq = (bytes: number, display: string, maxBytes = bytes) => ({ success: true, supported: true, adapter: 'bigquery', bytes, maxBytes, display }) +const OVER = bq(20 * GIB, '20.0 GiB') +const UNDER = bq(2 * GIB, '2.0 GiB') + +const bash = (command: string) => ({ tool: 'Bash' as const, command }) + +test('a model over the limit is refused with its estimate, and the command never runs', async ($, on) => { + const { clock, estimates, ran } = await start($, on, { answers: { orders: OVER } }) + const result: any = await $.tool.call(bash('dbt run --select orders')) + await clock.settle() + + expect(estimates).toEqual(['orders']) + expect(ran).toEqual([]) + expect(result.deny).toBe('≈ 20.0 GiB would be scanned, over the 10.0 GiB limit. Narrow the query, or ask the user to run /allow-scan and then run the same command again.') +}) + +test('a model under the limit runs, with the estimate in a line under the row', async ($, on) => { + const { clock, ran } = await start($, on, { answers: { orders: UNDER } }) + const result: any = await $.tool.call(bash('dbt build -s orders')) + await clock.settle() + + expect(ran).toEqual(['dbt build -s orders']) + expect(result.context).toEqual(['cost · orders would scan 2.0 GiB']) +}) + +test('dbt show --inline is estimated from its SQL', async ($, on) => { + const { clock, estimates } = await start($, on, { answers: { 'select * from big': OVER } }) + const result: any = await $.tool.call(bash(`dbt show --inline "select * from big"`)) + await clock.settle() + + expect(estimates).toEqual(['select * from big']) + expect(result.deny).toContain('20.0 GiB') +}) + +test('a BigQuery table name in backticks inside single quotes is estimated; a backtick outside them is not', async ($, on) => { + const q = 'select * from `p.d.t`' + const { clock, estimates } = await start($, on, { answers: { [q]: OVER } }) + const result: any = await $.tool.call(bash(`dbt show --inline '${q}'`)) + await clock.settle() + expect(estimates).toEqual([q]) + expect(result.deny).toContain('20.0 GiB') + const before = estimates.length + await $.tool.call(bash('dbt show --inline "select 1" --threads `echo 4`')) + await $.tool.call(bash('dbt show --inline "select `echo 1`"')) + await clock.settle() + expect(estimates.length).toBe(before) +}) + +test('a cd to another directory makes the no-dry-run verdict forgotten', async ($, on) => { + const options: Options = { answers: { orders: NO_DRY_RUN } } + const { clock } = await start($, on, options) + const run = () => $.tool.call(bash('dbt run -s orders')) as Promise + expect((await run()).deny).toBeUndefined() + options.answers = { orders: OVER } + await $.tool.call(bash('cd ../bigquery-project')) + await clock.settle() + expect((await run()).deny).toContain('20.0 GiB') +}) + +test('/allow-scan lets the next held command through once, then the one after is held again', async ($, on) => { + const { clock, estimates, ran, statuses } = await start($, on, { answers: { orders: OVER } }) + expect((await $.tool.call(bash('dbt run -s orders')) as any).deny).toBeDefined() + expect(statuses.filter(Boolean).at(-1)).toContain('held') + + const allowed = await $.command.run({ command: 'allow-scan', args: '' } as any) + expect(allowed.text).toContain('runs once') + const through: any = await $.tool.call(bash('dbt run -s orders')) + await clock.settle() + expect(through.deny).toBeUndefined() + expect(through.context).toEqual(['cost · orders would scan 20.0 GiB, over the 10.0 GiB limit: allowed once']) + expect(statuses.filter(Boolean).at(-1)).not.toContain('held') + expect(ran).toEqual(['dbt run -s orders']) + + expect(((await $.tool.call(bash('dbt run -s orders'))) as any).deny).toBeDefined() + expect(estimates).toHaveLength(3) +}) + +test('a selection that is not one plain model passes, saying there is no estimate, and asks nothing', async ($, on) => { + const { clock, estimates, ran } = await start($, on) + for (const [command, why] of [ + ['dbt run', 'no model named'], + ['dbt build -s +orders', 'graph, tag, path or package selector'], + ['dbt run -s tag:nightly', 'graph, tag, path or package selector'], + ['dbt run -s a b', 'several selectors'], + ] as const) { + const result: any = await $.tool.call(bash(command)) + expect(result.deny).toBeUndefined() + expect(result.context[0]).toContain('cost · no estimate for this selection') + expect(result.context[0]).toContain(why) + } + await clock.settle() + expect(estimates).toEqual([]) + expect(ran).toHaveLength(4) +}) + +test('commands that scan nothing are not looked at', async ($, on) => { + const { clock, estimates, ran } = await start($, on) + for (const command of ['dbt test -s orders', 'dbt compile -s orders', 'ls', 'git status']) await $.tool.call(bash(command)) + await clock.settle() + expect(estimates).toEqual([]) + expect(ran).toHaveLength(4) +}) + +test('another warehouse is told once, never held, and not asked again', async ($, on) => { + const { clock, estimates, ran } = await start($, on, { fallback: { success: true, supported: false, adapter: 'snowflake' } }) + const first: any = await $.tool.call(bash('dbt run -s orders')) + const second: any = await $.tool.call(bash('dbt run -s customers')) + await clock.settle() + + expect(first.deny).toBeUndefined() + expect(first.context).toEqual(['cost · no estimate on snowflake: only BigQuery has a dry run']) + expect(second.deny).toBeUndefined() + expect(estimates).toEqual(['orders']) + expect(ran).toEqual(['dbt run -s orders', 'dbt run -s customers']) +}) + +test('an altimate-code without the tool shows one line saying what is needed, and the command runs', async ($, on) => { + const { clock, ran } = await start($, on, { answers: { orders: 'unknown-tool' } }) + const result: any = await $.tool.call(bash('dbt run -s orders')) + await clock.settle() + + expect(ran).toEqual(['dbt run -s orders']) + expect(result.context).toHaveLength(1) + expect(result.context[0]).toContain('needs a newer altimate-code') +}) + +test('a dry run that failed passes the command with the reason', async ($, on) => { + const { clock, ran } = await start($, on, { answers: { orders: { success: false, error: 'Access Denied: BigQuery job' } } }) + const result: any = await $.tool.call(bash('dbt run -s orders')) + await clock.settle() + + expect(ran).toEqual(['dbt run -s orders']) + expect(result.context).toEqual(['cost · no estimate for orders: Access Denied: BigQuery job']) +}) + +test('/scan-cost estimates a model on demand and says how it stands against the limit', async ($, on) => { + const { clock, estimates, ran } = await start($, on, { answers: { orders: OVER, customers: UNDER } }) + expect((await $.command.run({ command: 'scan-cost', args: 'orders' } as any)).text).toBe( + 'orders would scan 20.0 GiB, over the 10.0 GiB limit: a dbt run of it is held until /allow-scan.', + ) + expect((await $.command.run({ command: 'scan-cost', args: 'customers' } as any)).text).toBe('customers would scan 2.0 GiB, within the 10.0 GiB limit.') + expect((await $.command.run({ command: 'scan-cost', args: '' } as any)).text).toContain('Last estimate: customers') + await clock.settle() + expect(estimates).toEqual(['orders', 'customers']) + expect(ran).toEqual([]) +}) + +test('a command naming two models over the limit is held whole, and one /allow-scan lets all of it through once', async ($, on) => { + const { clock, ran } = await start($, on, { answers: { orders: OVER, customers: bq(30 * GIB, '30.0 GiB') } }) + const command = 'dbt run -s orders && dbt run -s customers' + expect(((await $.tool.call(bash(command))) as any).deny).toContain('30.0 GiB') + await $.command.run({ command: 'allow-scan', args: '' } as any) + const through: any = await $.tool.call(bash(command)) + await clock.settle() + + expect(through.deny).toBeUndefined() + expect(through.context).toEqual([ + 'cost · orders would scan 20.0 GiB, over the 10.0 GiB limit: allowed once', + 'cost · customers would scan 30.0 GiB, over the 10.0 GiB limit: allowed once', + ]) + expect(ran).toEqual([command]) + expect(((await $.tool.call(bash(command))) as any).deny).toBeDefined() +}) + +test('/allow-scan with nothing held allows nothing', async ($, on) => { + const { clock } = await start($, on, { answers: { orders: OVER } }) + expect((await $.command.run({ command: 'allow-scan', args: '' } as any)).text).toContain('Nothing is held') + expect(((await $.tool.call(bash('dbt run -s orders'))) as any).deny).toBeDefined() + await clock.settle() +}) + +test('the allowance is for the command that was held: another over-limit command is held, and the allowance is gone', async ($, on) => { + const { clock, ran } = await start($, on, { answers: { orders: OVER, customers: OVER } }) + expect(((await $.tool.call(bash('dbt run -s orders'))) as any).deny).toBeDefined() + await $.command.run({ command: 'allow-scan', args: '' } as any) + + expect(((await $.tool.call(bash('dbt run -s customers'))) as any).deny).toBeDefined() + expect(((await $.tool.call(bash('dbt run -s orders'))) as any).deny).toBeDefined() + await clock.settle() + expect(ran).toEqual([]) +}) + +test('an estimate whose rounded size is within the limit but whose upper bound is over it is held', async ($, on) => { + const { clock } = await start($, on, { answers: { orders: bq(10 * GIB, '10.0 GiB', 10 * GIB + 52 * 1024 ** 2) } }) + expect(((await $.tool.call(bash('dbt run -s orders'))) as any).deny).toBe('up to 10.1 GiB (≈ 10.0 GiB estimated) would be scanned, over the 10.0 GiB limit. Narrow the query, or ask the user to run /allow-scan and then run the same command again.') + await clock.settle() +}) + +test('an upper bound that formats the same as the limit is not reported as equal to it', async ($, on) => { + const { clock } = await start($, on, { answers: { orders: bq(10 * GIB, '10.0 GiB', 10 * GIB + 30 * 1024 ** 2) } }) + expect(((await $.tool.call(bash('dbt run -s orders'))) as any).deny).toBe('just over 10.0 GiB would be scanned, over the 10.0 GiB limit. Narrow the query, or ask the user to run /allow-scan and then run the same command again.') + await clock.settle() +}) + +test('a warehouse found to have no dry run ends a hold from before, and its allowance', async ($, on) => { + const answers: Record = { orders: OVER } + const { clock, ran, statuses } = await start($, on, { answers }) + expect(((await $.tool.call(bash('dbt run -s orders'))) as any).deny).toBeDefined() + await $.command.run({ command: 'allow-scan', args: '' } as any) + answers.orders = { success: true, supported: false, adapter: 'snowflake' } + await $.tool.call(bash('dbt run -s orders')) + await clock.settle() + expect(ran).toEqual(['dbt run -s orders']) + expect(statuses.at(-1) ?? '').not.toContain('held') + expect((await $.command.run({ command: 'allow-scan', args: '' } as any)).text).toContain('Nothing is held') +}) + +test('a scan just over the limit reads as over it, not as equal to it', async ($, on) => { + const { clock } = await start($, on, { answers: { orders: bq(10 * GIB + 1024 ** 2, '10.0 GiB') } }) + expect(((await $.tool.call(bash('dbt run -s orders'))) as any).deny).toBe('just over 10.0 GiB would be scanned, over the 10.0 GiB limit. Narrow the query, or ask the user to run /allow-scan and then run the same command again.') + await clock.settle() +}) + +test('a command that sets its own target, defer state, vars or project is not estimated', async ($, on) => { + const { clock, estimates, ran } = await start($, on, { answers: { orders: OVER } }) + const commands = [ + 'dbt run -s orders --target prod', + 'dbt run -s orders --defer --state prod-run', + `dbt run -s orders --vars '{a: 1}'`, + 'dbt run -s orders --project-dir other', + 'DBT_PROFILES_DIR=ci dbt run -s orders', + 'cd other && dbt run -s orders', + ' cd other && dbt run -s orders', + 'export DBT_TARGET=prod; dbt run -s orders', + 'uv run --directory other dbt run -s orders', + "dbt run -s orders --'vars' x", + 'dbt run -s orders --full-refresh', + 'dbt run -s orders stg_*', + 'dbt run -s orders --full-\\\nrefresh', + 'if true; then cd other; fi; dbt run -s orders', + 'dbt --profiles-dir ci run -s orders', + 'FOO=1 dbt run -s orders', + "dbt run -s orders --threads $(printf '4 --target prod')", + 'dbt run -s orders --threads `echo 4 --vars x`', + 'dbt run -s orders 2>&${DBT_TARGET:=1}', + 'dbt --log-path "${DBT_TARGET:=prod}" deps; dbt run -s orders', + ] + for (const command of commands) { + const result: any = await $.tool.call(bash(command)) + expect(result.deny).toBeUndefined() + expect(result.context[0]).toContain('it sets its own target, defer state, vars, project or refresh mode') + } + await clock.settle() + expect(estimates).toEqual([]) + expect(ran).toEqual(commands) +}) + +test('dbt show --inline with its SQL on the next line after a backslash is estimated', async ($, on) => { + const { clock, estimates } = await start($, on, { answers: { 'select * from big': OVER } }) + const result: any = await $.tool.call(bash(`dbt show --inline \\\n 'select * from big'`)) + await clock.settle() + expect(estimates).toEqual(['select * from big']) + expect(result.deny).toContain('20.0 GiB') +}) + +test('letting a later command through clears the hold from the status line', async ($, on) => { + const { clock, statuses } = await start($, on, { answers: { orders: OVER, customers: UNDER } }) + await $.tool.call(bash('dbt run -s orders')) + await $.tool.call(bash('dbt run -s customers')) + await clock.settle() + expect(statuses.filter(Boolean).at(-1)).not.toContain('held') +}) + +test('a variant of the held command that is not estimated leaves the hold in place', async ($, on) => { + const { clock, ran } = await start($, on, { answers: { orders: OVER } }) + expect(((await $.tool.call(bash('dbt run -s orders'))) as any).deny).toBeDefined() + await $.tool.call(bash('dbt run -s +orders')) + await $.tool.call(bash('dbt run -s orders --full-refresh')) + await clock.settle() + await $.command.run({ command: 'allow-scan', args: '' } as any) + expect(((await $.tool.call(bash('dbt run -s orders'))) as any).deny).toBeUndefined() + expect(ran.at(-1)).toBe('dbt run -s orders') +}) + +test('a repeated --inline is estimated from the last one, as dbt reads it', async ($, on) => { + const { clock, estimates } = await start($, on, { answers: { 'select * from big': OVER } }) + const result: any = await $.tool.call(bash(`dbt show --inline 'select 1' --inline 'select * from big'`)) + await clock.settle() + expect(estimates).toEqual(['select * from big']) + expect(result.deny).toContain('20.0 GiB') +}) + +test('the lines under a command are the latest estimate of it, never an earlier one', async ($, on) => { + const answers: Record = { orders: UNDER } + const { clock } = await start($, on, { answers }) + expect(((await $.tool.call(bash('dbt run -s orders'))) as any).context).toEqual(['cost · orders would scan 2.0 GiB']) + answers.orders = bq(3 * GIB, '3.0 GiB') + expect(((await $.tool.call(bash('dbt run -s orders'))) as any).context).toEqual(['cost · orders would scan 3.0 GiB']) + await clock.settle() +}) + +test('/allow-scan allows only the hold it read: a hold replaced meanwhile is not allowed', async ($, on) => { + const { clock, ran } = await start($, on, { answers: { orders: OVER, customers: OVER } }) + expect(((await $.tool.call(bash('dbt run -s orders'))) as any).deny).toBeDefined() + const allowing = $.command.run({ command: 'allow-scan', args: '' } as any) + expect(((await $.tool.call(bash('dbt run -s customers'))) as any).deny).toBeDefined() + await allowing + await clock.settle() + // Whichever order the two steps took, customers (the later hold) is not allowed by a read of orders. + expect(((await $.tool.call(bash('dbt run -s customers'))) as any).deny).toBeDefined() + expect(ran).toEqual([]) +}) + +test('a non-dbt command in between neither clears the hold nor spends the allowance', async ($, on) => { + const { clock, ran } = await start($, on, { answers: { orders: OVER } }) + await $.tool.call(bash('dbt run -s orders')) + await $.command.run({ command: 'allow-scan', args: '' } as any) + await $.tool.call(bash('ls')) + const through: any = await $.tool.call(bash('dbt run -s orders')) + await clock.settle() + expect(through.deny).toBeUndefined() + expect(ran).toEqual(['ls', 'dbt run -s orders']) +}) + +test('inlineSql reads the query as the shell would', () => { + expect(inlineSql(`dbt show --inline "select 'a' as x"`)).toBe(`select 'a' as x`) + expect(inlineSql(`dbt show --inline='select 1'`)).toBe('select 1') + expect(inlineSql(`dbt show --inline select\\ 1 --limit 5`)).toBe('select 1') + expect(inlineSql(`dbt show --inline \\\n 'select 1'`)).toBe('select 1') + expect(inlineSql('dbt show -s orders')).toBeNull() + expect(inlineSql(`dbt show --inline 'select 1' --inline='select 2'`)).toBe('select 2') + // The value of --inline is the query, even when it looks like an option. + expect(inlineSql(`dbt show --inline '--inline=comment select 1'`)).toBe('--inline=comment select 1') +}) + +test('formatBytes writes the units the dry run does', () => { + expect(formatBytes(512)).toBe('512.0 Bytes') + expect(formatBytes(1536)).toBe('1.5 KiB') + expect(formatBytes(10 * GIB)).toBe('10.0 GiB') + expect(formatBytes(1024 ** 5)).toBe('1.0 PiB') +}) + +test('what leaves the query alone is still estimated: other dbt calls before it, thread and limit flags, a launcher', async ($, on) => { + const { clock, estimates } = await start($, on, { answers: { orders: UNDER, 'select 1': UNDER } }) + for (const command of ['dbt deps && dbt run -s orders --threads 4', 'uv run dbt build --select orders --fail-fast', `dbt show --inline "select 1" --limit 5`]) { + expect(((await $.tool.call(bash(command))) as any).context[0]).toContain('would scan 2.0 GiB') + } + await clock.settle() + expect(estimates).toEqual(['orders', 'orders', 'select 1']) +}) + +test('a very long run of blank lines before the command costs no noticeable time', async ($, on) => { + const { clock } = await start($, on, { answers: { orders: UNDER } }) + const began = Date.now() + await $.tool.call(bash(`${'\n'.repeat(64000)}dbt run -s orders`)) + await clock.settle() + expect(Date.now() - began).toBeLessThan(3000) +}) + +test('a line with a great many dbt calls is not read call by call', { timeoutMs: 20_000 }, async ($, on) => { + const { clock, estimates } = await start($, on, { answers: { orders: UNDER } }) + const began = Date.now() + const result: any = await $.tool.call(bash('dbt run -s orders;'.repeat(4000))) + await clock.settle() + expect(estimates).toEqual([]) + expect(result.context.length).toBe(6) + expect(result.context[0]).toContain('cost · no estimate') + expect(result.context[5]).toBe('cost · 3995 more notes not shown') + // Other features' hooks (runs reads the same line) share this call, so the bound is for the whole chain. + expect(Date.now() - began).toBeLessThan(10_000) +}) + +test('an estimate not done within the gate wait refuses the command, and the finished answer is used on the next try', async ($, on) => { + let release!: () => void + const { clock, estimates, ran } = await start($, on, { answers: { orders: OVER }, hold: new Promise(resolve => (release = resolve)) }) + const first = $.tool.call(bash('dbt run -s orders')) + await clock.advance(4_001) + expect(((await first) as any).deny).toContain('still being estimated') + expect(ran).toEqual([]) + + release() + await clock.settle() + const second: any = await $.tool.call(bash('dbt run -s orders')) + await clock.settle() + expect(second.deny).toContain('20.0 GiB') + // One estimate served both passes. + expect(estimates).toEqual(['orders']) + expect(ran).toEqual([]) +}) + +test('a held command stays held when its estimate cannot be made again, until /allow-scan', async ($, on) => { + const answers: Record = { orders: OVER } + const { clock, ran } = await start($, on, { answers }) + expect(((await $.tool.call(bash('dbt run -s orders'))) as any).deny).toContain('20.0 GiB') + + answers.orders = { success: false, error: 'Access Denied' } + const again: any = await $.tool.call(bash('dbt run -s orders')) + expect(again.deny).toContain('20.0 GiB') + expect(ran).toEqual([]) + + await $.command.run({ command: 'allow-scan', args: '' } as any) + const through: any = await $.tool.call(bash('dbt run -s orders')) + await clock.settle() + expect(through.deny).toBeUndefined() + expect(ran).toEqual(['dbt run -s orders']) +}) + +test('an estimate still running spends no allowance: the held command stays held until every estimate is in', async ($, on) => { + const options: Options = { answers: { a: OVER, b: OVER } } + const { clock, ran } = await start($, on, options) + const both = 'dbt run -s a && dbt run -s b' + expect(((await $.tool.call(bash(both))) as any).deny).toContain('20.0 GiB') + await $.command.run({ command: 'allow-scan', args: '' } as any) + + let release!: () => void + options.hold = new Promise(resolve => (release = resolve)) + options.holdFor = 'b' + const slow = $.tool.call(bash(both)) + await clock.advance(4_001) + expect(((await slow) as any).deny).toContain('still being estimated') + expect(ran).toEqual([]) + + // The allowance was not spent by the refusal: once b's estimate is in, the command goes through once. + release() + await clock.settle() + const through: any = await $.tool.call(bash(both)) + await clock.settle() + expect(through.deny).toBeUndefined() + expect(ran).toEqual([both]) +}) + +test('a command being retried keeps its finished answers however long the later estimate took', async ($, on) => { + const options: Options = { answers: { a: UNDER, b: UNDER } } + const { clock, estimates } = await start($, on, options) + const both = 'dbt run -s a && dbt run -s b' + let release!: () => void + options.hold = new Promise(resolve => (release = resolve)) + options.holdFor = 'b' + const first = $.tool.call(bash(both)) + await clock.advance(4_001) + expect(((await first) as any).deny).toContain('still being estimated') + // b takes longer than the answer for a is kept for a command nobody retries. + const waiting = [] as any[] + for (let i = 0; i < 7; i++) { + await clock.advance(10_000) + const again = $.tool.call(bash(both)) + await clock.advance(4_001) + waiting.push((await again) as any) + } + expect(waiting.every(w => w.deny.includes('still being estimated'))).toBe(true) + release() + await clock.settle() + options.hold = undefined + const retry: any = await $.tool.call(bash(both)) + await clock.settle() + expect(retry.deny).toBeUndefined() + expect(estimates).toEqual(['a', 'b']) +}) + +test('an answer nobody took is not used a minute later: the model may have changed', async ($, on) => { + const options: Options = { answers: { orders: OVER } } + const { clock, estimates, ran } = await start($, on, options) + let release!: () => void + options.hold = new Promise(resolve => (release = resolve)) + const first = $.tool.call(bash('dbt run -s orders')) + await clock.advance(4_001) + expect(((await first) as any).deny).toContain('still being estimated') + release() + await clock.settle() + + await clock.advance(61_000) + options.answers = { orders: UNDER } + const later: any = await $.tool.call(bash('dbt run -s orders')) + await clock.settle() + expect(later.deny).toBeUndefined() + expect(estimates).toEqual(['orders', 'orders']) + expect(ran).toEqual(['dbt run -s orders']) +}) + +test('the same model named twice on a line is estimated once, so a slow estimate does not refuse the retry forever', async ($, on) => { + const options: Options = { answers: { orders: UNDER } } + const { clock, estimates, ran } = await start($, on, options) + let release!: () => void + options.hold = new Promise(resolve => (release = resolve)) + const line = 'dbt run -s orders && dbt show -s orders' + const first = $.tool.call(bash(line)) + await clock.advance(4_001) + expect(((await first) as any).deny).toContain('still being estimated') + release() + await clock.settle() + options.hold = undefined + const retry: any = await $.tool.call(bash(line)) + await clock.settle() + expect(retry.deny).toBeUndefined() + expect(estimates).toEqual(['orders']) + expect(ran).toEqual([line]) +}) + +test('once the wait is spent no further estimate starts, so two adapters never start in one project at once', async ($, on) => { + const options: Options = { answers: { a: UNDER, b: UNDER } } + const { clock, estimates } = await start($, on, options) + let release!: () => void + options.hold = new Promise(resolve => (release = resolve)) + const first = $.tool.call(bash('dbt run -s a && dbt run -s b')) + await clock.advance(4_001) + const denied: any = await first + expect(denied.deny).toContain('The scan of a, b is still being estimated') + release() + await clock.settle() + // b was never started: only one adapter start runs at a time. + expect(estimates).toEqual(['a']) +}) + +test('a model named twice on a line is listed once as still being estimated', async ($, on) => { + const options: Options = { answers: { orders: UNDER } } + const { clock } = await start($, on, options) + let release!: () => void + options.hold = new Promise(resolve => (release = resolve)) + const first = $.tool.call(bash('dbt run -s orders && dbt show -s orders')) + await clock.advance(4_001) + expect(((await first) as any).deny).toContain('The scan of orders is still being estimated') + release() + await clock.settle() +}) + +test('a compound command gets every answer over its retries, however long each estimate takes', async ($, on) => { + const options: Options = { answers: { a: UNDER, b: UNDER, c: UNDER } } + const { clock, estimates, ran } = await start($, on, options) + let release!: () => void + options.hold = new Promise(resolve => (release = resolve)) + options.holdFor = 'c' + const all = 'dbt run -s a && dbt run -s b && dbt run -s c' + const first = $.tool.call(bash(all)) + await clock.advance(4_001) + expect(((await first) as any).deny).toContain('still being estimated') + release() + await clock.settle() + const through: any = await $.tool.call(bash(all)) + await clock.settle() + expect(through.deny).toBeUndefined() + // a and b were asked for once: the retry used their answers. + expect(estimates.filter(key => key === 'a')).toEqual(['a']) + expect(ran).toEqual([all]) +}) + +const EDIT = { tool: 'Edit', file_path: '/repo/models/orders.sql', old_string: '1', new_string: '2' } as any +const NO_DRY_RUN = { success: true, supported: false, adapter: 'duckdb' } + +test('an edit made while an estimate is running makes its answer void: the command is asked about again', async ($, on) => { + const options: Options = { answers: { orders: UNDER } } + const { clock, estimates, ran } = await start($, on, options) + let release!: () => void + options.hold = new Promise(resolve => (release = resolve)) + const first = $.tool.call(bash('dbt run -s orders')) + await clock.advance(1_000) + await $.tool.call(EDIT) + options.answers = { orders: OVER } + release() + await clock.settle() + expect(((await first) as any).deny).toContain('The project changed while the scan was being estimated') + expect(ran).toEqual([]) + + options.hold = undefined + const retry: any = await $.tool.call(bash('dbt run -s orders')) + await clock.settle() + expect(retry.deny).toContain('20.0 GiB') + expect(estimates).toEqual(['orders', 'orders']) +}) + +test('a shell command that can rewrite the model drops the estimates made before it', async ($, on) => { + const options: Options = { answers: { orders: UNDER } } + const { clock, estimates, ran } = await start($, on, options) + let release!: () => void + options.hold = new Promise(resolve => (release = resolve)) + const first = $.tool.call(bash('dbt run -s orders')) + await clock.advance(4_001) + expect(((await first) as any).deny).toContain('still being estimated') + release() + await clock.settle() + + await $.tool.call(bash('git restore models/orders.sql')) + options.hold = undefined + options.answers = { orders: OVER } + const retry: any = await $.tool.call(bash('dbt run -s orders')) + await clock.settle() + expect(retry.deny).toContain('20.0 GiB') + expect(ran).toEqual(['git restore models/orders.sql']) + expect(estimates).toEqual(['orders', 'orders']) +}) + +test('a read-only shell command keeps the answer waiting for the retry', async ($, on) => { + const options: Options = { answers: { orders: UNDER } } + const { clock, estimates } = await start($, on, options) + let release!: () => void + options.hold = new Promise(resolve => (release = resolve)) + const first = $.tool.call(bash('dbt run -s orders')) + await clock.advance(4_001) + expect(((await first) as any).deny).toContain('still being estimated') + release() + await clock.settle() + await $.tool.call(bash('ls models')) + options.hold = undefined + const retry: any = await $.tool.call(bash('dbt run -s orders')) + await clock.settle() + expect(retry.deny).toBeUndefined() + expect(estimates).toEqual(['orders']) +}) + +test('a cd drops the answer waiting for the retry: it may have selected another project', async ($, on) => { + const options: Options = { answers: { orders: UNDER } } + const { clock, estimates } = await start($, on, options) + let release!: () => void + options.hold = new Promise(resolve => (release = resolve)) + const first = $.tool.call(bash('dbt run -s orders')) + await clock.advance(4_001) + expect(((await first) as any).deny).toContain('still being estimated') + release() + await clock.settle() + await $.tool.call(bash('cd ../project-b')) + options.hold = undefined + options.answers = { orders: OVER } + const retry: any = await $.tool.call(bash('dbt run -s orders')) + await clock.settle() + expect(retry.deny).toContain('20.0 GiB') + expect(estimates).toEqual(['orders', 'orders']) +}) + +test('a dbt run that has finished drops the estimates made before it', async ($, on) => { + const options: Options = { answers: { orders: UNDER, customers: UNDER } } + const { clock, estimates } = await start($, on, options) + let release!: () => void + options.hold = new Promise(resolve => (release = resolve)) + const first = $.tool.call(bash('dbt run -s orders')) + await clock.advance(4_001) + expect(((await first) as any).deny).toContain('still being estimated') + release() + await clock.settle() + options.hold = undefined + await $.tool.call(bash('dbt run -s customers')) + await clock.settle() + options.answers = { orders: OVER } + const retry: any = await $.tool.call(bash('dbt run -s orders')) + await clock.settle() + expect(retry.deny).toContain('20.0 GiB') + expect(estimates.filter(e => e === 'orders').length).toBe(2) +}) + +test('a /cd to another project makes the no-dry-run verdict forgotten', async ($, on) => { + const options: Options = { answers: { orders: NO_DRY_RUN } } + const { clock } = await start($, on, options) + const run = () => $.tool.call(bash('dbt run -s orders')) as Promise + expect((await run()).deny).toBeUndefined() + options.answers = { orders: OVER } + options.cwd = '/elsewhere/bigquery-project' + await clock.settle() + expect((await run()).deny).toContain('20.0 GiB') +}) + +test('a /cd to another project drops the finished answer and the allowance', async ($, on) => { + const options: Options = { answers: { orders: UNDER } } + const { clock, estimates } = await start($, on, options) + let release!: () => void + options.hold = new Promise(resolve => (release = resolve)) + const first = $.tool.call(bash('dbt run -s orders')) + await clock.advance(4_001) + expect(((await first) as any).deny).toContain('still being estimated') + release() + await clock.settle() + options.hold = undefined + options.cwd = '/elsewhere/project-b' + options.answers = { orders: OVER } + const retry: any = await $.tool.call(bash('dbt run -s orders')) + await clock.settle() + expect(retry.deny).toContain('20.0 GiB') + expect(estimates).toEqual(['orders', 'orders']) + await $.command.run({ command: 'allow-scan', args: '' } as any) + options.cwd = '/elsewhere/project-c' + const again: any = await $.tool.call(bash('dbt run -s orders')) + await clock.settle() + expect(again.deny).toContain('20.0 GiB') +}) + +test('a dbt run that is let through drops the estimates kept from before it', async ($, on) => { + const options: Options = { answers: { orders: UNDER, customers: UNDER } } + const { clock, estimates } = await start($, on, options) + let release!: () => void + options.hold = new Promise(resolve => (release = resolve)) + options.holdFor = 'orders' + const first = $.tool.call(bash('dbt run -s orders')) + await clock.advance(4_001) + expect(((await first) as any).deny).toContain('still being estimated') + release() + await clock.settle() + options.hold = undefined + // Another command's run is let through, and is still running (not yet after-phase) when orders is retried. + let finish!: () => void + options.holdBash = { command: 'dbt run -s customers', until: new Promise(resolve => (finish = resolve)) } + const customers = $.tool.call(bash('dbt run -s customers')) + await clock.settle() + options.answers = { orders: OVER } + const retry: any = await $.tool.call(bash('dbt run -s orders')) + await clock.settle() + expect(retry.deny).toContain('20.0 GiB') + expect(estimates.filter(e => e === 'orders').length).toBe(2) + finish() + await customers +}) + +test('a second model waits for the estimate running in the project, and starts none while it runs', async ($, on) => { + const options: Options = { answers: { a: UNDER, b: UNDER } } + const { clock, estimates } = await start($, on, options) + let release!: () => void + options.hold = new Promise(resolve => (release = resolve)) + options.holdFor = 'a' + const first = $.tool.call(bash('dbt run -s a')) + await clock.advance(4_001) + expect(((await first) as any).deny).toContain('still being estimated') + const second = $.tool.call(bash('dbt run -s b')) + await clock.advance(4_001) + expect(((await second) as any).deny).toContain('still being estimated') + expect(estimates).toEqual([]) + release() + await clock.settle() + expect(estimates).toEqual(['a']) +}) + +test('an edit that voids a running estimate does not let another start beside it', async ($, on) => { + const options: Options = { answers: { a: UNDER, b: UNDER } } + const { clock, estimates } = await start($, on, options) + let release!: () => void + options.hold = new Promise(resolve => (release = resolve)) + options.holdFor = 'a' + const first = $.tool.call(bash('dbt run -s a')) + await clock.advance(4_001) + expect(((await first) as any).deny).toContain('still being estimated') + await $.tool.call(bash('git restore models/a.sql')) + const second = $.tool.call(bash('dbt run -s b')) + await clock.advance(4_001) + expect(((await second) as any).deny).toContain('still being estimated') + expect(estimates).toEqual([]) + release() + await clock.settle() + options.hold = undefined + const third: any = await $.tool.call(bash('dbt run -s b')) + await clock.settle() + expect(third.deny).toBeUndefined() + expect(estimates).toEqual(['a', 'b']) +}) + + +test('a run let through while an estimate is still running: the retry does not take that estimate\'s pre-run answer', async ($, on) => { + const options: Options = { answers: { orders: UNDER } } + const { clock, estimates } = await start($, on, options) + let release!: () => void + options.hold = new Promise(resolve => (release = resolve)) + const first = $.tool.call(bash('dbt run -s orders')) + await clock.advance(4_001) + expect(((await first) as any).deny).toContain('still being estimated') + // Not estimated (its own target), so it is let through at once and starts rebuilding. + expect(((await $.tool.call(bash('dbt run -s customers --target prod'))) as any).deny).toBeUndefined() + release() + await clock.settle() + options.hold = undefined + options.answers = { orders: OVER } + const retry: any = await $.tool.call(bash('dbt run -s orders')) + await clock.settle() + expect(retry.deny).toContain('20.0 GiB') + expect(estimates).toEqual(['orders', 'orders']) +}) + +test('a voided running estimate is not reused and holds the next one back until it settles', async ($, on) => { + const options: Options = { answers: { orders: UNDER } } + const { clock, estimates, started } = await start($, on, options) + let release!: () => void + options.hold = new Promise(resolve => (release = resolve)) + let finish!: () => void + options.holdBash = { command: 'dbt run -s customers --target prod', until: new Promise(resolve => (finish = resolve)) } + // Still inside the gate's wait when the run is let through (and still running): nothing has timed it out yet. + const first = $.tool.call(bash('dbt run -s orders')) + await clock.advance(1_000) + const rebuild = $.tool.call(bash('dbt run -s customers --target prod')) + await clock.advance(10) + const again = $.tool.call(bash('dbt run -s orders')) + await clock.advance(500) + expect(started).toEqual(['orders']) + await clock.advance(4_001) + expect(((await first) as any).deny).toContain('The project changed while the scan was being estimated') + expect(((await again) as any).deny).toContain('still being estimated') + expect(started).toEqual(['orders']) + finish() + await rebuild + release() + await clock.settle() + expect(estimates).toEqual(['orders']) +}) + +test('parallel dbt calls estimate one at a time: the second starts when the first is in', async ($, on) => { + const options: Options = { answers: { a: UNDER, b: UNDER } } + const { clock, started } = await start($, on, options) + let release!: () => void + options.hold = new Promise(resolve => (release = resolve)) + options.holdFor = 'a' + const both = Promise.all([$.tool.call(bash('dbt run -s a')), $.tool.call(bash('dbt run -s b'))]) + await clock.advance(1_000) + expect(started).toEqual(['a']) + release() + const results: any[] = await both + await clock.settle() + expect(started).toEqual(['a', 'b']) + // The first run may finish while the second is being estimated, which moves the project on: then the second asks again. + expect(results[0].deny).toBeUndefined() + expect(results[1].deny === undefined || results[1].deny.includes('Run the same command again')).toBe(true) +}) + +test('/scan-cost does not start beside a running gate estimate', async ($, on) => { + const options: Options = { answers: { a: UNDER, orders: UNDER } } + const { clock, started } = await start($, on, options) + let release!: () => void + options.hold = new Promise(resolve => (release = resolve)) + options.holdFor = 'a' + const gate = $.tool.call(bash('dbt run -s a')) + await clock.advance(500) + const scan = $.command.run({ command: 'scan-cost', args: 'orders' } as any) + await clock.advance(1_000) + expect(started).toEqual(['a']) + release() + await gate + expect(((await scan) as any).text).toContain('orders would scan 2.0 GiB') + expect(started).toEqual(['a', 'orders']) +}) + +test('a gate estimate does not start beside a running /scan-cost', async ($, on) => { + const options: Options = { answers: { a: UNDER, orders: UNDER } } + const { clock, started } = await start($, on, options) + let release!: () => void + options.hold = new Promise(resolve => (release = resolve)) + options.holdFor = 'orders' + const scan = $.command.run({ command: 'scan-cost', args: 'orders' } as any) + await clock.advance(500) + const gate = $.tool.call(bash('dbt run -s a')) + await clock.advance(1_000) + expect(started).toEqual(['orders']) + release() + await scan + expect(((await gate) as any).deny).toBeUndefined() + await clock.settle() + expect(started).toEqual(['orders', 'a']) +}) + +test('a /cd takes the old project\'s hold off the status line and forgets its last estimate', async ($, on) => { + const options: Options = { answers: { orders: OVER } } + const { clock, statuses } = await start($, on, options) + expect(((await $.tool.call(bash('dbt run -s orders'))) as any).deny).toBeDefined() + expect(statuses.filter(Boolean).at(-1)).toContain('held') + options.cwd = '/elsewhere/project-b' + await $.tool.call(bash('ls')) + await clock.settle() + expect(statuses.at(-1)).toBeUndefined() + const allowed = await $.command.run({ command: 'allow-scan', args: '' } as any) + expect(allowed.text).toContain('Nothing is held') + const last = await $.command.run({ command: 'scan-cost', args: '' } as any) + expect(last.text).toContain('Run /scan-cost ') +}) + +test('a quoted git option value is read: git -C "my repo" checkout . forgets the verdict', async ($, on) => { + const options: Options = { answers: { orders: NO_DRY_RUN } } + const { clock } = await start($, on, options) + const run = () => $.tool.call(bash('dbt run -s orders')) as Promise + expect((await run()).deny).toBeUndefined() + options.answers = { orders: OVER } + await $.tool.call(bash('git -C "my repo" checkout .')) + await clock.settle() + expect((await run()).deny).toContain('20.0 GiB') +}) + +test('a lower bound already over the limit holds the command', async ($, on) => { + const { clock } = await start($, on, { answers: { orders: { success: true, supported: true, adapter: 'bigquery', bytes: 20 * GIB, lowerBound: true, display: '20.0 GiB' } } }) + const result: any = await $.tool.call(bash('dbt run -s orders')) + await clock.settle() + expect(result.deny).toContain('20.0 GiB') +}) + +test('a lower-bound-only estimate holds nothing and keeps an existing hold', async ($, on) => { + const { clock } = await start($, on, { answers: { orders: OVER, cheap: { success: true, supported: true, adapter: 'bigquery', bytes: 0, lowerBound: true, display: '0 Bytes' } } }) + expect(((await $.tool.call(bash('dbt run -s orders'))) as any).deny).toContain('20.0 GiB') + const result: any = await $.tool.call(bash('dbt run -s cheap')) + await clock.settle() + expect(result.context?.join(' ')).toContain('no estimate') + expect(((await $.tool.call(bash('dbt run -s orders'))) as any).deny).toContain('20.0 GiB') +}) + +for (const restore of [ + 'git -C /repo checkout .', + 'git -c core.fsmonitor=false restore .', + 'git --git-dir /x --work-tree /y reset --hard', +]) { +test(`${restore} makes the no-dry-run verdict forgotten`, async ($, on) => { + const options: Options = { answers: { orders: NO_DRY_RUN } } + const { clock } = await start($, on, options) + const run = () => $.tool.call(bash('dbt run -s orders')) as Promise + expect((await run()).deny).toBeUndefined() + options.answers = { orders: OVER } + await $.tool.call(bash(restore)) + await clock.settle() + expect((await run()).deny).toContain('20.0 GiB') +}) +} + +test('a warehouse found to have no dry run is asked about again after an edit, a profile change or ten minutes', async ($, on) => { + const options: Options = { answers: { orders: NO_DRY_RUN } } + const { clock, ran } = await start($, on, options) + const run = () => $.tool.call(bash('dbt run -s orders')) as Promise + expect((await run()).deny).toBeUndefined() + + // The default target is now BigQuery: the next command is estimated and held. + options.answers = { orders: OVER } + expect((await run()).deny).toBeUndefined() + await $.tool.call(EDIT) + expect((await run()).deny).toContain('20.0 GiB') + + // The same by a shell command that names the profile. + options.answers = { orders: NO_DRY_RUN } + await $.command.run({ command: 'allow-scan', args: '' } as any) + expect((await run()).deny).toBeUndefined() + options.answers = { orders: OVER } + await $.tool.call(bash('sed -i s/duckdb/bigquery/ profiles.yml')) + expect((await run()).deny).toContain('20.0 GiB') + + // And by time alone. + options.answers = { orders: NO_DRY_RUN } + await $.command.run({ command: 'allow-scan', args: '' } as any) + expect((await run()).deny).toBeUndefined() + options.answers = { orders: OVER } + await clock.advance(10 * 60_000 + 1) + expect((await run()).deny).toContain('20.0 GiB') + expect(ran.length).toBeGreaterThan(0) +}) + +test('a git read or a redirect to another stream does not drop the estimate the retry is waiting for', async ($, on) => { + const options: Options = { answers: { orders: UNDER } } + const { clock, estimates } = await start($, on, options) + let release!: () => void + options.hold = new Promise(resolve => (release = resolve)) + const first = $.tool.call(bash('dbt run -s orders')) + await clock.advance(4_001) + expect(((await first) as any).deny).toContain('still being estimated') + release() + await clock.settle() + await $.tool.call(bash('git status')) + await $.tool.call(bash('git diff --stat 2>&1')) + await $.tool.call(bash('ls models 2>/dev/null')) + options.hold = undefined + const retry: any = await $.tool.call(bash('dbt run -s orders')) + await clock.settle() + expect(retry.deny).toBeUndefined() + expect(estimates).toEqual(['orders']) +}) + +test('a git restore that does not name the profile still makes the no-dry-run verdict forgotten', async ($, on) => { + const options: Options = { answers: { orders: NO_DRY_RUN } } + const { clock } = await start($, on, options) + const run = () => $.tool.call(bash('dbt run -s orders')) as Promise + expect((await run()).deny).toBeUndefined() + options.answers = { orders: OVER } + await $.tool.call(bash('git checkout .')) + await clock.settle() + expect((await run()).deny).toContain('20.0 GiB') +}) + +test('a shell line that rewrites the model and also runs dbt voids the estimates made before it', async ($, on) => { + const options: Options = { answers: { orders: UNDER, cheap: UNDER } } + const { clock, estimates } = await start($, on, options) + let release!: () => void + options.hold = new Promise(resolve => (release = resolve)) + options.holdFor = 'orders' + const first = $.tool.call(bash('dbt run -s orders')) + await clock.advance(4_001) + expect(((await first) as any).deny).toContain('still being estimated') + release() + await clock.settle() + + await $.tool.call(bash('git restore models/orders.sql && dbt run -s cheap')) + await clock.settle() + options.hold = undefined + options.answers = { orders: OVER, cheap: UNDER } + const retry: any = await $.tool.call(bash('dbt run -s orders')) + await clock.settle() + expect(retry.deny).toContain('20.0 GiB') + expect(estimates.filter(key => key === 'orders')).toEqual(['orders', 'orders']) +}) + +test('a command that is rewriting the model voids the estimates before it ends, not only after', async ($, on) => { + const options: Options = { answers: { orders: UNDER } } + const { clock, estimates } = await start($, on, options) + let release!: () => void + options.hold = new Promise(resolve => (release = resolve)) + const first = $.tool.call(bash('dbt run -s orders')) + await clock.advance(4_001) + expect(((await first) as any).deny).toContain('still being estimated') + release() + await clock.settle() + + // The rewrite is still running (its after-hook has not come) when the retry asks. + let finish!: () => void + options.holdBash = { command: 'sed -i s/a/b/ models/orders.sql; sleep 10', until: new Promise(resolve => (finish = resolve)) } + const rewrite = $.tool.call(bash('sed -i s/a/b/ models/orders.sql; sleep 10')) + await clock.advance(1_000) + options.hold = undefined + options.answers = { orders: OVER } + const retry: any = await $.tool.call(bash('dbt run -s orders')) + await clock.settle() + expect(retry.deny).toContain('20.0 GiB') + expect(estimates).toEqual(['orders', 'orders']) + finish() + await rewrite +}) + +test('a slow estimate for a line that also redirects its output is used on the retry, not asked for again', async ($, on) => { + const options: Options = { answers: { orders: UNDER } } + const { clock, estimates, ran } = await start($, on, options) + let release!: () => void + options.hold = new Promise(resolve => (release = resolve)) + const line = 'dbt run -s orders > run.log' + const first = $.tool.call(bash(line)) + await clock.advance(4_001) + expect(((await first) as any).deny).toContain('still being estimated') + release() + await clock.settle() + options.hold = undefined + const retry: any = await $.tool.call(bash(line)) + await clock.settle() + expect(retry.deny).toBeUndefined() + expect(estimates).toEqual(['orders']) + expect(ran).toEqual([line]) +}) + +test('a no-dry-run answer that an edit made void is not kept as the warehouse', async ($, on) => { + const options: Options = { answers: { orders: NO_DRY_RUN } } + const { clock } = await start($, on, options) + let release!: () => void + options.hold = new Promise(resolve => (release = resolve)) + const first = $.tool.call(bash('dbt run -s orders')) + await clock.advance(1_000) + await $.tool.call(EDIT) + release() + await clock.settle() + expect(((await first) as any).deny).toContain('The project changed') + options.hold = undefined + options.answers = { orders: OVER } + const retry: any = await $.tool.call(bash('dbt run -s orders')) + await clock.settle() + expect(retry.deny).toContain('20.0 GiB') +}) + +test('a status line that cannot be drawn does not let a scan over the limit through', async ($, on) => { + const { clock, ran } = await start($, on, { answers: { orders: OVER }, statusFails: true }) + const result: any = await $.tool.call(bash('dbt run -s orders')) + await clock.settle() + expect(result.deny).toContain('20.0 GiB') + expect(ran).toEqual([]) +}) + +test('an edit drops the estimates made before it', async ($, on) => { + const options: Options = { answers: { orders: UNDER } } + const { clock, estimates, ran } = await start($, on, options) + let release!: () => void + options.hold = new Promise(resolve => (release = resolve)) + const first = $.tool.call(bash('dbt run -s orders')) + await clock.advance(4_001) + expect(((await first) as any).deny).toContain('still being estimated') + release() + await clock.settle() + + options.answers = { orders: OVER } + await $.tool.call({ tool: 'Edit', file_path: '/repo/models/orders.sql', old_string: '1', new_string: '2' } as any) + await clock.settle() + const later: any = await $.tool.call(bash('dbt run -s orders')) + await clock.settle() + expect(later.deny).toContain('20.0 GiB') + expect(estimates).toEqual(['orders', 'orders']) + expect(ran).toEqual([]) +}) + +test('a pending estimate on another command does not carry /allow-scan over to it', async ($, on) => { + const options: Options = { answers: { a: OVER, b: OVER, c: UNDER } } + const { clock, ran } = await start($, on, options) + expect(((await $.tool.call(bash('dbt run -s a'))) as any).deny).toContain('20.0 GiB') + await $.command.run({ command: 'allow-scan', args: '' } as any) + + let release!: () => void + options.hold = new Promise(resolve => (release = resolve)) + options.holdFor = 'c' + const other = 'dbt run -s b && dbt run -s c' + const slow = $.tool.call(bash(other)) + await clock.advance(4_001) + expect(((await slow) as any).deny).toContain('still being estimated') + release() + await clock.settle() + const retry: any = await $.tool.call(bash(other)) + expect(retry.deny).toContain('20.0 GiB') + expect(ran).toEqual([]) +}) + +test('a /cd away and back does not forget the estimate still running in the project', async ($, on) => { + const options: Options = { answers: { orders: UNDER, b: UNDER, c: UNDER } } + const { clock, started } = await start($, on, options) + let release!: () => void + options.hold = new Promise(resolve => (release = resolve)) + options.holdFor = 'orders' + const first = $.command.run({ command: 'scan-cost', args: 'orders' } as any) + await clock.advance(500) + // Another project is not held back by it. + options.cwd = '/elsewhere/project-b' + const other = $.command.run({ command: 'scan-cost', args: 'b' } as any) + await clock.advance(500) + expect(((await other) as any).text).toContain('b would scan 2.0 GiB') + // Back in the first project, a second estimate waits its turn and gives up after the short wait. + options.cwd = undefined + const again = $.command.run({ command: 'scan-cost', args: 'c' } as any) + await clock.advance(4_001) + expect(((await again) as any).text).toContain('another estimate is still running in this project') + expect(started).toEqual(['orders', 'b']) + // The gate learns each project it passes through: a Bash pass in A, one in B (the /cd, while orders still runs), then back in A. + await $.tool.call(bash('ls')) + options.cwd = '/elsewhere/project-b' + await $.tool.call(bash('ls')) + options.cwd = undefined + await $.tool.call(bash('ls')) + const third = $.command.run({ command: 'scan-cost', args: 'd' } as any) + await clock.advance(4_001) + expect(((await third) as any).text).toContain('another estimate is still running in this project') + expect(started).toEqual(['orders', 'b']) + release() + await first +}) + +test('a queued /scan-cost that wakes after a /cd runs under the project now in use, behind its running estimate', async ($, on) => { + const options: Options = { answers: { orders: UNDER, b: UNDER, c: UNDER } } + const { clock, started } = await start($, on, options) + let releaseFirst!: () => void + let release!: () => void + options.holds = { orders: new Promise(resolve => (releaseFirst = resolve)), b: new Promise(resolve => (release = resolve)) } + const first = $.command.run({ command: 'scan-cost', args: 'orders' } as any) + await clock.advance(100) + const queued = $.command.run({ command: 'scan-cost', args: 'c' } as any) + await clock.advance(100) + options.cwd = '/elsewhere/project-b' + const inB = $.command.run({ command: 'scan-cost', args: 'b' } as any) + await clock.advance(100) + expect(started).toEqual(['orders', 'b']) + // The first project's estimate ends: the queued one must not start beside project b's running estimate. + releaseFirst() + await first + await clock.advance(1_000) + expect(started).toEqual(['orders', 'b']) + release() + await inB + expect(((await queued) as any).text).toContain('c would scan 2.0 GiB') + expect(started).toEqual(['orders', 'b', 'c']) + await first +}) + +test('a /cd while the gate waits for an estimate also drops the hold of the project left', async ($, on) => { + const options: Options = { answers: { a: OVER, b: UNDER } } + const { clock } = await start($, on, options) + expect(((await $.tool.call(bash('dbt run -s a'))) as any).deny).toContain('20.0 GiB') + let release!: () => void + options.hold = new Promise(resolve => (release = resolve)) + options.holdFor = 'b' + const call = $.tool.call(bash('dbt run -s b')) + await clock.advance(500) + options.cwd = '/elsewhere/project-b' + release() + expect(((await call) as any).deny).toContain('The project changed while the scan was being estimated') + await clock.settle() + expect(((await $.command.run({ command: 'allow-scan', args: '' } as any)) as any).text).toContain('Nothing is held') +}) + +test('a /cd while the gate waits for an estimate: its answer is of the project left, so the command is refused', async ($, on) => { + const options: Options = { answers: { orders: UNDER } } + const { clock, started } = await start($, on, options) + let release!: () => void + options.hold = new Promise(resolve => (release = resolve)) + options.holdFor = 'orders' + const call = $.tool.call(bash('dbt run -s orders')) + await clock.advance(500) + expect(started).toEqual(['orders']) + options.cwd = '/elsewhere/project-b' + release() + const result: any = await call + expect(result.deny).toContain('The project changed while the scan was being estimated') +}) + +test('a dbt run let through while the gate waits on its estimate voids that answer: the waiting command is not allowed on it', async ($, on) => { + const options: Options = { answers: { orders: UNDER } } + const { clock, ran } = await start($, on, options) + let release!: () => void + options.hold = new Promise(resolve => (release = resolve)) + let finish!: () => void + options.holdBash = { command: 'dbt run -s customers --target prod', until: new Promise(resolve => (finish = resolve)) } + const first = $.tool.call(bash('dbt run -s orders')) + await clock.advance(1_000) + const rebuild = $.tool.call(bash('dbt run -s customers --target prod')) + await clock.advance(10) + release() + await clock.advance(10) + expect(((await first) as any).deny).toContain('The project changed while the scan was being estimated') + finish() + await rebuild + expect(ran).toEqual(['dbt run -s customers --target prod']) +}) + +test('a parallel call whose estimate fails is held by the hold another call recorded meanwhile', async ($, on) => { + const options: Options = { sequence: { orders: [OVER, { success: false, error: 'Access Denied' }] } } + const { clock, ran, started } = await start($, on, options) + let release!: () => void + options.hold = new Promise(resolve => (release = resolve)) + const first = $.tool.call(bash('dbt run -s orders')) + const second = $.tool.call(bash('dbt run -s orders')) + await clock.advance(500) + release() + await clock.advance(100) + const results: any[] = [await first, await second] + await clock.settle() + // Both calls asked: the second got its own failing estimate, not the first one's answer. + expect(started).toEqual(['orders', 'orders']) + expect(results.every(r => typeof r?.deny === 'string')).toBe(true) + expect(ran).toEqual([]) +}) diff --git a/plugins/altimate-code/hooks/features/cost.tsx b/plugins/altimate-code/hooks/features/cost.tsx index a9010a50..f968d4a3 100644 --- a/plugins/altimate-code/hooks/features/cost.tsx +++ b/plugins/altimate-code/hooks/features/cost.tsx @@ -1,20 +1,155 @@ // Cost check: BigQuery scans over the limit are held until allowed. // -// The plugin calls each function below at its moment, in the order features/index.ts fixes; one with -// nothing to do stays a no-op. This feature's own hooks (its commands and panes, each under a matcher -// no other part uses) go in `register`. The events the whole plugin shares (session start and end, -// turns, Bash and file tool calls, the band) come through the other functions, since a plugin hooks -// each of those once. The status-line part is set with `setStatusPart($, 'cost', text)`. +// Before a `dbt run`, `dbt build` or `dbt show` Claude runs, the gate asks altimate-code's +// dbt_cost_estimate (a BigQuery dry run) what the query would scan. Over `maxScanBytes` the command is +// refused with the estimate until the user runs /allow-scan, which lets that same command through once. +// Other warehouses have no dry run: the first answer says so, and nothing is estimated or held after that. +// Only what names its query is estimated: `dbt show --inline ` and a single plain model name +// (`-s orders`); any other selection, and any command that sets its own target, defer state, vars or +// project, runs as it is, with one line saying so under the Bash row. +import { atom, read, update } from 'claude-code' import type { Args, EngineInterface, EventResult, On, RenderElement, ToolCallResult } from 'claude-code' +import { runTool, within } from '../altimate-cli' +import { parseDbtCommand, type DbtInvocation } from '../dbt-command' +import { setStatusPart } from '../status-line' import type { BashCall, ContextLines, EditedFile, FileCall } from '../tool-chain' +import type { CostState } from '../../types' import type { BandUi, FeatureContext } from './index' +/** Used when `maxScanBytes` is missing from the plugin's options: 10 GiB, as plugin.json's default. */ +const DEFAULT_MAX_SCAN_BYTES = 10 * 1024 ** 3 +/** The adapter may have to start (dbt deps, a parse) for the first estimate. */ +const ESTIMATE_BUDGET_MS = 120_000 +/** + * How long the gate waits for estimates, in all: inside the tool chain's 5 s phase deadline, past which the call + * runs unchecked. An estimate that is not done by then goes on, and the command is refused until it is. + */ +const GATE_WAIT_MS = 4_000 +/** The dbt subcommands that run SQL a person pays for. */ +const SCANNING = new Set(['run', 'build', 'show']) +/** A selector that is one plain model name: no graph operator, tag, path, package or wildcard. */ +const PLAIN_MODEL = /^[A-Za-z_][A-Za-z0-9_]*$/ +/** What the CLI answers for a tool it doesn't have (altimate-code's tool-cli runner). */ +const UNKNOWN_TOOL = /Unknown tool/i +/** The altimate-code that has dbt_cost_estimate; changing it is the one edit a release needs. */ +const NEEDED_VERSION = 'a newer altimate-code (the release with dbt_cost_estimate)' + +const costAtom = atom({ plugin: 'altimate-code', key: 'cost' } as const, null as CostState | null) + +const FRESH: CostState = { unsupportedAdapter: null, lastEstimate: null, held: null, allowOnce: false } + +/** The limit in force; set by `register` from the plugin's options. 0 never holds. */ +let limitBytes = DEFAULT_MAX_SCAN_BYTES +/** The lines to show under an estimated command's Bash row, by command; each gate pass replaces the last. Taken by bashAfter. */ +const notes = new Map() +/** Estimates still running past the gate's wait, by what they ask about; the next gate pass for the same target takes the answer. */ +const inflight = new Map; settled: Promise; doneAt: number | null; generation: number; dir: string | null }>() +/** Every estimate whose process is still running (gate or /scan-cost), whether or not an edit has voided its answer: one runs per project at a time. */ +const active = new Map, string | null>() +/** Commands the gate refused while their estimate was running, and when it last did: they have not run, so a retry changes nothing. */ +const refused = new Map() +/** Moves on whenever the project may have changed: an estimate started before it is never taken as an answer. */ +let generation = 0 +/** When the session found a warehouse with no dry run; null once forgotten. That finding is kept only so long, since the target can change. */ +let unsupportedSince: number | null = null +/** The directory the session was in at the last gate pass: estimates, a hold and an allowance belong to that project. */ +let seenDir: string | null = null +const UNSUPPORTED_KEEP_MS = 10 * 60_000 +/** Commands that can rewrite a model, the project or its profile from the shell: an estimate made before one is not reused. */ +const MAY_CHANGE_PROJECT = /(^|[\s;&|(])(git\s+((-[Cc]|--git-dir|--namespace|--work-tree)\s+(\S+|"[^"]*"|'[^']*')\s+|-\S+\s+)*(checkout|restore|reset|stash|apply|clean|switch|merge|rebase|pull|cherry-pick|revert|am|worktree)|sed|awk|perl|python3?|node|bun|patch|mv|cp|rm|tee|touch|make|sh|bash|dd|ed|ex)(\s|$)|>/ +/** Commands that can move the session to another project or target without rewriting a file. */ +const MOVES_DIRECTORY = /(^|[\s;&|(])(cd|pushd|popd)(\s|$)/ +/** The dbt subcommands that rebuild warehouse relations. */ +const REBUILDS = new Set(['run', 'build']) +/** The dbt subcommands that change nothing in the project. */ +const READS_ONLY = new Set(['run', 'build', 'show', 'compile', 'ls', 'list', 'debug', 'parse']) + +/** The command with the dbt calls that only read, and the redirects that write nothing, set aside. */ +function withoutReads(command: string): { all: DbtInvocation[]; rest: string } { + const all = parseDbtCommand(command) + let rest = command + for (const i of [...all].sort((a, b) => b.start - a.start)) rest = rest.slice(0, i.start) + ' ' + rest.slice(i.end) + // A redirect to another stream or to /dev/null writes nothing in the project. + return { all, rest: rest.replace(/\d*>&\d+|\d*>>?\s*\/dev\/null/g, ' ') } +} + +/** Whether the command can rewrite a model or the profile: any part of the line, with the dbt calls that only read set aside. */ +function mutates(all: DbtInvocation[], rest: string): boolean { + if (all.some(i => i.subcommand !== null && !READS_ONLY.has(i.subcommand))) return true + return MAY_CHANGE_PROJECT.test(rest) +} + +/** What a shell command did to the project: a rewrite voids the estimates and the verdict; a directory change the same, since it can select another project. */ +async function afterShellChange($: EngineInterface, command: string, ran = false) { + const { all, rest } = withoutReads(command) + // A dbt run that has finished may have rebuilt what another model reads: an estimate made before it is stale. + // Not before the command runs: the gate's own estimates, kept for the retry, would be dropped by every pass. + if (ran && all.some(i => i.subcommand !== null && REBUILDS.has(i.subcommand))) invalidate() + // A rewrite can bring back another target (a git restore does not name the profile), and a cd can select another project, + // which the estimates (keyed by model or SQL) do not name: nothing made before either is reused. + if (mutates(all, rest) || MOVES_DIRECTORY.test(rest)) { + invalidate() + await forgetUnsupported($) + } +} + +/** An answer nobody took within this long is dropped: the model may have changed since. */ +const ANSWER_KEEP_MS = 60_000 + /** Adds this feature's own hooks. */ -export function register(on: On, context: FeatureContext) {} +export function register(on: On, context: FeatureContext) { + const configured = Number((context.options as Record | undefined)?.maxScanBytes) + limitBytes = Number.isFinite(configured) && configured >= 0 ? configured : DEFAULT_MAX_SCAN_BYTES + + on('command.run', { command: 'allow-scan' }, async $ => { + // The allowance is set against the hold it was read from, in the same step: a hold replaced meanwhile is not allowed. + const held = (await read($, costAtom))?.held + let allowed = false + await update($, costAtom, prev => { + const current = prev ?? FRESH + allowed = !!held && current.held?.command === held.command + return allowed ? { ...current, allowOnce: true } : current + }) + if (!held) return { text: 'Nothing is held: no dbt command is waiting on its scan size.' } + if (!allowed) return { text: 'The held command changed meanwhile: run /allow-scan again to allow the one now held.' } + return { text: `Allowed: ${held.target} (≈ ${held.display}) runs once when Claude runs the same command again; after that it is held again.` } + }) + + on('command.run', { command: 'scan-cost' }, async ($, e) => { + const target = e.args.trim() + if (!target) { + const last = (await read($, costAtom))?.lastEstimate + return { text: last ? `Last estimate: ${last.target} would scan ${last.display}. Run /scan-cost for another.` : 'Run /scan-cost to estimate the scan of a model.' } + } + const began = await $.clock.now() + // The wait is short (the user is looking at the screen); the estimate's own budget starts when the wait ends, at the latest. + const started = await startWhenIdle($, { model: target }, began + GATE_WAIT_MS, () => undefined, one => one) + if (!started) return { text: 'No estimate: another estimate is still running in this project. Try again in a moment.' } + const answer = await started.answer + if (answer.kind === 'bytes') { + await forgetUnsupported($) + await rememberEstimate($, target, answer.bytes, answer.display) + return { text: `${target} would scan ${answer.display}${verdict(answer.maxBytes)}` } + } + return { text: answer.kind === 'unsupported' ? `No estimate: ${answer.adapter} has no dry run (only BigQuery has).` : `No estimate: ${answer.reason}` } + }) +} /** After the session started: the place to register commands. */ -export async function sessionStart($: EngineInterface, e: Args<'session.start'>) {} +export async function sessionStart($: EngineInterface, e: Args<'session.start'>) { + notes.clear() + inflight.clear() + refused.clear() + generation++ + unsupportedSince = null + seenDir = null + active.clear() + await update($, costAtom, () => ({ ...FRESH })) + await setStatusPart($, 'cost', null) + await $.command.register({ name: 'allow-scan', description: 'Let the dbt command held for its BigQuery scan size run once, when run again unchanged' }) + await $.command.register({ name: 'scan-cost', description: 'Estimate the BigQuery scan of a dbt model before running it', argumentHint: '' }) +} /** Also called for a subagent's turns, which carry `e.agentId`. */ export async function turnStart($: EngineInterface, e: Args<'turn.start'>) {} @@ -24,9 +159,11 @@ export async function turnComplete($: EngineInterface, e: Args<'turn.complete'>, export async function sessionEnd($: EngineInterface, e: Args<'session.end'>) {} -/** This feature's line in the band, under the lineage line; null when there is nothing to say. */ +/** This feature's line in the band: the command held, until it is allowed or a later dbt command is estimated under the limit. */ export async function bandLine($: EngineInterface, ui: BandUi): Promise { - return null + const held = (await read($, costAtom))?.held + if (!held) return null + return {`⛔ held: ${held.target} would scan ${held.display} (limit ${formatBytes(limitBytes)}) · /allow-scan`} } /** The command to run in place of `e.command`; null keeps it. */ @@ -39,15 +176,536 @@ export async function bashRewrite($: EngineInterface, e: BashCall): Promise { + const mine: { note?: object } = {} + const verdict = await gate($, e, mine).finally(() => { + // Only this pass's own note: a sibling pass for the same command may still be between noting and recording its answer. + if (overNow.get(e.command) === mine.note) overNow.delete(e.command) + }) + // A dbt run or build that is let through rebuilds relations while it runs: an estimate kept from before it is stale from here. + if (verdict === null && limitBytes > 0 && parseDbtCommand(e.command).some(i => i.subcommand !== null && REBUILDS.has(i.subcommand))) dropFinished() + return verdict +} + +/** Everything the gate learned in the project just left: its answers, hold, allowance, estimate and band line. */ +async function leaveProject($: EngineInterface) { + invalidate() + await forgetUnsupported($) + await update($, costAtom, prev => (prev ? { ...prev, held: null, allowOnce: false, lastEstimate: null } : prev)) + await showStatus($, null) +} + +/** Per command: the over-limit answer a pass has in hand but has not yet recorded as the hold. */ +const overNow = new Map() + +async function gate($: EngineInterface, e: BashCall, mine: { note?: object }): Promise<{ deny: string } | null> { + if (limitBytes <= 0) return null + // Before it runs, not only after: another call can ask while a long command is rewriting the project. + // A command refused for its estimate has not run: its retry is not a change, and keeps the answers it waited for. + if (!refused.has(e.command)) await afterShellChange($, e.command) + // A session moved to another project (a /cd) keeps nothing the gate learned in the last one. + const dir = await $.session.cwd().catch(() => seenDir) + if (seenDir !== null && dir !== seenDir) { + // What is still running belongs to the project left behind: it holds nothing back here, and still holds that project back on return. + await leaveProject($) + } + seenDir = dir ?? null + const all = parseDbtCommand(e.command) + const invocations = all.filter(i => i.subcommand !== null && SCANNING.has(i.subcommand)) + if (invocations.length === 0) return null + // An edit or a command that changed the project while the answers were coming in makes them void. + const startedAt = generation + const state = (await read($, costAtom)) ?? FRESH + // A warehouse with no dry run was found out already: the command is never held, and nothing is asked again, + // until an edit or a command that can move the target, or time, says it may have changed. + if (state.unsupportedAdapter) { + if (unsupportedSince != null && (await $.clock.now()) - unsupportedSince < UNSUPPORTED_KEEP_MS) return null + await forgetUnsupported($) + state.unsupportedAdapter = null + } + + const lines: string[] = [] + let estimated = false + const over: { label: string; bytes: number; point: number; display: string }[] = [] + // One wait for the whole command, however many queries it names. + const waitUntil = (await $.clock.now()) + GATE_WAIT_MS + const pending: string[] = [] + // What this pass already has, so the same query named twice on the line is asked for once. + const taken = new Map() + let failed = false + for (const invocation of invocations) { + const target = targetOf(e.command, invocation, all) + if (!target.model && target.sql === undefined) { + lines.push(`cost · no estimate for this selection (${target.why})`) + continue + } + const label = target.model ?? 'the inline query' + const answer = await estimateWithin($, target.model ? { model: target.model } : { sql: target.sql }, waitUntil, taken, refused.get(e.command)) + // A /cd while it was being asked: the answer is of the project left, and nothing here is allowed on it. + const now = (await $.session.cwd().catch(() => dir)) ?? null + if (now !== (dir ?? null)) { + await leaveProject($) + seenDir = now + } + // The project changed while this was being asked: nothing it says is published, and the command is refused below. + if (generation !== startedAt) break + if (answer === 'pending') { + if (!pending.includes(label)) pending.push(label) + continue + } + if (answer.kind === 'unsupported') { + // Nothing can be held on this warehouse, so a hold from before the verdict goes with it. + unsupportedSince = await $.clock.now() + if (generation !== startedAt) break + await update($, costAtom, prev => ({ ...(prev ?? FRESH), unsupportedAdapter: answer.adapter, held: null, allowOnce: false })) + await showStatus($, null) + lines.push(`cost · no estimate on ${answer.adapter}: only BigQuery has a dry run`) + over.length = 0 + pending.length = 0 + break + } + if (answer.kind === 'none') { + failed = true + lines.push(`cost · no estimate for ${label}: ${answer.reason}`) + continue + } + estimated = true + // Noted before any await: a parallel pass for this command whose own estimate fails must still see it. + if (answer.maxBytes > limitBytes) { + mine.note = { label, bytes: answer.maxBytes, display: answer.display } + overNow.set(e.command, mine.note as { label: string; bytes: number; display: string }) + } + await rememberEstimate($, label, answer.bytes, answer.display) + if (answer.maxBytes <= limitBytes) lines.push(`cost · ${label} would scan ${answer.display}`) + else over.push({ label, bytes: answer.maxBytes, point: answer.bytes, display: answer.display }) + } + + if (generation !== startedAt) { + // Nothing is cleared here: the change already cleared what was older, and a newer pass's estimates are its own. + return changedDeny(e.command) + } + + // A finished command's answers are taken: a later one asks again. A pending one keeps them for its retry. + if (pending.length === 0) for (const key of taken.keys()) inflight.delete(key) + + // An estimate still running decides nothing yet, and spends no allowance: the command is refused until it is in. + if (pending.length > 0) { + // What is already known to be over the limit stays held, so a failed estimate on the retry cannot free it. + if (over.length > 0) { + const worst = over.reduce((a, b) => (b.bytes > a.bytes ? b : a)) + await update($, costAtom, prev => { + const current = prev ?? FRESH + // An allowance belongs to the command it was given for: another command's hold does not inherit it. + const same = current.held?.command === e.command + return { ...current, allowOnce: same && current.allowOnce, held: { command: e.command, target: worst.label, bytes: worst.bytes, display: worst.display } } + }) + } + notes.delete(e.command) + refused.set(e.command, await $.clock.now()) + return { deny: `The scan of ${pending.join(', ')} is still being estimated (the dbt adapter may be starting). Run the same command again in a moment.` } + } + + // A command held earlier stays held while its estimate cannot be made again: only /allow-scan or an estimate under the limit frees it. + // The hold is read now, not from the state taken before the estimates: a parallel call may have recorded it meanwhile. + // Or from an over-limit answer a parallel pass for the same command has in hand and has not recorded yet. + const stored = over.length === 0 && failed ? (await read($, costAtom))?.held : null + const heldNow = stored && stored.command === e.command ? { label: stored.target, bytes: stored.bytes, display: stored.display } : overNow.get(e.command) + if (over.length === 0 && failed && heldNow && heldNow.bytes > limitBytes) { + over.push({ label: heldNow.label, bytes: heldNow.bytes, point: heldNow.bytes, display: heldNow.display }) + } + + if (over.length > 0) { + const worst = over.reduce((a, b) => (b.bytes > a.bytes ? b : a)) + // The allowance is spent in one step, and only by the command that was held for it. + let granted = false + try { + await update($, costAtom, prev => { + const current = prev ?? FRESH + granted = current.allowOnce && current.held?.command === e.command + return granted + ? { ...current, allowOnce: false, held: null } + : { ...current, allowOnce: false, held: { command: e.command, target: worst.label, bytes: worst.bytes, display: worst.display } } + }) + } catch { + // A hold that cannot be recorded must not become a pass: a gate that throws lets the call run. + notes.delete(e.command) + return { deny: `${worst.display} would be scanned, over the ${formatBytes(limitBytes)} limit, and the hold could not be recorded. Run the same command again.` } + } + if (granted) { + for (const one of over) lines.push(`cost · ${one.label} would scan ${one.display}, over the ${formatBytes(limitBytes)} limit: allowed once`) + keepNotes(e.command, lines) + // The hold is spent: the status line stops showing it. + await showLastEstimate($) + if (generation !== startedAt) return changedDeny(e.command) + return null + } + notes.delete(e.command) + await showStatus($, `scan held ≈ ${worst.display}`) + // The hold is decided on the upper bound, and the display is the rounded point estimate: either can read + // as not over the limit when the scan is just past it. + const scanned = + worst.point <= limitBytes && formatBytes(worst.bytes) !== formatBytes(limitBytes) + ? `up to ${formatBytes(worst.bytes)} (≈ ${worst.display} estimated)` + : worst.display === formatBytes(limitBytes) || worst.point <= limitBytes + ? `just over ${formatBytes(limitBytes)}` + : `≈ ${worst.display}` + return { + deny: `${scanned} would be scanned, over the ${formatBytes(limitBytes)} limit. Narrow the query, or ask the user to run /allow-scan and then run the same command again.`, + } + } + // A command that was estimated and is under the limit ends an earlier hold, and its allowance. One that + // was not estimated says nothing, so the hold stays. + if (estimated && (state.held || state.allowOnce)) { + await update($, costAtom, prev => (prev?.held || prev?.allowOnce ? { ...prev, held: null, allowOnce: false } : prev)) + await showLastEstimate($) + } + // An edit during the awaits above makes the estimate void: nothing is allowed on it. + if (generation !== startedAt) return changedDeny(e.command) + keepNotes(e.command, lines) return null } -export async function bashAfter($: EngineInterface, e: BashCall, result: ToolCallResult): Promise {} +function changedDeny(command: string) { + notes.delete(command) + return { deny: 'The project changed while the scan was being estimated. Run the same command again.' } +} + +/** Notes kept per command: a line of many dbt calls would otherwise put one note each into Claude's context. */ +const MAX_NOTES = 5 + +function keepNotes(command: string, lines: string[]) { + if (lines.length > MAX_NOTES) notes.set(command, [...lines.slice(0, MAX_NOTES), `cost · ${lines.length - MAX_NOTES} more notes not shown`]) + else if (lines.length > 0) notes.set(command, lines) + else notes.delete(command) +} + +/** What the gate said about the command, under its Bash row. */ +export async function bashAfter($: EngineInterface, e: BashCall, result: ToolCallResult): Promise { + const lines = notes.get(e.command) + notes.delete(e.command) + refused.delete(e.command) + // A shell command can rewrite a model or the profile (a git restore, a sed): no earlier estimate is reused after one. + await afterShellChange($, e.command, true) + return lines +} /** `file.before` holds the text from before the edit only when `editReadsBefore` is true. */ -export async function editAfter($: EngineInterface, e: FileCall, result: ToolCallResult, file: EditedFile): Promise {} +export async function editAfter($: EngineInterface, e: FileCall, result: ToolCallResult, file: EditedFile): Promise { + // An edit can change what a model scans, or the target it runs on: no estimate made before it is reused. + invalidate() + await forgetUnsupported($) +} + +/** Runs the estimate and counts it as running until it settles. */ +function startEstimate($: EngineInterface, args: { model?: string; sql?: string }, until: number, dir: string | null) { + const answer = estimate($, args, until) + const settled = answer.then(() => undefined, () => undefined) + active.set(settled, dir) + void settled.then(() => active.delete(settled)) + return { answer, settled, dir } +} + +/** + * Starts the estimate once no other is running, in the same step as that check: two waiters woken together never both + * start. `reuse` is asked at that step too, for an estimate of the same thing another pass started meanwhile. Null when + * `waitUntil` comes first. + */ +async function startWhenIdle( + $: EngineInterface, + args: { model?: string; sql?: string }, + waitUntil: number, + reuse: () => T | undefined, + begin: (started: { answer: Promise; settled: Promise; dir: string | null }) => T, +): Promise { + // The project is read again after each wait: a /cd meanwhile moves the check to the project now in use. + let dir: string | null = null + for (;;) { + dir = (await $.session.cwd().catch(() => seenDir)) ?? null + const running = [...active].filter(([, d]) => d === dir).map(([p]) => p) + if (running.length === 0) break + const left = waitUntil - (await $.clock.now()) + if (left <= 0 || !(await within($, left, Promise.all(running).then(() => true)))) return null + } + return reuse() ?? begin(startEstimate($, args, waitUntil + ESTIMATE_BUDGET_MS, dir)) +} + +/** + * A run or build about to start voids every answer: the finished ones are dropped, and the generation moves, so a pass that + * already took an answer (or waits on a running estimate) is refused rather than served, and asks again. An estimate still + * running is left to finish: its process counts toward the one-per-project queue. + */ +function dropFinished() { + generation++ + for (const [key, entry] of inflight) if (entry.doneAt != null) inflight.delete(key) +} + +function invalidate() { + generation++ + inflight.clear() +} + +/** Forgets a warehouse found to have no dry run, so the next dbt command is asked about again. */ +async function forgetUnsupported($: EngineInterface) { + unsupportedSince = null + await update($, costAtom, prev => (prev?.unsupportedAdapter ? { ...prev, unsupportedAdapter: null } : prev)) +} export async function readOrEditAfter($: EngineInterface, e: FileCall, result: ToolCallResult, file: { path: string }): Promise {} /** Whether `editAfter` needs the file's text from before the edit (one extra read per edit). */ export const editReadsBefore = false + +type Target = { model?: string; sql?: string; why: string } + +/** More dbt calls than this on one line are not estimated. */ +const MAX_INVOCATIONS = 20 +/** The dbt launchers whose project is the session's: nothing else on the line can move it. */ +const PLAIN_LAUNCHER = /^((uv|poetry) run )?(\S*\/)?dbt$/ +/** Flags with one value that leave the project, target and compiled SQL as they are. */ +const VALUE_FLAGS = new Set(['--inline', '--limit', '--threads']) +/** Flags that name the models; any number of values follow. */ +const SELECT_FLAGS = new Set(['-s', '--select', '-m', '--models', '--model']) +/** Flags without a value that leave the project, target and compiled SQL as they are. */ +const PLAIN_FLAGS = new Set(['--fail-fast', '-x', '--quiet', '-q']) + +/** Whether the shell would expand a `$` or a backtick on the line: inside single quotes (a BigQuery table name, say) they are literal. */ +function expands(command: string): boolean { + let quote: string | null = null + for (let i = 0; i < command.length; i++) { + const c = command[i]! + if (quote === "'") { + if (c === "'") quote = null + } else if (c === '\\') i++ + else if (quote === '"') { + if (c === '"') quote = null + else if (c === '$' || c === '`') return true + } else if (c === "'" || c === '"') quote = c + else if (c === '$' || c === '`') return true + } + return false +} + +/** + * Whether the estimate would be of what the invocation runs. It asks about the session's project at its + * default target, so it is only so when the line moves neither: the dbt launcher is plain, nothing + * precedes the invocation but other dbt invocations, and every flag is one known to leave the query + * alone, with no expansion on the line. Anything else (another target, defer state, vars, project, a + * full refresh) is not estimated. Every estimate is of the project as it is when the command starts: what an + * earlier step on the same line rebuilds is not known to it, which is the limit of any preflight. + */ +function isPlainInvocation(command: string, invocation: DbtInvocation, all: DbtInvocation[]): boolean { + // The shell expands `$` and backticks anywhere on the line (arguments, redirections, an earlier call) into + // flags or environment this check can't see; a line with many calls is not worth reading call by call. + if (all.length > MAX_INVOCATIONS || expands(command)) return false + // A glob or redirect in the arguments leaves the parsed selection incomplete. + if (invocation.dynamic || !PLAIN_LAUNCHER.test(invocation.binary)) return false + let before = command.slice(0, invocation.start) + for (const earlier of all.filter(one => one.end <= invocation.start).reverse()) { + before = before.slice(0, earlier.start) + ' ' + before.slice(earlier.end) + } + if (!/^[\s;&]*$/.test(before)) return false + const words = shellWords(command.slice(invocation.start, invocation.end)) + const launcher = shellWords(invocation.binary) + if (launcher.some((word, i) => words[i] !== word) || words[launcher.length] !== invocation.subcommand) return false + let selecting = false + let pending = false + for (const word of words.slice(launcher.length + 1)) { + if (pending) { + pending = false + continue + } + const flag = word.startsWith('-') ? word.split('=')[0]! : null + if (flag === null) { + if (selecting) continue + return false + } + selecting = SELECT_FLAGS.has(flag) + if (selecting || PLAIN_FLAGS.has(flag)) continue + if (VALUE_FLAGS.has(flag)) pending = !word.includes('=') + else return false + } + return true +} + +/** What a scanning invocation names to estimate: its inline SQL, or one plain model; else why not. */ +function targetOf(command: string, invocation: DbtInvocation, all: DbtInvocation[]): Target { + const text = command.slice(invocation.start, invocation.end) + if (!isPlainInvocation(command, invocation, all)) { + return { why: 'it sets its own target, defer state, vars, project or refresh mode, or runs after something that might' } + } + if (invocation.subcommand === 'show') { + const inline = inlineSql(text) + if (inline !== null) return inline.trim() ? { sql: inline, why: '' } : { why: 'empty inline query' } + } + const [only, ...rest] = invocation.select + if (only === undefined) return { why: 'no model named; selection resolves with dbt ls' } + if (rest.length > 0) return { why: 'several selectors' } + if (!PLAIN_MODEL.test(only)) return { why: `${only} is a graph, tag, path or package selector` } + return { model: only, why: '' } +} + +/** The SQL of `--inline` in a dbt command's text, unquoted as the shell would; null when there is none. */ +export function inlineSql(text: string): string | null { + const words = shellWords(text) + // dbt keeps the last of a repeated option. + let found: string | null = null + for (let i = 0; i < words.length; i++) { + const word = words[i]! + if (word === '--inline') found = words[++i] ?? null + else if (word.startsWith('--inline=')) found = word.slice('--inline='.length) + } + return found +} + +/** The words of a command's text with their quotes and backslashes resolved. */ +function shellWords(text: string): string[] { + const words: string[] = [] + let word = '' + let inWord = false + let quote: '"' | "'" | null = null + for (let i = 0; i < text.length; i++) { + const c = text[i]! + // A backslash before a line break joins the lines, in a word or in double quotes. + if (c === '\\' && quote !== "'" && (text[i + 1] === '\n' || (text[i + 1] === '\r' && text[i + 2] === '\n'))) { + i += text[i + 1] === '\n' ? 1 : 2 + continue + } + if (quote) { + if (c === quote) quote = null + else if (c === '\\' && quote === '"' && i + 1 < text.length && '"\\$`'.includes(text[i + 1]!)) word += text[++i] + else word += c + } else if (c === '"' || c === "'") { + quote = c + inWord = true + } else if (c === '\\' && i + 1 < text.length) { + word += text[++i] + inWord = true + } else if (/\s/.test(c)) { + if (inWord) words.push(word) + word = '' + inWord = false + } else { + word += c + inWord = true + } + } + if (inWord) words.push(word) + return words +} + +type Answer = { kind: 'bytes'; bytes: number; maxBytes: number; display: string } | { kind: 'unsupported'; adapter: string } | { kind: 'none'; reason: string } + +/** Asks altimate-code for the scan of a model or SQL; never throws: a failure is `none` with why. */ +async function estimate($: EngineInterface, args: { model?: string; sql?: string }, until: number): Promise { + try { + const run = await runTool($, 'dbt_cost_estimate', args, { write: true, until }) + if (!run.ok && UNKNOWN_TOOL.test(run.text)) return { kind: 'none', reason: `this altimate-code has no cost estimate: it needs ${NEEDED_VERSION}` } + const data = run.metadata as { supported?: unknown; adapter?: unknown; bytes?: unknown; maxBytes?: unknown; lowerBound?: unknown; display?: unknown; error?: unknown } | null + if (data?.supported === false) return { kind: 'unsupported', adapter: typeof data.adapter === 'string' ? data.adapter : 'this warehouse' } + // A dry run that can undercount (a zero, a script) gives no upper bound to hold a command on. + if (data?.supported === true && data.lowerBound === true) { + // A floor already over the limit is enough to hold; a floor under it says nothing about the cap. + if (typeof data.bytes === 'number' && Number.isFinite(data.bytes) && limitBytes > 0 && data.bytes > limitBytes) { + return { kind: 'bytes', bytes: data.bytes, maxBytes: data.bytes, display: typeof data.display === 'string' ? data.display : formatBytes(data.bytes) } + } + return { kind: 'none', reason: 'the dry run gave only a lower bound, not a cap' } + } + if (data?.supported === true && typeof data.bytes === 'number' && Number.isFinite(data.bytes)) { + // The tool's upper bound when it gives one: the display is rounded, and the limit is a hard one. + const maxBytes = typeof data.maxBytes === 'number' && Number.isFinite(data.maxBytes) && data.maxBytes >= data.bytes ? data.maxBytes : data.bytes + return { kind: 'bytes', bytes: data.bytes, maxBytes, display: typeof data.display === 'string' ? data.display : formatBytes(data.bytes) } + } + const why = typeof data?.error === 'string' ? data.error : run.text.trim().split('\n')[0] || 'altimate-code gave no estimate' + return { kind: 'none', reason: why } + } catch (error) { + return { kind: 'none', reason: error instanceof Error ? error.message : String(error) } + } +} + +/** The estimate, started on the long budget if none is running for these arguments; `pending` when it is not done by `waitUntil`. */ +async function estimateWithin($: EngineInterface, args: { model?: string; sql?: string }, waitUntil: number, taken: Map, refusedAt: number | undefined): Promise { + const key = JSON.stringify(args) + const had = taken.get(key) + if (had) return had + // The look-up and the insert below are one step: nothing is awaited between them, or two passes would each start one. + const now = await $.clock.now() + const here = (await $.session.cwd().catch(() => seenDir)) ?? null + let running = inflight.get(key) + // An answer made in another project is of that project's models, not this one's. + if (running && (running.generation !== generation || running.dir !== here)) running = undefined + // A command whose retries keep coming keeps what it waited for however long the others took: a later answer expiring an earlier one would never let it finish. + const retrying = refusedAt != null && now - refusedAt <= ANSWER_KEEP_MS + if (running?.doneAt != null && !retrying && now - running.doneAt > ANSWER_KEEP_MS) running = undefined + if (!running) { + // The wait is spent: nothing new starts. Each estimate may start the adapter (dbt deps), and two at once in one project collide. + if (waitUntil - now <= 0) return 'pending' + // Nor beside any running one, whichever command asks (a /scan-cost too), even when an edit voided its answer: + // this one waits its turn, within the same bound. + const entry = await startWhenIdle( + $, + args, + waitUntil, + () => { + const other = inflight.get(key) + return other && other.doneAt == null && other.generation === generation ? other : undefined + }, + started => { + const made = { answer: started.answer, settled: started.settled, doneAt: null as number | null, generation, dir: started.dir } + void made.answer.then(async () => void (made.doneAt = await $.clock.now())).catch(() => {}) + inflight.set(key, made) + return made + }, + ) + if (!entry) return 'pending' + running = entry + } + const left = waitUntil - (await $.clock.now()) + const done = left > 0 ? await within($, left, running.answer.then(answer => ({ answer }))) : null + if (!done) return 'pending' + // The project may have changed while it ran (an edit, a /cd, or a dbt run let through meanwhile): that answer is of the old one, and the next pass asks again. + if (running.generation !== generation || running.dir !== ((await $.session.cwd().catch(() => seenDir)) ?? null)) { + if (inflight.get(key) === running) inflight.delete(key) + return 'pending' + } + // Not dropped here: while another query of the command is pending, the retry still needs this answer. + taken.set(key, done.answer) + return done.answer +} + +async function rememberEstimate($: EngineInterface, target: string, bytes: number, display: string) { + // What is shown never decides the gate: a failure here must not let a scan the gate knows is over the limit through. + try { + await update($, costAtom, prev => ({ ...(prev ?? FRESH), lastEstimate: { target, bytes, display } })) + await setStatusPart($, 'cost', `scan ≈ ${display}`) + } catch {} +} + +/** The status line's cost part back to the last estimate, once no hold is left. */ +async function showLastEstimate($: EngineInterface) { + const last = (await read($, costAtom))?.lastEstimate + await showStatus($, last ? `scan ≈ ${last.display}` : null) +} + +/** The status line's cost part; a failure to draw it is dropped. */ +async function showStatus($: EngineInterface, text: string | null) { + try { + await setStatusPart($, 'cost', text) + } catch {} +} + +/** What an estimate means against the limit, as a sentence tail for /scan-cost. */ +function verdict(maxBytes: number): string { + if (limitBytes <= 0) return '.' + return maxBytes > limitBytes ? `, over the ${formatBytes(limitBytes)} limit: a dbt run of it is held until /allow-scan.` : `, within the ${formatBytes(limitBytes)} limit.` +} + +/** Bytes in the units BigQuery's dry run writes them in: "4.2 TiB". */ +export function formatBytes(bytes: number): string { + const units = ['Bytes', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB'] + let n = bytes + let i = 0 + while (Math.abs(n) >= 1024 && i < units.length - 1) { + n /= 1024 + i++ + } + return `${n.toFixed(1)} ${units[i]}` +} diff --git a/plugins/altimate-code/types/index.d.ts b/plugins/altimate-code/types/index.d.ts index 394ebd35..8a5f6733 100644 --- a/plugins/altimate-code/types/index.d.ts +++ b/plugins/altimate-code/types/index.d.ts @@ -282,7 +282,16 @@ export interface CommentsState {} // Cost check: the last scan estimate and any scan allowed once. // Filled in by that feature alone (hooks/features/); the other features never read or write it. // --------------------------------------------------------------------------------------------------- -export interface CostState {} +export interface CostState { + /** The warehouse that has no dry run (any but BigQuery), once the tool said so: nothing is estimated after that. */ + unsupportedAdapter: string | null + /** The newest estimate: what was estimated, in bytes, as the dry run wrote it. */ + lastEstimate: { target: string; bytes: number; display: string } | null + /** The command held for being over the limit, until it is allowed or a later dbt command is estimated under the limit; `bytes` is its scan's upper bound. */ + held: { command: string; target: string; bytes: number; display: string } | null + /** /allow-scan was given for the held command: that same command runs once. */ + allowOnce: boolean +} // --------------------------------------------------------------------------------------------------- // Project health: the health check issues on the changed models.