diff --git a/.opencode/skills/dbt-develop/SKILL.md b/.opencode/skills/dbt-develop/SKILL.md index 076c280b30..19bcd9ebae 100644 --- a/.opencode/skills/dbt-develop/SKILL.md +++ b/.opencode/skills/dbt-develop/SKILL.md @@ -170,7 +170,9 @@ altimate-dbt info # confirm every req **Diff column shape against the spec — use the `dbt-schema-verify` skill.** For each model the task touched, run `altimate-dbt schema-verify --model -` and treat any `mismatch` verdict as "not done." Full procedure, +` and treat a `mismatch` verdict (an enforced contract the table does +not match, or a declared column with tests that the table lacks) as "not +done." Columns the YAML does not list are not a mismatch. Full procedure, output interpretation, and fallback (when `altimate-dbt` is missing) live in the dedicated **dbt-schema-verify** skill, which auto-loads alongside this one. @@ -206,7 +208,7 @@ Use `altimate-dbt children` and `altimate-dbt parents` to verify the DAG is inta 5. **Fix ALL errors, not just yours.** After creating/modifying models, run a full `dbt build`. If ANY model fails — even pre-existing ones you didn't touch — fix them. Your job is to leave the project in a fully working state. 6. **Verify transformation correctness, not just mechanics.** For non-trivial models, generate and run dbt unit tests as part of the validate step (use the `dbt-unit-tests` skill). Passing `dbt build` only proves the SQL is syntactically valid — it doesn't prove the *values* are right. 7. **Enumerate deliverables, then check them off.** The task is not done until every model, column, test, and config change explicitly requested exists on disk and in the manifest. Re-read the prompt at the end and verify each requested item — don't trust your own intermediate "done" feeling. -8. **Match the column spec exactly — and verify it mechanically, not by inspection.** Use the dedicated **dbt-schema-verify** skill. Before declaring any model task done, run `altimate-dbt schema-verify --model ` and treat any `mismatch` verdict as "not done." Adding "helpful" extras (rank breakdowns, name-resolved fields, lineage metadata), reordering columns "more logically", or substituting synonyms (`supplier_id` for `supplier_company`, `transaction_type_name` for `transaction_type`) all break equality tests. The contract is what the spec says, not what you think would be useful. +8. **Produce the columns the task asks for — and check declared columns mechanically, not by inspection.** Use the dedicated **dbt-schema-verify** skill. Before declaring any model task done, run `altimate-dbt schema-verify --model ` and treat a `mismatch` verdict as "not done." Adding "helpful" extras (rank breakdowns, name-resolved fields, lineage metadata) or substituting synonyms (`supplier_id` for `supplier_company`) can break the task's own expectations: follow the columns the task and the existing consumers of the model call for. YAML that lists only some columns does not mean the others should be removed, and a YAML column nothing tests or enforces may be stale rather than required. ## Common Pitfalls in Transformation Logic diff --git a/.opencode/skills/dbt-schema-verify/SKILL.md b/.opencode/skills/dbt-schema-verify/SKILL.md index 14e62ae116..89ffb7dc1c 100644 --- a/.opencode/skills/dbt-schema-verify/SKILL.md +++ b/.opencode/skills/dbt-schema-verify/SKILL.md @@ -4,21 +4,17 @@ applyPaths: - "dbt_project.yml" - "**/dbt_project.yml" description: | - REQUIRED after building or modifying ANY dbt model that has columns declared - in `schema.yml` / `_models.yml`. Run `altimate-dbt schema-verify --model - ` to diff actual columns against the spec, and treat any `mismatch` - verdict as "not done." - - The most common reason "the build is green but the tests still fail" is - that the model produces the right *data values* in the wrong *column - shape* — extra columns, missing columns, wrong order, wrong types. Many - dbt equality tests grade the column tuple `(name, type, position)` - exactly, and the agent's prior bias is to add "helpful" extras - (`p1`/`p2`/`p3` rank breakdowns, name-resolved variants, lineage - metadata) or reorder columns "more logically." Both break the contract. - - This skill enforces the mechanical check that catches those bugs before - declaring done. Use it before declaring any model task complete. + Use after building or modifying a dbt model whose columns are declared in + YAML (`schema.yml` / `_models.yml`). Run `altimate-dbt schema-verify --model + ` to compare the built table with the declared columns, and treat a + `mismatch` verdict (a non-empty `findings` list) as "not done" until you have + decided whether the model or the YAML is wrong. + + YAML `columns:` is documentation plus the anchor for column tests. It is not + an exhaustive list of what a model may produce, so columns the YAML does not + mention are not an error unless a contract is enforced. What does count: an + enforced contract the table does not match, and a declared column that has + tests attached but is not produced. --- # dbt schema-verify @@ -59,8 +55,17 @@ Returns a structured JSON result: ```json { - "model": "int_asana__project_user_agg", + "model": "int_work_items__project_user_agg", "verdict": "mismatch", + "spec": { "declared_in": "models/schema.yml", "package": "work_items", "contract_enforced": false }, + "findings": [ + { + "kind": "tested-column-missing", + "columns": ["number_of_users_involved"], + "evidence": "models/schema.yml declares number_of_users_involved with tests (not_null_...), but the model does not produce it" + } + ], + "notes": [], "expected_columns": ["project_id", "users", "number_of_users_involved"], "actual_columns": ["project_id", "users"], "columns_extra": [], @@ -74,23 +79,37 @@ Returns a structured JSON result: | verdict | meaning | what to do | |---|---|---| -| `match` | actual columns match the spec exactly (case-insensitive on names) | DONE — proceed | -| `mismatch` | one or more of `columns_extra`, `columns_missing`, `columns_reordered`, `type_mismatches` is non-empty | NOT DONE — read the diff, fix the model SQL, rebuild, re-run schema-verify | -| `no-spec` | the model has no columns declared in `schema.yml` | DONE for shape-fidelity purposes — no contract to verify against | - -## How to act on a `mismatch` - -For each non-empty list, the fix is mechanical: - -| Field | What it means | What to change in the model SQL | -|---|---|---| -| `columns_extra` | columns in your model NOT in the spec | REMOVE them from the `SELECT` | -| `columns_missing` | columns in the spec NOT in your model | ADD them to the `SELECT` (compute them, or rename an existing column if you used a synonym) | -| `columns_reordered` | columns present in both but at different positions | REORDER the columns in your `SELECT` to match the spec's order | -| `type_mismatches` | declared `data_type` in spec disagrees with the warehouse's reported type | CAST in the `SELECT` or change the upstream source | - -Then run `altimate-dbt build --model ` again, then re-run -`altimate-dbt schema-verify --model ` until verdict is `match`. +| `match` | nothing dbt treats as an error was found (`notes` may still hold true observations) | proceed | +| `mismatch` | `findings` is non-empty: an enforced contract the table does not match, or a missing declared column with an attached test or a legacy test whose attachment could not be verified | NOT DONE until you decide which side is wrong | +| `no-spec` | the model has no columns declared in YAML | nothing to compare | + +`spec` says which YAML file declares the columns, in which package, and +whether a contract is enforced. Each entry in `findings` carries the +evidence. `columns_extra` / `columns_missing` / `columns_reordered` / +`type_mismatches` are the raw diff against the YAML, not instructions: +`columns_extra` only means "not listed in the YAML". + +## How to act on a finding + +A finding says two things disagree. It does not say which one is wrong. + +- **Contract enforced.** dbt will refuse to build a model whose columns differ + from its contract. Either the model should produce exactly the contract's + columns, or the contract is out of date. Decide from the task. +- **Declared column with tests, not produced.** The tests read a column that + is not there and will fail or error. Either the model should produce the + column, or the YAML entry (and its tests) is stale. Decide from the task. +- **Test attachment unverified.** The table lacks the declared column, and + the legacy manifest lists the model as a test dependency without identifying + which relation the test reads. Confirm the attachment in the test's YAML + before deciding whether to change the model or its declaration. +- **Never** delete or invent a column just because the YAML does or does not + list it. Undocumented columns are normal, and a declared column with no + tests and no contract (a `note`) may simply be stale. If the task states + which columns the output must have, follow the task. + +Then run `altimate-dbt build --model ` again and re-run +`altimate-dbt schema-verify --model `. ## Iron Rules @@ -98,18 +117,15 @@ Then run `altimate-dbt build --model ` again, then re-run columns yourself and concluding "looks right to me" does not count. Run the command and read its output. 2. **A `mismatch` is "not done", even if the build is green.** dbt build - only proves the SQL compiled and ran without errors. It does not prove - the column shape is correct. Equality tests grade shape AND values. -3. **Do not reinterpret the spec to make the model right.** The spec is - the contract. If the spec lists `supplier_company` and your model has - `supplier_id`, the answer is to fix your model, not to argue that - `supplier_id` is more useful. + does not validate the full declared column list unless a contract is enforced; + attached tests still run and check the columns they reference. +3. **Do not change the model to satisfy YAML that only documents some + columns.** The YAML is evidence about intent, not the spec of the whole + table. The task and the existing consumers of the model decide which + columns are wanted. 4. **Run schema-verify on every model touched, not just the last one.** - The most common "almost-pass" is N-1 models passing and the Nth one - silently failing on column shape. Walk the list. -5. **Skip only on `no-spec`.** Do not skip on the grounds that the model - is small, or trivial, or "obvious." The spec is small only because - the dbt project author already curated it. +5. **`no-spec` and `match` with notes need no action.** Read the notes; act + on them only if they match what the task asked for. ## Fallback when altimate-dbt is unavailable @@ -125,10 +141,9 @@ cat models/**/*.yml | grep -A 50 "name: " # or: yq eval '...' models/**/ dbt show --select --limit 0 ``` -Compare the two ordered lists. Produce the same four-bucket diff -(`columns_extra`, `columns_missing`, `columns_reordered`, -`type_mismatches`) in your head, and apply the same fix logic. The -mechanics don't change; only the tool name does. +Compare the two lists and apply the same rules: only an enforced +contract, or a declared column with tests that the table lacks, is a +problem. Columns the YAML does not list are not. ## What this skill does NOT cover diff --git a/bun.lock b/bun.lock index 268aeb726c..66ede6b403 100644 --- a/bun.lock +++ b/bun.lock @@ -151,6 +151,7 @@ }, "dependencies": { "@altimateai/dbt-integration": "^0.3.19", + "yaml": "2.8.3", }, "devDependencies": { "@tsconfig/bun": "catalog:", diff --git a/docs/docs/data-engineering/tools/dbt-tools.md b/docs/docs/data-engineering/tools/dbt-tools.md index 0b924da59e..e2acc35279 100644 --- a/docs/docs/data-engineering/tools/dbt-tools.md +++ b/docs/docs/data-engineering/tools/dbt-tools.md @@ -205,6 +205,14 @@ All commands provide friendly error diagnostics with actionable fix suggestions > **Tip:** In builder mode, the agent prefers `altimate-dbt` over the raw `dbt_run` tool for better error handling and auto-detection. +**Packages.** If the project declares packages (`packages.yml`, `dependencies.yml`), `altimate-dbt` installs them when an adapter-backed command starts (not `doctor`), but only when they are missing or out of date (a package directory is absent or has no `dbt_project.yml`, or the declarations changed since the last install). Installation is serialised across processes with a lock under `target/.altimate-dbt/`, so several `altimate-dbt` commands started together install at most once and cannot corrupt `dbt_packages/`. A lock left by a crashed process is broken automatically (owner process gone, or no heartbeat for 30 seconds) and the interrupted install is repeated. `altimate-dbt deps` always runs `dbt deps`, under the same lock. Because every command re-parses the project and dbt rewrites `target/manifest.json` in place, a command that reads the manifest while a sibling rewrites it re-reads it (up to six times, with backoff) before reporting it missing. + +For package install paths with Jinja beyond a literal `env_var(...)`, use `altimate-dbt deps` to let dbt resolve the path and install the packages. + +Automatic schema verification after `altimate-dbt build` uses a temporary target directory. Its compile artifacts leave the project's build results available for inspection. The build reports when it ran verification or reinstalled dependencies. Reinstalling dependencies resets the package directory, so changes made directly there may be lost. + +When `altimate-dbt test --model ` selects a model with no tests, it reports “Model has no tests.” Invalid selectors and failed tests retain dbt's diagnostics. + --- ## dbt Skills diff --git a/docs/docs/data-engineering/validators.md b/docs/docs/data-engineering/validators.md index 0aaab133ef..423844df71 100644 --- a/docs/docs/data-engineering/validators.md +++ b/docs/docs/data-engineering/validators.md @@ -53,7 +53,8 @@ deterministic-validators work and are currently soak-tested in shadow mode After the agent declares done, runs `altimate-dbt test --model ` against every dbt model the agent modified during this session. Refuses -to terminate if any model's tests fail or error. +to terminate if a model's tests fail, or if tests could not be run +because dbt itself errored. **Catches**: row-data correctness errors (`relationships`, `unique`, `not_null`, `accepted_values`, `AUTO_*_equality` tests) — the kind of @@ -61,16 +62,64 @@ bug that column-shape verification cannot detect because the schema can be green while the SELECT logic produces wrong values or wrong row counts. +**What counts as what** + +| dbt result for the model | Validator | +|---|---| +| Tests ran and all passed | passes | +| Tests ran and some failed or errored | **fails**, naming captured failing tests when available | +| dbt selected no tests (it prints `Nothing to do` and no summary) | passes, and sends nothing. A model with no tests has nothing to check. Recorded as `no_tests` in telemetry. | +| The selector matched no enabled node | passes (`unmatched_selector_models` in telemetry) | +| dbt aborted (connection, lock, compile or runtime error) before or while running tests | **fails**, with dbt's error text | + +A run error can be retried once in isolation to recover from warehouse contention, +including contention with another invocation. Each schema or test validator run retries +up to four models (`retried_serially` in telemetry). Compilation and parsing errors, +missing profiles, timeouts, and spawn failures keep their original diagnostics. + ### `dbt-schema-verify` After the agent declares done, runs `altimate-dbt schema-verify --model -` on every modified model. Reports a mismatch if the produced -column shape diverges from the `schema.yml` spec (extra, missing, -reordered, or type-mismatched columns). - -**Catches**: column-level drift that wouldn't be caught by `dbt build` -alone — equality tests against the spec would fail later but the -agent has already declared done. +` on every modified model and compares the built table with the +columns its YAML declares. It reports only what dbt's own semantics +establish as a problem, and says which file declares what: + +- **Enforced contract.** The model has `contract: {enforced: true}` and + the built table has columns the contract does not list, or lacks columns + it lists. dbt itself rejects such a model. Only column names are compared + here; `dbt build` is what checks a contract's data types. +- **Declared column with tests, not produced.** The YAML declares a column + that has at least one test attached, but the built table does not have it. + The test reads a column that does not exist. +- **Test attachment unverified.** The table lacks a declared column and a + legacy manifest lists the model as a dependency of a test on that column. + Check the test's YAML to establish which relation it reads. + +It does **not** report, and never tells the agent to remove, add or +reorder anything because of: + +- columns the model produces that the YAML does not list. YAML commonly + documents only some columns (those with tests or descriptions); that is + not an exhaustive list unless a contract is enforced; +- a declared column with no tests and no contract that the model lacks. + This may be a stale or aspirational entry, so it is only a note in the + `altimate-dbt schema-verify` output; +- column order, or a declared `data_type` that differs from the warehouse + type outside a contract (dbt does not enforce either, and adapters spell + types differently); +- a model with no YAML entry or no declared columns (`no-spec`). + +The message states the evidence and leaves the decision with the agent +(change the model if the YAML is right, or the YAML if its entry is +stale). The full classification appears in `altimate-dbt schema-verify` +and `altimate-dbt build --model` as `findings` (mismatches and evidence), +`notes` (true observations) and `spec` (declaring file, package, contract +flag). The project-wide `altimate-dbt build` returns these detailed fields +only for models with a mismatch; matching models are summarised as a count. +`columns_extra`, `columns_missing`, `columns_reordered` and +`type_mismatches` remain as the raw diff against the YAML and are not +instructions. A tool error is still a +failure (drift cannot be ruled out) and now includes the first error text. ## Completion gates (shadow mode) @@ -234,6 +283,10 @@ When a validator runs, it returns: // validator-specific extras: // dbt-tests-pass: passed?: number + no_tests?: number + no_tests_models?: string[] + unmatched_selector_models?: string[] + retried_serially?: number failed?: number errored?: number spawn_failures?: number @@ -244,13 +297,14 @@ When a validator runs, it returns: match?: number mismatch?: number no_spec?: number + diff_not_established?: number mismatch_models?: string[] } } ``` `reason` names the failing models inline (e.g. `"2 of 3 models you -edited have a column-shape mismatch against schema.yml: foo, bar"`). +edited contradict the YAML that declares their columns: foo, bar"`). ## Phased rollout plan @@ -284,9 +338,11 @@ default-on. Track progress in deeper (workspace layouts) are not detected. - Multiple `dbt_project.yml` candidates pick the alphabetically-first match deterministically. -- The validator surfaces "schema mismatch" even when the real cause - is "model never materialized" — distinguishing these requires - changes inside `altimate-dbt`. +- A model that was never materialized is reported as a tool error + ("has no warehouse table. Build it first"), not as a column finding. +- Retries consume the shared validator budget (three by default) whether + or not the failure text changed. A failure that is re-sent verbatim on + every retry spends the budget without giving the agent new information. ## Writing custom validators diff --git a/packages/dbt-tools/package.json b/packages/dbt-tools/package.json index 44174b14b5..34744d9fb1 100644 --- a/packages/dbt-tools/package.json +++ b/packages/dbt-tools/package.json @@ -9,11 +9,12 @@ "scripts": { "build": "bun build src/index.ts --outdir dist --target node --format esm && bun run script/copy-python.ts", "typecheck": "tsc --noEmit", - "test": "bun test test/cli.test.ts test/config.test.ts test/dbt-cli.test.ts test/dbt-resolve.test.ts test/adapter-bun.test.ts test/stale-manifest.test.ts test/build-integrity.test.ts test/adapter-target-path.test.ts --timeout 30000", - "test:e2e": "bun test test/e2e/ --timeout 300000" + "test": "bun test test/adapter-bun.test.ts --timeout 30000 && bun test test/cli.test.ts test/config.test.ts test/dbt-cli.test.ts test/dbt-resolve.test.ts test/stale-manifest.test.ts test/build-integrity.test.ts test/adapter-target-path.test.ts test/packages.test.ts test/manifest-retry.test.ts test/schema-verify.test.ts test/build.test.ts --timeout 30000", + "test:e2e": "bun run build && bun test test/e2e/ test/build-real-dbt.test.ts test/packages-real-dbt.test.ts --timeout 300000" }, "dependencies": { - "@altimateai/dbt-integration": "^0.3.19" + "@altimateai/dbt-integration": "^0.3.19", + "yaml": "2.8.3" }, "devDependencies": { "@tsconfig/bun": "catalog:", diff --git a/packages/dbt-tools/src/adapter.ts b/packages/dbt-tools/src/adapter.ts index 4786164c1a..2ff3bf21c6 100644 --- a/packages/dbt-tools/src/adapter.ts +++ b/packages/dbt-tools/src/adapter.ts @@ -1,6 +1,8 @@ import type { Config } from "./config" import { bufferLog } from "./log-buffer" import { recordManifest } from "./stale-manifest" +import { retryParseManifest } from "./manifest-retry" +import { dbtDepsInstaller, ensurePackages, PackageLockTimeoutError, trackedExplicitInstaller } from "./packages" export { getRecentDbtLogs, clearDbtLogs } from "./log-buffer" import { DBTProjectIntegrationAdapter, @@ -52,7 +54,10 @@ function configuration(cfg: Config): DBTConfiguration { getQueryLimit: () => cfg.queryLimit ?? DEFAULT_CONFIGURATION_VALUES.queryLimit, getEnableNotebooks: () => DEFAULT_CONFIGURATION_VALUES.enableNotebooks, getDisableQueryHistory: () => DEFAULT_CONFIGURATION_VALUES.disableQueryHistory, - getInstallDepsOnProjectInitialization: () => DEFAULT_CONFIGURATION_VALUES.installDepsOnProjectInitialization, + // altimate-dbt installs packages itself (packages.ts: only when missing or out of date, + // under a cross-process lock). The library's own install runs `dbt deps` unconditionally + // from every process, and concurrent runs corrupt dbt_packages. + getInstallDepsOnProjectInitialization: () => false, getDisableDepthsCalculation: () => true, getWorkingDirectory: () => cfg.projectRoot, getAltimateUrl: () => "", @@ -92,6 +97,7 @@ export type CreateOptions = { * would take the compiled SQL of every other model away from whoever reads that manifest. */ targetPath?: string + onPackagesInstalled?: () => void } function env(cfg: Config, options: CreateOptions): RuntimePythonEnvironment { @@ -126,6 +132,20 @@ export async function create(cfg: Config, options: CreateOptions = {}): Promise< onEnvironmentChanged: () => () => {}, } const exec = new CommandProcessExecutionFactory(term) + const createExecution = exec.createCommandProcessExecution.bind(exec) + exec.createCommandProcessExecution = (options) => { + const execution = createExecution(options) + const install = trackedExplicitInstaller(options) + if (install) { + execution.complete = () => install() + execution.completeWithTerminalOutput = () => install((text) => { + const formatted = execution.formatText(text) + term.log(formatted) + return formatted + }) + } + return execution + } const infra = new DBTCommandExecutionInfrastructure(runtime, term) const python = new PythonDBTCommandExecutionStrategy(exec, runtime, term, config) const cli = (cwd: string, path: string) => new CLIDBTCommandExecutionStrategy(exec, runtime, term, cwd, path) @@ -229,9 +249,23 @@ export async function create(cfg: Config, options: CreateOptions = {}): Promise< new SemanticModelParser(term), ) + // The library's own install is off (see `getInstallDepsOnProjectInitialization`), so every + // caller of `create` gets packages from here: only when missing or out of date, under a + // cross-process lock. Other failures are not fatal: the manifest parse reports a missing + // package with dbt's own error and the next call retries. Waiting too long for another + // process's install is fatal: continuing would read packages that are being rewritten. + try { + const packages = await ensurePackages(cfg.projectRoot, dbtDepsInstaller(cfg)) + if (packages.action === "installed") options.onPackagesInstalled?.() + } catch (err) { + if (err instanceof PackageLockTimeoutError) throw err + bufferLog(`[dbt-tools] package install check failed: ${err instanceof Error ? err.message : String(err)}`) + } + // Before `initialize`, whose parse is the one the commands answer from. adapter.on(DBTProjectIntegrationAdapterEvents.MANIFEST_PARSED, recordManifest) try { + retryParseManifest(adapter, { shouldRetry: () => adapter.getTargetPath() !== undefined }) await adapter.initialize() } catch (e) { // initialize() can fail after the integration started a Python bridge; stop it. diff --git a/packages/dbt-tools/src/commands/build.ts b/packages/dbt-tools/src/commands/build.ts index acaab9ee44..db8622c6c5 100644 --- a/packages/dbt-tools/src/commands/build.ts +++ b/packages/dbt-tools/src/commands/build.ts @@ -1,26 +1,33 @@ -import type { DBTProjectIntegrationAdapter, CommandProcessResult } from "@altimateai/dbt-integration" +import type { DBTProjectIntegrationAdapter } from "@altimateai/dbt-integration" +import { formatCommandResult } from "../failure-text" import { schemaVerify } from "./schema-verify" -export async function build(adapter: DBTProjectIntegrationAdapter, args: string[]) { +type WithVerifier = (verify: (adapter: DBTProjectIntegrationAdapter) => Promise) => Promise + +export async function build( + adapter: DBTProjectIntegrationAdapter, + args: string[], + withVerifier: WithVerifier = (verify) => verify(adapter), +) { const model = flag(args, "model") const downstream = args.includes("--downstream") if (!model) { if (downstream) return { error: "--downstream requires --model" } - return project(adapter) + return project(adapter, withVerifier) } const result = await adapter.unsafeBuildModelImmediately({ plusOperatorLeft: "", modelName: model, plusOperatorRight: downstream ? "+" : "", }) - const formatted = format(result) + const formatted = formatCommandResult(result) // Auto-run schema-verify after a successful single-model build. Surfacing // the verdict in the same tool result the agent just received is the // closest a CLI command can get to harness-level enforcement: the agent // cannot see a green build without also seeing the schema-verify diff. if (!("error" in formatted)) { - return { ...formatted, schema_verify: await safeVerify(adapter, model) } + return { ...formatted, schema_verify: await safeVerify(withVerifier, model) } } return formatted } @@ -34,19 +41,32 @@ export async function run(adapter: DBTProjectIntegrationAdapter, args: string[]) modelName: model, plusOperatorRight: downstream ? "+" : "", }) - return format(result) + return formatCommandResult(result) } export async function test(adapter: DBTProjectIntegrationAdapter, args: string[]) { const model = flag(args, "model") if (!model) return { error: "Missing --model" } const result = await adapter.unsafeRunModelTestImmediately(model) - return format(result) + const formatted = formatCommandResult(result) + if (result?.exitCode && result.exitCode !== 0) { + return { ...formatted, dbt_exit_code: result.exitCode } + } + if ( + "error" in formatted || + /does not match any (?:enabled )?nodes?|Encountered an error|Compilation Error|Parsing Error|Runtime Error|Database Error|Could not set lock|Done\.\s+PASS=/i.test(formatted.stdout) + ) { + return formatted + } + return { stdout: formatted.stdout.replace(/\bNothing to do\b[^\r\n]*/i, "Model has no tests.") } } -export async function project(adapter: DBTProjectIntegrationAdapter) { +export async function project( + adapter: DBTProjectIntegrationAdapter, + withVerifier: WithVerifier = (verify) => verify(adapter), +) { const result = await adapter.unsafeBuildProjectImmediately() - const formatted = format(result) + const formatted = formatCommandResult(result) if ("error" in formatted) return formatted // After a successful project-wide build, auto-run schema-verify on every @@ -56,49 +76,7 @@ export async function project(adapter: DBTProjectIntegrationAdapter) { // Only the mismatches and verify errors are reported back. `no-spec` // models are summarised as a count to keep the response compact. try { - const parsed = await adapter.parseManifest() - const nodes = parsed?.nodeMetaMap?.nodes ? Array.from(parsed.nodeMetaMap.nodes()) : [] - const verified: Array<{ model: string; verdict: string; columns_extra?: unknown; columns_missing?: unknown; columns_reordered?: unknown; type_mismatches?: unknown }> = [] - const errored: Array<{ model: string; error: string }> = [] - let nospec_count = 0 - for (const node of nodes) { - // Only models, only those with declared columns. Sources/seeds/snapshots/tests skipped. - const resType = (node as { resource_type?: string }).resource_type - if (resType !== "model") continue - const name = (node as { name?: string }).name - if (!name) continue - const cols = (node as { columns?: Record }).columns ?? {} - if (Object.keys(cols).length === 0) { - nospec_count++ - continue - } - try { - const v = await schemaVerify(adapter, ["--model", name]) - if ("error" in v) { - errored.push({ model: name, error: String((v as { error: unknown }).error) }) - } else if ((v as { verdict: string }).verdict === "no-spec") { - nospec_count++ - } else { - verified.push(v as { model: string; verdict: string }) - } - } catch (e) { - errored.push({ model: name, error: e instanceof Error ? e.message : String(e) }) - } - } - const mismatches = verified.filter((r) => r.verdict === "mismatch") - const matches = verified.filter((r) => r.verdict === "match") - return { - ...formatted, - schema_verify_summary: { - models_checked: verified.length + errored.length, - match: matches.length, - mismatch: mismatches.length, - no_spec: nospec_count, - errored: errored.length, - mismatches, - ...(errored.length > 0 && { errors: errored }), - }, - } + return await withVerifier(async (verifier) => ({ ...formatted, schema_verify_summary: await verifyProject(verifier) })) } catch (e) { return { ...formatted, @@ -109,9 +87,52 @@ export async function project(adapter: DBTProjectIntegrationAdapter) { } } -async function safeVerify(adapter: DBTProjectIntegrationAdapter, model: string) { +async function verifyProject(adapter: DBTProjectIntegrationAdapter) { + const parsed = await adapter.parseManifest() + const nodes = parsed?.nodeMetaMap?.nodes ? Array.from(parsed.nodeMetaMap.nodes()) : [] + const verified: Array<{ model: string; verdict: string; columns_extra?: unknown; columns_missing?: unknown; columns_reordered?: unknown; type_mismatches?: unknown }> = [] + const errored: Array<{ model: string; error: string }> = [] + let nospec_count = 0 + for (const node of nodes) { + // Only models, only those with declared columns. Sources/seeds/snapshots/tests skipped. + const resType = (node as { resource_type?: string }).resource_type + if (resType !== "model") continue + const name = (node as { name?: string }).name + if (!name) continue + const cols = (node as { columns?: Record }).columns ?? {} + if (Object.keys(cols).length === 0) { + nospec_count++ + continue + } + try { + const v = await schemaVerify(adapter, ["--model", name]) + if ("error" in v) { + errored.push({ model: name, error: String((v as { error: unknown }).error) }) + } else if ((v as { verdict: string }).verdict === "no-spec") { + nospec_count++ + } else { + verified.push(v as { model: string; verdict: string }) + } + } catch (e) { + errored.push({ model: name, error: e instanceof Error ? e.message : String(e) }) + } + } + const mismatches = verified.filter((r) => r.verdict === "mismatch") + const matches = verified.filter((r) => r.verdict === "match") + return { + models_checked: verified.length + errored.length, + match: matches.length, + mismatch: mismatches.length, + no_spec: nospec_count, + errored: errored.length, + mismatches, + ...(errored.length > 0 && { errors: errored }), + } +} + +async function safeVerify(withVerifier: WithVerifier, model: string) { try { - return await schemaVerify(adapter, ["--model", model]) + return await withVerifier((adapter) => schemaVerify(adapter, ["--model", model])) } catch (e) { return { error: `schema-verify failed: ${e instanceof Error ? e.message : String(e)}. Run \`altimate-dbt schema-verify --model ${model}\` manually to inspect.`, @@ -119,14 +140,6 @@ async function safeVerify(adapter: DBTProjectIntegrationAdapter, model: string) } } -// TODO: dbt writes info/progress logs to stderr even on success — checking stderr -// alone causes false failures. CommandProcessResult has no exit_code field, so we -// can't distinguish real errors yet. Revisit when the type is extended. -function format(result?: CommandProcessResult) { - if (result?.stderr) return { error: result.stderr, stdout: result.stdout } - return { stdout: result?.stdout ?? "" } -} - function flag(args: string[], name: string): string | undefined { const i = args.indexOf(`--${name}`) return i >= 0 ? args[i + 1] : undefined diff --git a/packages/dbt-tools/src/commands/deps.ts b/packages/dbt-tools/src/commands/deps.ts index feda088730..ab135cc40b 100644 --- a/packages/dbt-tools/src/commands/deps.ts +++ b/packages/dbt-tools/src/commands/deps.ts @@ -1,23 +1,19 @@ -import type { DBTProjectIntegrationAdapter, CommandProcessResult } from "@altimateai/dbt-integration" +import type { DBTProjectIntegrationAdapter } from "@altimateai/dbt-integration" +import { formatCommandResult } from "../failure-text" +import { installPackagesLocked } from "../packages" -export async function deps(adapter: DBTProjectIntegrationAdapter) { - const result = await adapter.installDeps() - return format(result) +// Explicit requests always run `dbt deps`, but under the same lock the automatic +// install uses so they cannot interleave with another process's install. +export async function deps(adapter: DBTProjectIntegrationAdapter, projectRoot: string) { + const result = await installPackagesLocked(projectRoot, () => adapter.installDeps()) + return formatCommandResult(result) } -export async function add(adapter: DBTProjectIntegrationAdapter, args: string[]) { +export async function add(adapter: DBTProjectIntegrationAdapter, args: string[], projectRoot: string) { const raw = flag(args, "packages") if (!raw) return { error: "Missing --packages" } - const result = await adapter.installDbtPackages(raw.split(",")) - return format(result) -} - -// TODO: dbt writes info/progress logs to stderr even on success — checking stderr -// alone causes false failures. CommandProcessResult has no exit_code field, so we -// can't distinguish real errors yet. Revisit when the type is extended. -function format(result?: CommandProcessResult) { - if (result?.stderr) return { error: result.stderr, stdout: result.stdout } - return { stdout: result?.stdout ?? "" } + const result = await installPackagesLocked(projectRoot, () => adapter.installDbtPackages(raw.split(","))) + return formatCommandResult(result) } function flag(args: string[], name: string): string | undefined { diff --git a/packages/dbt-tools/src/commands/schema-verify.ts b/packages/dbt-tools/src/commands/schema-verify.ts index 66ada8bf38..b3ad5818e3 100644 --- a/packages/dbt-tools/src/commands/schema-verify.ts +++ b/packages/dbt-tools/src/commands/schema-verify.ts @@ -1,30 +1,37 @@ -import type { ColumnMetaData, DBTProjectIntegrationAdapter } from "@altimateai/dbt-integration" +import type { ColumnMetaData, DBTProjectIntegrationAdapter, TestMetaData } from "@altimateai/dbt-integration" import { staleManifestNote } from "../stale-manifest" /** - * Verify that a model's actual produced columns match the spec declared in - * `schema.yml` (compiled into manifest.json as `node.columns`). + * Compare a model's materialized columns with the columns its YAML declares + * (`node.columns` in the manifest) and report only what dbt's own semantics + * establish. * - * Spec source: `adapter.nodeMetaMap.lookupByBaseName(model).columns` — these - * are the columns the schema.yml entry promised. Object insertion order is - * preserved through manifest parsing, so it carries the spec's column order. + * What YAML `columns:` means in dbt: + * - Without an enforced contract it is documentation plus the anchor for + * column tests. It is NOT an exhaustive list: models routinely produce + * columns the YAML never mentions, so "produced but not listed" says + * nothing about whether a column is wanted. + * - With `config.contract.enforced: true`, dbt itself requires the model to + * return exactly the declared columns and rejects the build otherwise. + * - A declared column the model does not produce is a real problem when a + * test is attached to it (the test reads a column that is not there), and + * when a contract is enforced. With neither, it is a stale or aspirational + * entry: worth knowing, not a failure. With unresolved test attachment in + * a legacy manifest, report the missing column and the uncertainty. + * - Column order in YAML never constrains the model (dbt aligns it itself + * under a contract), and `data_type` is only checked by dbt under a contract + * (adapters spell types differently, so string comparison is not evidence). * - * Actual source: `adapter.getColumnsOfModel(model)` — the columns the - * warehouse / catalog reports the materialized table actually has. - * - * Returns four lists the agent must treat as the contract: - * - columns_extra: in actual, not in spec → REMOVE from SELECT - * - columns_missing: in spec, not in actual → ADD to SELECT - * - columns_reordered: in both, wrong position → REORDER the SELECT - * - type_mismatches: same name, different declared types - * - * `verdict` is "match" iff all four lists are empty. - * - * Skip cases: - * - "no-spec": schema.yml doesn't declare columns for this model — nothing - * to verify; agent has no contract to fail against. - * - * Every answer rests on the manifest, its errors included, so each one says when that's a stale one. + * Output: + * - verdict "mismatch" iff `findings` is non-empty. Each finding names what is + * wrong, the YAML file that declares it, and why it is established. + * - `notes` carry true observations that are not problems (unlisted columns, + * declared-but-absent columns nothing depends on, type differences). + * - `columns_extra` / `columns_missing` / `columns_reordered` / + * `type_mismatches` are the raw diff against the YAML, for transparency. They + * are not instructions: `columns_extra` means "not listed in YAML". + * - verdict "no-spec": the model has no columns declared in YAML. + * Each answer records when its manifest may be stale. */ export async function schemaVerify(adapter: DBTProjectIntegrationAdapter, args: string[]) { const answer = await verify(adapter, args) @@ -56,7 +63,7 @@ async function verify(adapter: DBTProjectIntegrationAdapter, args: string[]) { error: `Failed to read actual columns for '${model}': ${e instanceof Error ? e.message : String(e)}. Build the model first: altimate-dbt build --model ${model}`, } } - if (!actual) { + if (!actual || (actual.length === 0 && expectedEntries.length > 0)) { return { error: `Model '${model}' is in the manifest but has no warehouse table. Build it first: altimate-dbt build --model ${model}`, } @@ -67,7 +74,7 @@ async function verify(adapter: DBTProjectIntegrationAdapter, args: string[]) { return { model, verdict: "no-spec" as const, - message: `Model '${model}' has no columns declared in schema.yml. There is no spec to verify against; the agent's column choices are unconstrained.`, + message: `Model '${model}' has no columns declared in YAML. There is no declaration to compare against.`, actual_columns: actual.map((c) => c.column), } } @@ -136,24 +143,176 @@ async function verify(adapter: DBTProjectIntegrationAdapter, args: string[]) { } } - const verdict = - columns_extra.length === 0 && - columns_missing.length === 0 && - columns_reordered.length === 0 && - type_mismatches.length === 0 - ? ("match" as const) - : ("mismatch" as const) + // 5. Decide what is established (see header). Raw diff lists above are kept as-is. + const contractEnforced = isContractEnforced(node as unknown as Record) + const { file: declaredIn, pkg: declaredPackage } = splitPatchPath(node.patch_path) + const where = describeSource(declaredIn, declaredPackage, node.package_name) + // Scanning every test of the project is only needed when a declared column is missing. + const tests = !contractEnforced && columns_missing.length > 0 ? columnTests(parsed?.testMetaMap, node.unique_id) : undefined + + const findings: Finding[] = [] + const notes: string[] = [] + + const listed = (names: string[]) => names.map((n) => `\`${n}\``).join(", ") + + if (contractEnforced) { + if (columns_missing.length > 0) + findings.push({ + kind: "contract-missing-columns", + columns: columns_missing, + evidence: `${where} declares an enforced contract that lists ${listed(columns_missing)}, but the built table does not have ${columns_missing.length === 1 ? "it" : "them"}. dbt rejects a contract-enforced model whose columns differ from the contract.`, + }) + if (columns_extra.length > 0) + findings.push({ + kind: "contract-extra-columns", + columns: columns_extra, + evidence: `${where} declares an enforced contract that does not list ${listed(columns_extra)}, but the built table has ${columns_extra.length === 1 ? "it" : "them"}. dbt rejects a contract-enforced model whose columns differ from the contract.`, + }) + } else { + const missingWithTests = columns_missing.filter((c) => tests?.attached.has(c.toLowerCase())) + const missingWithUnverifiedTests = columns_missing.filter( + (c) => !tests?.attached.has(c.toLowerCase()) && tests?.unverified.has(c.toLowerCase()), + ) + const missingUntested = columns_missing.filter( + (c) => !missingWithTests.includes(c) && !missingWithUnverifiedTests.includes(c), + ) + if (missingWithTests.length > 0) { + const detail = missingWithTests + .map((c) => `\`${c}\` (tests: ${(tests?.attached.get(c.toLowerCase()) ?? []).join(", ")})`) + .join("; ") + findings.push({ + kind: "tested-column-missing", + columns: missingWithTests, + evidence: `${where} declares column(s) with tests attached that the built table does not have: ${detail}. Those tests read a column that does not exist.`, + }) + } + if (missingWithUnverifiedTests.length > 0) { + const detail = missingWithUnverifiedTests + .map((c) => `\`${c}\` (tests: ${tests?.unverified.get(c.toLowerCase())?.join(", ")})`) + .join("; ") + findings.push({ + kind: "test-attachment-unverified", + columns: missingWithUnverifiedTests, + evidence: `${where} declares column(s) the built table does not have: ${detail}. The legacy manifest cannot establish which dependency each test reads.`, + }) + } + if (missingUntested.length > 0) + notes.push( + `${where} declares ${listed(missingUntested)}, which the built table does not have. No test or contract depends on ${missingUntested.length === 1 ? "it" : "them"}, so this may be a stale or aspirational YAML entry rather than a defect in the model.`, + ) + if (columns_extra.length > 0) + notes.push( + `The built table has ${listed(columns_extra)}, not listed in ${where}. dbt does not require YAML to list every column, so this is not an error.`, + ) + } + if (type_mismatches.length > 0 && contractEnforced) + notes.push( + `Declared data_type differs from the warehouse type for ${listed(type_mismatches.map((t) => t.column))}; dbt compares types itself when it builds a contract-enforced model and rejects a real violation, so confirm with a build rather than from this string comparison (adapters spell types differently).`, + ) + + const verdict = findings.length === 0 ? ("match" as const) : ("mismatch" as const) return { model, verdict, + spec: { declared_in: declaredIn, package: declaredPackage ?? node.package_name, contract_enforced: contractEnforced }, expected_columns: expectedNames, actual_columns: actualNames, columns_extra, columns_missing, columns_reordered, type_mismatches, + findings, + notes, + } +} + +export interface Finding { + kind: "contract-missing-columns" | "contract-extra-columns" | "tested-column-missing" | "test-attachment-unverified" + columns: string[] + /** Which YAML declares what, and why this is a problem. */ + evidence: string +} + +function isContractEnforced(node: Record): boolean { + const fromConfig = (node.config as { contract?: { enforced?: unknown } } | undefined)?.contract?.enforced + const fromNode = (node.contract as { enforced?: unknown } | undefined)?.enforced + return fromConfig === true || fromNode === true +} + +/** dbt's `patch_path` looks like `://`. */ +function splitPatchPath(patchPath: string | undefined): { file?: string; pkg?: string } { + if (!patchPath) return {} + const m = /^([^:/]+):\/\/(.*)$/.exec(patchPath) + return m ? { pkg: m[1], file: m[2] } : { file: patchPath } +} + +function describeSource(file: string | undefined, declaringPackage: string | undefined, modelPackage: string | undefined): string { + const f = file ? `\`${file}\`` : "the YAML entry" + if (declaringPackage && modelPackage && declaringPackage !== modelPackage) + return `${f} in package \`${declaringPackage}\` (not this project)` + return f +} + +// Older manifests lack attached_node. In a generic test's metadata, model is +// the relation under test and other kwargs (e.g. relationships.to) may add refs. +function legacyTestModel(model: unknown, dependencies: string[]): string | undefined { + if (typeof model === "string") { + const expression = /^\s*{{\s*(.*?)\s*}}\s*$/s.exec(model)?.[1] + if (!expression) return undefined + const relation = /^get_where_subquery\(\s*(.*?)\s*\)$/s.exec(expression)?.[1] ?? expression + const ref = /^ref\(\s*(['"])([\w-]+)\1(?:\s*,\s*(['"])([\w-]+)\3)?(?:\s*,\s*(?:v|version)\s*=\s*(\d+))?\s*\)$/.exec(relation) + if (ref) { + const [, , first, , second, version] = ref + const candidates = dependencies.filter((id) => { + const [resource, pkg, name, modelVersion] = id.split(".") + if (resource !== "model" && resource !== "seed" && resource !== "snapshot") return false + return (second ? pkg === first && name === second : name === first) && + (version === undefined || modelVersion === `v${version}`) + }) + return candidates.length === 1 ? candidates[0] : undefined + } + const source = /^source\(\s*(['"])([\w-]+)\1\s*,\s*(['"])([\w-]+)\3\s*\)$/.exec(relation) + if (!source) return undefined + const candidates = dependencies.filter((id) => { + const [resource, , name, table] = id.split(".") + return resource === "source" && name === source[2] && table === source[4] + }) + return candidates.length === 1 ? candidates[0] : undefined + } + return dependencies.length === 1 ? dependencies[0] : undefined +} + +function testColumnName(column: string): string { + if (column.length < 2) return column + if (column.startsWith("`") && column.endsWith("`")) return column.slice(1, -1).replace(/\\`/g, "`") + if (column.startsWith('"') && column.endsWith('"')) return column.slice(1, -1).replace(/""/g, '"') + if (column.startsWith("[") && column.endsWith("]")) return column.slice(1, -1).replace(/\]\]/g, "]") + return column +} + +function columnTests(testMetaMap: Map | undefined, modelUniqueId: string) { + const attached = new Map() + const unverified = new Map() + for (const [name, test] of testMetaMap?.entries?.() ?? []) { + if (!test.column_name) continue + let destination: Map + if (test.attached_node) { + if (test.attached_node !== modelUniqueId) continue + destination = attached + } else { + const dependencies = test.depends_on?.nodes ?? [] + if (!dependencies.includes(modelUniqueId)) continue + const kwargs = test.test_metadata?.kwargs as { model?: unknown } | undefined + const relations = dependencies.filter((id) => !id.startsWith("macro.")) + const model = legacyTestModel(kwargs?.model, relations) + if (model && model !== modelUniqueId) continue + destination = model ? attached : unverified + } + const column = testColumnName(test.column_name).toLowerCase() + destination.set(column, [...(destination.get(column) ?? []), name]) } + return { attached, unverified } } function flag(args: string[], name: string): string | undefined { diff --git a/packages/dbt-tools/src/failure-text.ts b/packages/dbt-tools/src/failure-text.ts index f506791896..0e1429be82 100644 --- a/packages/dbt-tools/src/failure-text.ts +++ b/packages/dbt-tools/src/failure-text.ts @@ -1,3 +1,5 @@ +import type { CommandProcessResult } from "@altimateai/dbt-integration" + /** What `--format text` prints on stderr for an error answer. */ export function failureLines(failed: Record): string[] { const lines: string[] = [] @@ -9,3 +11,12 @@ export function failureLines(failed: Record): string[] { if (Array.isArray(failed.logs)) lines.push(failed.logs.join("\n")) return lines } + +export function formatCommandResult(result?: CommandProcessResult) { + const stdout = result?.stdout ?? "" + if (result?.exitCode && result.exitCode !== 0) { + return { error: result.stderr || stdout || `dbt exited with code ${result.exitCode}`, stdout } + } + if (result?.exitCode !== 0 && result?.stderr) return { error: result.stderr, stdout } + return { stdout } +} diff --git a/packages/dbt-tools/src/index.ts b/packages/dbt-tools/src/index.ts index 93875fbab3..bb22ce92bd 100644 --- a/packages/dbt-tools/src/index.ts +++ b/packages/dbt-tools/src/index.ts @@ -1,5 +1,7 @@ import { join, resolve } from "path" import { existsSync } from "fs" +import { mkdtemp, rm } from "node:fs/promises" +import { tmpdir } from "node:os" import { read, type Config } from "./config" import { init } from "./commands/init" import { validate } from "./check" @@ -71,6 +73,9 @@ function output(result: unknown) { const fmt = rest.includes("--format") && rest[rest.indexOf("--format") + 1] === "text" if (fmt && typeof result === "object" && result !== null) { const obj = result as Record + if (cmd === "build" && Array.isArray(obj.notes)) { + for (const note of obj.notes) console.log(note) + } if ("error" in obj) { console.error(failureLines(obj).join("\n")) process.exit(1) @@ -142,15 +147,25 @@ async function main() { const { configure } = await import("./dbt-cli") configure({ pythonPath: cfg.pythonPath, projectRoot: cfg.projectRoot }) + const { withUnverifiedPackagesLock } = await import("./packages") + return withUnverifiedPackagesLock(cfg.projectRoot, () => runAdapterCommand(cfg)) +} + +async function runAdapterCommand(cfg: Config) { // Lazy import: the adapter loads dbt-integration, which commands above don't need + const { create } = await import("./adapter") let adapter + const buildNotes: string[] = [] + const onPackagesInstalled = () => { + buildNotes.push("Build re-installed packages and reset the package directory (dbt_packages by default). Edits inside it may have been lost.") + } try { - const { create } = await import("./adapter") - adapter = await create(cfg) + adapter = await create(cfg, cmd === "build" ? { onPackagesInstalled } : {}) } catch (err) { - return failure(err) + return { ...failure(err), ...(buildNotes.length > 0 && { notes: buildNotes }) } } + let adapterDisposed = false let result: unknown try { switch (cmd) { @@ -164,7 +179,23 @@ async function main() { result = await (await import("./commands/compile")).query(adapter, rest) break case "build": - result = await (await import("./commands/build")).build(adapter, rest) + result = await (await import("./commands/build")).build(adapter, rest, async (verify) => { + await adapter.dispose() + adapterDisposed = true + const targetPath = await mkdtemp(join(tmpdir(), "altimate-dbt-verify-")) + try { + const verifier = await create(cfg, { targetPath, onPackagesInstalled }) + try { + const answer = await verify(verifier) + buildNotes.push("Schema verification ran after build. Its compile artifacts use a separate target directory so the build artifacts remain available.") + return answer + } finally { + await verifier.dispose() + } + } finally { + await rm(targetPath, { recursive: true, force: true }) + } + }) break case "run": result = await (await import("./commands/build")).run(adapter, rest) @@ -194,19 +225,21 @@ async function main() { result = await (await import("./commands/graph")).parents(adapter, rest) break case "deps": - result = await (await import("./commands/deps")).deps(adapter) + result = await (await import("./commands/deps")).deps(adapter, cfg.projectRoot) break case "add-packages": - result = await (await import("./commands/deps")).add(adapter, rest) + result = await (await import("./commands/deps")).add(adapter, rest, cfg.projectRoot) break default: result = { error: `Unknown command: ${cmd}`, usage: USAGE } } } finally { - try { await adapter.dispose() } catch {} + if (!adapterDisposed) { + try { await adapter.dispose() } catch {} + } } - return result + return buildNotes.length && result && typeof result === "object" ? { ...result, notes: buildNotes } : result } main().then(output).catch(bail) diff --git a/packages/dbt-tools/src/manifest-retry.ts b/packages/dbt-tools/src/manifest-retry.ts new file mode 100644 index 0000000000..027eebd9fa --- /dev/null +++ b/packages/dbt-tools/src/manifest-retry.ts @@ -0,0 +1,36 @@ +/** + * Re-read the manifest when another process is rewriting it. + * + * Every `altimate-dbt` process runs `dbt parse` when it starts, and dbt rewrites + * `target/manifest.json` in place. A process that reads the file while a sibling + * is rewriting it gets nothing back: the library's `parseManifest` returns + * `undefined` and later calls fail with "not found in manifest" or "No manifest + * has been generated". The window is a few milliseconds, but processes started + * together move in lockstep (previously the unconditional `dbt deps` of each + * process spread them out), so it was hit in about 1 call in 9 when four ran at once. + * The file is complete again moments later, so reading again is the repair. + */ + +export interface ManifestSource { + parseManifest(): Promise +} + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) + +/** Replace `source.parseManifest` with a version that retries while the result is missing. */ +export function retryParseManifest( + source: ManifestSource, + opts: { attempts?: number; baseDelayMs?: number; shouldRetry?: () => boolean } = {}, +): void { + const attempts = opts.attempts ?? 6 + const base = opts.baseDelayMs ?? 150 + const original = source.parseManifest.bind(source) + source.parseManifest = async () => { + let result = await original() + for (let i = 1; result === undefined && i < attempts && (opts.shouldRetry?.() ?? true); i++) { + await sleep(base * i + Math.floor(Math.random() * base)) + result = await original() + } + return result + } +} diff --git a/packages/dbt-tools/src/packages.ts b/packages/dbt-tools/src/packages.ts new file mode 100644 index 0000000000..5731c65c2a --- /dev/null +++ b/packages/dbt-tools/src/packages.ts @@ -0,0 +1,682 @@ +/** + * dbt package installation that is safe to run from many `altimate-dbt` + * processes at once. + * + * Why this exists: `@altimateai/dbt-integration` runs `dbt deps` unconditionally + * the first time an adapter parses the manifest (its + * `installDepsOnProjectInitialization` setting, which `adapter.ts` now turns + * off). `dbt deps` deletes and re-extracts every package directory, so two + * `altimate-dbt` processes started together (the finish-time validators start + * several) could leave `dbt_packages//dbt_project.yml` missing or half + * written, which makes every later dbt command fail with "No dbt_project.yml + * found". Installation is now (1) skipped when the installed packages already + * satisfy what the project declares, and (2) serialised by a cross-process + * lock with a stale-owner story when it is needed. + * + * State lives under `/target/.altimate-dbt/`: + * packages.lock/ mkdir-based lock; owner.json inside; mtime = heartbeat + * packages.lock.break/ short-lived guard so only one process breaks a stale lock + * packages.stamp.json fingerprint of the declaration files + package dirs of the last good install + * packages.dirty present while an install is running or after one did not finish + */ + +import { spawn, spawnSync } from "child_process" +import { createHash, randomUUID } from "crypto" +import { + existsSync, + mkdirSync, + readFileSync, + readdirSync, + renameSync, + rmSync, + statSync, + utimesSync, + writeFileSync, +} from "fs" +import { AsyncLocalStorage } from "node:async_hooks" +import { hostname } from "os" +import { isAbsolute, join, relative, resolve } from "path" +import { parse as parseYaml } from "yaml" +import { bufferLog } from "./log-buffer" +import { buildDbtEnv, resolveDbt } from "./dbt-resolve" + +const STATE_DIR = join("target", ".altimate-dbt") +const DECLARATION_FILES = ["packages.yml", "dependencies.yml", "package-lock.yml"] as const +const DEFAULT_INSTALL_PATH = "dbt_packages" + +/** Fall back to the heartbeat for locks whose process lifetime cannot be checked locally. */ +export const DEFAULT_STALE_MS = 30_000 +const DEFAULT_HEARTBEAT_MS = 2_000 +/** Upper bound on waiting for another process's install before giving up on the lock. */ +const DEFAULT_MAX_WAIT_MS = 10 * 60_000 +const DEPS_TIMEOUT_MS = 5 * 60_000 +const BREAK_GUARD_STALE_MS = 30_000 + +export interface LockOptions { + staleMs?: number + heartbeatMs?: number + maxWaitMs?: number +} + +/** Thrown when another process holds the package install lock for longer than we are willing to wait. */ +export class PackageLockTimeoutError extends Error {} + +export type EnsureAction = "none" | "skipped" | "skipped-after-wait" | "installed" + +export interface EnsureResult { + action: EnsureAction + /** Why an install was (or was not) needed. */ + reason: string +} + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const jitter = (base: number) => base + Math.floor(Math.random() * base) + +// --------------------------------------------------------------------------- +// Declared vs installed +// --------------------------------------------------------------------------- + +interface Declared { + /** Hub packages carry a predictable install directory name; others do not. */ + hubDir?: string +} + +function readYaml(path: string): unknown { + try { + return parseYaml(readFileSync(path, "utf-8")) + } catch { + return undefined + } +} + +function projectInstallPath(root: string): string | undefined { + const cfg = readYaml(join(root, "dbt_project.yml")) as Record | undefined + const raw = cfg?.["packages-install-path"] + if (typeof raw !== "string" || raw.length === 0) return resolve(root, DEFAULT_INSTALL_PATH) + if (!raw.includes("{{") && !raw.includes("{%")) return resolve(root, raw) + + // Resolve dbt's common env_var form without guessing the result of other Jinja. + const envVar = /^\s*{{\s*env_var\s*\(\s*(['"])([A-Za-z_][A-Za-z0-9_]*)\1\s*(?:,\s*(['"])(.*?)\3\s*)?\)\s*}}\s*$/.exec(raw) + if (!envVar) return + const value = process.env[envVar[2]!] ?? envVar[4] + if (value) return resolve(root, value) +} + +/** Check current declarations as well as the lock, which can lag the project. */ +function declaredPackages(root: string): Declared[] { + const packages = new Map() + for (const file of ["package-lock.yml", "packages.yml", "dependencies.yml"]) { + const doc = readYaml(join(root, file)) as { packages?: unknown } | undefined + if (!Array.isArray(doc?.packages)) continue + for (const entry of doc.packages) { + if (!entry || typeof entry !== "object") continue + const spec = entry as Record + if (typeof spec.package === "string") { + packages.set(`hub:${spec.package}`, { hubDir: spec.package.split("/").pop() ?? spec.package }) + } else { + const kind = ["local", "git", "tarball", "private"].find((key) => typeof spec[key] === "string") + const identity = kind ? JSON.stringify([kind, spec[kind], spec.subdirectory ?? ""]) : JSON.stringify(spec) + packages.set(identity, {}) + } + } + } + return [...packages.values()] +} + +function packageEntries(installPath: string): string[] | undefined { + try { + return readdirSync(installPath) + } catch (e) { + if ((e as NodeJS.ErrnoException).code === "ENOENT") return [] + bufferLog(`[dbt-tools] could not inspect the package directory: ${String(e)}`) + } +} + +function inspectPackages(installPath: string) { + const entries = packageEntries(installPath) + if (!entries) return undefined + const complete = new Set() + const incomplete: string[] = [] + for (const name of entries) { + if (name.startsWith(".")) continue + const dir = join(installPath, name) + try { + if (!statSync(dir).isDirectory()) continue + } catch { + continue // dangling symlink + } + // A truncated or half-written file is not a complete package. + const project = readYaml(join(dir, "dbt_project.yml")) + if (project && typeof project === "object" && !Array.isArray(project)) complete.add(name) + else incomplete.push(name) + } + return { complete, incomplete } +} + +const PACKAGE_ENV_VAR = /\benv_var\s*\(\s*(['"])([A-Za-z_][A-Za-z0-9_]*)\1/g + +function declarationContents(root: string) { + return DECLARATION_FILES.flatMap((file) => { + const path = join(root, file) + return existsSync(path) ? [{ file, text: readFileSync(path, "utf8") }] : [] + }) +} + +function fingerprint( + root: string, + installPath: string, + declarations = declarationContents(root), +): string { + const h = createHash("sha256") + // The install path decides which directories hold the packages, so a change of it invalidates the stamp. + // Relative to the project when inside it, so reaching the same project through a symlink gives + // the same fingerprint; a path outside the project is kept as written. + const rel = relative(root, installPath) + h.update(rel.startsWith("..") || isAbsolute(rel) ? installPath : rel).update("\0") + for (const { file, text } of declarations) { + h.update(file).update("\0").update(text).update("\0") + for (const match of text.matchAll(PACKAGE_ENV_VAR)) { + const name = match[2]! + h.update(JSON.stringify([name, process.env[name]])).update("\0") + } + } + return h.digest("hex") +} + +function stateDir(root: string): string { + return join(root, STATE_DIR) +} + +interface Stamp { + fingerprint: string + packages: string[] +} + +function readStamp(root: string): Stamp | undefined { + try { + return JSON.parse(readFileSync(join(stateDir(root), "packages.stamp.json"), "utf-8")) as Stamp + } catch { + return undefined + } +} + +export interface SatisfiedResult { + ok: boolean + reason: string +} + +/** + * Do the installed packages satisfy what the project declares? Read-only and + * cheap; this is what lets the common case skip `dbt deps` entirely. + */ +export function packagesSatisfied(root: string): SatisfiedResult { + const declared = declaredPackages(root) + if (declared.length === 0) return { ok: true, reason: "no packages declared" } + if (existsSync(join(stateDir(root), "packages.dirty"))) + return { ok: false, reason: "a previous or concurrent install has not finished" } + + const installPath = projectInstallPath(root) + if (!installPath) return { ok: false, reason: "packages-install-path cannot be determined before dbt runs" } + const inspection = inspectPackages(installPath) + if (!inspection) return { ok: false, reason: "packages-install-path cannot be inspected" } + const { complete: installed, incomplete } = inspection + if (incomplete.length > 0) return { ok: false, reason: `incomplete package directory: ${incomplete.join(", ")}` } + + const declarations = declarationContents(root) + const stamp = readStamp(root) + if (stamp) { + if (stamp.fingerprint !== fingerprint(root, installPath, declarations)) + return { ok: false, reason: "package declarations changed since the last install" } + const gone = stamp.packages.filter((p) => !installed.has(p)) + if (gone.length > 0) return { ok: false, reason: `installed package missing: ${gone.join(", ")}` } + return { ok: true, reason: "installed packages match the last install of these declarations" } + } + + if (declarations.some(({ text }) => !text.matchAll(PACKAGE_ENV_VAR).next().done)) { + return { ok: false, reason: "environment-dependent packages have no install stamp" } + } + + // A lock older than the declarations, or an installation older than either, + // may represent a different set of versions even if the directory names match. + const declaredAt = Math.max( + ...["packages.yml", "dependencies.yml"].map((file) => statSync(join(root, file), { throwIfNoEntry: false })?.mtimeMs ?? 0), + ) + const lockedAt = statSync(join(root, "package-lock.yml"), { throwIfNoEntry: false })?.mtimeMs ?? 0 + const installedAt = statSync(installPath, { throwIfNoEntry: false })?.mtimeMs + if (lockedAt && declaredAt > lockedAt) return { ok: false, reason: "packages.yml is newer than package-lock.yml" } + if (installedAt !== undefined && (declaredAt > installedAt || lockedAt > installedAt + 1000)) { + return { ok: false, reason: "package declarations are newer than the installed package directory" } + } + + // No stamp: packages were installed by something else (image build, `dbt deps` by hand). + // Hub packages can be verified by name. The version is not compared: the `version:` in an + // installed hub package's dbt_project.yml is not kept in step with its release (a package + // locked at 1.3.0 can report 0.1.0), so comparing it would reinstall on every start. + let unverifiable = 0 + for (const d of declared) { + if (!d.hubDir) { + unverifiable++ + continue + } + if (!installed.has(d.hubDir)) return { ok: false, reason: `package ${d.hubDir} is not installed` } + } + const nonHubInstalled = [...installed.keys()].filter((n) => !declared.some((d) => d.hubDir === n)).length + if (nonHubInstalled < unverifiable) return { ok: false, reason: "fewer non-hub packages installed than declared" } + return { ok: true, reason: "installed packages satisfy the declarations" } +} + +// --------------------------------------------------------------------------- +// Cross-process lock +// --------------------------------------------------------------------------- + +interface Owner { + pid: number + host: string + token: string + startedAt: number + /** Null means the installer is starting or its process is managed by the adapter. */ + installerPid?: number | null + installerStarted?: string + processStarted?: string +} + +type TrackInstaller = (pid: number | null | undefined) => void + +function processIdentity(pid: number): { started: string; exited: boolean } | undefined { + if (process.platform === "linux") { + try { + // comm is parenthesized and may contain spaces or parentheses. + const stat = readFileSync(`/proc/${pid}/stat`, "utf8") + const fields = stat.slice(stat.lastIndexOf(")") + 2).split(" ") + if (fields[19]) return { started: fields[19], exited: fields[0] === "Z" || fields[0] === "X" } + } catch {} + } + if (process.platform === "darwin") { + const result = spawnSync("/bin/ps", ["-p", String(pid), "-o", "lstart=", "-o", "stat="], { encoding: "utf8", timeout: 500 }) + const match = result.stdout?.trim().match(/^(.*?)\s+(\S+)$/) + if (result.status === 0 && match?.[1]) return { started: match[1], exited: match[2]?.startsWith("Z") === true } + } +} + +function pidAlive(pid: number, started?: string): boolean { + if (!Number.isSafeInteger(pid) || pid <= 0) return false + try { + process.kill(pid, 0) + } catch (e) { + return (e as NodeJS.ErrnoException).code === "EPERM" + } + if (started === undefined) return true + const identity = processIdentity(pid) + return !identity || (!identity.exited && identity.started === started) +} + +function readOwner(lockDir: string): Owner | undefined { + try { + return JSON.parse(readFileSync(join(lockDir, "owner.json"), "utf-8")) as Owner + } catch { + return undefined + } +} + +function lockIsStale(lockDir: string, staleMs: number): boolean { + let mtime: number + try { + mtime = statSync(lockDir).mtimeMs + } catch { + return false // already gone + } + const owner = readOwner(lockDir) + if (owner?.host === hostname()) { + if (pidAlive(owner.pid, owner.processStarted) || owner.installerPid === null) return false + return owner.installerPid === undefined || !pidAlive(owner.installerPid, owner.installerStarted) + } + return Date.now() - mtime > staleMs +} + +/** Remove a stale lock. A guard directory makes sure only one process does it at a time. */ +function breakStaleLock(lockDir: string, staleMs: number, observedToken: string | undefined): void { + const guard = `${lockDir}.break` + try { + mkdirSync(guard) + } catch { + try { + if (Date.now() - statSync(guard).mtimeMs > BREAK_GUARD_STALE_MS) rmSync(guard, { recursive: true, force: true }) + } catch {} + return + } + try { + // Re-check under the guard: the lock may have been released and re-taken since we looked. + // A new owner has a new token, so only the lock we judged stale is removed. + if (readOwner(lockDir)?.token === observedToken && lockIsStale(lockDir, staleMs)) { + bufferLog(`[dbt-tools] breaking stale package lock ${lockDir}`) + rmSync(lockDir, { recursive: true, force: true }) + } + } finally { + rmSync(guard, { recursive: true, force: true }) + } +} + +async function acquire(root: string, opts: Required): Promise<{ release: () => void; trackInstaller: TrackInstaller }> { + const dir = stateDir(root) + const lockDir = join(dir, "packages.lock") + mkdirSync(dir, { recursive: true }) + const deadline = Date.now() + opts.maxWaitMs + for (;;) { + try { + mkdirSync(lockDir) + break + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== "EEXIST") throw e + if (lockIsStale(lockDir, opts.staleMs)) breakStaleLock(lockDir, opts.staleMs, readOwner(lockDir)?.token) + if (Date.now() > deadline) { + const owner = readOwner(lockDir) + const unknownInstaller = owner?.host === hostname() && owner.installerPid === null && !pidAlive(owner.pid, owner.processStarted) + const recovery = unknownInstaller ? " Verify the previous dbt install has stopped before removing this lock." : "" + throw new PackageLockTimeoutError(`timed out after ${opts.maxWaitMs}ms waiting for the package install lock at ${lockDir}.${recovery}`) + } + await sleep(jitter(50)) + } + } + const owner: Owner = { + pid: process.pid, + host: hostname(), + token: randomUUID(), + startedAt: Date.now(), + processStarted: processIdentity(process.pid)?.started, + } + writeFileSync(join(lockDir, "owner.json"), JSON.stringify(owner)) + const beat = setInterval(() => { + try { + if (readOwner(lockDir)?.token !== owner.token) { + bufferLog("[dbt-tools] package install lock was taken over; continuing without it") + clearInterval(beat) + return + } + const now = new Date() + utimesSync(lockDir, now, now) + } catch {} + }, opts.heartbeatMs) + beat.unref() + const trackInstaller: TrackInstaller = (pid) => { + if (readOwner(lockDir)?.token !== owner.token) throw new Error("Lost the package install lock") + // Publish each update atomically so readers always see the owner or its installer. + const temp = join(lockDir, `owner.${owner.token}.tmp`) + const installerStarted = typeof pid === "number" ? processIdentity(pid)?.started : undefined + writeFileSync(temp, JSON.stringify({ ...owner, installerPid: pid, installerStarted })) + renameSync(temp, join(lockDir, "owner.json")) + } + const release = () => { + clearInterval(beat) + // Only remove the lock if it is still ours. Releasing is best-effort: a failure to delete + // must not replace the result of the install that already finished. + try { + if (readOwner(lockDir)?.token === owner.token) rmSync(lockDir, { recursive: true, force: true }) + } catch (e) { + bufferLog(`[dbt-tools] could not release the package lock: ${String(e)}`) + } + } + return { release, trackInstaller } +} + +function withDefaults(o?: LockOptions): Required { + return { + staleMs: o?.staleMs ?? DEFAULT_STALE_MS, + heartbeatMs: o?.heartbeatMs ?? DEFAULT_HEARTBEAT_MS, + maxWaitMs: o?.maxWaitMs ?? DEFAULT_MAX_WAIT_MS, + } +} + +// A CLI command with an unverified destination holds the lock across all +// adapter work, including nested installs and the post-build verification. +const commandPackageLock = new AsyncLocalStorage<{ root: string; trackInstaller: TrackInstaller }>() + +async function underPackagesLock( + root: string, + fn: (trackInstaller: TrackInstaller) => Promise, + opts?: LockOptions, +): Promise { + const held = commandPackageLock.getStore() + if (held?.root === root) return fn(held.trackInstaller) + + const { release, trackInstaller } = await acquire(root, withDefaults(opts)) + try { + return await fn(trackInstaller) + } finally { + release() + } +} + +/** Run `fn` while holding the project's package lock. */ +export async function withPackagesLock(root: string, fn: () => Promise, opts?: LockOptions): Promise { + return underPackagesLock(root, fn, opts) +} + +export async function withUnverifiedPackagesLock(root: string, fn: () => Promise): Promise { + if (declaredPackages(root).length === 0) return fn() + const installPath = projectInstallPath(root) + if (installPath && packageEntries(installPath)) return fn() + // A waiting caller will have to run deps again. Keep it from deleting the + // first caller's packages until that caller has finished reading them. + return underPackagesLock(root, (trackInstaller) => commandPackageLock.run({ root, trackInstaller }, fn)) +} + +// --------------------------------------------------------------------------- +// Install +// --------------------------------------------------------------------------- + +// dbt writes package-lock.yml during deps. The project declarations and the +// destination it read must stay stable while the installer is running. +function installInputs(root: string, installPath: string, declarations = declarationContents(root)): string { + return fingerprint(root, installPath, declarations.filter(({ file }) => file !== "package-lock.yml")) +} + +/** + * Run an install while holding the lock, bracketed by the `dirty` marker so a + * crash or failure mid-install is noticed (and repaired) by the next caller. + * Caller must already hold the lock. + */ +async function installUnderLock(root: string, install: () => Promise, record: boolean): Promise { + const dir = stateDir(root) + const dirty = join(dir, "packages.dirty") + mkdirSync(dir, { recursive: true }) // `dbt clean` may have removed target/ while we waited + writeFileSync(dirty, JSON.stringify({ pid: process.pid, startedAt: Date.now() })) + const originalPath = projectInstallPath(root) + const inputsBefore = originalPath && installInputs(root, originalPath) + const result = await install() + const installPath = projectInstallPath(root) + const stampFile = join(dir, "packages.stamp.json") + const inspection = installPath ? inspectPackages(installPath) : undefined + if (!installPath || !inspection) { + // dbt can resolve Jinja itself. Its output is authoritative + // when we cannot inspect the resulting directory. + rmSync(stampFile, { force: true }) + if (!failedCommand(result)) rmSync(dirty, { force: true }) + return result + } + const declarationsAfter = declarationContents(root) + if (inputsBefore !== installInputs(root, installPath, declarationsAfter)) { + // The install may have consumed the earlier declarations. Leave the dirty + // marker so the next adapter-backed command repairs the current inputs. + rmSync(stampFile, { force: true }) + return result + } + const complete = [...inspection.complete] + const broken = inspection.incomplete + // An explicit install (record = false) returns dbt's own failure result instead, so the + // caller can show dbt's message rather than this one; the dirty marker stays for the next check. + const incomplete = declaredPackages(root).length > 0 && (complete.length === 0 || broken.length > 0) + if (incomplete && record) { + throw new Error(`dbt deps finished but the package directory is incomplete${broken.length ? `: ${broken.join(", ")}` : ""}`) + } + if (record || (!incomplete && commandExitCode(result) === 0)) { + writeFileSync(stampFile, JSON.stringify({ fingerprint: fingerprint(root, installPath, declarationsAfter), packages: complete } satisfies Stamp)) + } else { + // An explicit install may return no exit status. The next call verifies + // the directories rather than trusting the old stamp. + rmSync(stampFile, { force: true }) + } + if (!incomplete && !failedCommand(result)) rmSync(dirty, { force: true }) + return result +} + +function commandExitCode(result: unknown): number | undefined { + if (result && typeof result === "object" && "exitCode" in result && (typeof result.exitCode === "number" || result.exitCode === null)) { + return result.exitCode ?? 1 + } +} + +function failedCommand(result: unknown): boolean { + if (!result || typeof result !== "object") return false + const exitCode = commandExitCode(result) + if (exitCode !== undefined) return exitCode !== 0 + const output = result as { stderr?: string; stdout?: string } + // Older engines can omit the status. dbt also emits progress on stderr, so + // that stream by itself does not establish a failure. + return /Encountered an error|(?:Compilation|Parsing|Runtime|Database) Error/i.test( + `${output.stderr ?? ""}\n${output.stdout ?? ""}`, + ) +} + +/** + * Make sure the declared packages are installed. Installs at most once per + * change of declarations, however many processes ask at the same time. + */ +export async function ensurePackages( + root: string, + install: (trackInstaller: TrackInstaller) => Promise, + opts?: LockOptions, +): Promise { + if (declaredPackages(root).length === 0) return { action: "none", reason: "no packages declared" } + const fast = packagesSatisfied(root) + if (fast.ok) return { action: "skipped", reason: fast.reason } + + const waitedFor = existsSync(join(stateDir(root), "packages.lock")) + return underPackagesLock(root, async (trackInstaller) => { + // Another process may have finished the install while we waited. + const again = packagesSatisfied(root) + if (again.ok) return { action: "skipped-after-wait", reason: waitedFor ? `${again.reason} (after waiting for the lock)` : again.reason } + await installUnderLock(root, () => install(trackInstaller), true) + return { action: "installed", reason: again.reason } + }, opts) +} + +/** + * Run an explicit install request (`altimate-dbt deps`, `add-packages`) under the lock. The + * library can report failure as a result. Stamp only a reported exit status of 0 + * with complete package directories. + */ +export async function installPackagesLocked(root: string, install: () => Promise, opts?: LockOptions): Promise { + return underPackagesLock(root, (trackInstaller) => + installUnderLock(root, () => explicitInstaller.run(trackInstaller, install), false), opts) +} + +const explicitInstaller = new AsyncLocalStorage() + +type InstallProcess = { + command: string + args?: string[] + cwd?: string + envVars?: NodeJS.ProcessEnv + stdin?: string + signal?: AbortSignal + timeoutMs?: number + maxBuffer?: number +} + +// The integration selects the appropriate command and flags for each dbt +// engine. Only its explicit package installs need our process tracker. +export function trackedExplicitInstaller(options: InstallProcess) { + const track = explicitInstaller.getStore() + if (track) return (formatOutput?: (text: string) => string) => runInstallProcess(options, track, formatOutput) +} + +function runInstallProcess(options: InstallProcess, track: TrackInstaller, formatOutput?: (text: string) => string) { + return new Promise<{ exitCode: number | null; stdout: string; stderr: string; fullOutput: string }>((resolve, reject) => { + let stdout = "" + let stderr = "" + let fullOutput = "" + if (options.signal?.aborted) { + reject(new Error("dbt deps was cancelled")) + return + } + + // A crash between spawning and publishing the PID must block reclaim. + track(null) + const child = spawn(options.command, options.args, { + cwd: options.cwd, + env: options.envVars, + windowsHide: true, + }) + let error: unknown + let forceKill: ReturnType | undefined + const abort = () => { + if (forceKill) return + child.kill("SIGTERM") + forceKill = setTimeout(() => child.kill("SIGKILL"), 5_000) + forceKill.unref() + } + const timeout = options.timeoutMs ? setTimeout(() => { + error ??= new Error("dbt deps timed out") + abort() + }, options.timeoutMs) : undefined + let bytes = 0 + const collect = (chunk: Buffer, stream: "stdout" | "stderr") => { + bytes += chunk.length + if (options.maxBuffer !== undefined && bytes > options.maxBuffer) { + error ??= new Error("dbt deps output exceeded its buffer limit") + abort() + return + } + const raw = chunk.toString() + const text = formatOutput ? formatOutput(raw) : raw + if (stream === "stdout") stdout += text + else stderr += text + fullOutput += text + } + options.signal?.addEventListener("abort", abort, { once: true }) + child.stdout.on("data", (chunk: Buffer) => collect(chunk, "stdout")) + child.stderr.on("data", (chunk: Buffer) => collect(chunk, "stderr")) + child.once("error", (err) => { error ??= err }) + child.stdin.on("error", (err) => { + if ((err as NodeJS.ErrnoException).code !== "EPIPE") error ??= err + }) + child.once("close", (exitCode, signal) => { + clearTimeout(timeout) + clearTimeout(forceKill) + options.signal?.removeEventListener("abort", abort) + try { + track(undefined) + } catch (err) { + error ??= err + } + if (error) reject(error) + else if (signal) reject(new Error(`dbt deps terminated by ${signal}`)) + else resolve({ exitCode, stdout, stderr, fullOutput }) + }) + try { + if (child.pid !== undefined) track(child.pid) + child.stdin.end(options.stdin) + } catch (err) { + error = err + abort() + } + }) +} + +/** An installer that shells out to `dbt deps` with the project's resolved dbt binary. */ +export function dbtDepsInstaller(cfg: { pythonPath?: string; projectRoot: string }): (trackInstaller: TrackInstaller) => Promise { + return async (trackInstaller) => { + const dbt = resolveDbt(cfg.pythonPath, cfg.projectRoot) + const result = await runInstallProcess({ + command: dbt.path, + args: ["deps"], + cwd: cfg.projectRoot, + envVars: buildDbtEnv(dbt), + timeoutMs: DEPS_TIMEOUT_MS, + maxBuffer: 10 * 1024 * 1024, + }, trackInstaller) + if (result.exitCode !== 0) { + throw new Error(`dbt deps failed: ${(result.stderr || result.stdout || "process terminated").slice(-500)}`) + } + } +} diff --git a/packages/dbt-tools/test/build-real-dbt.test.ts b/packages/dbt-tools/test/build-real-dbt.test.ts new file mode 100644 index 0000000000..d2b6c9b4a5 --- /dev/null +++ b/packages/dbt-tools/test/build-real-dbt.test.ts @@ -0,0 +1,68 @@ +import { expect, test } from "bun:test" +import { spawnSync } from "node:child_process" +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { dirname, join } from "node:path" +import { findRealDbt } from "./fixture/find-real-dbt" +import { DbtBuildGreenValidator } from "../../opencode/src/altimate/validators/dbt-build-green" + +const dist = join(import.meta.dir, "../dist/index.js") +const dbt = existsSync(dist) ? findRealDbt() : undefined +const realTest = dbt ? test : test.skip + +for (const mode of ["model", "project", "red"] as const) { + realTest(`real dbt ${mode} build retains its execution artifact`, async () => { + const root = mkdtempSync(join(tmpdir(), "dbt-build-artifact-")) + try { + mkdirSync(join(root, "models")) + writeFileSync(join(root, "dbt_project.yml"), "name: artifact\nversion: '1.0'\nconfig-version: 2\nprofile: artifact\ntarget-path: artifacts\n") + writeFileSync(join(root, "profiles.yml"), "artifact:\n target: dev\n outputs:\n dev:\n type: duckdb\n path: artifact.duckdb\n threads: 1\n") + writeFileSync(join(root, "models/orders.sql"), mode === "red" ? "select missing_column as id\n" : "select 1 as id\n") + writeFileSync(join(root, "models/orders.yml"), "version: 2\nmodels:\n - name: orders\n columns:\n - name: id\n tests: [not_null]\n") + const run = (...args: string[]) => spawnSync("node", [dist, ...args], { + cwd: root, + encoding: "utf8", + env: { ...process.env, HOME: root, XDG_CONFIG_HOME: join(root, ".config"), DBT_PROFILES_DIR: root, ALTIMATE_DBT_PATH: dbt ?? undefined }, + timeout: 90_000, + }) + if (mode === "model") { + mkdirSync(join(root, "local_pkg")) + writeFileSync(join(root, "local_pkg/dbt_project.yml"), "name: local_pkg\nversion: '1.0'\nconfig-version: 2\n") + writeFileSync(join(root, "packages.yml"), "packages:\n - local: local_pkg\n") + } + const init = run("init", "--project-root", root, "--python-path", join(dirname(dbt!), "python3")) + expect(init.status, init.stdout + init.stderr).toBe(0) + const output = run("build", ...(mode === "project" ? [] : ["--model", "orders"])) + const artifact = JSON.parse(readFileSync(join(root, "artifacts/run_results.json"), "utf8")) + expect(artifact.args.which, output.stdout + output.stderr).toBe("build") + expect(artifact.results.some((row: { status: string }) => row.status === (mode === "red" ? "error" : "pass"))).toBe(true) + if (mode === "red") { + expect(output.status, output.stdout + output.stderr).toBe(1) + } else { + expect(output.status, output.stdout + output.stderr).toBe(0) + const response = JSON.parse(output.stdout) + if (mode === "project") expect(response.schema_verify_summary.match).toBe(1) + else expect(response.schema_verify.verdict).toBe("match") + expect(response.notes.join(" ")).toContain("Schema verification ran after build") + if (mode === "model") expect(response.notes.join(" ")).toContain("re-installed packages and reset") + } + const validation = await DbtBuildGreenValidator.check({ + sessionID: "artifact", + workingDirectory: root, + sessionStartMs: 0, + step: 1, + retryCount: 0, + }) + expect(validation.ok, validation.reason).toBe(mode !== "red") + if (mode === "model") { + writeFileSync(join(root, "models/untested.sql"), "select 2 as id\n") + const untested = run("test", "--model", "untested", "--format", "text") + expect(untested.status, untested.stdout + untested.stderr).toBe(0) + expect(untested.stdout).toContain("Model has no tests") + expect(untested.stdout).not.toContain("Nothing to do") + } + } finally { + rmSync(root, { recursive: true, force: true }) + } + }, 120_000) +} diff --git a/packages/dbt-tools/test/build.test.ts b/packages/dbt-tools/test/build.test.ts index 1067870e3c..7149dd488b 100644 --- a/packages/dbt-tools/test/build.test.ts +++ b/packages/dbt-tools/test/build.test.ts @@ -1,3 +1,7 @@ +import { add, deps } from "../src/commands/deps" +import { mkdtempSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" import { describe, test, expect, mock } from "bun:test" import { build } from "../src/commands/build" import type { DBTProjectIntegrationAdapter } from "@altimateai/dbt-integration" @@ -71,7 +75,8 @@ describe("build command", () => { }) test("project-wide build collects per-model schema-verify mismatches", async () => { - // Mock manifest with 3 models: one matching spec, one mismatch (extra col), one no-spec. + // Mock manifest with 3 models: one matching spec, one mismatch (enforced contract violated), one no-spec. + // A column the YAML does not list is NOT a mismatch on its own (see schema-verify.test.ts). const matchingNode = { resource_type: "model", name: "users_dim", @@ -80,6 +85,7 @@ describe("build command", () => { const mismatchNode = { resource_type: "model", name: "products_dim", + config: { contract: { enforced: true } }, columns: { id: { name: "id", description: "", data_type: "INT" } }, } const nospecNode = { resource_type: "model", name: "legacy_facts", columns: {} } @@ -114,6 +120,30 @@ describe("build command", () => { expect(summary.errored).toBe(0) expect(summary.mismatches[0]?.model).toBe("products_dim") expect(summary.mismatches[0]?.columns_extra).toContain("extra_col") + expect((summary.mismatches[0] as unknown as { findings: Array<{ kind: string }> }).findings[0]?.kind).toBe("contract-extra-columns") + }) + + test("project-wide build does not report a model whose only difference is columns the YAML does not list", async () => { + const node = { + resource_type: "model", + name: "orders", + columns: { id: { name: "id", description: "", data_type: "" } }, + } + const adapter = makeAdapter({ + parseManifest: mock(() => Promise.resolve({ + nodeMetaMap: { + lookupByBaseName: mock(() => node), + lookupByUniqueId: mock(() => undefined), + nodes: mock(() => [node][Symbol.iterator]()), + }, + } as never)), + getColumnsOfModel: mock(() => Promise.resolve([{ column: "id", dtype: "INT" }, { column: "total", dtype: "INT" }])), + }) + const result = await build(adapter, []) + const summary = (result as unknown as { schema_verify_summary: { match: number; mismatch: number; mismatches: unknown[] } }).schema_verify_summary + expect(summary.match).toBe(1) + expect(summary.mismatch).toBe(0) + expect(summary.mismatches).toEqual([]) }) test("build surfaces stderr as error", async () => { @@ -126,3 +156,18 @@ describe("build command", () => { expect(result).toEqual({ error: "compilation error", stdout: "partial output" }) }) }) + +test("deps and add-packages keep successful stderr output from becoming an error", async () => { + const root = mkdtempSync(join(tmpdir(), "dbt-deps-output-")) + try { + const success = { stdout: "Installed package", stderr: "Progress logged to stderr", exitCode: 0 } + const adapter = { + installDeps: async () => success, + installDbtPackages: async () => success, + } as unknown as DBTProjectIntegrationAdapter + expect(await deps(adapter, root)).toEqual({ stdout: success.stdout }) + expect(await add(adapter, ["--packages", "example/util"], root)).toEqual({ stdout: success.stdout }) + } finally { + rmSync(root, { recursive: true, force: true }) + } +}) diff --git a/packages/dbt-tools/test/cli.test.ts b/packages/dbt-tools/test/cli.test.ts index 659bd12db7..d08454cf85 100644 --- a/packages/dbt-tools/test/cli.test.ts +++ b/packages/dbt-tools/test/cli.test.ts @@ -1,11 +1,24 @@ -import { describe, test, expect } from "bun:test" +import { afterEach, beforeEach, describe, test, expect } from "bun:test" +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "fs" +import { tmpdir } from "os" import { join } from "path" const entry = join(import.meta.dir, "../src/index.ts") describe("cli", () => { + let home: string + const env = () => ({ ...process.env, HOME: home, XDG_CONFIG_HOME: join(home, ".config") }) + + beforeEach(() => { + home = mkdtempSync(join(tmpdir(), "dbt-tools-cli-")) + }) + + afterEach(() => { + rmSync(home, { recursive: true, force: true }) + }) + test("no args prints usage", async () => { - const result = Bun.spawnSync(["bun", entry], { env: { ...process.env, HOME: "/tmp/dbt-tools-test-home" } }) + const result = Bun.spawnSync(["bun", entry], { env: env() }) const out = result.stdout.toString() const parsed = JSON.parse(out) expect(parsed.commands).toBeDefined() @@ -14,29 +27,29 @@ describe("cli", () => { }) test("help prints usage", async () => { - const result = Bun.spawnSync(["bun", entry, "help"], { env: { ...process.env, HOME: "/tmp/dbt-tools-test-home" } }) + const result = Bun.spawnSync(["bun", entry, "help"], { env: env() }) const out = result.stdout.toString() const parsed = JSON.parse(out) expect(parsed.commands).toBeDefined() }) test("unknown command without config produces config error", async () => { - const result = Bun.spawnSync(["bun", entry, "nonexistent"], { env: { ...process.env, HOME: "/tmp/dbt-tools-test-home" } }) + const result = Bun.spawnSync(["bun", entry, "nonexistent"], { env: env() }) const out = result.stdout.toString() const parsed = JSON.parse(out) expect(parsed.error).toContain("altimate-dbt init") }) test("missing config produces init hint", async () => { - const result = Bun.spawnSync(["bun", entry, "info"], { env: { ...process.env, HOME: "/tmp/dbt-tools-test-home" } }) + const result = Bun.spawnSync(["bun", entry, "info"], { env: env() }) const out = result.stdout.toString() const parsed = JSON.parse(out) expect(parsed.error).toContain("altimate-dbt init") }) test("init without dbt_project.yml produces error", async () => { - const result = Bun.spawnSync(["bun", entry, "init", "--project-root", "/tmp/nonexistent-dbt-project"], { - env: { ...process.env, HOME: "/tmp/dbt-tools-test-home" }, + const result = Bun.spawnSync(["bun", entry, "init", "--project-root", join(home, "nonexistent-dbt-project")], { + env: env(), }) const out = result.stdout.toString() const parsed = JSON.parse(out) @@ -44,30 +57,26 @@ describe("cli", () => { }) test("usage gives children and parents' answer on a stale manifest", async () => { - const result = Bun.spawnSync(["bun", entry], { env: { ...process.env, HOME: "/tmp/dbt-tools-test-home" } }) + const result = Bun.spawnSync(["bun", entry], { env: env() }) const parsed = JSON.parse(result.stdout.toString()) expect(parsed.commands.children).toContain("{ stale_manifest, models }") expect(parsed.commands.parents).toContain("{ stale_manifest, models }") }) test("usage includes doctor command", async () => { - const result = Bun.spawnSync(["bun", entry], { env: { ...process.env, HOME: "/tmp/dbt-tools-test-home" } }) + const result = Bun.spawnSync(["bun", entry], { env: env() }) const parsed = JSON.parse(result.stdout.toString()) expect(parsed.commands.doctor).toBeDefined() }) test("doctor without config produces init hint", async () => { - const result = Bun.spawnSync(["bun", entry, "doctor"], { env: { ...process.env, HOME: "/tmp/dbt-tools-test-home" } }) + const result = Bun.spawnSync(["bun", entry, "doctor"], { env: env() }) const parsed = JSON.parse(result.stdout.toString()) expect(parsed.error).toContain("altimate-dbt init") }) test("info with bad prerequisites shows actionable error", async () => { - const { mkdtempSync, writeFileSync } = await import("fs") - const { join } = await import("path") - const home = mkdtempSync("/tmp/dbt-tools-err-") const dir = join(home, ".altimate-code") - const { mkdirSync } = await import("fs") mkdirSync(dir, { recursive: true }) writeFileSync(join(dir, "dbt.json"), JSON.stringify({ projectRoot: "/tmp/definitely-not-a-project", @@ -75,7 +84,7 @@ describe("cli", () => { dbtIntegration: "core", queryLimit: 500, })) - const result = Bun.spawnSync(["bun", entry, "info"], { env: { ...process.env, HOME: home } }) + const result = Bun.spawnSync(["bun", entry, "info"], { env: env() }) const parsed = JSON.parse(result.stdout.toString()) expect(parsed.error).toContain("Prerequisites not met") expect(parsed.error).toContain("project") diff --git a/packages/dbt-tools/test/e2e/dbt-versions.test.ts b/packages/dbt-tools/test/e2e/dbt-versions.test.ts index a90586f872..123b95a204 100644 --- a/packages/dbt-tools/test/e2e/dbt-versions.test.ts +++ b/packages/dbt-tools/test/e2e/dbt-versions.test.ts @@ -127,6 +127,8 @@ function dbt( timeout, env: { ...process.env, + HOME: workDir, + XDG_CONFIG_HOME: join(workDir, ".config"), DBT_PROFILES_DIR: workDir, // Ensure the venv's python is first on PATH so dbt finds it PATH: `${join(version.dbtPath, "..")}:${process.env.PATH}`, @@ -162,6 +164,8 @@ function altDbt( ...process.env, // Point altimate-dbt config at our temp project HOME: projectRoot, + XDG_CONFIG_HOME: join(projectRoot, ".config"), + DBT_PROFILES_DIR: projectRoot, }, timeout, }) @@ -175,8 +179,7 @@ function altDbt( } /** Create a temp copy of the fixture project and bootstrap it for a specific dbt version. */ -function setupProject(version: DbtVersion): string { - const workDir = mkdtempSync(join(tmpdir(), `dbt-e2e-${version.label}-`)) +function setupProject(version: DbtVersion, workDir: string): void { cpSync(FIXTURE_DIR, workDir, { recursive: true }) // Write profiles.yml with absolute DuckDB path (dbt 1.7 resolves relative paths from CWD) @@ -204,8 +207,6 @@ function setupProject(version: DbtVersion): string { // Seed + build so models exist in the database dbt(version, workDir, ["seed"]) dbt(version, workDir, ["build"]) - - return workDir } // --------------------------------------------------------------------------- @@ -216,10 +217,19 @@ describe.skipIf(!HAS_DBT)("altimate-dbt e2e", () => { for (const version of VERSIONS) { describe(`dbt ${version.label} (${version.full})`, () => { let workDir: string + let savedHome: string | undefined + let savedConfigHome: string | undefined + let savedProfilesDir: string | undefined beforeAll(async () => { console.log(`\n→ Setting up dbt ${version.full} project...`) - workDir = setupProject(version) + savedHome = process.env.HOME + savedConfigHome = process.env.XDG_CONFIG_HOME + savedProfilesDir = process.env.DBT_PROFILES_DIR + workDir = mkdtempSync(join(tmpdir(), `dbt-e2e-${version.label}-`)) + process.env.HOME = workDir + process.env.XDG_CONFIG_HOME = join(workDir, ".config") + setupProject(version, workDir) console.log(` Project: ${workDir}`) // dbt-cli.ts caches the dbt it resolves on first use. configure() // clears that cache and points it at this version's venv; without it @@ -229,6 +239,12 @@ describe.skipIf(!HAS_DBT)("altimate-dbt e2e", () => { }, DBT_TIMEOUT * 2) afterAll(() => { + if (savedHome === undefined) delete process.env.HOME + else process.env.HOME = savedHome + if (savedConfigHome === undefined) delete process.env.XDG_CONFIG_HOME + else process.env.XDG_CONFIG_HOME = savedConfigHome + if (savedProfilesDir === undefined) delete process.env.DBT_PROFILES_DIR + else process.env.DBT_PROFILES_DIR = savedProfilesDir if (workDir) { try { rmSync(workDir, { recursive: true, force: true }) diff --git a/packages/dbt-tools/test/fixture/find-real-dbt.ts b/packages/dbt-tools/test/fixture/find-real-dbt.ts new file mode 100644 index 0000000000..525972acd5 --- /dev/null +++ b/packages/dbt-tools/test/fixture/find-real-dbt.ts @@ -0,0 +1,12 @@ +import { spawnSync } from "node:child_process" +import { existsSync } from "node:fs" +import { join } from "node:path" + +export function findRealDbt(): string | undefined { + if (process.platform === "win32" || process.env.DBT_E2E_SKIP === "1") return + const pinned = join(import.meta.dir, "../.dbt-venvs/1.10/bin/dbt") + const dbt = existsSync(pinned) ? pinned : Bun.which("dbt") + if (!dbt) return + const version = spawnSync(dbt, ["--version"], { encoding: "utf8", timeout: 20_000 }) + if (version.status === 0 && /duckdb/i.test(version.stdout + version.stderr)) return dbt +} diff --git a/packages/dbt-tools/test/fixture/packages-worker.ts b/packages/dbt-tools/test/fixture/packages-worker.ts new file mode 100644 index 0000000000..44b58e1483 --- /dev/null +++ b/packages/dbt-tools/test/fixture/packages-worker.ts @@ -0,0 +1,42 @@ +// Child process used by test/packages.test.ts: runs ensurePackages against a +// project with a deliberately slow, non-atomic fake `dbt deps`, then reports +// what it observed once ensurePackages returned. +// +// usage: bun packages-worker.ts [crash|dbt|explicit] +import { appendFileSync, existsSync, mkdirSync, rmSync, writeFileSync } from "fs" +import { join } from "path" +import { dbtDepsInstaller, ensurePackages, installPackagesLocked, trackedExplicitInstaller } from "../../src/packages" + +const [root, log, pkg, mode] = process.argv.slice(2) as [string, string, string, string | undefined] +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) + +// What dbt deps does to a package: delete it, re-create the directory, then +// write its files. A reader that looks in between sees a missing or empty package. +async function slowInstall() { + appendFileSync(log, `install ${process.pid}\n`) + const dir = join(root, "dbt_packages", pkg) + rmSync(dir, { recursive: true, force: true }) + await sleep(150) + mkdirSync(dir, { recursive: true }) + if (mode === "crash") { + // Die with the package directory present but incomplete. + process.kill(process.pid, "SIGKILL") + } + await sleep(150) + writeFileSync(join(dir, "dbt_project.yml"), `name: ${pkg}\nversion: '1.0.0'\n`) + await sleep(50) +} + +const opts = { staleMs: 1500, heartbeatMs: 100 } +if (mode === "explicit") { + await installPackagesLocked(root, async () => { + const install = trackedExplicitInstaller({ command: process.env.ALTIMATE_DBT_PATH!, args: ["deps"], cwd: root }) + return install!() + }, opts) +} else { + const install = mode === "dbt" ? dbtDepsInstaller({ projectRoot: root }) : slowInstall + await ensurePackages(root, install, opts) +} +// This is the moment a real command would go on to read the packages. +const ok = existsSync(join(root, "dbt_packages", pkg, "dbt_project.yml")) +appendFileSync(log, `${ok ? "ready" : "BROKEN"} ${process.pid}\n`) diff --git a/packages/dbt-tools/test/manifest-retry.test.ts b/packages/dbt-tools/test/manifest-retry.test.ts new file mode 100644 index 0000000000..bb6d05cfb0 --- /dev/null +++ b/packages/dbt-tools/test/manifest-retry.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, test } from "bun:test" +import { retryParseManifest } from "../src/manifest-retry" + +function source(results: Array) { + let calls = 0 + return { + get calls() { + return calls + }, + parseManifest: async () => results[Math.min(calls++, results.length - 1)] as { nodes: number } | undefined, + } +} + +describe("retryParseManifest", () => { + test("a manifest that is whole on the first read is read once", async () => { + const s = source([{ nodes: 1 }]) + retryParseManifest(s, { baseDelayMs: 1 }) + expect(await s.parseManifest()).toEqual({ nodes: 1 }) + expect(s.calls).toBe(1) + }) + + test("a manifest caught mid-rewrite is read again until it is whole", async () => { + const s = source([undefined, undefined, { nodes: 2 }]) + retryParseManifest(s, { baseDelayMs: 1 }) + expect(await s.parseManifest()).toEqual({ nodes: 2 }) + expect(s.calls).toBe(3) + }) + + test("a failed project check without a target does not wait for a missing manifest", async () => { + const s = source([undefined, { nodes: 2 }]) + retryParseManifest(s, { baseDelayMs: 1, shouldRetry: () => false }) + expect(await s.parseManifest()).toBeUndefined() + expect(s.calls).toBe(1) + }) + + test("gives up after the attempt limit and reports the absence", async () => { + const s = source([undefined]) + retryParseManifest(s, { attempts: 4, baseDelayMs: 1 }) + expect(await s.parseManifest()).toBeUndefined() + expect(s.calls).toBe(4) + }) +}) diff --git a/packages/dbt-tools/test/packages-real-dbt.test.ts b/packages/dbt-tools/test/packages-real-dbt.test.ts new file mode 100644 index 0000000000..f14e678256 --- /dev/null +++ b/packages/dbt-tools/test/packages-real-dbt.test.ts @@ -0,0 +1,205 @@ +// Real dbt + real built altimate-dbt (dist/index.js): concurrent invocations must +// not each run `dbt deps`. Skipped when dist/ is not built or dbt-duckdb is not +// installed. Uses a local package, so it needs no network. +import { afterAll, beforeAll, describe, expect, test } from "bun:test" +import { spawn, spawnSync } from "child_process" +import { chmodSync, copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "fs" +import { tmpdir } from "os" +import { dirname, join } from "path" +import { findRealDbt } from "./fixture/find-real-dbt" + +const DIST = process.env.ALTIMATE_DBT_DIST ?? join(import.meta.dir, "..", "dist", "index.js") + +const REAL_DBT = existsSync(DIST) ? findRealDbt() : undefined +const suite = REAL_DBT ? describe : describe.skip + +let work = "" +let proj = "" +let shim = "" +let log = "" +let env: Record = {} + +const CONCURRENCY = 4 + +function dbtSync(args: string[]) { + return spawnSync(REAL_DBT!, args, { cwd: proj, encoding: "utf-8", env: { ...process.env, ...env }, timeout: 120_000 }) +} + +function runAltimateDbt(i: number, args = ["schema-verify", "--model", "m"], overrides: Record = {}) { + return new Promise<{ code: number | null; out: string }>((resolve) => { + const c = spawn( + "node", + [DIST, ...args], + { cwd: proj, env: { ...process.env, ...env, E2E_DB: join(proj, `db${i}.duckdb`), BUILD_ROOT: "build", ...overrides } }, + ) + let out = "" + c.stdout.on("data", (d) => (out += d)) + c.stderr.on("data", (d) => (out += d)) + c.on("close", (code) => resolve({ code, out })) + }) +} + +const depsCalls = () => + existsSync(log) ? readFileSync(log, "utf-8").split("\n").filter((l) => l.split(" ")[0] === "deps").length : 0 + +const packageHealthy = () => existsSync(join(proj, "dbt_packages", "localpkg", "dbt_project.yml")) + +suite("real dbt: concurrent altimate-dbt invocations and `dbt deps`", () => { + beforeAll(() => { + work = mkdtempSync(join(tmpdir(), "dbt-pkgs-real-")) + proj = join(work, "proj") + const pkg = join(work, "localpkg") + mkdirSync(join(pkg, "macros"), { recursive: true }) + writeFileSync(join(pkg, "dbt_project.yml"), "name: localpkg\nversion: '0.1.0'\nconfig-version: 2\n") + writeFileSync(join(pkg, "macros", "one.sql"), "{% macro one() %}1{% endmacro %}\n") + mkdirSync(join(proj, "models"), { recursive: true }) + mkdirSync(join(proj, "seeds"), { recursive: true }) + writeFileSync(join(proj, "dbt_project.yml"), "name: proj\nversion: '1.0'\nconfig-version: 2\nprofile: proj\n") + writeFileSync( + join(proj, "profiles.yml"), + "proj:\n target: dev\n outputs:\n dev:\n type: duckdb\n path: \"{{ env_var('E2E_DB', 'e2e.duckdb') }}\"\n threads: 1\n", + ) + writeFileSync(join(proj, "packages.yml"), "packages:\n - local: ../localpkg\n") + writeFileSync(join(proj, "seeds", "raw.csv"), "id,name\n1,a\n2,b\n") + writeFileSync(join(proj, "models", "m.sql"), "select id, name, {{ localpkg.one() }} as one from {{ ref('raw') }}\n") + writeFileSync(join(proj, "models", "m.yml"), "version: 2\nmodels:\n - name: m\n columns:\n - name: id\n") + + // A `dbt` that records its subcommand, then runs the real one. Both the dbt-tools resolver + // (ALTIMATE_DBT_PATH) and the integration library (PATH) will find it. + log = join(work, "dbt-calls.log") + const bin = join(work, "bin") + mkdirSync(bin) + shim = join(bin, "dbt") + writeFileSync(shim, `#!/bin/sh\necho "$@" >> "${log}"\nexec "${REAL_DBT}" "$@"\n`) + chmodSync(shim, 0o755) + const home = join(work, "home") + mkdirSync(home) + env = { + HOME: home, + XDG_CONFIG_HOME: join(home, ".config"), + DBT_PROFILES_DIR: proj, + PATH: `${bin}${process.platform === "win32" ? ";" : ":"}${process.env.PATH ?? ""}`, + ALTIMATE_DBT_PATH: shim, + } + // The library looks for dbt next to the configured python, so point it at a directory + // whose `dbt` is the recording shim. + const venvBin = join(work, "venv", "bin") + mkdirSync(venvBin, { recursive: true }) + copyFileSync(shim, join(venvBin, "dbt")) + chmodSync(join(venvBin, "dbt"), 0o755) + const realPython = join(dirname(REAL_DBT!), "python3") + // A wrapper, not a symlink: python finds its site-packages relative to the real executable. + writeFileSync(join(venvBin, "python3"), `#!/bin/sh\nexec "${existsSync(realPython) ? realPython : "python3"}" "$@"\n`) + chmodSync(join(venvBin, "python3"), 0o755) + const init = spawnSync("node", [DIST, "init", "--project-root", proj, "--python-path", join(venvBin, "python3")], { + env: { ...process.env, ...env }, + encoding: "utf-8", + }) + expect(init.status).toBe(0) + + // Prepare a built project (the shim records these calls too; the log is cleared below). + expect(dbtSync(["deps"]).status).toBe(0) + expect(dbtSync(["build"]).status).toBe(0) + for (let i = 0; i < CONCURRENCY; i++) copyFileSync(join(proj, "e2e.duckdb"), join(proj, `db${i}.duckdb`)) + }, 180_000) + + afterAll(() => { + if (work) rmSync(work, { recursive: true, force: true }) + }) + + test("packages already installed: concurrent invocations run `dbt deps` zero times", async () => { + writeFileSync(log, "") + expect(packageHealthy()).toBe(true) + const runs = await Promise.all(Array.from({ length: CONCURRENCY }, (_, i) => runAltimateDbt(i))) + expect(runs.every((r) => r.code === 0 && !/No dbt_project\.yml found/.test(r.out))).toBe(true) + expect(depsCalls()).toBe(0) + expect(packageHealthy()).toBe(true) + }, 120_000) + + test("packages missing: concurrent invocations install exactly once and leave a healthy directory", async () => { + rmSync(join(proj, "dbt_packages"), { recursive: true, force: true }) + rmSync(join(proj, "target", ".altimate-dbt"), { recursive: true, force: true }) + writeFileSync(log, "") + const runs = await Promise.all(Array.from({ length: CONCURRENCY }, (_, i) => runAltimateDbt(i))) + expect(runs.every((r) => r.code === 0 && !/No dbt_project\.yml found/.test(r.out))).toBe(true) + expect(depsCalls()).toBe(1) + expect(packageHealthy()).toBe(true) + }, 120_000) + + test("a package directory left without dbt_project.yml is repaired by the next invocation", async () => { + rmSync(join(proj, "dbt_packages", "localpkg"), { recursive: true, force: true }) + mkdirSync(join(proj, "dbt_packages", "localpkg"), { recursive: true }) // present but empty + writeFileSync(log, "") + expect((await runAltimateDbt(0)).code).toBe(0) + expect(depsCalls()).toBe(1) + expect(packageHealthy()).toBe(true) + }, 120_000) + test("explicit deps succeeds with the integration's selected command under the lock", async () => { + const result = await runAltimateDbt(0, ["deps"]) + expect(result.code, result.out).toBe(0) + expect(packageHealthy()).toBe(true) + expect(existsSync(join(proj, "target", ".altimate-dbt", "packages.lock"))).toBe(false) + }, 120_000) + + test("Python Core explicit deps publishes the PID of the selected installer", async () => { + const home = join(work, "core-home") + const cfg = join(home, ".altimate-code") + mkdirSync(cfg, { recursive: true }) + const ready = join(work, "explicit-installer.pid") + const proceed = join(work, "continue-explicit-install") + const python = join(work, "tracked-python") + const interpreter = join(work, "venv", "bin", "python3") + writeFileSync(python, `#!/bin/sh +case "$2" in + *'dbt_cli.invoke([r"""deps"""'*) + echo "$$" > "${ready}" + i=0 + while [ ! -f "${proceed}" ] && [ "$i" -lt 400 ]; do + sleep 0.05 + i=$((i + 1)) + done + ;; +esac +exec "${interpreter}" "$@" +`) + chmodSync(python, 0o755) + writeFileSync(join(cfg, "dbt.json"), JSON.stringify({ projectRoot: proj, pythonPath: python, dbtIntegration: "core" })) + const ownerFile = join(proj, "target", ".altimate-dbt", "packages.lock", "owner.json") + const run = runAltimateDbt(0, ["deps"], { HOME: home, XDG_CONFIG_HOME: join(home, ".config") }) + try { + const deadline = Date.now() + 15_000 + let tracked = false + while (!tracked && Date.now() < deadline) { + if (existsSync(ready) && existsSync(ownerFile)) { + const pid = Number(readFileSync(ready, "utf-8")) + tracked = pid > 0 && JSON.parse(readFileSync(ownerFile, "utf-8")).installerPid === pid + } + if (!tracked) await Bun.sleep(25) + } + expect(tracked).toBe(true) + } finally { + writeFileSync(proceed, "") + const result = await run + expect(result.code, result.out).toBe(0) + } + expect(packageHealthy()).toBe(true) + }, 120_000) + + test("concurrent commands install and read a compound Jinja package path without overlap", async () => { + const config = join(proj, "dbt_project.yml") + const original = readFileSync(config, "utf-8") + const buildRoot = join(proj, "build") + try { + writeFileSync(config, original + `packages-install-path: "{{ env_var('BUILD_ROOT') }}/dbt_packages"\n`) + writeFileSync(log, "") + const runs = await Promise.all(Array.from({ length: CONCURRENCY }, (_, i) => runAltimateDbt(i))) + for (const result of runs) expect(result.code, result.out).toBe(0) + expect(depsCalls()).toBe(CONCURRENCY) + expect(existsSync(join(buildRoot, "dbt_packages", "localpkg", "dbt_project.yml"))).toBe(true) + } finally { + writeFileSync(config, original) + rmSync(buildRoot, { recursive: true, force: true }) + } + }, 120_000) + +}) diff --git a/packages/dbt-tools/test/packages.test.ts b/packages/dbt-tools/test/packages.test.ts new file mode 100644 index 0000000000..782945e3f1 --- /dev/null +++ b/packages/dbt-tools/test/packages.test.ts @@ -0,0 +1,678 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test" +import { spawn, spawnSync } from "child_process" +import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, utimesSync, writeFileSync } from "fs" +import { tmpdir } from "os" +import { join } from "path" +import { + ensurePackages, + PackageLockTimeoutError, + installPackagesLocked, + packagesSatisfied, + withPackagesLock, + withUnverifiedPackagesLock, +} from "../src/packages" + +const WORKER = join(import.meta.dir, "fixture", "packages-worker.ts") +const PKG = "mypkg" + +let root = "" + +function writeProject(opts: { lock?: string; packages?: string } = {}) { + writeFileSync(join(root, "dbt_project.yml"), "name: proj\nversion: '1.0'\nconfig-version: 2\nprofile: proj\n") + writeFileSync(join(root, "packages.yml"), opts.packages ?? `packages:\n - package: acme/${PKG}\n version: 1.0.0\n`) + if (opts.lock) writeFileSync(join(root, "package-lock.yml"), opts.lock) +} + +function lockFor(version: string) { + return `packages:\n- package: acme/${PKG}\n version: ${version}\nsha1_hash: abc\n` +} + +/** What `dbt deps` leaves behind for a healthy install. */ +function installPackage(version = "1.0.0", name = PKG) { + const dir = join(root, "dbt_packages", name) + mkdirSync(dir, { recursive: true }) + writeFileSync(join(dir, "dbt_project.yml"), `name: ${name}\nversion: '${version}'\n`) +} + +function counter() { + let calls = 0 + return { + install: async () => { + calls++ + installPackage() + }, + get calls() { + return calls + }, + } +} + +const stateDir = () => join(root, "target", ".altimate-dbt") + +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "dbt-pkgs-")) +}) +afterEach(() => { + rmSync(root, { recursive: true, force: true }) +}) + +describe("packagesSatisfied / ensurePackages: when to install", () => { + test("no packages declared: nothing to do, installer never called", async () => { + writeFileSync(join(root, "dbt_project.yml"), "name: proj\n") + const c = counter() + expect((await ensurePackages(root, c.install)).action).toBe("none") + expect(c.calls).toBe(0) + }) + + test("declared but not installed: installs once, then skips", async () => { + writeProject() + const c = counter() + expect((await ensurePackages(root, c.install)).action).toBe("installed") + expect((await ensurePackages(root, c.install)).action).toBe("skipped") + expect((await ensurePackages(root, c.install)).action).toBe("skipped") + expect(c.calls).toBe(1) + }) + + test("packages already installed by something else and matching the lock: not reinstalled", async () => { + // e.g. an image built with `dbt deps`: no stamp from us, but the install is current. + writeProject({ lock: lockFor("1.0.0") }) + installPackage("1.0.0") + const c = counter() + expect((await ensurePackages(root, c.install)).action).toBe("skipped") + expect(c.calls).toBe(0) + }) + + test("changed declarations invalidate an older install without a lock or stamp", async () => { + writeProject() + installPackage("1.0.0") + writeProject({ packages: `packages:\n - package: acme/${PKG}\n version: 2.0.0\n` }) + const now = Date.now() + utimesSync(join(root, "dbt_packages"), new Date(now - 250), new Date(now - 250)) + utimesSync(join(root, "packages.yml"), new Date(now), new Date(now)) + const c = counter() + expect((await ensurePackages(root, c.install)).action).toBe("installed") + expect(c.calls).toBe(1) + }) + + test("a recent lock does not hide a newly declared package", async () => { + writeProject({ + lock: lockFor("1.0.0"), + packages: `packages:\n - package: acme/${PKG}\n version: 1.0.0\n - package: acme/new_package\n version: 1.0.0\n`, + }) + installPackage() + const sameTime = new Date() + utimesSync(join(root, "packages.yml"), sameTime, sameTime) + utimesSync(join(root, "package-lock.yml"), sameTime, sameTime) + let calls = 0 + expect((await ensurePackages(root, async () => { + calls++ + installPackage("1.0.0", "new_package") + })).action).toBe("installed") + expect(calls).toBe(1) + }) + + test("a newly checked out lock invalidates an older preinstalled directory", async () => { + writeProject({ lock: lockFor("2.0.0"), packages: `packages:\n - package: acme/${PKG}\n version: 2.0.0\n` }) + installPackage("1.0.0") + const older = new Date(Date.now() - 60_000) + utimesSync(join(root, "dbt_packages"), older, older) + const c = counter() + expect((await ensurePackages(root, c.install)).action).toBe("installed") + expect(c.calls).toBe(1) + }) + + test("a preinstalled environment-dependent package is refreshed once to record its inputs", async () => { + writeProject({ packages: `packages:\n - local: "{{ env_var('TEST_ALTIMATE_PREINSTALLED_PACKAGE', '../local_pkg') }}"\n` }) + installPackage() + const c = counter() + expect((await ensurePackages(root, c.install)).action).toBe("installed") + expect((await ensurePackages(root, c.install)).action).toBe("skipped") + expect(c.calls).toBe(1) + }) + + test("a changed environment-dependent dependency invalidates the install stamp", async () => { + const variable = "TEST_ALTIMATE_LOCAL_PACKAGE" + const saved = process.env[variable] + try { + writeProject({ packages: `packages:\n - local: "{{ env_var ('${variable}') }}"\n` }) + process.env[variable] = "../package_v1" + const c = counter() + expect((await ensurePackages(root, c.install)).action).toBe("installed") + expect((await ensurePackages(root, c.install)).action).toBe("skipped") + process.env[variable] = "../package_v2" + expect((await ensurePackages(root, c.install)).action).toBe("installed") + expect(c.calls).toBe(2) + expect(readFileSync(join(stateDir(), "packages.stamp.json"), "utf8")).not.toContain("../package_v2") + } finally { + if (saved === undefined) delete process.env[variable] + else process.env[variable] = saved + } + }) + + test("an env_var package path is stamped and does not get reset on the next command", async () => { + writeProject() + writeFileSync(join(root, "dbt_project.yml"), `name: proj\npackages-install-path: "{{ env_var('TEST_ALTIMATE_PACKAGE_PATH', 'custom_packages') }}"\n`) + const installPath = join(root, "custom_packages", PKG) + let calls = 0 + const install = async () => { + calls++ + mkdirSync(installPath, { recursive: true }) + writeFileSync(join(installPath, "dbt_project.yml"), `name: ${PKG}\n`) + } + expect((await ensurePackages(root, install)).action).toBe("installed") + expect((await ensurePackages(root, install)).action).toBe("skipped") + expect(calls).toBe(1) + }) + + test("a compound Jinja package path falls back to installing under the lock", async () => { + writeProject() + writeFileSync(join(root, "dbt_project.yml"), `name: proj\npackages-install-path: "{{ env_var('BUILD_ROOT') }}/dbt_packages"\n`) + const installPath = join(root, "build", "dbt_packages", PKG) + const result = await ensurePackages(root, async () => { + expect(existsSync(join(stateDir(), "packages.lock"))).toBe(true) + mkdirSync(installPath, { recursive: true }) + writeFileSync(join(installPath, "dbt_project.yml"), `name: ${PKG}\n`) + }) + expect(result.action).toBe("installed") + expect(existsSync(join(installPath, "dbt_project.yml"))).toBe(true) + expect(existsSync(join(stateDir(), "packages.stamp.json"))).toBe(false) + }) + + test("an unverified package path stays locked while the command reads the packages", async () => { + writeProject() + writeFileSync(join(root, "dbt_project.yml"), `name: proj\npackages-install-path: "{{ env_var('BUILD_ROOT') }}/dbt_packages"\n`) + const reading = Promise.withResolvers() + const done = Promise.withResolvers() + const c = counter() + const reader = withUnverifiedPackagesLock(root, async () => { + await ensurePackages(root, c.install) + reading.resolve() + await done.promise + }) + try { + await Promise.race([reading.promise, reader]) + await expect(ensurePackages(root, c.install, { maxWaitMs: 100 })).rejects.toBeInstanceOf(PackageLockTimeoutError) + expect(c.calls).toBe(1) + } finally { + done.resolve() + await reader + } + expect((await ensurePackages(root, c.install)).action).toBe("installed") + expect(c.calls).toBe(2) + }) + + test("an unreadable install destination falls back to running dbt", async () => { + writeProject() + writeFileSync(join(root, "dbt_packages"), "not a directory") + const c = counter() + const result = await ensurePackages(root, async () => { + rmSync(join(root, "dbt_packages")) + await c.install() + }) + expect(result.action).toBe("installed") + expect(c.calls).toBe(1) + }) + + test("the version inside an installed hub package is not compared with the lock (hub packages report stale versions)", async () => { + // dbt_utils locked at 1.3.0 installs with `version: 0.1.0` in its dbt_project.yml. + writeProject({ lock: lockFor("1.3.0") }) + installPackage("0.1.0") + const c = counter() + expect((await ensurePackages(root, c.install)).action).toBe("skipped") + expect(c.calls).toBe(0) + }) + + test("a declared hub package whose directory is missing: reinstalls", async () => { + writeProject({ lock: lockFor("1.0.0") }) + installPackage("1.0.0", "other") + expect(packagesSatisfied(root).ok).toBe(false) + }) + + test("package directory without dbt_project.yml: reinstalls", async () => { + writeProject({ lock: lockFor("1.0.0") }) + installPackage() + const c = counter() + await ensurePackages(root, c.install) // stamps a good install + rmSync(join(root, "dbt_packages", PKG, "dbt_project.yml")) + expect(packagesSatisfied(root).ok).toBe(false) + expect((await ensurePackages(root, c.install)).action).toBe("installed") + expect(existsSync(join(root, "dbt_packages", PKG, "dbt_project.yml"))).toBe(true) + }) + + test("a changed packages-install-path invalidates the stamp", async () => { + writeProject({ lock: lockFor("1.0.0") }) + const c = counter() + await ensurePackages(root, c.install) + writeFileSync(join(root, "dbt_project.yml"), "name: proj\nversion: '1.0'\nconfig-version: 2\nprofile: proj\npackages-install-path: other_dir\n") + mkdirSync(join(root, "other_dir", PKG), { recursive: true }) + writeFileSync(join(root, "other_dir", PKG, "dbt_project.yml"), `name: ${PKG}\nversion: '1.0.0'\n`) + expect(packagesSatisfied(root).ok).toBe(false) + }) + + test("the same project reached through a symlink keeps its stamp", async () => { + writeProject({ lock: lockFor("1.0.0") }) + const c = counter() + await ensurePackages(root, c.install) + const link = `${root}-link` + symlinkSync(root, link) + try { + expect((await ensurePackages(link, c.install)).action).toBe("skipped") + expect(c.calls).toBe(1) + } finally { + rmSync(link, { force: true }) + } + }) + + test("a truncated dbt_project.yml is not a complete package", () => { + writeProject({ lock: lockFor("1.0.0") }) + installPackage() + writeFileSync(join(root, "dbt_packages", PKG, "dbt_project.yml"), "") + expect(packagesSatisfied(root).ok).toBe(false) + }) + + test("an empty package-lock.yml does not hide the packages in packages.yml", async () => { + writeProject({ lock: "packages: []\nsha1_hash: abc\n" }) + const c = counter() + expect((await ensurePackages(root, c.install)).action).toBe("installed") + expect(c.calls).toBe(1) + }) + + test("a declaration changed during installation is not stamped as installed", async () => { + writeProject() + await ensurePackages(root, async () => { + installPackage("1.0.0") + writeFileSync(join(root, "packages.yml"), `packages:\n - package: acme/${PKG}\n version: 2.0.0\n`) + }) + expect(existsSync(join(stateDir(), "packages.stamp.json"))).toBe(false) + expect(packagesSatisfied(root).ok).toBe(false) + + let repairs = 0 + const repair = async () => { + repairs++ + installPackage("2.0.0") + } + expect((await ensurePackages(root, repair)).action).toBe("installed") + expect((await ensurePackages(root, repair)).action).toBe("skipped") + expect(repairs).toBe(1) + }) + + test("declarations changed since the last install: reinstalls", async () => { + writeProject() + const c = counter() + await ensurePackages(root, c.install) + writeFileSync(join(root, "packages.yml"), `packages:\n - package: acme/${PKG}\n version: 1.1.0\n`) + expect((await ensurePackages(root, c.install)).action).toBe("installed") + expect(c.calls).toBe(2) + }) + + test("non-hub packages are verified by count before an install and by name from the stamp afterwards", async () => { + writeProject({ packages: "packages:\n - local: ../elsewhere\n" }) + const pkg = join(root, "dbt_packages", "elsewhere") + const stamp = join(stateDir(), "packages.stamp.json") + expect(packagesSatisfied(root).ok).toBe(false) + installPackage("0.1.0", "elsewhere") + expect(packagesSatisfied(root).ok).toBe(true) + expect(existsSync(stamp)).toBe(false) + + const install = async () => installPackage("0.1.0", "elsewhere") + rmSync(pkg, { recursive: true }) + expect((await ensurePackages(root, install)).action).toBe("installed") + expect(existsSync(stamp)).toBe(true) + expect((await ensurePackages(root, install)).action).toBe("skipped") + + rmSync(pkg, { recursive: true }) + installPackage("0.1.0", "another_package") + expect(packagesSatisfied(root).reason).toContain("installed package missing: elsewhere") + expect((await ensurePackages(root, install)).action).toBe("installed") + expect(existsSync(join(pkg, "dbt_project.yml"))).toBe(true) + }) + + test("honours packages-install-path from dbt_project.yml", async () => { + writeProject() + writeFileSync( + join(root, "dbt_project.yml"), + "name: proj\nconfig-version: 2\nprofile: proj\npackages-install-path: deps_here\n", + ) + const dir = join(root, "deps_here", PKG) + mkdirSync(dir, { recursive: true }) + writeFileSync(join(dir, "dbt_project.yml"), "name: mypkg\nversion: '1.0.0'\n") + expect(packagesSatisfied(root).ok).toBe(true) + }) +}) + +describe("crash and failure recovery", () => { + test("an install that throws leaves the project marked unfinished; the next call repairs it", async () => { + writeProject() + await expect( + ensurePackages(root, async () => { + mkdirSync(join(root, "dbt_packages", PKG), { recursive: true }) // half-installed + throw new Error("network down") + }), + ).rejects.toThrow("network down") + expect(existsSync(join(stateDir(), "packages.dirty"))).toBe(true) + expect(packagesSatisfied(root).ok).toBe(false) + + const c = counter() + expect((await ensurePackages(root, c.install)).action).toBe("installed") + expect(existsSync(join(stateDir(), "packages.dirty"))).toBe(false) + expect(packagesSatisfied(root).ok).toBe(true) + }) + + test("dirty marker forces a reinstall even when every package directory looks complete", async () => { + writeProject({ lock: lockFor("1.0.0") }) + installPackage("1.0.0") + mkdirSync(stateDir(), { recursive: true }) + writeFileSync(join(stateDir(), "packages.dirty"), "{}") + const c = counter() + expect((await ensurePackages(root, c.install)).action).toBe("installed") + expect(c.calls).toBe(1) + }) + + test("a deps run that finishes with an empty or incomplete package directory is not recorded as good", async () => { + writeProject() + await expect(ensurePackages(root, async () => {})).rejects.toThrow("incomplete") + expect(existsSync(join(stateDir(), "packages.stamp.json"))).toBe(false) + expect(existsSync(join(stateDir(), "packages.dirty"))).toBe(true) + }) +}) + +describe("lock", () => { + test("is exclusive and released afterwards", async () => { + writeProject() + const order: string[] = [] + const a = withPackagesLock(root, async () => { + order.push("a-start") + await new Promise((r) => setTimeout(r, 200)) + order.push("a-end") + }) + await new Promise((r) => setTimeout(r, 50)) + const b = withPackagesLock(root, async () => { + order.push("b-start") + }) + await Promise.all([a, b]) + expect(order).toEqual(["a-start", "a-end", "b-start"]) + expect(existsSync(join(stateDir(), "packages.lock"))).toBe(false) + }) + + test("a lock left by a process that no longer exists is broken immediately", async () => { + writeProject() + const lockDir = join(stateDir(), "packages.lock") + mkdirSync(lockDir, { recursive: true }) + // A pid that is certainly not running: spawn and reap a child. + const dead = spawn(process.execPath, ["-e", "0"]) + await new Promise((r) => dead.on("close", r)) + writeFileSync( + join(lockDir, "owner.json"), + JSON.stringify({ pid: dead.pid, host: require("os").hostname(), token: "x", startedAt: Date.now() }), + ) + const started = Date.now() + await withPackagesLock(root, async () => {}, { staleMs: 60_000 }) + expect(Date.now() - started).toBeLessThan(2_000) + }) + + test("a live installer keeps an abandoned lock until the installer exits", async () => { + writeProject() + const lockDir = join(stateDir(), "packages.lock") + mkdirSync(lockDir, { recursive: true }) + const parent = spawn(process.execPath, ["-e", "0"]) + await new Promise((resolve) => parent.once("close", resolve)) + const installer = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"]) + const finished = new Promise((resolve) => installer.once("close", resolve)) + try { + writeFileSync( + join(lockDir, "owner.json"), + JSON.stringify({ pid: parent.pid, installerPid: installer.pid, host: require("os").hostname(), token: "x" }), + ) + const old = new Date(Date.now() - 10_000) + utimesSync(lockDir, old, old) + const c = counter() + await expect(ensurePackages(root, c.install, { staleMs: 1, maxWaitMs: 100 })).rejects.toBeInstanceOf(PackageLockTimeoutError) + expect(c.calls).toBe(0) + } finally { + installer.kill() + await finished + } + const c = counter() + expect((await ensurePackages(root, c.install)).action).toBe("installed") + expect(c.calls).toBe(1) + }) + + const canReadProcessIdentity = process.platform === "linux" || ( + process.platform === "darwin" && spawnSync("/bin/ps", ["-p", String(process.pid)]).status === 0 + ) + test.skipIf(!canReadProcessIdentity)("reclaims a lock when the installer PID belongs to a different process", async () => { + writeProject() + const lockDir = join(stateDir(), "packages.lock") + mkdirSync(lockDir, { recursive: true }) + const parent = spawn(process.execPath, ["-e", "0"]) + await new Promise((resolve) => parent.once("close", resolve)) + writeFileSync(join(lockDir, "owner.json"), JSON.stringify({ + pid: parent.pid, + installerPid: process.pid, + installerStarted: "previous installer", + host: require("os").hostname(), + token: "x", + })) + const c = counter() + expect((await ensurePackages(root, c.install, { maxWaitMs: 500 })).action).toBe("installed") + expect(c.calls).toBe(1) + }) + + test("a lock whose owner is alive but has stopped heartbeating is broken after staleMs", async () => { + writeProject() + const lockDir = join(stateDir(), "packages.lock") + mkdirSync(lockDir, { recursive: true }) + writeFileSync( + join(lockDir, "owner.json"), + JSON.stringify({ pid: process.pid + 1, host: "some-other-host", token: "x", startedAt: 0 }), + ) + const old = new Date(Date.now() - 10_000) + utimesSync(lockDir, old, old) + await withPackagesLock(root, async () => {}, { staleMs: 1_000 }) + }) + + test("a live, heartbeating owner is waited for, not broken", async () => { + writeProject() + let holderDone = false + const holder = withPackagesLock( + root, + async () => { + await new Promise((r) => setTimeout(r, 600)) // longer than staleMs: only the heartbeat keeps it alive + holderDone = true + }, + { staleMs: 300, heartbeatMs: 50 }, + ) + await new Promise((r) => setTimeout(r, 100)) + let waiterSawHolderDone = false + await withPackagesLock(root, async () => (waiterSawHolderDone = holderDone), { staleMs: 300, heartbeatMs: 50 }) + await holder + expect(waiterSawHolderDone).toBe(true) + }) + + test("gives up with a clear error when the lock is never released", async () => { + writeProject() + const lockDir = join(stateDir(), "packages.lock") + mkdirSync(lockDir, { recursive: true }) + writeFileSync(join(lockDir, "owner.json"), JSON.stringify({ pid: process.pid, host: "h", token: "x", startedAt: 0 })) + await expect(withPackagesLock(root, async () => {}, { staleMs: 60_000, maxWaitMs: 300 })).rejects.toThrow( + "timed out", + ) + }) + + test("explicit install (altimate-dbt deps) runs under the lock, clears the dirty marker and does not claim success", async () => { + writeProject({ lock: lockFor("1.0.0") }) + const install = counter() + await withPackagesLock(root, async () => { + await expect(installPackagesLocked(root, install.install, { maxWaitMs: 100 })).rejects.toThrow("timed out") + expect(install.calls).toBe(0) + }) + await installPackagesLocked(root, async () => installPackage()) + expect(packagesSatisfied(root).ok).toBe(true) + expect(existsSync(join(stateDir(), "packages.dirty"))).toBe(false) + expect(existsSync(join(stateDir(), "packages.stamp.json"))).toBe(false) + }) + + test("successful explicit deps stamps an environment-dependent installation", async () => { + writeProject({ packages: `packages:\n - local: "{{ env_var('TEST_ALTIMATE_EXPLICIT_PACKAGE', '../local_pkg') }}"\n` }) + const setup = counter() + await installPackagesLocked(root, async () => { + await setup.install() + return { exitCode: 0, stdout: "Installed package", stderr: "Progress logged to stderr" } + }) + const unexpected = counter() + expect((await ensurePackages(root, unexpected.install)).action).toBe("skipped") + expect(unexpected.calls).toBe(0) + }) + + test("a failed explicit install leaves repair pending even when package YAML looks complete", async () => { + writeProject({ lock: lockFor("1.0.0") }) + const failed = { stdout: "dbt could not finish extraction", stderr: "", exitCode: 1 } + await expect(installPackagesLocked(root, async () => { + installPackage() + return failed + })).resolves.toBe(failed) + expect(packagesSatisfied(root).ok).toBe(false) + const repair = counter() + expect((await ensurePackages(root, repair.install)).action).toBe("installed") + expect(repair.calls).toBe(1) + }) + + test("an explicit install whose dbt deps failed on an empty directory returns dbt's result and leaves the dirty marker", async () => { + writeProject({ lock: lockFor("1.0.0") }) + const failed = { stderr: "Runtime Error: could not reach hub.getdbt.com" } + const result = await installPackagesLocked(root, async () => failed) + expect(result).toBe(failed) + expect(existsSync(join(stateDir(), "packages.dirty"))).toBe(true) + expect(packagesSatisfied(root).ok).toBe(false) + }) + + test("a plain file in the install path is not an incomplete package", async () => { + writeProject({ lock: lockFor("1.0.0") }) + installPackage("1.0.0") + writeFileSync(join(root, "dbt_packages", "README.md"), "notes") + expect(packagesSatisfied(root).ok).toBe(true) + }) + + test("a packages.yml edited after the lock was written is not satisfied by the old install", () => { + writeProject({ lock: lockFor("1.0.0") }) + installPackage("1.0.0") + expect(packagesSatisfied(root).ok).toBe(true) + const later = new Date(Date.now() + 10_000) + utimesSync(join(root, "packages.yml"), later, later) + expect(packagesSatisfied(root)).toMatchObject({ ok: false, reason: expect.stringContaining("newer") }) + }) +}) + +describe("concurrent callers", () => { + test("many in-process callers install exactly once and all return with packages complete", async () => { + writeProject() + let installs = 0 + const seen: boolean[] = [] + const slow = async () => { + installs++ + const dir = join(root, "dbt_packages", PKG) + mkdirSync(dir, { recursive: true }) + await new Promise((r) => setTimeout(r, 150)) + writeFileSync(join(dir, "dbt_project.yml"), "name: mypkg\nversion: '1.0.0'\n") + } + await Promise.all( + Array.from({ length: 8 }, async () => { + await ensurePackages(root, slow) + seen.push(existsSync(join(root, "dbt_packages", PKG, "dbt_project.yml"))) + }), + ) + expect(installs).toBe(1) + expect(seen).toEqual(Array(8).fill(true)) + }) + + function runWorker(mode?: "crash") { + const logFile = join(root, "worker.log") + return new Promise<{ code: number | null; signal: NodeJS.Signals | null }>((resolve) => { + const c = spawn(process.execPath, [WORKER, root, logFile, PKG, ...(mode ? [mode] : [])], { stdio: "ignore" }) + c.on("close", (code, signal) => resolve({ code, signal })) + }) + } + const readLog = () => (existsSync(join(root, "worker.log")) ? readFileSync(join(root, "worker.log"), "utf-8").trim().split("\n") : []) + + test("separate processes started together: one install, nobody sees a broken package directory", async () => { + writeProject() + const results = await Promise.all(Array.from({ length: 8 }, () => runWorker())) + expect(results.every((r) => r.code === 0)).toBe(true) + const lines = readLog() + expect(lines.filter((l) => l.startsWith("install")).length).toBe(1) + expect(lines.filter((l) => l.startsWith("ready")).length).toBe(8) + expect(lines.filter((l) => l.startsWith("BROKEN")).length).toBe(0) + }, 30_000) + + test.skipIf(process.platform === "win32").each(["dbt", "explicit"])("killing the %s caller does not unlock its live installer", async (mode) => { + writeProject() + const shim = join(root, "dbt") + const ready = join(root, "installer.pid") + writeFileSync(shim, '#!/bin/sh\necho "$$" > installer.pid\nexec sleep 10\n') + chmodSync(shim, 0o755) + const parent = spawn(process.execPath, [WORKER, root, join(root, "worker.log"), PKG, mode], { + env: { ...process.env, ALTIMATE_DBT_PATH: shim }, + stdio: "ignore", + }) + const stopped = new Promise((resolve) => parent.once("close", (_code, signal) => resolve(signal))) + try { + const lockDir = join(stateDir(), "packages.lock") + const deadline = Date.now() + 3_000 + while ((!existsSync(ready) || !readFileSync(ready, "utf8").trim()) && Date.now() < deadline) { + await Bun.sleep(10) + } + const pid = Number(readFileSync(ready, "utf8")) + // Wait for the parent to finish publishing the recorded installer PID. + const owner = join(lockDir, "owner.json") + while (JSON.parse(readFileSync(owner, "utf8")).installerPid === null && Date.now() < deadline) { + await Bun.sleep(10) + } + expect(JSON.parse(readFileSync(owner, "utf8")).installerPid).toBe(pid) + parent.kill("SIGKILL") + expect(await stopped).toBe("SIGKILL") + const c = counter() + await expect(ensurePackages(root, c.install, { staleMs: 1, maxWaitMs: 100 })).rejects.toBeInstanceOf(PackageLockTimeoutError) + expect(c.calls).toBe(0) + } finally { + parent.kill("SIGKILL") + await stopped + const pid = existsSync(ready) ? Number(readFileSync(ready, "utf8")) : 0 + if (pid > 0) { + try { + process.kill(pid, "SIGKILL") + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error + } + } + } + }, 10_000) + + test("a process killed in the middle of an install does not poison the project", async () => { + writeProject() + const crashed = await runWorker("crash") + expect(crashed.signal).toBe("SIGKILL") + // Package directory exists but is incomplete, lock and dirty marker are left behind. + expect(existsSync(join(root, "dbt_packages", PKG))).toBe(true) + expect(existsSync(join(root, "dbt_packages", PKG, "dbt_project.yml"))).toBe(false) + expect(existsSync(join(stateDir(), "packages.lock"))).toBe(true) + + // Recovery: several processes arrive at once; the dead owner's lock is broken, the install repeats once. + const results = await Promise.all(Array.from({ length: 4 }, () => runWorker())) + expect(results.every((r) => r.code === 0)).toBe(true) + const lines = readLog() + expect(lines.filter((l) => l.startsWith("BROKEN")).length).toBe(0) + expect(lines.filter((l) => l.startsWith("ready")).length).toBe(4) + expect(lines.filter((l) => l.startsWith("install")).length).toBe(2) // the crashed one + one repair + expect(packagesSatisfied(root).ok).toBe(true) + }, 30_000) +}) + +describe("adapter wiring", () => { + test("the dbt integration library is told not to run `dbt deps` itself", () => { + // adapter.ts cannot be imported under bun (python-bridge), so assert on the source. The + // library default is true; leaving it on reintroduces the unsynchronised install from + // every process. The behavioural check is test/packages-real-dbt.test.ts. + const src = readFileSync(join(import.meta.dir, "..", "src", "adapter.ts"), "utf-8") + expect(src).toMatch(/getInstallDepsOnProjectInitialization:\s*\(\)\s*=>\s*false/) + }) +}) diff --git a/packages/dbt-tools/test/schema-verify.test.ts b/packages/dbt-tools/test/schema-verify.test.ts index 04094c31ce..5d01c9e4e8 100644 --- a/packages/dbt-tools/test/schema-verify.test.ts +++ b/packages/dbt-tools/test/schema-verify.test.ts @@ -8,6 +8,15 @@ type AdapterOverrides = { nodeFound?: boolean parseManifestError?: Error getColumnsError?: Error + /** `config.contract.enforced` on the model. */ + contractEnforced?: boolean + /** Tests attached to columns of the model: column name -> test names. */ + columnTests?: Record + /** Describe the tests as an older manifest does: no attached_node, the model is in depends_on. */ + legacyTests?: boolean + legacyRelationship?: { dependencies: string[]; model?: string } + patchPath?: string + packageName?: string } function makeAdapter(o: AdapterOverrides = {}): DBTProjectIntegrationAdapter { @@ -20,11 +29,11 @@ function makeAdapter(o: AdapterOverrides = {}): DBTProjectIntegrationAdapter { schema: "main", alias: "target", name: "target", - package_name: "proj", + package_name: o.packageName ?? "proj", description: "", - patch_path: "schema.yml", + patch_path: o.patchPath ?? "schema.yml", columns: o.expectedColumns ?? {}, - config: {} as never, + config: (o.contractEnforced === undefined ? {} : { contract: { enforced: o.contractEnforced } }) as never, resource_type: "model", depends_on: { nodes: [], macros: [] } as never, is_external_project: false, @@ -32,6 +41,25 @@ function makeAdapter(o: AdapterOverrides = {}): DBTProjectIntegrationAdapter { meta: {}, } as unknown as NodeMetaData) + const testMetaMap = new Map() + for (const [column, names] of Object.entries(o.columnTests ?? {})) + for (const name of names) + testMetaMap.set( + name, + o.legacyTests + ? { depends_on: { nodes: ["macro.x", "model.proj.target"] }, column_name: column } + : { attached_node: "model.proj.target", column_name: column }, + ) + if (o.legacyRelationship) { + testMetaMap.set("relationships_orders_customer_id", { + column_name: "customer_id", + depends_on: { nodes: o.legacyRelationship.dependencies }, + test_metadata: { name: "relationships", kwargs: { model: o.legacyRelationship.model } }, + }) + } + // A test attached to some other model must never count. + testMetaMap.set("not_null_other_email", { attached_node: "model.proj.other", column_name: "email" }) + const parseManifest = o.parseManifestError ? mock(() => Promise.reject(o.parseManifestError)) : mock(() => Promise.resolve({ @@ -40,6 +68,7 @@ function makeAdapter(o: AdapterOverrides = {}): DBTProjectIntegrationAdapter { lookupByUniqueId: mock(() => node), nodes: mock(() => []), }, + testMetaMap, } as never)) const getColumnsOfModel = o.getColumnsError @@ -67,6 +96,12 @@ describe("schema-verify command", () => { expect(result).toEqual({ error: "Missing --model" }) }) + test("a missing relation stays an error when its YAML columns are only documented", async () => { + const adapter = makeAdapter({ actualColumns: [], expectedColumns: { id: col("id") } }) + const result = await schemaVerify(adapter, ["--model", "target"]) + expect(result).toMatchObject({ error: expect.stringContaining("has no warehouse table") }) + }) + test("model not found in manifest", async () => { const adapter = makeAdapter({ nodeFound: false }) const result = await schemaVerify(adapter, ["--model", "missing_model"]) @@ -95,29 +130,179 @@ describe("schema-verify command", () => { expect(result.columns_reordered).toEqual([]) }) - test("detects extra columns in actual not in spec", async () => { + test("extra columns the YAML does not list are reported in the raw diff but are not a mismatch", async () => { const adapter = makeAdapter({ expectedColumns: { id: col("id"), name: col("name") }, actualColumns: [db("id"), db("name"), db("extra1"), db("extra2")], }) const result = await schemaVerify(adapter, ["--model", "target"]) as Record - expect(result.verdict).toBe("mismatch") + expect(result.verdict).toBe("match") + expect(result.findings).toEqual([]) expect(result.columns_extra).toEqual(["extra1", "extra2"]) - expect(result.columns_missing).toEqual([]) + expect((result.notes as string[]).join(" ")).toContain("not an error") }) - test("detects missing columns in actual that spec requires", async () => { + test("YAML that documents only the tested columns: undocumented columns are never an error", async () => { + // monthly_agg_reviews: YAML lists DATE_SENTIMENT_ID and REVIEW_SENTIMENT, the model produces six columns. const adapter = makeAdapter({ - expectedColumns: { id: col("id"), name: col("name"), email: col("email") }, - actualColumns: [db("id"), db("name")], + expectedColumns: { DATE_SENTIMENT_ID: col("DATE_SENTIMENT_ID"), REVIEW_SENTIMENT: col("REVIEW_SENTIMENT") }, + columnTests: { DATE_SENTIMENT_ID: ["unique_x", "not_null_x"], REVIEW_SENTIMENT: ["accepted_values_x"] }, + actualColumns: [ + db("REVIEW_TOTALS"), db("REVIEW_SENTIMENT"), db("MONTH_YEAR"), db("MONTH"), db("YEAR"), db("DATE_SENTIMENT_ID"), + ], }) - const result = await schemaVerify(adapter, ["--model", "target"]) as Record + const result = await schemaVerify(adapter, ["--model", "monthly_agg_reviews"]) as Record + expect(result.verdict).toBe("match") + expect(result.findings).toEqual([]) + // Column order differs from the YAML too; that is not an error either. + expect((result.columns_reordered as unknown[]).length).toBeGreaterThan(0) + }) + + test("a declared column the model does not produce, with a test attached, is a finding naming the file and the test", async () => { + const adapter = makeAdapter({ + expectedColumns: { id: col("id"), email: col("email") }, + columnTests: { email: ["not_null_target_email"] }, + patchPath: "proj://models/schema.yml", + actualColumns: [db("id")], + }) + const result = await schemaVerify(adapter, ["--model", "target"]) as Record expect(result.verdict).toBe("mismatch") expect(result.columns_missing).toEqual(["email"]) - expect(result.columns_extra).toEqual([]) + expect(result.findings).toHaveLength(1) + expect(result.findings[0].kind).toBe("tested-column-missing") + expect(result.findings[0].columns).toEqual(["email"]) + expect(result.findings[0].evidence).toContain("models/schema.yml") + expect(result.findings[0].evidence).toContain("not_null_target_email") + expect(result.spec).toEqual({ declared_in: "models/schema.yml", package: "proj", contract_enforced: false }) + }) + + test.each(["`email`", '"email"', "[email]"])("a missing column with a quoted test column %s is a mismatch", async (name) => { + const adapter = makeAdapter({ + expectedColumns: { id: col("id"), email: col("email") }, + actualColumns: [db("id")], + columnTests: { [name]: ["not_null_target_email"] }, + }) + const result = await schemaVerify(adapter, ["--model", "target"]) + expect(result).toMatchObject({ + verdict: "mismatch", + findings: [{ kind: "tested-column-missing", columns: ["email"] }], + }) + }) + + test("a manifest without attached_node (older dbt) still finds the tested model through depends_on", async () => { + const adapter = makeAdapter({ + expectedColumns: { id: col("id"), email: col("email") }, + columnTests: { email: ["not_null_target_email"] }, + legacyTests: true, + patchPath: "proj://models/schema.yml", + actualColumns: [db("id")], + }) + const result = await schemaVerify(adapter, ["--model", "target"]) as Record + expect(result.findings).toHaveLength(1) + expect(result.findings[0].kind).toBe("tested-column-missing") + }) + + test.each([ + ["{{ get_where_subquery(ref('target')) }}", "model.proj.customers", "mismatch", "tested-column-missing"], + ["{{ ref('proj', 'customers') }}", "model.proj.customers", "match", undefined], + ["{{ get_where_subquery(source('raw', 'customers')) }}", "source.proj.raw.customers", "match", undefined], + [undefined, "model.proj.customers", "mismatch", "test-attachment-unverified"], + ] as const)("a legacy relationship test with two dependencies and model %s", async (model, otherDependency, verdict, kind) => { + const adapter = makeAdapter({ + expectedColumns: { id: col("id"), customer_id: col("customer_id") }, + actualColumns: [db("id")], + legacyRelationship: { dependencies: [otherDependency, "model.proj.target"], model }, + }) + const result = await schemaVerify(adapter, ["--model", "target"]) + expect(result).toMatchObject({ + verdict, + columns_missing: ["customer_id"], + findings: kind ? [expect.objectContaining({ kind, columns: ["customer_id"] })] : [], + }) + }) + + test("a declared column the model does not produce, with nothing attached, is only a note", async () => { + // Package YAML declares assignee_status (description only); the model never produced it. + const adapter = makeAdapter({ + expectedColumns: { task_id: col("task_id"), assignee_status: col("assignee_status") }, + columnTests: { task_id: ["unique_x"] }, + patchPath: "work_items_source://models/stg_work_items.yml", + packageName: "proj", + actualColumns: [db("task_id"), db("name")], + }) + const result = await schemaVerify(adapter, ["--model", "target"]) as Record + expect(result.verdict).toBe("match") + expect(result.findings).toEqual([]) + const notes = (result.notes as string[]).join("\n") + expect(notes).toContain("assignee_status") + expect(notes).toContain("models/stg_work_items.yml") + expect(notes).toContain("package `work_items_source`") + expect(notes).toContain("stale or aspirational") + }) + + test("tests attached to a different model do not make a missing column a finding", async () => { + const adapter = makeAdapter({ + expectedColumns: { id: col("id"), email: col("email") }, + columnTests: {}, + actualColumns: [db("id")], + }) + // makeAdapter always adds not_null_other_email on model.proj.other; this model's `email` is missing. + const result = await schemaVerify(adapter, ["--model", "target"]) as Record + expect(result.verdict).toBe("match") + }) + + test("enforced contract: a missing column is a finding", async () => { + const adapter = makeAdapter({ + contractEnforced: true, + expectedColumns: { id: col("id", "integer"), name: col("name", "varchar") }, + actualColumns: [db("id", "INTEGER")], + }) + const result = await schemaVerify(adapter, ["--model", "target"]) as Record + expect(result.verdict).toBe("mismatch") + expect(result.findings.map((f: any) => f.kind)).toEqual(["contract-missing-columns"]) + expect(result.findings[0].evidence).toContain("enforced contract") + expect(result.spec.contract_enforced).toBe(true) + }) + + test("enforced contract: an extra column is a finding (dbt rejects it)", async () => { + const adapter = makeAdapter({ + contractEnforced: true, + expectedColumns: { id: col("id"), name: col("name") }, + actualColumns: [db("id"), db("name"), db("amount")], + }) + const result = await schemaVerify(adapter, ["--model", "target"]) as Record + expect(result.verdict).toBe("mismatch") + expect(result.findings.map((f: any) => f.kind)).toEqual(["contract-extra-columns"]) + expect(result.findings[0].columns).toEqual(["amount"]) + }) + + test("enforced contract that the table satisfies: match, even when the order differs", async () => { + const adapter = makeAdapter({ + contractEnforced: true, + expectedColumns: { id: col("id", "integer"), name: col("name", "varchar") }, + actualColumns: [db("name", "VARCHAR"), db("id", "INTEGER")], + }) + const result = await schemaVerify(adapter, ["--model", "target"]) as Record + expect(result.verdict).toBe("match") + }) + + test("contract = false is not a contract", async () => { + const adapter = makeAdapter({ + contractEnforced: false, + expectedColumns: { id: col("id") }, + actualColumns: [db("id"), db("extra")], + }) + expect(((await schemaVerify(adapter, ["--model", "target"])) as Record).verdict).toBe("match") + }) + + test("model without any YAML columns: no-spec, whatever the table looks like", async () => { + const adapter = makeAdapter({ expectedColumns: {}, actualColumns: [db("a"), db("b"), db("c")], patchPath: "" }) + const result = await schemaVerify(adapter, ["--model", "target"]) as Record + expect(result.verdict).toBe("no-spec") + expect(result.findings).toBeUndefined() }) - test("detects column reordering when same set but different position", async () => { + test("column order that differs from the YAML is reported in the raw diff, never as a mismatch", async () => { const adapter = makeAdapter({ // schema.yml order: id, name, email expectedColumns: { id: col("id"), name: col("name"), email: col("email") }, @@ -125,13 +310,11 @@ describe("schema-verify command", () => { actualColumns: [db("name"), db("id"), db("email")], }) const result = await schemaVerify(adapter, ["--model", "target"]) as Record - expect(result.verdict).toBe("mismatch") + expect(result.verdict).toBe("match") expect(result.columns_extra).toEqual([]) expect(result.columns_missing).toEqual([]) const reordered = result.columns_reordered as Array<{ column: string }> - expect(reordered.length).toBeGreaterThan(0) - const reorderedNames = reordered.map((r) => r.column) - expect(reorderedNames).toContain("id") + expect(reordered.map((r) => r.column)).toContain("id") }) test("case-insensitive name comparison (dbt convention)", async () => { @@ -143,16 +326,26 @@ describe("schema-verify command", () => { expect(result.verdict).toBe("match") }) - test("detects type mismatch when spec declares a different data_type", async () => { + test("declared data_type that differs without a contract: raw diff only, no mismatch", async () => { const adapter = makeAdapter({ expectedColumns: { id: col("id", "INTEGER"), name: col("name", "VARCHAR") }, actualColumns: [db("id", "BIGINT"), db("name", "VARCHAR")], }) const result = await schemaVerify(adapter, ["--model", "target"]) as Record - expect(result.verdict).toBe("mismatch") - const mm = result.type_mismatches as Array<{ column: string; actual_type: string; expected_type: string }> - expect(mm.length).toBe(1) - expect(mm[0]?.column).toBe("id") + expect(result.verdict).toBe("match") + const mm = result.type_mismatches as Array<{ column: string }> + expect(mm.map((t) => t.column)).toEqual(["id"]) + }) + + test("declared data_type that differs under a contract: noted, because dbt checks types itself at build", async () => { + const adapter = makeAdapter({ + contractEnforced: true, + expectedColumns: { id: col("id", "INTEGER") }, + actualColumns: [db("id", "BIGINT")], + }) + const result = await schemaVerify(adapter, ["--model", "target"]) as Record + expect(result.verdict).toBe("match") + expect(result.notes.join(" ")).toContain("data_type") }) test("ignores type mismatch when spec does not declare data_type", async () => { @@ -175,9 +368,9 @@ describe("schema-verify command", () => { expect((result as { error: string }).error).toContain("Build the model first") }) - test("realistic ade-bench f1002 pattern — extra rank-breakdown columns", async () => { - // Spec: just rank, driver_full_name, podiums - // Actual: agent helpfully added p1, p2, p3 breakdowns + test("extra rank-breakdown columns beyond the YAML are not an error by themselves", async () => { + // YAML: rank, driver_full_name, podiums. Model also returns p1, p2, p3. dbt is happy with that; + // whether the extras are wanted is a question for the task, not for the YAML. const adapter = makeAdapter({ expectedColumns: { rank: col("rank"), @@ -187,25 +380,18 @@ describe("schema-verify command", () => { actualColumns: [db("rank"), db("driver_full_name"), db("podiums"), db("p1"), db("p2"), db("p3")], }) const result = await schemaVerify(adapter, ["--model", "most_podiums"]) as Record - expect(result.verdict).toBe("mismatch") + expect(result.verdict).toBe("match") expect(result.columns_extra).toEqual(["p1", "p2", "p3"]) expect(result.columns_missing).toEqual([]) }) - test("realistic ade-bench pattern — column-order divergence (product_id-first vs inventory_id-first)", async () => { + test("no output of schema-verify tells the reader to remove, add or reorder anything", async () => { const adapter = makeAdapter({ - // Spec leads with product_id - expectedColumns: { - product_id: col("product_id"), - product_code: col("product_code"), - inventory_id: col("inventory_id"), - }, - // Actual leads with inventory_id - actualColumns: [db("inventory_id"), db("product_id"), db("product_code")], + expectedColumns: { id: col("id"), gone: col("gone") }, + columnTests: { gone: ["not_null_target_gone"] }, + actualColumns: [db("name"), db("id")], }) - const result = await schemaVerify(adapter, ["--model", "obt_product_inventory"]) as Record - expect(result.verdict).toBe("mismatch") - const reordered = result.columns_reordered as Array<{ column: string }> - expect(reordered.length).toBeGreaterThan(0) + const text = JSON.stringify(await schemaVerify(adapter, ["--model", "target"])) + expect(text).not.toMatch(/\b(remove|add|reorder|cast|drop|delete)\b/i) }) }) diff --git a/packages/opencode/src/altimate/native/connections/register.ts b/packages/opencode/src/altimate/native/connections/register.ts index 39fcf6139f..9b3fd08397 100644 --- a/packages/opencode/src/altimate/native/connections/register.ts +++ b/packages/opencode/src/altimate/native/connections/register.ts @@ -688,7 +688,12 @@ register("sql.execute", async (params: SqlExecuteParams, context): Promise` or check the project's dbt profile." + : errorMsg + return { columns: [], rows: [], row_count: 0, truncated: false, error } as SqlExecuteResult & { error: string } + // altimate_change end } }) diff --git a/packages/opencode/src/altimate/validators/dbt-schema-verify.ts b/packages/opencode/src/altimate/validators/dbt-schema-verify.ts index df629694df..8d2493350c 100644 --- a/packages/opencode/src/altimate/validators/dbt-schema-verify.ts +++ b/packages/opencode/src/altimate/validators/dbt-schema-verify.ts @@ -4,8 +4,15 @@ * * Fires after the agent declares done. Detects whether the session touched * any dbt models, runs `altimate-dbt schema-verify` against each touched - * model, and reports a mismatch if the produced column shape diverges from - * the schema.yml spec. + * model, and reports the disagreements between the built table and the YAML + * that declares its columns that dbt's own semantics establish as problems: + * an enforced contract that the table does not match, or a declared column + * that has tests attached but is not produced. + * + * It deliberately says nothing about columns the model produces that the YAML + * does not list (YAML commonly documents only some columns), and never tells + * the agent to remove or add columns. See `packages/dbt-tools/src/commands/ + * schema-verify.ts` for the rule. * * The agent does not see this validator existing — it runs in the harness * AFTER `finishReason === "stop"`. Its output is surfaced to the agent only @@ -24,12 +31,23 @@ import { modelNameFromPath, extractLastJsonObject, runWithConcurrencyLimit, + retryErroredSerially, sanitizeForPrompt, } from "./validator-utils" +interface SchemaVerifyFinding { + kind?: string + columns?: string[] + /** Which YAML declares what, and why it is a problem. Written by altimate-dbt. */ + evidence?: string +} + interface SchemaVerifyOutput { model?: string verdict?: "match" | "mismatch" | "no-spec" + /** Present from the altimate-dbt that classifies findings; absent from older builds. */ + findings?: SchemaVerifyFinding[] + notes?: string[] columns_extra?: string[] columns_missing?: string[] columns_reordered?: unknown[] @@ -39,6 +57,17 @@ interface SchemaVerifyOutput { stale_manifest?: { written_at?: unknown } } +/** + * Established problems for one result. Only `findings` count: the raw + * extra/missing/reordered/type lists are an unfiltered diff against the YAML + * and prove nothing on their own. An older altimate-dbt that does not emit + * `findings` cannot say whether a contract or a test is involved, so its + * "mismatch" verdict is not treated as established. + */ +function establishedFindings(r: SchemaVerifyOutput): SchemaVerifyFinding[] { + return r.verdict === "mismatch" && Array.isArray(r.findings) ? r.findings.filter((f) => f.evidence) : [] +} + /** * Extract a SchemaVerifyOutput JSON object from mixed stdout. * `altimate-dbt schema-verify` may emit dbt log noise (ANSI codes, parser @@ -119,24 +148,17 @@ async function runSchemaVerify(model: string, cwd: string): Promise 0) { - lines.push(` • Columns in your model NOT in spec — REMOVE: ${m.columns_extra.join(", ")}`) - } - if (m.columns_missing && m.columns_missing.length > 0) { - lines.push(` • Columns in spec NOT in your model — ADD: ${m.columns_missing.join(", ")}`) - } - if (m.columns_reordered && m.columns_reordered.length > 0) { - lines.push(` • Columns in wrong order — REORDER the SELECT to match schema.yml`) - } - if (m.type_mismatches && m.type_mismatches.length > 0) { - lines.push(` • Type mismatches — CAST or change the upstream source`) - } + for (const f of establishedFindings(m)) lines.push(` • ${sanitizeForPrompt(f.evidence ?? "", 1200)}`) } return lines.join("\n") } @@ -156,7 +178,7 @@ function leadWithStaleManifestNote(results: SchemaVerifyOutput[], hint: string): export const DbtSchemaVerifyValidator: Validator = { name: "dbt-schema-verify", description: - "After the agent declares done, runs `altimate-dbt schema-verify` on every dbt model the agent modified during this session and refuses to terminate if any model's actual columns diverge from the schema.yml spec (extra, missing, reordered, or type-mismatched).", + "After the agent declares done, runs `altimate-dbt schema-verify` on every dbt model the agent modified during this session and refuses to terminate if the built table contradicts its YAML in a way dbt itself treats as an error: an enforced contract the columns do not match, or a declared column with tests attached that the model does not produce. Columns the model produces that the YAML does not list are not reported.", async appliesTo(ctx: ValidatorContext): Promise { // Only run for sessions that took place inside a dbt project. Quick check. @@ -194,11 +216,19 @@ export const DbtSchemaVerifyValidator: Validator = { // Run schema-verify calls with a bounded concurrency limit to prevent // resource contention from too many simultaneous dbt processes. let spawnFailures = 0 - const outputs = await runWithConcurrencyLimit( + const parallel = await runWithConcurrencyLimit( touched, (path) => runSchemaVerify(modelNameFromPath(path), dbtRoot), VALIDATOR_CONCURRENCY, ) + // An error from a parallel run may be the processes contending for the warehouse + // (single-writer DuckDB), not a fact about the model: retry those one at a time. + const { outputs, retried } = await retryErroredSerially( + touched, + parallel, + (path) => runSchemaVerify(modelNameFromPath(path), dbtRoot), + (o) => (o.verdict ? undefined : o.error), + ) const results: SchemaVerifyOutput[] = [] for (let i = 0; i < outputs.length; i++) { const out = outputs[i]! @@ -213,9 +243,11 @@ export const DbtSchemaVerifyValidator: Validator = { } } - const mismatches = results.filter((r) => r.verdict === "mismatch") + const mismatches = results.filter((r) => establishedFindings(r).length > 0) const noSpec = results.filter((r) => r.verdict === "no-spec").length - const matches = results.filter((r) => r.verdict === "match").length + // "match" or a mismatch verdict with nothing established (nothing to act on). + const matches = results.filter((r) => !r.error && (r.verdict === "match" || r.verdict === "mismatch") && establishedFindings(r).length === 0).length + const unestablished = results.filter((r) => !r.error && r.verdict === "mismatch" && establishedFindings(r).length === 0).length const errored = results.filter((r) => r.error).length const baseDetails = { @@ -223,7 +255,11 @@ export const DbtSchemaVerifyValidator: Validator = { verified: results.length, match: matches, no_spec: noSpec, + // Results whose raw diff differs from the YAML but where nothing is established + // (unlisted columns, untested declared-but-absent columns). Telemetry only. + diff_not_established: unestablished, errored, + retried_serially: retried, spawn_failures: spawnFailures, dbt_root: dbtRoot, session_id: ctx.sessionID, @@ -244,10 +280,13 @@ export const DbtSchemaVerifyValidator: Validator = { // (set at push time), so the operator can see WHICH models couldn't be verified // instead of an anonymous count. const erroredNames = results.filter((r) => r.error).map((r) => r.model).filter(Boolean) as string[] + // The cause, not just a count: the first error, trimmed to one line. + const firstErrorRaw = results.find((r) => r.error)?.error + const firstError = firstErrorRaw ? sanitizeForPrompt(firstErrorRaw, 240) : undefined const reason = mismatches.length > 0 - ? `${mismatches.length} of ${results.length} models you edited have a column-shape mismatch against schema.yml${mismatchNames.length ? `: ${mismatchNames.join(", ")}` : ""}. The build may be green, but equality tests will fail.` - : `${errored} model(s) could not be schema-verified (spawn or tool errors)${erroredNames.length ? `: ${erroredNames.join(", ")}` : ""} — schema drift cannot be ruled out. Investigate before declaring done.` + ? `${mismatches.length} of ${results.length} models you edited contradict the YAML that declares their columns${mismatchNames.length ? `: ${mismatchNames.join(", ")}` : ""}.${errored > 0 ? ` ${errored} more could not be schema-verified${erroredNames.length ? ` (${erroredNames.join(", ")})` : ""}${firstError ? `, first error: ${firstError}` : ""}.` : ""}` + : `${errored} model(s) could not be schema-verified (spawn or tool errors)${erroredNames.length ? `: ${erroredNames.join(", ")}` : ""} — schema drift cannot be ruled out${firstError ? ` (first error: ${firstError})` : ""}. Investigate before declaring done.` // altimate_change end return { @@ -257,7 +296,7 @@ export const DbtSchemaVerifyValidator: Validator = { results, mismatches.length > 0 ? formatFixHint(mismatches) + - `\n\nFix the model SQL to match the schema.yml spec (do not edit the spec), rebuild, and the harness will re-check before declaring done.` + `\n\nThese are facts about the built table and the YAML, not a verdict on which side is wrong: change the model if the YAML is right, or the YAML if its entry is stale. A column the model produces that the YAML does not list is not an error unless a contract is enforced.` : `Run \`altimate-dbt schema-verify \` manually to diagnose the error. Check that altimate-dbt is on PATH and that the dbt project compiles cleanly.`, ), details: { diff --git a/packages/opencode/src/altimate/validators/dbt-tests-pass.ts b/packages/opencode/src/altimate/validators/dbt-tests-pass.ts index ecc9282908..f706130459 100644 --- a/packages/opencode/src/altimate/validators/dbt-tests-pass.ts +++ b/packages/opencode/src/altimate/validators/dbt-tests-pass.ts @@ -27,6 +27,7 @@ import { modelNameFromPath, extractLastJsonObject, runWithConcurrencyLimit, + retryErroredSerially, } from "./validator-utils" export interface TestSummary { @@ -47,6 +48,13 @@ interface TestRunOutput { summary?: TestSummary /** Top-level error from altimate-dbt (manifest missing, compile error, etc.). */ error?: string + /** + * dbt ran and selected no tests for this model (it printed "Nothing to do" + * and no summary). That is "nothing to check", not a failed check. + */ + noTests?: boolean + /** Subset of `noTests`: dbt said the selector matched no enabled node at all. */ + unmatchedSelector?: boolean } /** @@ -116,6 +124,25 @@ export function parseDbtTestOutput(stdout: string): TestSummary | null { return { total, pass, error, failingTests } } +// dbt prints these instead of a `Done. PASS=...` summary when the selection has +// no tests (every dbt 1.x we have seen; a selector that matches no node adds +// "does not match any enabled nodes" before it). +const NOTHING_TO_DO_RE = /\bNothing to do\b|\bNo nodes selected\b|\bModel has no tests\b/i +const UNMATCHED_SELECTOR_RE = /does not match any (?:enabled )?nodes?/i +// Markers of a run that failed before or while selecting. If any is present +// "Nothing to do" is not trustworthy, so the run stays an error. +const DBT_ABORT_RE = /Encountered an error|Compilation Error|Parsing Error|Runtime Error|Database Error|Could not set lock/i + +/** + * Did dbt run and find no tests for the selection? Only true when dbt printed + * its "Nothing to do" notice, no run summary, and no sign of an abort. + */ +export function isNothingToTest(dbtLog: string): { noTests: boolean; unmatchedSelector: boolean } { + const cleaned = stripAnsi(dbtLog ?? "") + const noTests = NOTHING_TO_DO_RE.test(cleaned) && !DBT_ABORT_RE.test(cleaned) && parseDbtTestOutput(cleaned) === null + return { noTests, unmatchedSelector: noTests && UNMATCHED_SELECTOR_RE.test(cleaned) } +} + /** Strip ANSI CSI/colour escape sequences from a string. */ function stripAnsi(s: string): string { // Matches CSI sequences (most common: \x1b[...m for colours). @@ -166,16 +193,38 @@ async function runDbtTest(model: string, cwd: string): Promise