From 2c7c746fc02a8744344cd0201a64e14f53e760a3 Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Sat, 21 Nov 2026 10:07:22 -0800 Subject: [PATCH 01/12] fix: make inconclusive completion checks advisory --- docs/docs/data-engineering/validators.md | 19 +++-- docs/docs/reference/telemetry.md | 2 +- .../altimate/validators/dbt-build-green.ts | 2 + .../validators/dbt-deliverable-names.ts | 1 + .../altimate/validators/dbt-dialect-guard.ts | 1 + .../validators/dbt-incremental-config.ts | 1 + .../altimate/validators/dbt-nothing-built.ts | 1 + .../altimate/validators/dbt-schema-verify.ts | 12 +++ .../src/altimate/validators/dbt-tests-pass.ts | 21 +++++- packages/opencode/src/session/prompt.ts | 66 ++++++++++++++-- .../src/session/validators/enforcement.ts | 29 +++++++ .../opencode/src/session/validators/types.ts | 8 +- .../validators/dbt-build-green.test.ts | 1 + .../validators/dbt-incremental-config.test.ts | 1 + .../dbt-schema-verify-stale.test.ts | 27 ++++++- packages/opencode/test/session/prompt.test.ts | 75 +++++++++++++++++++ 16 files changed, 248 insertions(+), 19 deletions(-) create mode 100644 packages/opencode/src/session/validators/enforcement.ts diff --git a/docs/docs/data-engineering/validators.md b/docs/docs/data-engineering/validators.md index 0aaab133ef..8b4427268f 100644 --- a/docs/docs/data-engineering/validators.md +++ b/docs/docs/data-engineering/validators.md @@ -7,10 +7,11 @@ description: "Harness-side validators run after the agent declares done, catchin Validators are harness-side checks that run **after** the agent declares done. Unlike tools, they are not visible to the agent — the framework dispatches -them automatically once the LLM emits `finishReason === "stop"`. If a -validator fails, the framework injects a synthetic user turn with the -failure body, giving the agent one more turn to fix the issue (bounded by -a per-session retry budget). +them automatically once the LLM emits `finishReason === "stop"`. A confirmed finding injects a synthetic user turn with the failure details, +giving the agent one more turn to fix the issue within the retry budget. +An inconclusive check, such as a schema tool failure, leaves an advisory in +the next model request's context. An advisory alone does not start a request +or spend a retry. This catches a class of error that prompt engineering alone cannot: the agent claiming "done" while the work is actually broken (failing dbt @@ -53,7 +54,8 @@ deterministic-validators work and are currently soak-tested in shadow mode After the agent declares done, runs `altimate-dbt test --model ` against every dbt model the agent modified during this session. Refuses -to terminate if any model's tests fail or error. +to terminate if a model's test run reports failures. If the test tool cannot +return a usable result, the validator reports an advisory. **Catches**: row-data correctness errors (`relationships`, `unique`, `not_null`, `accepted_values`, `AUTO_*_equality` tests) — the kind of @@ -137,7 +139,7 @@ Two opt-in modes: | Env var | Effect | |---|---| -| `ALTIMATE_VALIDATORS_ENABLED=1` | Full enforcement. Failing validators inject a synthetic user turn for the agent to fix (bounded by retries). | +| `ALTIMATE_VALIDATORS_ENABLED=1` | Enforces findings within the retry budget and adds inconclusive checks as advisories. | | `ALTIMATE_VALIDATORS_SHADOW=1` | Telemetry-only. Validators run and emit `validator_check` events (with `enforced: false`), but do **not** block the session. Use this to measure "would have caught a real bug" rates against production traffic. | Set in your shell, your `~/.altimate-code/altimate-code.json`'s `env` @@ -158,6 +160,7 @@ export ALTIMATE_VALIDATORS_SHADOW=1 |---|---|---| | `ALTIMATE_VALIDATORS_ENABLED` | unset (off) | Master enforcement switch | | `ALTIMATE_VALIDATORS_SHADOW` | unset (off) | Telemetry-only mode | +| `ALTIMATE_VALIDATORS_STRICT` | unset (off) | Set to `1` to restore the prior retry policy for inconclusive checks and suppress advisories. Requires enforcement mode. | | `ALTIMATE_VALIDATORS_MAX_RETRIES` | `3` | How many synthetic-message retries per session before giving up | | `ALTIMATE_VALIDATORS_TIMEOUT_MS` | `60000` | Per-subprocess kill timeout (NaN/0/negative falls back to default) | | `ALTIMATE_VALIDATORS_CONCURRENCY` | `4` | Max concurrent `altimate-dbt` subprocesses (clamped to `items.length`) | @@ -186,7 +189,9 @@ minute of wall time. ## Telemetry When validators run (either mode), they emit one -`validator_check` event per applied validator: +`validator_check` event per applied validator. The optional `classification` +field is `finding` or `unverifiable`. The `ok` field retains its legacy +meaning. `enforced` is false for advisory-only results in advisory mode. ```json { diff --git a/docs/docs/reference/telemetry.md b/docs/docs/reference/telemetry.md index c5e1639caf..3a1576b04a 100644 --- a/docs/docs/reference/telemetry.md +++ b/docs/docs/reference/telemetry.md @@ -54,7 +54,7 @@ We collect the following categories of events: | `error_fingerprint` | Error pattern for grouping — SHA-256 hash of the masked error message, error class, tool name, whether recovery succeeded, plus `error_message`. `error_message` contains only words from a fixed list of common error terms taken from the first line of the error; every other word is replaced with ``, so no user text, identifiers, paths, values or SQL can appear (max 200 characters). | | `sql_fingerprint` | SQL structural shape via AST parsing — statement types, table count, function count, subquery/aggregation/window function presence, and AST node count. No table names, column names, or SQL content. | | `schema_complexity` | Warehouse schema structural metrics from introspection — bucketed table, column, and schema counts plus average columns per table. No schema names or content. | -| `validator_check` | A completion-gate validator ran on session end — validator name, `ok` boolean, step, retry count, `enforced` flag (false in shadow mode), and structured `details` (model counts, elapsed time, concurrency limit — no SQL or model content). Only emitted when `ALTIMATE_VALIDATORS_ENABLED=1` or `ALTIMATE_VALIDATORS_SHADOW=1`. See [Validators](../data-engineering/validators.md). | +| `validator_check` | A completion-gate validator ran on session end — validator name, `ok` boolean, step, retry count, `enforced` flag (false in shadow mode and for advisory-only results in advisory mode), optional `classification` (`finding` or `unverifiable`), and structured `details` (model counts, elapsed time, concurrency limit — no SQL or model content). Only emitted when `ALTIMATE_VALIDATORS_ENABLED=1` or `ALTIMATE_VALIDATORS_SHADOW=1`. See [Validators](../data-engineering/validators.md). | | `validator_retries_exhausted` | A session terminated with unresolved validator failures after exhausting the synthetic-retry budget — names of the failing validators (no failure body content). | | `onboarding_started` | The first-run setup gate opened (fresh launch with no usable model). | | `model_picker_shown` | The provider picker was displayed. `trigger` distinguishes the first run from `/connect` and from the prompt gate (`altimate_base_back` is defined in the schema but no longer emitted — it recorded backing out of the old consent dialog, which no longer exists). | diff --git a/packages/opencode/src/altimate/validators/dbt-build-green.ts b/packages/opencode/src/altimate/validators/dbt-build-green.ts index dc470fdcbc..54c69a863e 100644 --- a/packages/opencode/src/altimate/validators/dbt-build-green.ts +++ b/packages/opencode/src/altimate/validators/dbt-build-green.ts @@ -219,6 +219,7 @@ export const DbtBuildGreenValidator: Validator = { : `You edited ${touchedPaths.length} model(s) but the only build artifact (${sanitizeForPrompt(artifact.path, 160)}) predates this session. Your edits have never been built.` return { ok: false, + classification: "finding", reason, fixHint: freshButNotABuild ? `That command only parses and renders SQL — it never runs it against the warehouse. Run \`dbt build\` for the models you changed and confirm it finishes without errors before declaring done.` @@ -523,6 +524,7 @@ export const DbtBuildGreenValidator: Validator = { return { ok: false, + classification: "finding", reason: `The build is not green: ${reasonParts.join("; ")}.`, fixHint: hintLines.join("\n"), details, diff --git a/packages/opencode/src/altimate/validators/dbt-deliverable-names.ts b/packages/opencode/src/altimate/validators/dbt-deliverable-names.ts index 6ff113ec79..cc497646a7 100644 --- a/packages/opencode/src/altimate/validators/dbt-deliverable-names.ts +++ b/packages/opencode/src/altimate/validators/dbt-deliverable-names.ts @@ -220,6 +220,7 @@ export const DbtDeliverableNamesValidator: Validator = { return { ok: false, + classification: "finding", reason: `Deliverable-name mismatch: ${reasonParts.join("; ")}.`, fixHint: hintLines.join("\n"), details: { ...details, unrequested_models: unrequested }, diff --git a/packages/opencode/src/altimate/validators/dbt-dialect-guard.ts b/packages/opencode/src/altimate/validators/dbt-dialect-guard.ts index 758809de9f..474596e751 100644 --- a/packages/opencode/src/altimate/validators/dbt-dialect-guard.ts +++ b/packages/opencode/src/altimate/validators/dbt-dialect-guard.ts @@ -293,6 +293,7 @@ export const DbtDialectGuardValidator: Validator = { return { ok: false, + classification: "finding", // Repository-derived model names reach a synthetic `role: "user"` turn // through dispatch; sanitized as in `dbt-build-green`. reason: `${findings.length} unguarded warehouse-specific construct(s) in ${byModel.size} model(s) you edited: ${Array.from(byModel.keys()).map((m) => sanitizeForPrompt(m, 80)).join(", ")}.`, diff --git a/packages/opencode/src/altimate/validators/dbt-incremental-config.ts b/packages/opencode/src/altimate/validators/dbt-incremental-config.ts index 64fb6c2064..f3d009f028 100644 --- a/packages/opencode/src/altimate/validators/dbt-incremental-config.ts +++ b/packages/opencode/src/altimate/validators/dbt-incremental-config.ts @@ -473,6 +473,7 @@ export const DbtIncrementalConfigValidator: Validator = { return { ok: false, + classification: "finding", // Model names come from repository filenames and reach a synthetic // `role: "user"` turn through dispatch, so they are sanitized exactly as // `dbt-build-green` sanitizes node names and artifact paths. diff --git a/packages/opencode/src/altimate/validators/dbt-nothing-built.ts b/packages/opencode/src/altimate/validators/dbt-nothing-built.ts index a9591e5a7d..90fbd04aa2 100644 --- a/packages/opencode/src/altimate/validators/dbt-nothing-built.ts +++ b/packages/opencode/src/altimate/validators/dbt-nothing-built.ts @@ -432,6 +432,7 @@ export const DbtNothingBuiltValidator: Validator = { return { ok: false, + classification: "finding", reason, fixHint: [ diff --git a/packages/opencode/src/altimate/validators/dbt-schema-verify.ts b/packages/opencode/src/altimate/validators/dbt-schema-verify.ts index df629694df..dc25dff7c9 100644 --- a/packages/opencode/src/altimate/validators/dbt-schema-verify.ts +++ b/packages/opencode/src/altimate/validators/dbt-schema-verify.ts @@ -244,6 +244,14 @@ export const DbtSchemaVerifyValidator: Validator = { // (set at push time), so the operator can see WHICH models couldn't be verified // instead of an anonymous count. const erroredNames = results.filter((r) => r.error).map((r) => r.model).filter(Boolean) as string[] + const advisoryNames = results.flatMap((result, index) => + result.error ? [modelNameFromPath(touched[index]!)] : [], + ) + const advisoryCause = + spawnFailures === errored ? "the schema verification subprocess did not start or returned no output" + : spawnFailures > 0 ? "the schema verification subprocess could not run or the tool failed" + : "the schema verification tool failed" + const reason = mismatches.length > 0 ? `${mismatches.length} of ${results.length} models you edited have a column-shape mismatch against schema.yml${mismatchNames.length ? `: ${mismatchNames.join(", ")}` : ""}. The build may be green, but equality tests will fail.` @@ -252,6 +260,10 @@ export const DbtSchemaVerifyValidator: Validator = { return { ok: false, + classification: mismatches.length > 0 ? "finding" : "unverifiable", + advisory: errored > 0 + ? `Could not verify the schema for ${errored} model(s) because ${advisoryCause}: ${advisoryNames.slice(0, 10).map((name) => sanitizeForPrompt(name.replace(/[^\w.-]/g, "?"), 80)).join(", ")}.` + : undefined, reason, fixHint: leadWithStaleManifestNote( results, diff --git a/packages/opencode/src/altimate/validators/dbt-tests-pass.ts b/packages/opencode/src/altimate/validators/dbt-tests-pass.ts index ecc9282908..cef8506390 100644 --- a/packages/opencode/src/altimate/validators/dbt-tests-pass.ts +++ b/packages/opencode/src/altimate/validators/dbt-tests-pass.ts @@ -27,6 +27,7 @@ import { modelNameFromPath, extractLastJsonObject, runWithConcurrencyLimit, + sanitizeForPrompt, } from "./validator-utils" export interface TestSummary { @@ -166,12 +167,17 @@ async function runDbtTest(model: string, cwd: string): Promise 0) { + resolve({ model, summary }) + return + } if (typeof envelope.error === "string") { resolve({ model, error: envelope.error.slice(0, 500) }) return } - const dbtLog = typeof envelope.stdout === "string" ? envelope.stdout : "" - const summary = parseDbtTestOutput(dbtLog) if (!summary) { resolve({ model, error: `no PASS/ERROR summary in dbt output: ${dbtLog.slice(-300)}` }) return @@ -269,9 +275,18 @@ export const DbtTestsPassValidator: Validator = { elapsed_ms: Date.now() - startedAt, } + const unverifiedNames = [ + ...errored.map((r) => r.model), + ...touched.filter((_, index) => !outputs[index]).map(modelNameFromPath), + ] + const advisory = unverifiedNames.length > 0 + ? `Could not verify dbt tests for ${unverifiedNames.length} model(s) because the test tool did not return a usable result: ${unverifiedNames.slice(0, 10).map((name) => sanitizeForPrompt(name.replace(/[^\w.-]/g, "?"), 80)).join(", ")}.` + : undefined + if (failures.length === 0 && errored.length === 0) { return { ok: true, + ...(advisory && { classification: "unverifiable", advisory }), details: { ...baseDetails, passed: passed.length, @@ -290,6 +305,8 @@ export const DbtTestsPassValidator: Validator = { : `${errored.length} of ${results.length} models could not be tested${erroredNames.length ? `: ${erroredNames.join(", ")}` : ""}. Investigate before declaring done.` return { ok: false, + classification: failures.length > 0 ? "finding" : "unverifiable", + advisory, reason, fixHint: formatFixHint(hintBlocks) + diff --git a/packages/opencode/src/session/prompt.ts b/packages/opencode/src/session/prompt.ts index 93187d0201..df123c755b 100644 --- a/packages/opencode/src/session/prompt.ts +++ b/packages/opencode/src/session/prompt.ts @@ -115,6 +115,7 @@ import { Fingerprint } from "../altimate/fingerprint" // altimate_change end // altimate_change start - validator framework (see session/validators/types.ts header) +import { validatorEnforcement } from "./validators/enforcement" import { ValidatorRegistry } from "./validators/registry" import { registerAltimateValidators } from "../altimate/validators" import { sanitizeTelemetryDetails } from "../altimate/validators/validator-utils" @@ -900,6 +901,10 @@ export namespace SessionPrompt { let compactionCount = 0 // altimate_change start — validator framework retry counter let validatorRetryCount = 0 + const validatorStrict = process.env.ALTIMATE_VALIDATORS_STRICT === "1" + const validatorAdvisoriesEnabled = process.env.ALTIMATE_VALIDATORS_ENABLED === "1" && !validatorStrict + const seenValidatorAdvisories = new Set() + let pendingValidatorAdvisories: MessageV2.Assistant | undefined // altimate_change end let sessionAgentName = "" let sessionHadError = false @@ -2229,6 +2234,24 @@ export namespace SessionPrompt { // altimate_change end // altimate_change start — only a request actually sent to the model counts toward the cap modelSteps++ + // A queued note survives normal termination and history compaction. Read + // it once when resuming this session, or use the note queued in this loop. + if (validatorAdvisoriesEnabled && modelSteps === 1) { + for (const message of MessageV2.stream(sessionID)) { + if (message.info.role !== "assistant" || message.info.id === processor.message.id || message.info.summary) continue + const { validatorAdvisories, validatorAdvisoriesShown } = message.info.metadata ?? {} + if (Array.isArray(validatorAdvisories) && validatorAdvisories.length > 0) pendingValidatorAdvisories = message.info + if (Array.isArray(validatorAdvisoriesShown)) { + for (const advisory of validatorAdvisoriesShown) seenValidatorAdvisories.add(advisory) + } + break + } + } + const validatorAdvisories = pendingValidatorAdvisories?.metadata?.validatorAdvisories + const advisoryContext = + Array.isArray(validatorAdvisories) && validatorAdvisories.length > 0 + ? ["Completion checks that could not be verified (advisory only):", ...validatorAdvisories].join("\n") + : undefined // altimate_change end const result = await processor.process({ user: lastUser, @@ -2238,7 +2261,9 @@ export namespace SessionPrompt { agent, abort, sessionID, - system, + // altimate_change start — one-shot advisory context for the existing turn + system: advisoryContext ? [...system, advisoryContext] : system, + // altimate_change end // altimate_change start — upstream_fix: append ambient context after the cached instructions environment: await SystemPrompt.environment(model), // altimate_change end @@ -2284,6 +2309,20 @@ export namespace SessionPrompt { // altimate_change end }) + // altimate_change start — deliver advisory context at most once + if (pendingValidatorAdvisories) { + const { validatorAdvisories, ...metadata } = pendingValidatorAdvisories.metadata ?? {} + if (Array.isArray(validatorAdvisories)) { + for (const advisory of validatorAdvisories) seenValidatorAdvisories.add(advisory) + } + await Session.updateMessage({ + ...pendingValidatorAdvisories, + metadata: { ...metadata, validatorAdvisoriesShown: [...seenValidatorAdvisories] }, + }) + pendingValidatorAdvisories = undefined + } + // altimate_change end + // altimate_change start - finalize buffered echoes before publishing any text to clients const stepParts = MessageV2.parts(processor.message.id) // Capture before post-processing can fail or request compaction. @@ -2512,7 +2551,12 @@ export namespace SessionPrompt { }), ) } - const failures = checks.filter((c) => !c.result.ok) + const { failures, advisories, retry, exhausted } = validatorEnforcement(checks, { + enabled: validatorsEnabled, + strict: validatorStrict, + retryCount: validatorRetryCount, + maxRetries: maxValidatorRetries, + }) // Telemetry: emit one event per validator that ran, plus a session // rollup. Always emitted, even when the feature flag is off, so we @@ -2536,12 +2580,24 @@ export namespace SessionPrompt { ok: vRes.ok, step, retry_count: validatorRetryCount, - enforced: validatorsEnabled, + enforced: validatorsEnabled && (validatorStrict || vRes.classification !== "unverifiable"), + ...(vRes.classification && { classification: vRes.classification }), ...(vRes.details && { details: sanitizeTelemetryDetails(vRes.details) }), } as any) } - if (failures.length > 0 && validatorsEnabled && validatorRetryCount < maxValidatorRetries) { + if (advisories.length > 0) { + const pending = advisories.filter((advisory) => !seenValidatorAdvisories.has(advisory)) + processor.message.metadata = { + ...processor.message.metadata, + validatorAdvisories: pending, + validatorAdvisoriesShown: advisories.filter((advisory) => seenValidatorAdvisories.has(advisory)), + } + await Session.updateMessage(processor.message) + if (pending.length > 0) pendingValidatorAdvisories = processor.message + } + + if (retry) { // Build a single synthetic user-turn body that aggregates every // failing validator's reason + fixHint. The agent sees this as // the next user message and gets one more turn to address it. @@ -2585,7 +2641,7 @@ export namespace SessionPrompt { validatorRetryCount++ continue - } else if (failures.length > 0 && validatorsEnabled && validatorRetryCount >= maxValidatorRetries) { + } else if (exhausted) { // Retry budget exhausted with outstanding failures. Session will // terminate on the natural break below. Emit an explicit signal so // the operator dashboard can distinguish "completed cleanly" from diff --git a/packages/opencode/src/session/validators/enforcement.ts b/packages/opencode/src/session/validators/enforcement.ts new file mode 100644 index 0000000000..42684d8d48 --- /dev/null +++ b/packages/opencode/src/session/validators/enforcement.ts @@ -0,0 +1,29 @@ +import type { Validator, ValidatorResult } from "./types" + +interface Check { + validator: Validator + result: ValidatorResult +} + +export function validatorEnforcement( + checks: Check[], + options: { enabled: boolean; strict: boolean; retryCount: number; maxRetries: number }, +) { + const failures: Check[] = [] + const advisories: string[] = [] + if (options.enabled) { + for (const check of checks) { + const { validator, result } = check + if (!result.ok && (options.strict || result.classification !== "unverifiable")) failures.push(check) + if (!options.strict && result.advisory) { + advisories.push(`[altimate-validator advisory: ${validator.name}] ${result.advisory}`) + } + } + } + return { + failures, + advisories, + retry: failures.length > 0 && options.retryCount < options.maxRetries, + exhausted: failures.length > 0 && options.retryCount >= options.maxRetries, + } +} diff --git a/packages/opencode/src/session/validators/types.ts b/packages/opencode/src/session/validators/types.ts index eb86eaef3a..7e7ff504f9 100644 --- a/packages/opencode/src/session/validators/types.ts +++ b/packages/opencode/src/session/validators/types.ts @@ -66,9 +66,15 @@ export interface ValidatorContext { * Result of running a validator against a session context. */ export interface ValidatorResult { - /** `true` means the agent's work passes this check. */ + /** Whether the previous enforcement policy accepted the result. */ ok: boolean + /** Set by the validator when it has a finding or could not complete the check. */ + classification?: "finding" | "unverifiable" + + /** A prompt-safe description of what could not be checked and why. May accompany a finding. */ + advisory?: string + /** Short human-readable explanation of what's wrong. Surfaced in the * synthetic user message; should fit on one or two lines. */ reason?: string diff --git a/packages/opencode/test/altimate/validators/dbt-build-green.test.ts b/packages/opencode/test/altimate/validators/dbt-build-green.test.ts index fbacda0529..2a70d1cffb 100644 --- a/packages/opencode/test/altimate/validators/dbt-build-green.test.ts +++ b/packages/opencode/test/altimate/validators/dbt-build-green.test.ts @@ -141,6 +141,7 @@ describe("DbtBuildGreenValidator — fresh artifact", () => { ]) const r = await DbtBuildGreenValidator.check(ctx()) expect(r.ok).toBe(false) + expect(r.classification).toBe("finding") expect(r.reason).toContain("stg_orders") expect(r.fixHint).toContain("Compilation Error in model") }) diff --git a/packages/opencode/test/altimate/validators/dbt-incremental-config.test.ts b/packages/opencode/test/altimate/validators/dbt-incremental-config.test.ts index dd8dd199dc..8a9d42d00f 100644 --- a/packages/opencode/test/altimate/validators/dbt-incremental-config.test.ts +++ b/packages/opencode/test/altimate/validators/dbt-incremental-config.test.ts @@ -76,6 +76,7 @@ describe("DbtIncrementalConfigValidator — upsert without a key", () => { ) const r = await DbtIncrementalConfigValidator.check(ctx()) expect(r.ok).toBe(false) + expect(r.classification).toBe("finding") expect(r.fixHint).toContain("unique_key") expect((r.details!["findings"] as Array<{ kind: string }>)[0]!.kind).toBe( "upsert-without-unique-key", diff --git a/packages/opencode/test/altimate/validators/dbt-schema-verify-stale.test.ts b/packages/opencode/test/altimate/validators/dbt-schema-verify-stale.test.ts index efbcf47aeb..ef755343f7 100644 --- a/packages/opencode/test/altimate/validators/dbt-schema-verify-stale.test.ts +++ b/packages/opencode/test/altimate/validators/dbt-schema-verify-stale.test.ts @@ -7,7 +7,8 @@ import fs from "fs/promises" import { tmpdir } from "os" import { join } from "path" import { DbtSchemaVerifyValidator } from "../../../src/altimate/validators/dbt-schema-verify" -import type { ValidatorContext } from "../../../src/session/validators/types" +import { DbtTestsPassValidator } from "../../../src/altimate/validators/dbt-tests-pass" +import type { Validator, ValidatorContext } from "../../../src/session/validators/types" const WRITTEN_AT = "2026-11-05T08:00:00.000Z" const STALE = { @@ -54,7 +55,7 @@ afterEach(async () => { }) /** Runs the validator over one model edited this session, with `answer` as schema-verify's answer. */ -async function check(answer: object) { +async function check(answer: object, validator: Validator = DbtSchemaVerifyValidator) { await fs.writeFile(answerFile, JSON.stringify(answer)) const ctx: ValidatorContext = { sessionID: "sv-stale", @@ -64,13 +65,14 @@ async function check(answer: object) { retryCount: 0, } await fs.writeFile(join(project, "models", "orders.sql"), "select 1 as id, 2 as note\n") - return DbtSchemaVerifyValidator.check(ctx) + return validator.check(ctx) } describe("dbt-schema-verify on a stale manifest", () => { test("a mismatch's message leads with the note and the parse time", async () => { const result = await check({ stale_manifest: STALE, ...MISMATCH }) expect(result.ok).toBe(false) + expect(result.classification).toBe("finding") expect(result.fixHint?.startsWith(`${NOTE_START} (written «${WRITTEN_AT}»).`)).toBe(true) expect(result.fixHint).toContain("Columns in your model NOT in spec — REMOVE: note") }) @@ -78,6 +80,8 @@ describe("dbt-schema-verify on a stale manifest", () => { test("an error's message leads with the note too", async () => { const result = await check({ stale_manifest: STALE, error: "Model 'orders' not found in manifest." }) expect(result.ok).toBe(false) + expect(result.classification).toBe("unverifiable") + expect(result.advisory).toContain("orders") expect(result.fixHint?.startsWith(NOTE_START)).toBe(true) expect(result.fixHint).toContain("altimate-dbt schema-verify ") }) @@ -95,3 +99,20 @@ describe("dbt-schema-verify on a stale manifest", () => { ) }) }) + +describe("dbt-tests-pass result classification", () => { + test("a tool error before the tests run is unverifiable", async () => { + const result = await check({ error: "Could not set lock on file" }, DbtTestsPassValidator) + expect(result).toMatchObject({ ok: false, classification: "unverifiable" }) + expect(result.advisory).toContain("orders") + }) + + test("a real test failure is a finding even when the wrapper also reports an error", async () => { + const result = await check({ + error: "dbt exited with code 1", + stdout: "1 of 1 FAIL 1 not_null_orders_id [FAIL 1 in 0.10s]\nDone. PASS=0 WARN=0 ERROR=1 SKIP=0 TOTAL=1", + }, DbtTestsPassValidator) + expect(result).toMatchObject({ ok: false, classification: "finding" }) + expect(result.fixHint).toContain("not_null_orders_id") + }) +}) diff --git a/packages/opencode/test/session/prompt.test.ts b/packages/opencode/test/session/prompt.test.ts index eef6c80017..2599ec833b 100644 --- a/packages/opencode/test/session/prompt.test.ts +++ b/packages/opencode/test/session/prompt.test.ts @@ -43,6 +43,7 @@ import { SessionSummary } from "../../src/session/summary" import { Instruction } from "../../src/session/instruction" import { SessionProcessor } from "../../src/session/processor" import { SessionPrompt } from "../../src/session/prompt" +import { ValidatorRegistry } from "../../src/session/validators/registry" import { SessionRevert } from "../../src/session/revert" import { SessionRunState } from "../../src/session/run-state" import { MessageID, PartID, SessionID } from "../../src/session/schema" @@ -574,6 +575,80 @@ for (const downgrade of [true, false]) { } // altimate_change end +// altimate_change start — completion validator enforcement through the real prompt loop +for (const mode of ["advisory", "finding", "strict"] as const) { + noLLMServer.instance( + `completion validators use the ${mode} policy`, + () => Effect.gen(function* () { + const flagNames = ["ALTIMATE_VALIDATORS_ENABLED", "ALTIMATE_VALIDATORS_STRICT", "ALTIMATE_VALIDATORS_MAX_RETRIES"] + const previousFlags = flagNames.map((name) => process.env[name]) + process.env.ALTIMATE_VALIDATORS_ENABLED = "1" + process.env.ALTIMATE_VALIDATORS_STRICT = mode === "strict" ? "1" : "0" + delete process.env.ALTIMATE_VALIDATORS_MAX_RETRIES + const requests: string[] = [] + const note = "Could not verify orders because the schema tool failed." + const runAll = spyOn(ValidatorRegistry, "runAll").mockImplementation(async () => [{ + validator: ValidatorRegistry.list()[0]!, + result: mode === "finding" + ? { ok: false, classification: "finding", reason: "Confirmed failure." } + : { ok: false, classification: "unverifiable", reason: note, advisory: note }, + }]) + const stream = spyOn(LLM, "stream").mockImplementation(async (input) => { + requests.push(input.system.join("\n")) + const usage = { inputTokens: 10, outputTokens: 5, totalTokens: 15 } + async function* fullStream() { + yield { type: "start" } + yield { type: "start-step" } + yield { type: "text-start", id: "reply" } + yield { type: "text-delta", id: "reply", text: "Done." } + yield { type: "text-end", id: "reply" } + yield { type: "finish-step", finishReason: "stop", usage } + yield { type: "finish", finishReason: "stop", totalUsage: usage } + } + return { fullStream: fullStream() } as unknown as Awaited> + }) + try { + const { prompt, chat } = yield* boot() + const reply = (text: string) => prompt.prompt({ + sessionID: chat.id, + agent: "build", + model: promptRef, + parts: [{ type: "text", text }], + }) + yield* reply("Review invoice totals.") + const retries = runAll.mock.calls.map(([ctx]) => ctx.retryCount) + if (mode === "advisory") { + expect(requests).toHaveLength(1) + expect(retries).toEqual([0]) + expect(requests[0]).not.toContain(note) + yield* reply("Follow-up on invoice totals.") + yield* reply("Another question about invoices.") + yield* reply("A final invoice question.") + expect(requests).toHaveLength(4) + expect(requests[1]).toContain(note) + expect(requests[2]).not.toContain(note) + expect(requests[3]).not.toContain(note) + } else { + expect(requests).toHaveLength(4) + expect(retries).toEqual([0, 1, 2, 3]) + expect(requests.every((request) => !request.includes(note))).toBe(true) + } + } finally { + stream.mockRestore() + runAll.mockRestore() + for (const [index, name] of flagNames.entries()) { + const value = previousFlags[index] + if (value === undefined) delete process.env[name] + else process.env[name] = value + } + } + }), + { config: { ...cfg, snapshot: false, learn: { enabled: false } } }, + 30_000, + ) +} +// altimate_change end + // altimate_change start - ambient environment stays outside the cached prompt prefix noLLMServer.instance( "prompt loop separates ambient environment from cached instructions and user content", From b9719421d48088d6b8302d52f8da0f5a2d51f430 Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Sat, 21 Nov 2026 10:15:23 -0800 Subject: [PATCH 02/12] fix: retain validator advisory delivery state across turns --- docs/docs/data-engineering/tools/dbt-tools.md | 6 ++-- docs/docs/data-engineering/validators.md | 3 +- .../altimate/validators/dbt-schema-verify.ts | 6 ++-- .../src/altimate/validators/dbt-tests-pass.ts | 4 +-- packages/opencode/src/session/llm.ts | 13 +++++++- packages/opencode/src/session/prompt.ts | 30 ++++++++--------- .../src/session/validators/enforcement.ts | 21 +++++++++++- packages/opencode/test/session/llm.test.ts | 27 ++++++++++++---- packages/opencode/test/session/prompt.test.ts | 32 ++++++++++++++----- 9 files changed, 100 insertions(+), 42 deletions(-) diff --git a/docs/docs/data-engineering/tools/dbt-tools.md b/docs/docs/data-engineering/tools/dbt-tools.md index 0b924da59e..09f543617a 100644 --- a/docs/docs/data-engineering/tools/dbt-tools.md +++ b/docs/docs/data-engineering/tools/dbt-tools.md @@ -373,11 +373,11 @@ WHERE updated_at > (SELECT MAX(updated_at) FROM {{ this }}) Beyond the agent-facing tools above, altimate-code ships **harness-side validators** that fire automatically after the agent declares done. They run `altimate-dbt test` and `altimate-dbt schema-verify` against every model -modified during the session and block "done" if anything failed. +modified during the session. Confirmed findings block completion and +inconclusive checks are advisory by default. This is **opt-in** today via either `ALTIMATE_VALIDATORS_ENABLED=1` -(enforcement mode — failing validators block "done" with synthetic -retries) or `ALTIMATE_VALIDATORS_SHADOW=1` (telemetry-only mode — runs +(enforcement mode, with synthetic retries for confirmed findings) or `ALTIMATE_VALIDATORS_SHADOW=1` (telemetry-only mode — runs without blocking, useful for measuring "would have caught" rates). When neither flag is set the dispatch path is completely skipped and there is zero overhead. See the [Validators page](../validators.md) for the diff --git a/docs/docs/data-engineering/validators.md b/docs/docs/data-engineering/validators.md index 8b4427268f..cd8e0dd5d5 100644 --- a/docs/docs/data-engineering/validators.md +++ b/docs/docs/data-engineering/validators.md @@ -7,7 +7,8 @@ description: "Harness-side validators run after the agent declares done, catchin Validators are harness-side checks that run **after** the agent declares done. Unlike tools, they are not visible to the agent — the framework dispatches -them automatically once the LLM emits `finishReason === "stop"`. A confirmed finding injects a synthetic user turn with the failure details, +them automatically once the LLM emits `finishReason === "stop"`. A confirmed +finding injects a synthetic user turn with the failure details, giving the agent one more turn to fix the issue within the retry budget. An inconclusive check, such as a schema tool failure, leaves an advisory in the next model request's context. An advisory alone does not start a request diff --git a/packages/opencode/src/altimate/validators/dbt-schema-verify.ts b/packages/opencode/src/altimate/validators/dbt-schema-verify.ts index dc25dff7c9..1114bccc19 100644 --- a/packages/opencode/src/altimate/validators/dbt-schema-verify.ts +++ b/packages/opencode/src/altimate/validators/dbt-schema-verify.ts @@ -156,7 +156,7 @@ function leadWithStaleManifestNote(results: SchemaVerifyOutput[], hint: string): export const DbtSchemaVerifyValidator: Validator = { name: "dbt-schema-verify", description: - "After the agent declares done, runs `altimate-dbt schema-verify` on every dbt model the agent modified during this session and refuses to terminate if any model's actual columns diverge from the schema.yml spec (extra, missing, reordered, or type-mismatched).", + "After the agent declares done, runs `altimate-dbt schema-verify` on every dbt model the agent modified during this session and refuses to terminate if any model's actual columns diverge from the schema.yml spec (extra, missing, reordered, or type-mismatched). Inconclusive checks are advisory by default.", async appliesTo(ctx: ValidatorContext): Promise { // Only run for sessions that took place inside a dbt project. Quick check. @@ -231,9 +231,7 @@ export const DbtSchemaVerifyValidator: Validator = { elapsed_ms: Date.now() - startedAt, } - // Fail closed: return ok only when every model was verified and none mismatched. - // Errors (spawn failures, schema-verify tool errors) prevent a clean pass because - // we cannot rule out drift on models we failed to inspect. + // Retain the legacy ok value for strict mode. Classification controls advisory mode. if (mismatches.length === 0 && errored === 0) { return { ok: true, details: baseDetails } } diff --git a/packages/opencode/src/altimate/validators/dbt-tests-pass.ts b/packages/opencode/src/altimate/validators/dbt-tests-pass.ts index cef8506390..1315359c28 100644 --- a/packages/opencode/src/altimate/validators/dbt-tests-pass.ts +++ b/packages/opencode/src/altimate/validators/dbt-tests-pass.ts @@ -4,7 +4,7 @@ * * Fires after the agent declares done. Detects which dbt models the session * touched, runs `altimate-dbt test --model ` against each, and refuses - * to terminate if any dbt test fails or errors. + * to terminate if the test run reports failures. Inconclusive checks are advisory by default. * * Catches row-data correctness errors that the column-shape validator does * not: a model whose schema.yml matches the actual columns can still fail @@ -213,7 +213,7 @@ function formatFixHint(failures: TestRunOutput[]): string { export const DbtTestsPassValidator: Validator = { name: "dbt-tests-pass", description: - "After the agent declares done, runs `altimate-dbt test` against every dbt model the agent modified during this session and refuses to terminate if any model's tests fail or error. Catches row-data correctness errors (relationships, unique, not_null, accepted_values, AUTO_*_equality) that column-shape verification cannot detect.", + "After the agent declares done, runs `altimate-dbt test` against every dbt model the agent modified during this session. Reported test failures block completion, and inconclusive checks are advisory by default. Catches row-data correctness errors (relationships, unique, not_null, accepted_values, AUTO_*_equality) that column-shape verification cannot detect.", async appliesTo(ctx: ValidatorContext): Promise { return (await findDbtProjectRoot(ctx.workingDirectory)) !== null diff --git a/packages/opencode/src/session/llm.ts b/packages/opencode/src/session/llm.ts index b806e6270c..80fd9aa94e 100644 --- a/packages/opencode/src/session/llm.ts +++ b/packages/opencode/src/session/llm.ts @@ -75,6 +75,9 @@ export namespace LLM { // altimate_change start — upstream_fix: keep ambient context outside the reusable system prefix environment?: string[] // altimate_change end + // altimate_change start — per-request context excluded from prompt cache breakpoints + uncachedTurnContext?: string + // altimate_change end abort: AbortSignal messages: ModelMessage[] small?: boolean @@ -360,7 +363,7 @@ export namespace LLM { }), inputTokens: () => estimateInputTokens({ - system, + system: input.uncachedTurnContext ? [...system, input.uncachedTurnContext] : system, messages: ProviderTransform.messagesForInputEstimate(input.messages, input.model), tools, instructions: params.options.instructions, @@ -499,6 +502,14 @@ export namespace LLM { // @ts-expect-error args.params.prompt = ProviderTransform.message(args.params.prompt, input.model, requestOptions) // altimate_change end + // altimate_change start — append advisory context after the normal cache breakpoints + if (input.uncachedTurnContext) { + args.params.prompt.push({ + role: "user", + content: [{ type: "text", text: input.uncachedTurnContext }], + }) + } + // altimate_change end } return args.params }, diff --git a/packages/opencode/src/session/prompt.ts b/packages/opencode/src/session/prompt.ts index df123c755b..61deab5eac 100644 --- a/packages/opencode/src/session/prompt.ts +++ b/packages/opencode/src/session/prompt.ts @@ -115,7 +115,7 @@ import { Fingerprint } from "../altimate/fingerprint" // altimate_change end // altimate_change start - validator framework (see session/validators/types.ts header) -import { validatorEnforcement } from "./validators/enforcement" +import { ValidatorEnforcement } from "./validators/enforcement" import { ValidatorRegistry } from "./validators/registry" import { registerAltimateValidators } from "../altimate/validators" import { sanitizeTelemetryDetails } from "../altimate/validators/validator-utils" @@ -2234,17 +2234,11 @@ export namespace SessionPrompt { // altimate_change end // altimate_change start — only a request actually sent to the model counts toward the cap modelSteps++ - // A queued note survives normal termination and history compaction. Read - // it once when resuming this session, or use the note queued in this loop. if (validatorAdvisoriesEnabled && modelSteps === 1) { - for (const message of MessageV2.stream(sessionID)) { - if (message.info.role !== "assistant" || message.info.id === processor.message.id || message.info.summary) continue - const { validatorAdvisories, validatorAdvisoriesShown } = message.info.metadata ?? {} - if (Array.isArray(validatorAdvisories) && validatorAdvisories.length > 0) pendingValidatorAdvisories = message.info - if (Array.isArray(validatorAdvisoriesShown)) { - for (const advisory of validatorAdvisoriesShown) seenValidatorAdvisories.add(advisory) - } - break + pendingValidatorAdvisories = ValidatorEnforcement.latestAdvisoryMessage(sessionID) + const shown = pendingValidatorAdvisories?.metadata?.validatorAdvisoriesShown + if (Array.isArray(shown)) { + for (const advisory of shown) seenValidatorAdvisories.add(advisory) } } const validatorAdvisories = pendingValidatorAdvisories?.metadata?.validatorAdvisories @@ -2261,8 +2255,9 @@ export namespace SessionPrompt { agent, abort, sessionID, + system, // altimate_change start — one-shot advisory context for the existing turn - system: advisoryContext ? [...system, advisoryContext] : system, + uncachedTurnContext: advisoryContext, // altimate_change end // altimate_change start — upstream_fix: append ambient context after the cached instructions environment: await SystemPrompt.environment(model), @@ -2310,14 +2305,17 @@ export namespace SessionPrompt { }) // altimate_change start — deliver advisory context at most once - if (pendingValidatorAdvisories) { + if ( + pendingValidatorAdvisories && advisoryContext && processor.message.finish && + !processor.message.error && !abort.aborted + ) { const { validatorAdvisories, ...metadata } = pendingValidatorAdvisories.metadata ?? {} if (Array.isArray(validatorAdvisories)) { for (const advisory of validatorAdvisories) seenValidatorAdvisories.add(advisory) } await Session.updateMessage({ ...pendingValidatorAdvisories, - metadata: { ...metadata, validatorAdvisoriesShown: [...seenValidatorAdvisories] }, + metadata: { ...metadata, validatorAdvisories: [], validatorAdvisoriesShown: [...seenValidatorAdvisories] }, }) pendingValidatorAdvisories = undefined } @@ -2551,7 +2549,7 @@ export namespace SessionPrompt { }), ) } - const { failures, advisories, retry, exhausted } = validatorEnforcement(checks, { + const { failures, advisories, retry, exhausted } = ValidatorEnforcement.decide(checks, { enabled: validatorsEnabled, strict: validatorStrict, retryCount: validatorRetryCount, @@ -2591,7 +2589,7 @@ export namespace SessionPrompt { processor.message.metadata = { ...processor.message.metadata, validatorAdvisories: pending, - validatorAdvisoriesShown: advisories.filter((advisory) => seenValidatorAdvisories.has(advisory)), + validatorAdvisoriesShown: [...seenValidatorAdvisories], } await Session.updateMessage(processor.message) if (pending.length > 0) pendingValidatorAdvisories = processor.message diff --git a/packages/opencode/src/session/validators/enforcement.ts b/packages/opencode/src/session/validators/enforcement.ts index 42684d8d48..66b65b82ad 100644 --- a/packages/opencode/src/session/validators/enforcement.ts +++ b/packages/opencode/src/session/validators/enforcement.ts @@ -1,3 +1,7 @@ +import { MessageTable } from "@opencode-ai/core/session/sql" +import { Database, sql } from "../../storage/db" +import type { MessageV2 } from "../message-v2" +import type { SessionID } from "../schema" import type { Validator, ValidatorResult } from "./types" interface Check { @@ -5,7 +9,7 @@ interface Check { result: ValidatorResult } -export function validatorEnforcement( +export function decide( checks: Check[], options: { enabled: boolean; strict: boolean; retryCount: number; maxRetries: number }, ) { @@ -27,3 +31,18 @@ export function validatorEnforcement( exhausted: failures.length > 0 && options.retryCount >= options.maxRetries, } } + +// Look up only the advisory record, independent of clean turns and history compaction. +export function latestAdvisoryMessage(sessionID: SessionID): MessageV2.Assistant | undefined { + const row = Database.use((db) => + db.select({ id: MessageTable.id, data: MessageTable.data }) + .from(MessageTable) + .where(sql`${MessageTable.session_id} = ${sessionID} and json_type(${MessageTable.data}, '$.metadata.validatorAdvisories') = 'array'`) + .orderBy(sql`${MessageTable.time_created} desc, rowid desc`) + .limit(1) + .get(), + ) + if (row?.data.role === "assistant") return { ...row.data, id: row.id, sessionID } as MessageV2.Assistant +} + +export * as ValidatorEnforcement from "./enforcement" diff --git a/packages/opencode/test/session/llm.test.ts b/packages/opencode/test/session/llm.test.ts index 779e95346f..8d607ad6fe 100644 --- a/packages/opencode/test/session/llm.test.ts +++ b/packages/opencode/test/session/llm.test.ts @@ -1153,7 +1153,8 @@ describe("session.llm.stream client trace", () => { userID: string traceparent?: string configHeaders?: Record - }): Promise { + uncachedTurnContext?: string + }) { const server = state.server if (!server) throw new Error("Server not initialized") const { model } = await loadFixture("openai", "gpt-5.2") @@ -1225,6 +1226,7 @@ describe("session.llm.stream client trace", () => { model: resolved, agent, system: ["You are a helpful assistant."], + uncachedTurnContext: input.uncachedTurnContext, abort: new AbortController().signal, messages: [{ role: "user", content: "Hello" }], tools: {}, @@ -1233,11 +1235,24 @@ describe("session.llm.stream client trace", () => { } }, }) - return (await request).headers + return request } + test("sends one-shot context after the ordinary request content", async () => { + const note = "The schema verification tool did not run." + const { body } = await captureTurn({ + providerID: "altimate-backend", + userID: "msg_trace_uncached_context", + uncachedTurnContext: note, + }) + const messages = body.input as Array<{ role: string; content: unknown }> + expect(messages.at(-1)).toMatchObject({ role: "user", content: [{ text: note }] }) + expect(messages.at(-2)).toMatchObject({ role: "user", content: [{ text: "Hello" }] }) + expect(JSON.stringify(messages.slice(0, -1))).not.toContain(note) + }) + test("an Altimate provider receives the turn's trace with a fresh child span", async () => { - const headers = await captureTurn({ + const { headers } = await captureTurn({ providerID: "altimate-backend", userID: "msg_trace_altimate", traceparent: `00-${TRACE_ID}-${CLIENT_SPAN}-01`, @@ -1249,13 +1264,13 @@ describe("session.llm.stream client trace", () => { }) test("an untraced turn sends no trace headers", async () => { - const headers = await captureTurn({ providerID: "altimate-backend", userID: "msg_trace_none" }) + const { headers } = await captureTurn({ providerID: "altimate-backend", userID: "msg_trace_none" }) expect(headers.get("traceparent")).toBeNull() expect(headers.get("x-request-id")).toBeNull() }) test("a non-Altimate provider never receives the trace", async () => { - const headers = await captureTurn({ + const { headers } = await captureTurn({ providerID: "custom-gateway", userID: "msg_trace_other", traceparent: `00-${TRACE_ID}-${CLIENT_SPAN}-01`, @@ -1265,7 +1280,7 @@ describe("session.llm.stream client trace", () => { }) test("configuration that sets either header keeps the pair out entirely", async () => { - const headers = await captureTurn({ + const { headers } = await captureTurn({ providerID: "altimate-backend", userID: "msg_trace_config", traceparent: `00-${TRACE_ID}-${CLIENT_SPAN}-01`, diff --git a/packages/opencode/test/session/prompt.test.ts b/packages/opencode/test/session/prompt.test.ts index 2599ec833b..dfd72ac4d4 100644 --- a/packages/opencode/test/session/prompt.test.ts +++ b/packages/opencode/test/session/prompt.test.ts @@ -576,7 +576,7 @@ for (const downgrade of [true, false]) { // altimate_change end // altimate_change start — completion validator enforcement through the real prompt loop -for (const mode of ["advisory", "finding", "strict"] as const) { +for (const mode of ["advisory", "advisory-after-error", "finding", "strict"] as const) { noLLMServer.instance( `completion validators use the ${mode} policy`, () => Effect.gen(function* () { @@ -587,14 +587,20 @@ for (const mode of ["advisory", "finding", "strict"] as const) { delete process.env.ALTIMATE_VALIDATORS_MAX_RETRIES const requests: string[] = [] const note = "Could not verify orders because the schema tool failed." - const runAll = spyOn(ValidatorRegistry, "runAll").mockImplementation(async () => [{ - validator: ValidatorRegistry.list()[0]!, - result: mode === "finding" - ? { ok: false, classification: "finding", reason: "Confirmed failure." } - : { ok: false, classification: "unverifiable", reason: note, advisory: note }, - }]) + let checkCount = 0 + const runAll = spyOn(ValidatorRegistry, "runAll").mockImplementation(async () => { + checkCount++ + if (mode === "advisory" && checkCount === 2) return [] + return [{ + validator: ValidatorRegistry.list()[0]!, + result: mode === "finding" + ? { ok: false, classification: "finding", reason: "Confirmed failure." } + : { ok: false, classification: "unverifiable", reason: note, advisory: note }, + }] + }) const stream = spyOn(LLM, "stream").mockImplementation(async (input) => { - requests.push(input.system.join("\n")) + requests.push(input.uncachedTurnContext ?? "") + if (mode === "advisory-after-error" && requests.length === 2) throw new Error("Provider unavailable") const usage = { inputTokens: 10, outputTokens: 5, totalTokens: 15 } async function* fullStream() { yield { type: "start" } @@ -628,6 +634,16 @@ for (const mode of ["advisory", "finding", "strict"] as const) { expect(requests[1]).toContain(note) expect(requests[2]).not.toContain(note) expect(requests[3]).not.toContain(note) + } else if (mode === "advisory-after-error") { + expect(requests).toHaveLength(1) + const failed = yield* reply("Follow-up while the provider is unavailable.") + expect(failed.info).toHaveProperty("error") + yield* reply("Try again.") + yield* reply("One more question.") + expect(requests).toHaveLength(4) + expect(requests[1]).toContain(note) + expect(requests[2]).toContain(note) + expect(requests[3]).not.toContain(note) } else { expect(requests).toHaveLength(4) expect(retries).toEqual([0, 1, 2, 3]) From 505bef87b13d0364a9693bc10308e1d6528eb31c Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Sat, 21 Nov 2026 10:20:28 -0800 Subject: [PATCH 03/12] fix: accept completed dbt tests with diagnostic stderr --- .../src/altimate/validators/dbt-tests-pass.ts | 36 +++++++++---------- packages/opencode/src/session/prompt.ts | 2 +- .../src/session/validators/enforcement.ts | 5 ++- .../opencode/src/session/validators/types.ts | 5 ++- .../dbt-schema-verify-stale.test.ts | 15 ++++++++ 5 files changed, 40 insertions(+), 23 deletions(-) diff --git a/packages/opencode/src/altimate/validators/dbt-tests-pass.ts b/packages/opencode/src/altimate/validators/dbt-tests-pass.ts index 1315359c28..d301da536b 100644 --- a/packages/opencode/src/altimate/validators/dbt-tests-pass.ts +++ b/packages/opencode/src/altimate/validators/dbt-tests-pass.ts @@ -169,20 +169,13 @@ async function runDbtTest(model: string, cwd: string): Promise 0) { - resolve({ model, summary }) + // A complete summary is usable even when the wrapper reports diagnostic stderr. + const error = typeof envelope.error === "string" ? envelope.error.slice(0, 500) : undefined + if (summary) { + resolve({ model, summary, error: summary.error === 0 ? error : undefined }) return } - if (typeof envelope.error === "string") { - resolve({ model, error: envelope.error.slice(0, 500) }) - return - } - if (!summary) { - resolve({ model, error: `no PASS/ERROR summary in dbt output: ${dbtLog.slice(-300)}` }) - return - } - resolve({ model, summary }) + resolve({ model, error: error ?? `no PASS/ERROR summary in dbt output: ${dbtLog.slice(-300)}` }) }) }) } @@ -261,7 +254,8 @@ export const DbtTestsPassValidator: Validator = { } const failures = results.filter((r) => r.summary && r.summary.error > 0) - const errored = results.filter((r) => r.error && !r.summary) + const toolErrors = results.filter((r) => r.error) + const errored = toolErrors.filter((r) => !r.summary) const passed = results.filter((r) => r.summary && r.summary.error === 0) // A model with no tests at all isn't a failure — it's just nothing to verify. const noTests = results.filter((r) => r.summary && r.summary.total === 0) @@ -283,7 +277,7 @@ export const DbtTestsPassValidator: Validator = { ? `Could not verify dbt tests for ${unverifiedNames.length} model(s) because the test tool did not return a usable result: ${unverifiedNames.slice(0, 10).map((name) => sanitizeForPrompt(name.replace(/[^\w.-]/g, "?"), 80)).join(", ")}.` : undefined - if (failures.length === 0 && errored.length === 0) { + if (failures.length === 0 && toolErrors.length === 0) { return { ok: true, ...(advisory && { classification: "unverifiable", advisory }), @@ -296,16 +290,18 @@ export const DbtTestsPassValidator: Validator = { } } - const hintBlocks: TestRunOutput[] = [...failures, ...errored] + const hintBlocks: TestRunOutput[] = [...failures, ...toolErrors.map(({ model, error }) => ({ model, error }))] const failingNames = failures.map((f) => f.model).filter(Boolean) - const erroredNames = errored.map((f) => f.model).filter(Boolean) + const erroredNames = toolErrors.map((f) => f.model).filter(Boolean) const reason = failures.length > 0 ? `${failures.length} of ${results.length} models you edited have failing dbt tests${failingNames.length ? `: ${failingNames.join(", ")}` : ""}.` - : `${errored.length} of ${results.length} models could not be tested${erroredNames.length ? `: ${erroredNames.join(", ")}` : ""}. Investigate before declaring done.` + : `${toolErrors.length} of ${results.length} models could not be tested${erroredNames.length ? `: ${erroredNames.join(", ")}` : ""}. Investigate before declaring done.` + const ok = failures.length === 0 && errored.length === 0 return { - ok: false, - classification: failures.length > 0 ? "finding" : "unverifiable", + ok, + ...(ok && { strictFailure: true }), + classification: failures.length > 0 ? "finding" : advisory ? "unverifiable" : undefined, advisory, reason, fixHint: @@ -318,7 +314,7 @@ export const DbtTestsPassValidator: Validator = { errored: errored.length, spawn_failures: spawnFailures, failing_models: failingNames, - errored_models: erroredNames, + errored_models: errored.map((r) => r.model), }, } }, diff --git a/packages/opencode/src/session/prompt.ts b/packages/opencode/src/session/prompt.ts index 61deab5eac..32a332af4e 100644 --- a/packages/opencode/src/session/prompt.ts +++ b/packages/opencode/src/session/prompt.ts @@ -2575,7 +2575,7 @@ export namespace SessionPrompt { timestamp: Date.now(), session_id: sessionID, validator_name: validator.name, - ok: vRes.ok, + ok: vRes.ok && !vRes.strictFailure, step, retry_count: validatorRetryCount, enforced: validatorsEnabled && (validatorStrict || vRes.classification !== "unverifiable"), diff --git a/packages/opencode/src/session/validators/enforcement.ts b/packages/opencode/src/session/validators/enforcement.ts index 66b65b82ad..04a848e396 100644 --- a/packages/opencode/src/session/validators/enforcement.ts +++ b/packages/opencode/src/session/validators/enforcement.ts @@ -18,7 +18,10 @@ export function decide( if (options.enabled) { for (const check of checks) { const { validator, result } = check - if (!result.ok && (options.strict || result.classification !== "unverifiable")) failures.push(check) + const blocks = options.strict + ? result.strictFailure || !result.ok + : !result.ok && result.classification !== "unverifiable" + if (blocks) failures.push(check) if (!options.strict && result.advisory) { advisories.push(`[altimate-validator advisory: ${validator.name}] ${result.advisory}`) } diff --git a/packages/opencode/src/session/validators/types.ts b/packages/opencode/src/session/validators/types.ts index 7e7ff504f9..43c174f122 100644 --- a/packages/opencode/src/session/validators/types.ts +++ b/packages/opencode/src/session/validators/types.ts @@ -66,9 +66,12 @@ export interface ValidatorContext { * Result of running a validator against a session context. */ export interface ValidatorResult { - /** Whether the previous enforcement policy accepted the result. */ + /** Whether the validator reported a failure. */ ok: boolean + /** A legacy failure retained solely for the strict-mode control policy. */ + strictFailure?: true + /** Set by the validator when it has a finding or could not complete the check. */ classification?: "finding" | "unverifiable" diff --git a/packages/opencode/test/altimate/validators/dbt-schema-verify-stale.test.ts b/packages/opencode/test/altimate/validators/dbt-schema-verify-stale.test.ts index ef755343f7..020e2e336b 100644 --- a/packages/opencode/test/altimate/validators/dbt-schema-verify-stale.test.ts +++ b/packages/opencode/test/altimate/validators/dbt-schema-verify-stale.test.ts @@ -8,6 +8,7 @@ import { tmpdir } from "os" import { join } from "path" import { DbtSchemaVerifyValidator } from "../../../src/altimate/validators/dbt-schema-verify" import { DbtTestsPassValidator } from "../../../src/altimate/validators/dbt-tests-pass" +import { ValidatorEnforcement } from "../../../src/session/validators/enforcement" import type { Validator, ValidatorContext } from "../../../src/session/validators/types" const WRITTEN_AT = "2026-11-05T08:00:00.000Z" @@ -107,6 +108,20 @@ describe("dbt-tests-pass result classification", () => { expect(result.advisory).toContain("orders") }) + test("a successful summary with stderr is advisory-free and preserves the strict control", async () => { + const result = await check({ + error: "Adapter diagnostic", + stdout: "1 of 1 PASS not_null_orders_id [PASS in 0.10s]\nDone. PASS=1 WARN=0 ERROR=0 SKIP=0 TOTAL=1", + }, DbtTestsPassValidator) + const checks = [{ validator: DbtTestsPassValidator, result }] + const options = { enabled: true, retryCount: 0, maxRetries: 3 } + expect(ValidatorEnforcement.decide(checks, { ...options, strict: false })).toMatchObject({ + retry: false, + advisories: [], + }) + expect(ValidatorEnforcement.decide(checks, { ...options, strict: true }).retry).toBe(true) + }) + test("a real test failure is a finding even when the wrapper also reports an error", async () => { const result = await check({ error: "dbt exited with code 1", From a24149d1a36f81bc1757925e4f7a43d030f6eb9f Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Sat, 21 Nov 2026 10:26:14 -0800 Subject: [PATCH 04/12] fix: clarify advisory outcomes and trace delivery context --- docs/docs/data-engineering/tools/dbt-tools.md | 10 +++---- docs/docs/data-engineering/validators.md | 22 ++++++++++------ docs/docs/reference/telemetry.md | 2 +- .../src/altimate/observability/tracing.ts | 4 +++ .../validators/dbt-deliverable-names.ts | 26 ++++++++++++------- .../src/altimate/validators/dbt-tests-pass.ts | 9 ++++--- .../altimate/validators/validator-utils.ts | 13 +++++++--- packages/opencode/src/session/llm.ts | 1 + .../opencode/src/session/validators/types.ts | 12 ++++----- .../test/altimate/tracing-run-content.test.ts | 7 ++++- .../validators/dbt-deliverable-names.test.ts | 15 +++++++++++ 11 files changed, 84 insertions(+), 37 deletions(-) diff --git a/docs/docs/data-engineering/tools/dbt-tools.md b/docs/docs/data-engineering/tools/dbt-tools.md index 09f543617a..d3e1df74ae 100644 --- a/docs/docs/data-engineering/tools/dbt-tools.md +++ b/docs/docs/data-engineering/tools/dbt-tools.md @@ -373,12 +373,12 @@ WHERE updated_at > (SELECT MAX(updated_at) FROM {{ this }}) Beyond the agent-facing tools above, altimate-code ships **harness-side validators** that fire automatically after the agent declares done. They run `altimate-dbt test` and `altimate-dbt schema-verify` against every model -modified during the session. Confirmed findings block completion and -inconclusive checks are advisory by default. +modified during the session. -This is **opt-in** today via either `ALTIMATE_VALIDATORS_ENABLED=1` -(enforcement mode, with synthetic retries for confirmed findings) or `ALTIMATE_VALIDATORS_SHADOW=1` (telemetry-only mode — runs -without blocking, useful for measuring "would have caught" rates). When +With `ALTIMATE_VALIDATORS_ENABLED=1`, confirmed findings block completion +with synthetic retries and inconclusive checks leave advisories. Set +`ALTIMATE_VALIDATORS_STRICT=1` to restore the previous retry policy. +`ALTIMATE_VALIDATORS_SHADOW=1` alone runs the checks for telemetry. When neither flag is set the dispatch path is completely skipped and there is zero overhead. See the [Validators page](../validators.md) for the full reference, env var catalogue, performance characteristics, and diff --git a/docs/docs/data-engineering/validators.md b/docs/docs/data-engineering/validators.md index cd8e0dd5d5..9aa2b0fb6a 100644 --- a/docs/docs/data-engineering/validators.md +++ b/docs/docs/data-engineering/validators.md @@ -7,12 +7,12 @@ description: "Harness-side validators run after the agent declares done, catchin Validators are harness-side checks that run **after** the agent declares done. Unlike tools, they are not visible to the agent — the framework dispatches -them automatically once the LLM emits `finishReason === "stop"`. A confirmed -finding injects a synthetic user turn with the failure details, -giving the agent one more turn to fix the issue within the retry budget. -An inconclusive check, such as a schema tool failure, leaves an advisory in -the next model request's context. An advisory alone does not start a request -or spend a retry. +them automatically once the LLM emits `finishReason === "stop"`. +With `ALTIMATE_VALIDATORS_ENABLED=1`, a confirmed finding gives the agent +one more turn to fix the issue within the retry budget. An inconclusive +check leaves a note in the next model request's context. Set +`ALTIMATE_VALIDATORS_STRICT=1` to restore the previous retry policy. +Shadow-only mode records results in telemetry without injecting messages. This catches a class of error that prompt engineering alone cannot: the agent claiming "done" while the work is actually broken (failing dbt @@ -69,7 +69,8 @@ counts. After the agent declares done, runs `altimate-dbt schema-verify --model ` on every modified model. Reports a mismatch if the produced column shape diverges from the `schema.yml` spec (extra, missing, -reordered, or type-mismatched columns). +reordered, or type-mismatched columns). If the schema verification tool +cannot complete the check, the validator reports an advisory. **Catches**: column-level drift that wouldn't be caught by `dbt build` alone — equality tests against the spec would fail later but the @@ -192,7 +193,9 @@ minute of wall time. When validators run (either mode), they emit one `validator_check` event per applied validator. The optional `classification` field is `finding` or `unverifiable`. The `ok` field retains its legacy -meaning. `enforced` is false for advisory-only results in advisory mode. +meaning. `enforced` is false for advisory-only results when +`ALTIMATE_VALIDATORS_ENABLED=1` is set without +`ALTIMATE_VALIDATORS_STRICT=1`. ```json { @@ -227,6 +230,9 @@ When a validator runs, it returns: ```ts { ok: boolean + classification?: "finding" | "unverifiable" + advisory?: string // what could not be checked and why + strictFailure?: true // preserves a legacy failure in strict mode reason?: string // human-readable failure summary fixHint?: string // the body injected into the synthetic user turn details: { diff --git a/docs/docs/reference/telemetry.md b/docs/docs/reference/telemetry.md index 3a1576b04a..493f88c7e6 100644 --- a/docs/docs/reference/telemetry.md +++ b/docs/docs/reference/telemetry.md @@ -54,7 +54,7 @@ We collect the following categories of events: | `error_fingerprint` | Error pattern for grouping — SHA-256 hash of the masked error message, error class, tool name, whether recovery succeeded, plus `error_message`. `error_message` contains only words from a fixed list of common error terms taken from the first line of the error; every other word is replaced with ``, so no user text, identifiers, paths, values or SQL can appear (max 200 characters). | | `sql_fingerprint` | SQL structural shape via AST parsing — statement types, table count, function count, subquery/aggregation/window function presence, and AST node count. No table names, column names, or SQL content. | | `schema_complexity` | Warehouse schema structural metrics from introspection — bucketed table, column, and schema counts plus average columns per table. No schema names or content. | -| `validator_check` | A completion-gate validator ran on session end — validator name, `ok` boolean, step, retry count, `enforced` flag (false in shadow mode and for advisory-only results in advisory mode), optional `classification` (`finding` or `unverifiable`), and structured `details` (model counts, elapsed time, concurrency limit — no SQL or model content). Only emitted when `ALTIMATE_VALIDATORS_ENABLED=1` or `ALTIMATE_VALIDATORS_SHADOW=1`. See [Validators](../data-engineering/validators.md). | +| `validator_check` | A completion-gate validator ran on session end — validator name, `ok` boolean, step, retry count, `enforced` flag (false in shadow-only mode and for `unverifiable` results when `ALTIMATE_VALIDATORS_STRICT=1` is unset), optional `classification` (`finding` or `unverifiable`), and structured `details` (model counts, elapsed time, concurrency limit — no SQL or model content). Only emitted when `ALTIMATE_VALIDATORS_ENABLED=1` or `ALTIMATE_VALIDATORS_SHADOW=1`. See [Validators](../data-engineering/validators.md). | | `validator_retries_exhausted` | A session terminated with unresolved validator failures after exhausting the synthetic-retry budget — names of the failing validators (no failure body content). | | `onboarding_started` | The first-run setup gate opened (fresh launch with no usable model). | | `model_picker_shown` | The provider picker was displayed. `trigger` distinguishes the first run from `/connect` and from the prompt gate (`altimate_base_back` is defined in the schema but no longer emitted — it recorded backing out of the old consent dialog, which no longer exists). | diff --git a/packages/opencode/src/altimate/observability/tracing.ts b/packages/opencode/src/altimate/observability/tracing.ts index 1218d559cf..0af6acfc07 100644 --- a/packages/opencode/src/altimate/observability/tracing.ts +++ b/packages/opencode/src/altimate/observability/tracing.ts @@ -41,6 +41,7 @@ export const LLMRequestEvent = BusEvent.define( variant: z.string().optional(), startTime: z.number(), system: z.string(), + uncachedTurnContext: z.string().optional(), instructions: z.string().optional(), tools: z.array( z.object({ @@ -1363,6 +1364,9 @@ export class Trace { }, input: { system: snapshot("system", request.system), + ...(request.uncachedTurnContext !== undefined && { + uncachedTurnContext: snapshot("uncachedTurnContext", request.uncachedTurnContext), + }), tools: snapshot("tools", tools), ...(request.instructions !== undefined && { instructions: snapshot("instructions", request.instructions) }), }, diff --git a/packages/opencode/src/altimate/validators/dbt-deliverable-names.ts b/packages/opencode/src/altimate/validators/dbt-deliverable-names.ts index cc497646a7..347a733810 100644 --- a/packages/opencode/src/altimate/validators/dbt-deliverable-names.ts +++ b/packages/opencode/src/altimate/validators/dbt-deliverable-names.ts @@ -39,7 +39,7 @@ import { findDbtProjectRoot, findTaskInstructionFiles, extractRequiredDeliverables, - collectProducedNodeNames, + collectProducedNodeInventory, modelsModifiedSince, modelNameFromPath, resolveWithinRoot, @@ -138,7 +138,7 @@ export const DbtDeliverableNamesValidator: Validator = { return { ok: true, details: { skipped: "no literal contract", session_id: ctx.sessionID } } } - const produced = await collectProducedNodeNames(dbtRoot) + const { names: produced, manifestUnreadable } = await collectProducedNodeInventory(dbtRoot) const missingModels = contract.required.models.filter((name) => !produced.has(name)) const missingFiles: string[] = [] for (const relative of contract.required.files) { @@ -176,8 +176,14 @@ export const DbtDeliverableNamesValidator: Validator = { ), ) + // An unreadable manifest leaves alias claims unresolved. A literal file + // absence is independently verifiable and still has its own retry. + const advisory = manifestUnreadable && missingModels.length > 0 + ? "Could not verify the missing dbt deliverable names against aliases because manifest.json could not be read or parsed." + : undefined + const includeModelFindings = missingModels.length > 0 && (!advisory || missingFiles.length === 0) const reasonParts: string[] = [] - if (missingModels.length > 0) { + if (includeModelFindings) { reasonParts.push( `the task names ${missingModels.length} deliverable(s) this project does not define: ${missingModels.join(", ")}`, ) @@ -195,7 +201,7 @@ export const DbtDeliverableNamesValidator: Validator = { const hintLines: string[] = [ `The task document (${safeTaskFile}) states these names literally. A model that does the right thing under a different name does not satisfy the task, and self-verification against the renamed output will not detect it.`, ] - if (missingModels.length > 0) { + if (includeModelFindings) { hintLines.push(` • Create or rename to exactly: ${missingModels.join(", ")}`) } if (missingFiles.length > 0) { @@ -214,13 +220,15 @@ export const DbtDeliverableNamesValidator: Validator = { ` • Models you created this session that the task did not name: ${safeUnrequested.join(", ")}. If one of them is a renamed version of a required deliverable, rename the file (and any \`ref()\` to it) back to the required name.`, ) } - hintLines.push( - " • If a required deliverable is produced under an alias, set `alias` in its config so the required name is the relation name.", - ) - + if (includeModelFindings) { + hintLines.push( + " • If a required deliverable is produced under an alias, set `alias` in its config so the required name is the relation name.", + ) + } return { ok: false, - classification: "finding", + classification: advisory && missingFiles.length === 0 ? "unverifiable" : "finding", + advisory, reason: `Deliverable-name mismatch: ${reasonParts.join("; ")}.`, fixHint: hintLines.join("\n"), details: { ...details, unrequested_models: unrequested }, diff --git a/packages/opencode/src/altimate/validators/dbt-tests-pass.ts b/packages/opencode/src/altimate/validators/dbt-tests-pass.ts index d301da536b..fb47382078 100644 --- a/packages/opencode/src/altimate/validators/dbt-tests-pass.ts +++ b/packages/opencode/src/altimate/validators/dbt-tests-pass.ts @@ -12,9 +12,8 @@ * tests because the SELECT logic produces wrong values or wrong row counts. * * The agent does not see this validator existing — it runs in the harness - * AFTER `finishReason === "stop"`. Its output is surfaced to the agent only - * if there are failures, via a synthetic user message the framework injects - * to force one more turn. See types.ts header for the rationale. + * AFTER `finishReason === "stop"`. Findings can trigger a retry. The harness + * adds an inconclusive check as an advisory on the next ordinary request. */ import { spawn } from "child_process" @@ -290,7 +289,9 @@ export const DbtTestsPassValidator: Validator = { } } - const hintBlocks: TestRunOutput[] = [...failures, ...toolErrors.map(({ model, error }) => ({ model, error }))] + const hintBlocks = failures.length > 0 + ? failures + : toolErrors.map(({ model, error }) => ({ model, error })) const failingNames = failures.map((f) => f.model).filter(Boolean) const erroredNames = toolErrors.map((f) => f.model).filter(Boolean) const reason = diff --git a/packages/opencode/src/altimate/validators/validator-utils.ts b/packages/opencode/src/altimate/validators/validator-utils.ts index caa99988f0..70bf9ce565 100644 --- a/packages/opencode/src/altimate/validators/validator-utils.ts +++ b/packages/opencode/src/altimate/validators/validator-utils.ts @@ -2039,6 +2039,10 @@ const INVENTORY_MAX_DEPTH = 8 * from literals, so a project on custom `model-paths` is inventoried too. */ export async function collectProducedNodeNames(dbtRoot: string): Promise> { + return (await collectProducedNodeInventory(dbtRoot)).names +} + +export async function collectProducedNodeInventory(dbtRoot: string) { const names = new Set() async function scan(dir: string, depth: number, extensions: string[]): Promise { if (depth > INVENTORY_MAX_DEPTH) return @@ -2116,10 +2120,13 @@ export async function collectProducedNodeNames(dbtRoot: string): Promise 0) names.add(value.toLowerCase()) } } - } catch { - // no manifest — the fs inventory stands alone + } catch (error) { + // A missing optional manifest uses the filesystem inventory. Read or + // parse errors mean the alias inventory could not be verified. + const manifestUnreadable = (error as NodeJS.ErrnoException).code !== "ENOENT" + return { names, manifestUnreadable } } - return names + return { names, manifestUnreadable: false } } /** diff --git a/packages/opencode/src/session/llm.ts b/packages/opencode/src/session/llm.ts index 80fd9aa94e..95dbd22295 100644 --- a/packages/opencode/src/session/llm.ts +++ b/packages/opencode/src/session/llm.ts @@ -400,6 +400,7 @@ export namespace LLM { ...system, ...input.messages.filter((message) => message.role === "system").map((message) => message.content), ].join("\n\n"), + uncachedTurnContext: input.uncachedTurnContext, instructions: typeof requestOptions.instructions === "string" ? requestOptions.instructions : undefined, tools: await Promise.all( Object.entries(tools) diff --git a/packages/opencode/src/session/validators/types.ts b/packages/opencode/src/session/validators/types.ts index 43c174f122..f254ab11d1 100644 --- a/packages/opencode/src/session/validators/types.ts +++ b/packages/opencode/src/session/validators/types.ts @@ -22,10 +22,9 @@ * framework is generic. The dispatch hook lives in the session step loop * (prompt.ts); registered validators are evaluated when `finishReason` * resolves to a non-tool stop. - * - A failed validator does not throw. It returns `{ok: false, ...}` and the - * framework inserts a synthetic user message describing the gap. The loop - * continues with that message in context; the model gets one more turn to - * fix the issue. A retry budget prevents runaway loops. + * - A finding uses the retry budget to give the model another turn to fix + * the issue. An unverifiable check adds an advisory to the next ordinary + * request. Strict mode restores the previous enforcement policy. * * Generalisable: this is not a dbt-specific change. The hook fires for every * session regardless of workload; per-validator `appliesTo()` decides whether @@ -66,13 +65,14 @@ export interface ValidatorContext { * Result of running a validator against a session context. */ export interface ValidatorResult { - /** Whether the validator reported a failure. */ + /** A pass has `ok: true`; a failed or inconclusive check can return `ok: false`. */ ok: boolean /** A legacy failure retained solely for the strict-mode control policy. */ strictFailure?: true - /** Set by the validator when it has a finding or could not complete the check. */ + /** A failed result with no classification or with `finding` can spend a retry. + * `unverifiable` is advisory unless `ALTIMATE_VALIDATORS_STRICT=1`. */ classification?: "finding" | "unverifiable" /** A prompt-safe description of what could not be checked and why. May accompany a finding. */ diff --git a/packages/opencode/test/altimate/tracing-run-content.test.ts b/packages/opencode/test/altimate/tracing-run-content.test.ts index 34f9674d15..3d9a37da43 100644 --- a/packages/opencode/test/altimate/tracing-run-content.test.ts +++ b/packages/opencode/test/altimate/tracing-run-content.test.ts @@ -76,7 +76,8 @@ function sha256(value: string) { describe("run trace content", () => { test("request snapshots preserve effective inputs and hash-dedupe canonical tool schemas", async () => { const { root, files } = recording() - const first = request() + const note = "The schema verification tool did not run." + const first = request({ uncachedTurnContext: note }) root.logLLMRequest(first) root.logLLMRequest( request({ @@ -96,6 +97,10 @@ describe("run trace content", () => { await root.endTrace() const spans = files.get("root")!.spans.filter((span) => span.name === "llm-request") expect(spans).toHaveLength(2) + expect(spans[0]?.input).toMatchObject({ + uncachedTurnContext: { hash: sha256(note), content: note }, + }) + expect(spans[1]?.input).not.toHaveProperty("uncachedTurnContext") const systemHash = sha256(first.system) const instructionsHash = sha256(first.instructions!) const toolsHash = sha256( diff --git a/packages/opencode/test/altimate/validators/dbt-deliverable-names.test.ts b/packages/opencode/test/altimate/validators/dbt-deliverable-names.test.ts index 07dd49118a..17b3582c66 100644 --- a/packages/opencode/test/altimate/validators/dbt-deliverable-names.test.ts +++ b/packages/opencode/test/altimate/validators/dbt-deliverable-names.test.ts @@ -92,6 +92,7 @@ describe("DbtDeliverableNamesValidator — check", () => { await writeModel("fct_orders_v2.sql") const r = await DbtDeliverableNamesValidator.check(ctx()) expect(r.ok).toBe(false) + expect(r.classification).toBe("finding") expect(r.details!["missing_models"]).toEqual(["fct_orders"]) expect(r.fixHint).toContain("fct_orders_v2") }) @@ -119,6 +120,20 @@ describe("DbtDeliverableNamesValidator — check", () => { expect((await DbtDeliverableNamesValidator.check(ctx())).ok).toBe(true) }) + test.each([ + { contract: "Create the model `fct_orders`.", classification: "unverifiable" }, + { contract: "Create the model `fct_orders` and the file `models/orders.sql`.", classification: "finding" }, + ])("classifies a broken manifest as $classification", async ({ contract, classification }) => { + await makeProject() + await writeTask(contract) + await writeModel("orders_fact.sql") + await fs.mkdir(join(dir, "target"), { recursive: true }) + await fs.writeFile(join(dir, "target", "manifest.json"), "invalid JSON") + const result = await DbtDeliverableNamesValidator.check(ctx()) + expect(result).toMatchObject({ ok: false, classification }) + expect(result.advisory).toContain("manifest.json") + }) + test("a seed satisfies a required name", async () => { await makeProject() await writeTask("## Deliverables\n- `country_codes`\n") From 308059568f80dadaf0c18f59b732334251b369dd Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Sat, 21 Nov 2026 10:29:41 -0800 Subject: [PATCH 05/12] fix: exclude one-shot context from saved history checkpoints --- .../provider/claude-agent/sync/decide.ts | 7 +++-- .../provider/claude-agent/sync/fingerprint.ts | 5 ++-- packages/opencode/src/session/llm.ts | 6 ++-- packages/opencode/src/session/turn-context.ts | 17 +++++++++++ .../provider/claude-agent/sync/sync.test.ts | 30 +++++++++++++++++++ 5 files changed, 57 insertions(+), 8 deletions(-) create mode 100644 packages/opencode/src/session/turn-context.ts diff --git a/packages/opencode/src/altimate/provider/claude-agent/sync/decide.ts b/packages/opencode/src/altimate/provider/claude-agent/sync/decide.ts index 30c9258a1d..2e4d2a881f 100644 --- a/packages/opencode/src/altimate/provider/claude-agent/sync/decide.ts +++ b/packages/opencode/src/altimate/provider/claude-agent/sync/decide.ts @@ -2,6 +2,7 @@ export * as SyncDecision from "./decide" import type { LanguageModelV3Prompt } from "@ai-sdk/provider" import { Fingerprint } from "./fingerprint" +import { TurnContext } from "@/session/turn-context" import type { SessionRecord } from "./record" export function decide(prompt: LanguageModelV3Prompt, record?: SessionRecord.Record) { @@ -24,8 +25,10 @@ export function decide(prompt: LanguageModelV3Prompt, record?: SessionRecord.Rec if (reply.role !== "assistant" || reply.content.some((part) => part.type === "tool-call")) break matched++ } - const start = groups[matched]?.start ?? prompt.length - if (record?.interrupted && start === prompt.length) return { kind: "changed", start: 0 } as const + const next = groups[matched] + if (record?.interrupted && !next) return { kind: "changed", start: 0 } as const + const tail = prompt.at(-1) + const start = next?.start ?? (tail && TurnContext.isTransient(tail) ? prompt.length - 1 : prompt.length) const missed = groups.slice(matched) const kind = missed.some((group) => group.role !== "user") ? "catch-up" : "continue" return { kind, start } as const diff --git a/packages/opencode/src/altimate/provider/claude-agent/sync/fingerprint.ts b/packages/opencode/src/altimate/provider/claude-agent/sync/fingerprint.ts index e68ada7147..5cfbeb85fc 100644 --- a/packages/opencode/src/altimate/provider/claude-agent/sync/fingerprint.ts +++ b/packages/opencode/src/altimate/provider/claude-agent/sync/fingerprint.ts @@ -1,6 +1,7 @@ export * as Fingerprint from "./fingerprint" import { createHash } from "node:crypto" +import { TurnContext } from "@/session/turn-context" import type { LanguageModelV3Message, LanguageModelV3Prompt } from "@ai-sdk/provider" export interface Group { @@ -92,7 +93,7 @@ export function groups(prompt: LanguageModelV3Prompt, prefix: readonly { readonl const message = prompt[i]! const start = i i++ - if (message.role === "system") continue + if (message.role === "system" || TurnContext.isTransient(message)) continue const parts = visible(message) if (parts.length === 0) continue const members = [hash(parts)] @@ -102,7 +103,7 @@ export function groups(prompt: LanguageModelV3Prompt, prefix: readonly { readonl const checkpoint = prefix[groups.length] if (checkpoint && hash({ role: message.role, members: members.toSorted() }) === checkpoint.hash) break const next = prompt[i]! - if (next.role !== "user") break + if (next.role !== "user" || TurnContext.isTransient(next)) break members.push(hash(visible(next))) i++ } diff --git a/packages/opencode/src/session/llm.ts b/packages/opencode/src/session/llm.ts index 95dbd22295..d93165a8a7 100644 --- a/packages/opencode/src/session/llm.ts +++ b/packages/opencode/src/session/llm.ts @@ -47,6 +47,7 @@ import { normalizeBigInts } from "@/altimate/util/json" import { randomUUIDv7 } from "bun" import { Bus } from "@/bus" import { LLMRequestEvent } from "@/altimate/observability/tracing" +import { TurnContext } from "./turn-context" // altimate_change end // altimate_change start — run deadlines override reasoning effort for each remaining request import { applyBudgetEffort, budgetMessages } from "@/altimate/session/budget-effort" @@ -505,10 +506,7 @@ export namespace LLM { // altimate_change end // altimate_change start — append advisory context after the normal cache breakpoints if (input.uncachedTurnContext) { - args.params.prompt.push({ - role: "user", - content: [{ type: "text", text: input.uncachedTurnContext }], - }) + args.params.prompt.push(TurnContext.make(input.uncachedTurnContext)) } // altimate_change end } diff --git a/packages/opencode/src/session/turn-context.ts b/packages/opencode/src/session/turn-context.ts new file mode 100644 index 0000000000..2734aeb40e --- /dev/null +++ b/packages/opencode/src/session/turn-context.ts @@ -0,0 +1,17 @@ +import type { LanguageModelV3Message } from "@ai-sdk/provider" + +// These messages belong to one outgoing request, outside the saved conversation. +// Keep the marker on the in-process message identity so it never reaches the wire. +const transient = new WeakSet() + +export function make(text: string): LanguageModelV3Message { + const message: LanguageModelV3Message = { role: "user", content: [{ type: "text", text }] } + transient.add(message) + return message +} + +export function isTransient(message: LanguageModelV3Message): boolean { + return transient.has(message) +} + +export * as TurnContext from "./turn-context" diff --git a/packages/opencode/test/altimate/provider/claude-agent/sync/sync.test.ts b/packages/opencode/test/altimate/provider/claude-agent/sync/sync.test.ts index 4e65b7c67e..f0cb161054 100644 --- a/packages/opencode/test/altimate/provider/claude-agent/sync/sync.test.ts +++ b/packages/opencode/test/altimate/provider/claude-agent/sync/sync.test.ts @@ -3,6 +3,8 @@ import type { LanguageModelV3Prompt } from "@ai-sdk/provider" import { Fingerprint } from "@/altimate/provider/claude-agent/sync/fingerprint" import { SyncDecision } from "@/altimate/provider/claude-agent/sync/decide" import { CatchUp } from "@/altimate/provider/claude-agent/sync/catch-up" +import { Checkpoints } from "@/altimate/provider/claude-agent/sync/checkpoints" +import { TurnContext } from "@/session/turn-context" const user = (text: string): LanguageModelV3Prompt[number] => ({ role: "user", content: [{ type: "text", text }] }) const assistant = (text: string): LanguageModelV3Prompt[number] => ({ @@ -100,6 +102,34 @@ describe("history reconciliation", () => { expect(hashes([user("Changed reminder"), ...past])).not.toEqual(hashes(original)) }) + test("one-shot turn context reaches the model and keeps the saved tool continuation", () => { + const prompt = [...past, user("Read the file")] + const note = TurnContext.make("Schema verification could not run.") + const request = [...prompt, note] + const initial = SyncDecision.decide(request, record) + expect(initial).toEqual({ kind: "continue", start: 2 }) + expect(CatchUp.content(request, initial.start)).toContainEqual({ + type: "text", text: "Schema verification could not run.", + }) + const call = { type: "tool-call", toolCallId: "a", toolName: "read", input: { path: "orders.sql" } } as const + const history = Checkpoints.build({ prompt: request, previous: record, reply: [call], toolMessages: new Map() }) + const followup: LanguageModelV3Prompt = [ + ...prompt, + { role: "assistant", content: [call] }, + { + role: "tool", + content: [{ type: "tool-result", toolCallId: "a", toolName: "read", output: { type: "text", value: "OK" } }], + }, + ] + const saved = { ...record, history } + const continuation = SyncDecision.decide(followup, saved) + expect(continuation).toEqual({ kind: "catch-up", start: 4 }) + expect(JSON.stringify(CatchUp.content(followup, continuation.start))).not.toContain("Schema verification") + expect(SyncDecision.decide([...prompt, note], { ...record, history: Fingerprint.groups(prompt) })).toEqual({ + kind: "continue", start: 3, + }) + }) + test("compares tool inputs independent of key order and keeps tool images with their result", () => { const prompt: LanguageModelV3Prompt = [ { From 643c66923b866c034c2bc261f8ae569a66d83f66 Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Sat, 21 Nov 2026 10:33:56 -0800 Subject: [PATCH 06/12] fix: keep validator advisories pending during finding retries --- packages/opencode/src/session/prompt.ts | 9 ++++-- packages/opencode/test/session/prompt.test.ts | 31 ++++++++++++++----- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/packages/opencode/src/session/prompt.ts b/packages/opencode/src/session/prompt.ts index 32a332af4e..e5b40dc737 100644 --- a/packages/opencode/src/session/prompt.ts +++ b/packages/opencode/src/session/prompt.ts @@ -2243,7 +2243,10 @@ export namespace SessionPrompt { } const validatorAdvisories = pendingValidatorAdvisories?.metadata?.validatorAdvisories const advisoryContext = - Array.isArray(validatorAdvisories) && validatorAdvisories.length > 0 + Array.isArray(validatorAdvisories) && + validatorAdvisories.length > 0 && + !echoCommittedAnswer && + msgs.some((message) => message.info.id === lastUser.id && UserRequest.requestID(message) !== undefined) ? ["Completion checks that could not be verified (advisory only):", ...validatorAdvisories].join("\n") : undefined // altimate_change end @@ -2585,7 +2588,9 @@ export namespace SessionPrompt { } if (advisories.length > 0) { - const pending = advisories.filter((advisory) => !seenValidatorAdvisories.has(advisory)) + const previous = pendingValidatorAdvisories?.metadata?.validatorAdvisories + const pending = [...new Set([...(Array.isArray(previous) ? previous : []), ...advisories])] + .filter((advisory) => !seenValidatorAdvisories.has(advisory)) processor.message.metadata = { ...processor.message.metadata, validatorAdvisories: pending, diff --git a/packages/opencode/test/session/prompt.test.ts b/packages/opencode/test/session/prompt.test.ts index dfd72ac4d4..7089b3d2a5 100644 --- a/packages/opencode/test/session/prompt.test.ts +++ b/packages/opencode/test/session/prompt.test.ts @@ -576,7 +576,7 @@ for (const downgrade of [true, false]) { // altimate_change end // altimate_change start — completion validator enforcement through the real prompt loop -for (const mode of ["advisory", "advisory-after-error", "finding", "strict"] as const) { +for (const mode of ["advisory", "advisory-after-error", "mixed", "finding", "strict"] as const) { noLLMServer.instance( `completion validators use the ${mode} policy`, () => Effect.gen(function* () { @@ -587,16 +587,24 @@ for (const mode of ["advisory", "advisory-after-error", "finding", "strict"] as delete process.env.ALTIMATE_VALIDATORS_MAX_RETRIES const requests: string[] = [] const note = "Could not verify orders because the schema tool failed." + const retryNote = "Could not verify customers because the test tool failed." let checkCount = 0 const runAll = spyOn(ValidatorRegistry, "runAll").mockImplementation(async () => { checkCount++ if (mode === "advisory" && checkCount === 2) return [] - return [{ - validator: ValidatorRegistry.list()[0]!, - result: mode === "finding" - ? { ok: false, classification: "finding", reason: "Confirmed failure." } - : { ok: false, classification: "unverifiable", reason: note, advisory: note }, - }] + const validator = ValidatorRegistry.list()[0]! + const finding = { validator, result: { ok: false, classification: "finding" as const, reason: "Confirmed failure." } } + const advisory = (text: string) => ({ + validator, + result: { ok: false, classification: "unverifiable" as const, reason: text, advisory: text }, + }) + if (mode === "finding") return [finding] + if (mode === "mixed") { + if (checkCount === 1) return [finding, advisory(note)] + if (checkCount === 2) return [advisory(retryNote)] + return [] + } + return [advisory(note)] }) const stream = spyOn(LLM, "stream").mockImplementation(async (input) => { requests.push(input.uncachedTurnContext ?? "") @@ -634,6 +642,15 @@ for (const mode of ["advisory", "advisory-after-error", "finding", "strict"] as expect(requests[1]).toContain(note) expect(requests[2]).not.toContain(note) expect(requests[3]).not.toContain(note) + } else if (mode === "mixed") { + expect(retries).toEqual([0, 1]) + expect(requests).toEqual(["", ""]) + yield* reply("Follow-up on invoice totals.") + expect(requests).toHaveLength(3) + expect(requests[2]).toContain(note) + expect(requests[2]).toContain(retryNote) + yield* reply("A final invoice question.") + expect(requests[3]).toBe("") } else if (mode === "advisory-after-error") { expect(requests).toHaveLength(1) const failed = yield* reply("Follow-up while the provider is unavailable.") From 842fa78e69f6f3a57ce2ff2512d76366e74d74f7 Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Sat, 21 Nov 2026 10:39:02 -0800 Subject: [PATCH 07/12] fix: keep validator advisory state out of shared conversations --- packages/opencode/src/session/index.ts | 6 +- packages/opencode/src/session/prompt.ts | 55 ++++++----------- packages/opencode/src/session/session.ts | 6 +- .../src/session/validators/enforcement.ts | 59 ++++++++++++++----- packages/opencode/src/util/filesystem.ts | 5 +- packages/opencode/test/session/prompt.test.ts | 3 + 6 files changed, 79 insertions(+), 55 deletions(-) diff --git a/packages/opencode/src/session/index.ts b/packages/opencode/src/session/index.ts index aee9ca8898..3c4d4989da 100644 --- a/packages/opencode/src/session/index.ts +++ b/packages/opencode/src/session/index.ts @@ -1,3 +1,6 @@ +// altimate_change start — clean up private validator state with its session +import { ValidatorEnforcement } from "./validators/enforcement" +// altimate_change end // altimate_change start — release session-owned scratch stores on deletion import { closeScratch } from "@/altimate/native/connections/scratch" // altimate_change end @@ -753,8 +756,9 @@ export namespace Session { await remove(child.id) } await unshare(sessionID).catch(() => {}) - // altimate_change start — release only the deleted session's scratch store + // altimate_change start — release the deleted session's scratch and advisory state await closeScratch(sessionID) + await ValidatorEnforcement.removeAdvisories(sessionID) // altimate_change end // CASCADE delete handles messages and parts automatically Database.use((db) => { diff --git a/packages/opencode/src/session/prompt.ts b/packages/opencode/src/session/prompt.ts index e5b40dc737..a18faf6865 100644 --- a/packages/opencode/src/session/prompt.ts +++ b/packages/opencode/src/session/prompt.ts @@ -902,9 +902,10 @@ export namespace SessionPrompt { // altimate_change start — validator framework retry counter let validatorRetryCount = 0 const validatorStrict = process.env.ALTIMATE_VALIDATORS_STRICT === "1" - const validatorAdvisoriesEnabled = process.env.ALTIMATE_VALIDATORS_ENABLED === "1" && !validatorStrict - const seenValidatorAdvisories = new Set() - let pendingValidatorAdvisories: MessageV2.Assistant | undefined + const validatorAdvisories = + process.env.ALTIMATE_VALIDATORS_ENABLED === "1" && !validatorStrict + ? await ValidatorEnforcement.loadAdvisories(sessionID) + : undefined // altimate_change end let sessionAgentName = "" let sessionHadError = false @@ -2234,20 +2235,11 @@ export namespace SessionPrompt { // altimate_change end // altimate_change start — only a request actually sent to the model counts toward the cap modelSteps++ - if (validatorAdvisoriesEnabled && modelSteps === 1) { - pendingValidatorAdvisories = ValidatorEnforcement.latestAdvisoryMessage(sessionID) - const shown = pendingValidatorAdvisories?.metadata?.validatorAdvisoriesShown - if (Array.isArray(shown)) { - for (const advisory of shown) seenValidatorAdvisories.add(advisory) - } - } - const validatorAdvisories = pendingValidatorAdvisories?.metadata?.validatorAdvisories const advisoryContext = - Array.isArray(validatorAdvisories) && - validatorAdvisories.length > 0 && + validatorAdvisories?.pending.length && !echoCommittedAnswer && msgs.some((message) => message.info.id === lastUser.id && UserRequest.requestID(message) !== undefined) - ? ["Completion checks that could not be verified (advisory only):", ...validatorAdvisories].join("\n") + ? ["Completion checks that could not be verified (advisory only):", ...validatorAdvisories.pending].join("\n") : undefined // altimate_change end const result = await processor.process({ @@ -2308,19 +2300,10 @@ export namespace SessionPrompt { }) // altimate_change start — deliver advisory context at most once - if ( - pendingValidatorAdvisories && advisoryContext && processor.message.finish && - !processor.message.error && !abort.aborted - ) { - const { validatorAdvisories, ...metadata } = pendingValidatorAdvisories.metadata ?? {} - if (Array.isArray(validatorAdvisories)) { - for (const advisory of validatorAdvisories) seenValidatorAdvisories.add(advisory) - } - await Session.updateMessage({ - ...pendingValidatorAdvisories, - metadata: { ...metadata, validatorAdvisories: [], validatorAdvisoriesShown: [...seenValidatorAdvisories] }, - }) - pendingValidatorAdvisories = undefined + if (validatorAdvisories && advisoryContext && processor.message.finish && !processor.message.error && !abort.aborted) { + validatorAdvisories.shown.push(...validatorAdvisories.pending.map(ValidatorEnforcement.advisoryFingerprint)) + validatorAdvisories.pending = [] + ValidatorEnforcement.saveAdvisories(sessionID, validatorAdvisories) } // altimate_change end @@ -2587,17 +2570,15 @@ export namespace SessionPrompt { } as any) } - if (advisories.length > 0) { - const previous = pendingValidatorAdvisories?.metadata?.validatorAdvisories - const pending = [...new Set([...(Array.isArray(previous) ? previous : []), ...advisories])] - .filter((advisory) => !seenValidatorAdvisories.has(advisory)) - processor.message.metadata = { - ...processor.message.metadata, - validatorAdvisories: pending, - validatorAdvisoriesShown: [...seenValidatorAdvisories], + if (validatorAdvisories) { + const additional = advisories.filter((note) => + !validatorAdvisories.pending.includes(note) && + !validatorAdvisories.shown.includes(ValidatorEnforcement.advisoryFingerprint(note)), + ) + if (additional.length > 0) { + validatorAdvisories.pending.push(...new Set(additional)) + ValidatorEnforcement.saveAdvisories(sessionID, validatorAdvisories) } - await Session.updateMessage(processor.message) - if (pending.length > 0) pendingValidatorAdvisories = processor.message } if (retry) { diff --git a/packages/opencode/src/session/session.ts b/packages/opencode/src/session/session.ts index d77b027c97..84a9306293 100644 --- a/packages/opencode/src/session/session.ts +++ b/packages/opencode/src/session/session.ts @@ -1,3 +1,6 @@ +// altimate_change start — clean up private validator state with its session +import { ValidatorEnforcement } from "./validators/enforcement" +// altimate_change end // altimate_change start — release session-owned scratch stores on deletion import { closeScratch } from "@/altimate/native/connections/scratch" // altimate_change end @@ -675,8 +678,9 @@ export const layer: Layer.Layer< yield* remove(child.id) } - // altimate_change start — release only the deleted session's scratch store + // altimate_change start — release the deleted session's scratch and advisory state yield* Effect.promise(() => closeScratch(sessionID)) + yield* Effect.promise(() => ValidatorEnforcement.removeAdvisories(sessionID)) // altimate_change end yield* events.publish(SessionV1.Event.Deleted, { sessionID, info: session }) yield* events.remove(sessionID) diff --git a/packages/opencode/src/session/validators/enforcement.ts b/packages/opencode/src/session/validators/enforcement.ts index 04a848e396..24101048c1 100644 --- a/packages/opencode/src/session/validators/enforcement.ts +++ b/packages/opencode/src/session/validators/enforcement.ts @@ -1,6 +1,9 @@ -import { MessageTable } from "@opencode-ai/core/session/sql" -import { Database, sql } from "../../storage/db" -import type { MessageV2 } from "../message-v2" +import { createHash } from "node:crypto" +import { rm } from "node:fs/promises" +import path from "node:path" +import { Global } from "../../global" +import { Filesystem } from "../../util/filesystem" +import { Log } from "../../util/log" import type { SessionID } from "../schema" import type { Validator, ValidatorResult } from "./types" @@ -35,17 +38,45 @@ export function decide( } } -// Look up only the advisory record, independent of clean turns and history compaction. -export function latestAdvisoryMessage(sessionID: SessionID): MessageV2.Assistant | undefined { - const row = Database.use((db) => - db.select({ id: MessageTable.id, data: MessageTable.data }) - .from(MessageTable) - .where(sql`${MessageTable.session_id} = ${sessionID} and json_type(${MessageTable.data}, '$.metadata.validatorAdvisories') = 'array'`) - .orderBy(sql`${MessageTable.time_created} desc, rowid desc`) - .limit(1) - .get(), - ) - if (row?.data.role === "assistant") return { ...row.data, id: row.id, sessionID } as MessageV2.Assistant +interface AdvisoryState { + pending: string[] + shown: string[] +} + +const log = Log.create({ service: "validator-advisories" }) + +export function advisoryFingerprint(text: string): string { + return createHash("sha256").update(text).digest("hex") +} + +function advisoryFile(sessionID: SessionID): string { + return path.join(Global.Path.state, "validator-advisories", advisoryFingerprint(sessionID) + ".json") +} + +// A local file avoids putting advisories in the conversation and share stream. +export async function loadAdvisories(sessionID: SessionID): Promise { + try { + const value = await Filesystem.readJson(advisoryFile(sessionID)) + if ( + Array.isArray(value?.pending) && value.pending.every((entry) => typeof entry === "string") && + Array.isArray(value?.shown) && value.shown.every((entry) => typeof entry === "string") + ) return value + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") log.warn("failed to read validator advisories") + } + return { pending: [], shown: [] } +} + +export function saveAdvisories(sessionID: SessionID, state: AdvisoryState): void { + try { + Filesystem.writeJsonAtomic(advisoryFile(sessionID), state, 0o600) + } catch { + log.warn("failed to save validator advisories") + } +} + +export async function removeAdvisories(sessionID: SessionID): Promise { + await rm(advisoryFile(sessionID), { force: true }).catch(() => log.warn("failed to remove validator advisories")) } export * as ValidatorEnforcement from "./enforcement" diff --git a/packages/opencode/src/util/filesystem.ts b/packages/opencode/src/util/filesystem.ts index 7831bf18f7..0eab2a4727 100644 --- a/packages/opencode/src/util/filesystem.ts +++ b/packages/opencode/src/util/filesystem.ts @@ -345,11 +345,12 @@ export namespace Filesystem { * * @param targetPath Destination path. Parent directory is created if missing. * @param value Value to serialize as JSON. + * @param mode Permissions for the new file, when explicitly set. */ - export function writeJsonAtomic(targetPath: string, value: unknown): void { + export function writeJsonAtomic(targetPath: string, value: unknown, mode?: number): void { const tmpPath = `${targetPath}.tmp-${randomBytes(6).toString("hex")}` mkdirSync(dirname(targetPath), { recursive: true }) - writeFileSync(tmpPath, JSON.stringify(value, null, 2) + "\n") + writeFileSync(tmpPath, JSON.stringify(value, null, 2) + "\n", { mode }) renameSync(tmpPath, targetPath) } // altimate_change end diff --git a/packages/opencode/test/session/prompt.test.ts b/packages/opencode/test/session/prompt.test.ts index 7089b3d2a5..b869b1f45b 100644 --- a/packages/opencode/test/session/prompt.test.ts +++ b/packages/opencode/test/session/prompt.test.ts @@ -642,6 +642,9 @@ for (const mode of ["advisory", "advisory-after-error", "mixed", "finding", "str expect(requests[1]).toContain(note) expect(requests[2]).not.toContain(note) expect(requests[3]).not.toContain(note) + const storedConversation = JSON.stringify([...MessageV2.stream(chat.id)]) + expect(storedConversation).not.toContain(note) + expect(storedConversation).not.toContain("validatorAdvisories") } else if (mode === "mixed") { expect(retries).toEqual([0, 1]) expect(requests).toEqual(["", ""]) From d5ca5aa67f8d98aeb323ea7f2e4653c30bea5cf8 Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Sat, 21 Nov 2026 10:54:58 -0800 Subject: [PATCH 08/12] fix: retain strict diagnostics and recover advisory state writes --- docs/docs/data-engineering/validators.md | 1 + docs/docs/reference/telemetry.md | 2 +- .../altimate/validators/dbt-nothing-built.ts | 14 ++++++++----- .../src/altimate/validators/dbt-tests-pass.ts | 13 ++++++------ packages/opencode/src/session/prompt.ts | 3 ++- .../src/session/validators/enforcement.ts | 19 +++++++++++++---- .../opencode/src/session/validators/types.ts | 3 +++ .../provider/claude-agent/sync/sync.test.ts | 5 +++-- .../validators/dbt-nothing-built.test.ts | 11 ++++++++++ .../dbt-schema-verify-stale.test.ts | 21 ++++++++++++++++++- 10 files changed, 72 insertions(+), 20 deletions(-) diff --git a/docs/docs/data-engineering/validators.md b/docs/docs/data-engineering/validators.md index 9aa2b0fb6a..a7a491ee36 100644 --- a/docs/docs/data-engineering/validators.md +++ b/docs/docs/data-engineering/validators.md @@ -235,6 +235,7 @@ When a validator runs, it returns: strictFailure?: true // preserves a legacy failure in strict mode reason?: string // human-readable failure summary fixHint?: string // the body injected into the synthetic user turn + strictFixHint?: string // legacy retry guidance in strict mode details: { models_touched: number dbt_root: string | null diff --git a/docs/docs/reference/telemetry.md b/docs/docs/reference/telemetry.md index 493f88c7e6..8e0f52f5be 100644 --- a/docs/docs/reference/telemetry.md +++ b/docs/docs/reference/telemetry.md @@ -54,7 +54,7 @@ We collect the following categories of events: | `error_fingerprint` | Error pattern for grouping — SHA-256 hash of the masked error message, error class, tool name, whether recovery succeeded, plus `error_message`. `error_message` contains only words from a fixed list of common error terms taken from the first line of the error; every other word is replaced with ``, so no user text, identifiers, paths, values or SQL can appear (max 200 characters). | | `sql_fingerprint` | SQL structural shape via AST parsing — statement types, table count, function count, subquery/aggregation/window function presence, and AST node count. No table names, column names, or SQL content. | | `schema_complexity` | Warehouse schema structural metrics from introspection — bucketed table, column, and schema counts plus average columns per table. No schema names or content. | -| `validator_check` | A completion-gate validator ran on session end — validator name, `ok` boolean, step, retry count, `enforced` flag (false in shadow-only mode and for `unverifiable` results when `ALTIMATE_VALIDATORS_STRICT=1` is unset), optional `classification` (`finding` or `unverifiable`), and structured `details` (model counts, elapsed time, concurrency limit — no SQL or model content). Only emitted when `ALTIMATE_VALIDATORS_ENABLED=1` or `ALTIMATE_VALIDATORS_SHADOW=1`. See [Validators](../data-engineering/validators.md). | +| `validator_check` | A completion-gate validator ran after the model attempted to finish — validator name, `ok` boolean, step, retry count, `enforced` flag (false in shadow-only mode and for `unverifiable` results when `ALTIMATE_VALIDATORS_STRICT=1` is unset), optional `classification` (`finding` or `unverifiable`), and structured `details` (model counts, elapsed time, concurrency limit — no SQL or model content). Only emitted when `ALTIMATE_VALIDATORS_ENABLED=1` or `ALTIMATE_VALIDATORS_SHADOW=1`. See [Validators](../data-engineering/validators.md). | | `validator_retries_exhausted` | A session terminated with unresolved validator failures after exhausting the synthetic-retry budget — names of the failing validators (no failure body content). | | `onboarding_started` | The first-run setup gate opened (fresh launch with no usable model). | | `model_picker_shown` | The provider picker was displayed. `trigger` distinguishes the first run from `/connect` and from the prompt gate (`altimate_base_back` is defined in the schema but no longer emitted — it recorded backing out of the old consent dialog, which no longer exists). | diff --git a/packages/opencode/src/altimate/validators/dbt-nothing-built.ts b/packages/opencode/src/altimate/validators/dbt-nothing-built.ts index 90fbd04aa2..bbf5906725 100644 --- a/packages/opencode/src/altimate/validators/dbt-nothing-built.ts +++ b/packages/opencode/src/altimate/validators/dbt-nothing-built.ts @@ -37,7 +37,7 @@ import { isFailedRunStatus, resolveDbtSourcePaths, runResultsProducedNodes, - collectProducedNodeNames, + collectProducedNodeInventory, resolveWithinRoot, sanitizeForPrompt, MODEL_NODE_EXTENSIONS, @@ -298,9 +298,9 @@ export const DbtNothingBuiltValidator: Validator = { // Existence rather than authorship is also what `dbt-deliverable-names` // already asserts, so the two contract gates now agree on what "delivered" // means instead of disagreeing by one session. - const existingNodes = hasNamedDeliverables - ? await collectProducedNodeNames(dbtRoot) - : new Set() + const { names: existingNodes, manifestUnreadable } = hasNamedDeliverables + ? await collectProducedNodeInventory(dbtRoot) + : { names: new Set(), manifestUnreadable: false } // Same modification/creation split as `modificationModels`: "Update the // file `models/schema.yml`" cannot be satisfied by the file merely // existing from before this session — the task is asking for a change to @@ -430,9 +430,13 @@ export const DbtNothingBuiltValidator: Validator = { : `The task document at ${safeTaskFile} names required deliverables${namedText}, but this session wrote no project files and produced no fresh successful build artifact. Nothing was built, so the task is not done.` : `This session wrote no project files and produced no fresh successful build artifact, but this project is configured to require artifacts. Nothing was built, so the task is not done.` + const advisory = manifestUnreadable && namedModels.some((name) => !modificationSet.has(name.toLowerCase())) + ? "Could not verify whether dbt deliverables already exist under aliases because manifest.json could not be read or parsed." + : undefined return { ok: false, - classification: "finding", + classification: advisory ? "unverifiable" : "finding", + advisory, reason, fixHint: [ diff --git a/packages/opencode/src/altimate/validators/dbt-tests-pass.ts b/packages/opencode/src/altimate/validators/dbt-tests-pass.ts index fb47382078..e06606b28b 100644 --- a/packages/opencode/src/altimate/validators/dbt-tests-pass.ts +++ b/packages/opencode/src/altimate/validators/dbt-tests-pass.ts @@ -289,9 +289,9 @@ export const DbtTestsPassValidator: Validator = { } } - const hintBlocks = failures.length > 0 - ? failures - : toolErrors.map(({ model, error }) => ({ model, error })) + const errorHints = toolErrors.map(({ model, error }) => ({ model, error })) + const hintBlocks = failures.length > 0 ? failures : errorHints + const remediation = `\n\nFix the model SQL (not the tests). Common causes: wrong JOIN type (LEFT vs INNER changing row counts), missing GROUP BY columns, dropped/added rows from filters, type coercion mismatch on join keys. Rebuild and the harness will re-check before declaring done.` const failingNames = failures.map((f) => f.model).filter(Boolean) const erroredNames = toolErrors.map((f) => f.model).filter(Boolean) const reason = @@ -305,9 +305,10 @@ export const DbtTestsPassValidator: Validator = { classification: failures.length > 0 ? "finding" : advisory ? "unverifiable" : undefined, advisory, reason, - fixHint: - formatFixHint(hintBlocks) + - `\n\nFix the model SQL (not the tests). Common causes: wrong JOIN type (LEFT vs INNER changing row counts), missing GROUP BY columns, dropped/added rows from filters, type coercion mismatch on join keys. Rebuild and the harness will re-check before declaring done.`, + fixHint: formatFixHint(hintBlocks) + remediation, + ...(failures.length > 0 && errorHints.length > 0 && { + strictFixHint: formatFixHint([...failures, ...errorHints]) + remediation, + }), details: { ...baseDetails, passed: passed.length, diff --git a/packages/opencode/src/session/prompt.ts b/packages/opencode/src/session/prompt.ts index a18faf6865..e15e4663f2 100644 --- a/packages/opencode/src/session/prompt.ts +++ b/packages/opencode/src/session/prompt.ts @@ -2588,7 +2588,8 @@ export namespace SessionPrompt { const body = failures .map(({ validator, result: vRes }) => { const head = `[altimate-validator: ${validator.name}] ${vRes.reason ?? "validation failed"}` - const tail = vRes.fixHint ? `\n${vRes.fixHint}` : "" + const hint = (validatorStrict ? vRes.strictFixHint : undefined) ?? vRes.fixHint + const tail = hint ? `\n${hint}` : "" return head + tail }) .join("\n\n") diff --git a/packages/opencode/src/session/validators/enforcement.ts b/packages/opencode/src/session/validators/enforcement.ts index 24101048c1..3839f028f7 100644 --- a/packages/opencode/src/session/validators/enforcement.ts +++ b/packages/opencode/src/session/validators/enforcement.ts @@ -25,8 +25,9 @@ export function decide( ? result.strictFailure || !result.ok : !result.ok && result.classification !== "unverifiable" if (blocks) failures.push(check) - if (!options.strict && result.advisory) { - advisories.push(`[altimate-validator advisory: ${validator.name}] ${result.advisory}`) + if (!options.strict && (result.advisory || result.classification === "unverifiable")) { + const note = result.advisory ?? "The validator could not complete this check." + advisories.push(`[altimate-validator advisory: ${validator.name}] ${note}`) } } } @@ -44,6 +45,8 @@ interface AdvisoryState { } const log = Log.create({ service: "validator-advisories" }) +// A transient write failure must not replay a delivered note on the next turn. +const unsaved = new Map() export function advisoryFingerprint(text: string): string { return createHash("sha256").update(text).digest("hex") @@ -55,6 +58,11 @@ function advisoryFile(sessionID: SessionID): string { // A local file avoids putting advisories in the conversation and share stream. export async function loadAdvisories(sessionID: SessionID): Promise { + const recovery = unsaved.get(sessionID) + if (recovery) { + saveAdvisories(sessionID, recovery) + return recovery + } try { const value = await Filesystem.readJson(advisoryFile(sessionID)) if ( @@ -70,12 +78,15 @@ export async function loadAdvisories(sessionID: SessionID): Promise { + unsaved.delete(sessionID) await rm(advisoryFile(sessionID), { force: true }).catch(() => log.warn("failed to remove validator advisories")) } diff --git a/packages/opencode/src/session/validators/types.ts b/packages/opencode/src/session/validators/types.ts index f254ab11d1..ed465169e9 100644 --- a/packages/opencode/src/session/validators/types.ts +++ b/packages/opencode/src/session/validators/types.ts @@ -86,6 +86,9 @@ export interface ValidatorResult { * message verbatim. */ fixHint?: string + /** Overrides the fix hint in strict mode when prior diagnostics must be preserved. */ + strictFixHint?: string + /** Structured detail for telemetry; not surfaced to the agent unless the * validator explicitly includes it in `reason`/`fixHint`. */ details?: Record diff --git a/packages/opencode/test/altimate/provider/claude-agent/sync/sync.test.ts b/packages/opencode/test/altimate/provider/claude-agent/sync/sync.test.ts index f0cb161054..03a0050cb2 100644 --- a/packages/opencode/test/altimate/provider/claude-agent/sync/sync.test.ts +++ b/packages/opencode/test/altimate/provider/claude-agent/sync/sync.test.ts @@ -112,7 +112,9 @@ describe("history reconciliation", () => { type: "text", text: "Schema verification could not run.", }) const call = { type: "tool-call", toolCallId: "a", toolName: "read", input: { path: "orders.sql" } } as const - const history = Checkpoints.build({ prompt: request, previous: record, reply: [call], toolMessages: new Map() }) + const checkpoint = { previous: record, reply: [call], toolMessages: new Map() } + const history = Checkpoints.build({ ...checkpoint, prompt: request }) + expect(history).toEqual(Checkpoints.build({ ...checkpoint, prompt })) const followup: LanguageModelV3Prompt = [ ...prompt, { role: "assistant", content: [call] }, @@ -124,7 +126,6 @@ describe("history reconciliation", () => { const saved = { ...record, history } const continuation = SyncDecision.decide(followup, saved) expect(continuation).toEqual({ kind: "catch-up", start: 4 }) - expect(JSON.stringify(CatchUp.content(followup, continuation.start))).not.toContain("Schema verification") expect(SyncDecision.decide([...prompt, note], { ...record, history: Fingerprint.groups(prompt) })).toEqual({ kind: "continue", start: 3, }) diff --git a/packages/opencode/test/altimate/validators/dbt-nothing-built.test.ts b/packages/opencode/test/altimate/validators/dbt-nothing-built.test.ts index ac0579c023..ddac934032 100644 --- a/packages/opencode/test/altimate/validators/dbt-nothing-built.test.ts +++ b/packages/opencode/test/altimate/validators/dbt-nothing-built.test.ts @@ -384,6 +384,17 @@ describe("run_results and inventory helpers", () => { expect(isFailedRunStatus("")).toBe(true) }) + test("an unreadable manifest leaves a possible alias unverifiable", async () => { + await makeProject() + await writeModel("orders_fact") + await fs.writeFile(join(dir, "TASK.md"), "Create the model `fct_orders`.\n") + await fs.mkdir(join(dir, "target"), { recursive: true }) + await fs.writeFile(join(dir, "target", "manifest.json"), "{ invalid") + const result = await DbtNothingBuiltValidator.check(ctxFuture()) + expect(result).toMatchObject({ ok: false, classification: "unverifiable" }) + expect(result.advisory).toContain("manifest.json") + }) + test("collectProducedNodeNames unions filesystem names and manifest aliases", async () => { await makeProject() await writeModel("stg_orders") diff --git a/packages/opencode/test/altimate/validators/dbt-schema-verify-stale.test.ts b/packages/opencode/test/altimate/validators/dbt-schema-verify-stale.test.ts index 020e2e336b..2d4a0ad3d2 100644 --- a/packages/opencode/test/altimate/validators/dbt-schema-verify-stale.test.ts +++ b/packages/opencode/test/altimate/validators/dbt-schema-verify-stale.test.ts @@ -38,7 +38,11 @@ beforeAll(async () => { const bin = join(root, "bin") await fs.mkdir(bin) answerFile = join(root, "answer.json") - await fs.writeFile(join(bin, "altimate-dbt"), `#!/bin/sh\ncat "${answerFile}"\n`, { mode: 0o755 }) + await fs.writeFile( + join(bin, "altimate-dbt"), + `#!/bin/sh\nif [ "$3" = customers ]; then cat "${answerFile}.customers"; else cat "${answerFile}"; fi\n`, + { mode: 0o755 }, + ) originalPath = process.env.PATH ?? "" process.env.PATH = `${bin}:${originalPath}` project = join(root, "project") @@ -53,6 +57,8 @@ afterAll(async () => { afterEach(async () => { await fs.rm(answerFile, { force: true }) + await fs.rm(answerFile + ".customers", { force: true }) + await fs.rm(join(project, "models", "customers.sql"), { force: true }) }) /** Runs the validator over one model edited this session, with `answer` as schema-verify's answer. */ @@ -122,6 +128,19 @@ describe("dbt-tests-pass result classification", () => { expect(ValidatorEnforcement.decide(checks, { ...options, strict: true }).retry).toBe(true) }) + test("a mixed retry includes tool errors only in the strict hint", async () => { + await fs.writeFile(join(project, "models", "customers.sql"), "select 1 as id\n") + await fs.writeFile(answerFile + ".customers", JSON.stringify({ error: "Adapter unavailable" })) + const result = await check({ + stdout: "1 of 1 FAIL 1 not_null_orders_id [FAIL 1 in 0.10s]\nDone. PASS=0 WARN=0 ERROR=1 SKIP=0 TOTAL=1", + }, DbtTestsPassValidator) + expect(result).toMatchObject({ ok: false, classification: "finding" }) + expect(result.advisory).toContain("customers") + expect(result.fixHint).toContain("not_null_orders_id") + expect(result.fixHint).not.toContain("Adapter unavailable") + expect(result.strictFixHint).toContain("Adapter unavailable") + }) + test("a real test failure is a finding even when the wrapper also reports an error", async () => { const result = await check({ error: "dbt exited with code 1", From f5829da3d6d6f9acb8ff6d2b70b96b80bc7120e3 Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Sat, 21 Nov 2026 10:58:53 -0800 Subject: [PATCH 09/12] fix: preserve original mixed-result messages in strict mode --- docs/docs/data-engineering/validators.md | 1 + .../validators/dbt-deliverable-names.ts | 61 +++++++------------ .../src/altimate/validators/dbt-tests-pass.ts | 14 +++-- packages/opencode/src/session/prompt.ts | 3 +- .../opencode/src/session/validators/types.ts | 3 + .../validators/dbt-deliverable-names.test.ts | 6 ++ .../dbt-schema-verify-stale.test.ts | 2 + 7 files changed, 43 insertions(+), 47 deletions(-) diff --git a/docs/docs/data-engineering/validators.md b/docs/docs/data-engineering/validators.md index a7a491ee36..2a0e1856e8 100644 --- a/docs/docs/data-engineering/validators.md +++ b/docs/docs/data-engineering/validators.md @@ -235,6 +235,7 @@ When a validator runs, it returns: strictFailure?: true // preserves a legacy failure in strict mode reason?: string // human-readable failure summary fixHint?: string // the body injected into the synthetic user turn + strictReason?: string // legacy failure summary in strict mode strictFixHint?: string // legacy retry guidance in strict mode details: { models_touched: number diff --git a/packages/opencode/src/altimate/validators/dbt-deliverable-names.ts b/packages/opencode/src/altimate/validators/dbt-deliverable-names.ts index 347a733810..e0939f1992 100644 --- a/packages/opencode/src/altimate/validators/dbt-deliverable-names.ts +++ b/packages/opencode/src/altimate/validators/dbt-deliverable-names.ts @@ -182,55 +182,36 @@ export const DbtDeliverableNamesValidator: Validator = { ? "Could not verify the missing dbt deliverable names against aliases because manifest.json could not be read or parsed." : undefined const includeModelFindings = missingModels.length > 0 && (!advisory || missingFiles.length === 0) - const reasonParts: string[] = [] - if (includeModelFindings) { - reasonParts.push( - `the task names ${missingModels.length} deliverable(s) this project does not define: ${missingModels.join(", ")}`, - ) - } - if (missingFiles.length > 0) { - reasonParts.push(`required file(s) missing: ${missingFiles.join(", ")}`) - } + const modelReason = missingModels.length > 0 + ? `the task names ${missingModels.length} deliverable(s) this project does not define: ${missingModels.join(", ")}` + : undefined + const fileReason = missingFiles.length > 0 + ? `required file(s) missing: ${missingFiles.join(", ")}` + : undefined + const reason = (includeModels: boolean) => + `Deliverable-name mismatch: ${[includeModels && modelReason, fileReason].filter(Boolean).join("; ")}.` - // The workspace directory is repository/environment-controlled and can in - // principle carry adversarial text; `contract.taskFile` is built from it. - // `dbt-nothing-built` already routes its own task-file path through - // `sanitizeForPrompt` for this reason — this interpolation is the - // matching gap in this file. + // Task paths and model filenames come from the repository. Quote and + // bound them before including them in synthetic retry messages. const safeTaskFile = sanitizeForPrompt(contract.taskFile, 160) - const hintLines: string[] = [ + const safeUnrequested = unrequested.map((name) => sanitizeForPrompt(name, 80)) + const fixHint = (includeModels: boolean) => [ `The task document (${safeTaskFile}) states these names literally. A model that does the right thing under a different name does not satisfy the task, and self-verification against the renamed output will not detect it.`, - ] - if (includeModelFindings) { - hintLines.push(` • Create or rename to exactly: ${missingModels.join(", ")}`) - } - if (missingFiles.length > 0) { - hintLines.push(` • Create at exactly these paths: ${missingFiles.join(", ")}`) - } - if (unrequested.length > 0) { - // `unrequested` is derived from ACTUAL FILENAMES ON DISK, which — unlike - // `missingModels`/`missingFiles` (constrained to identifier/path-shaped - // code-span tokens by `extractRequiredDeliverables`) — can contain - // arbitrary bytes, including newlines, on a POSIX filesystem. Spliced - // verbatim into `fixHint`, a hostile or merely adversarial filename - // breaks out of the bullet it is quoted in and lands at instruction - // position in the synthetic retry turn `prompt.ts` builds from this. - const safeUnrequested = unrequested.map((n) => sanitizeForPrompt(n, 80)) - hintLines.push( + includeModels && modelReason && ` • Create or rename to exactly: ${missingModels.join(", ")}`, + fileReason && ` • Create at exactly these paths: ${missingFiles.join(", ")}`, + includeModels && safeUnrequested.length > 0 && ` • Models you created this session that the task did not name: ${safeUnrequested.join(", ")}. If one of them is a renamed version of a required deliverable, rename the file (and any \`ref()\` to it) back to the required name.`, - ) - } - if (includeModelFindings) { - hintLines.push( + includeModels && " • If a required deliverable is produced under an alias, set `alias` in its config so the required name is the relation name.", - ) - } + ].filter(Boolean).join("\n") + return { ok: false, classification: advisory && missingFiles.length === 0 ? "unverifiable" : "finding", advisory, - reason: `Deliverable-name mismatch: ${reasonParts.join("; ")}.`, - fixHint: hintLines.join("\n"), + reason: reason(includeModelFindings), + fixHint: fixHint(includeModelFindings), + ...(!includeModelFindings && { strictReason: reason(true), strictFixHint: fixHint(true) }), details: { ...details, unrequested_models: unrequested }, } }, diff --git a/packages/opencode/src/altimate/validators/dbt-tests-pass.ts b/packages/opencode/src/altimate/validators/dbt-tests-pass.ts index e06606b28b..76413d0b57 100644 --- a/packages/opencode/src/altimate/validators/dbt-tests-pass.ts +++ b/packages/opencode/src/altimate/validators/dbt-tests-pass.ts @@ -171,7 +171,7 @@ async function runDbtTest(model: string, cwd: string): Promise f.model).filter(Boolean) const erroredNames = toolErrors.map((f) => f.model).filter(Boolean) - const reason = - failures.length > 0 - ? `${failures.length} of ${results.length} models you edited have failing dbt tests${failingNames.length ? `: ${failingNames.join(", ")}` : ""}.` + const describeFailure = (names: string[]) => + names.length > 0 + ? `${names.length} of ${results.length} models you edited have failing dbt tests: ${names.join(", ")}.` : `${toolErrors.length} of ${results.length} models could not be tested${erroredNames.length ? `: ${erroredNames.join(", ")}` : ""}. Investigate before declaring done.` + const strictFailures = failures.filter((result) => !result.error) const ok = failures.length === 0 && errored.length === 0 return { ok, ...(ok && { strictFailure: true }), classification: failures.length > 0 ? "finding" : advisory ? "unverifiable" : undefined, advisory, - reason, + reason: describeFailure(failingNames), fixHint: formatFixHint(hintBlocks) + remediation, ...(failures.length > 0 && errorHints.length > 0 && { - strictFixHint: formatFixHint([...failures, ...errorHints]) + remediation, + strictReason: describeFailure(strictFailures.map((result) => result.model)), + strictFixHint: formatFixHint([...strictFailures, ...errorHints]) + remediation, }), details: { ...baseDetails, diff --git a/packages/opencode/src/session/prompt.ts b/packages/opencode/src/session/prompt.ts index e15e4663f2..8e648c941d 100644 --- a/packages/opencode/src/session/prompt.ts +++ b/packages/opencode/src/session/prompt.ts @@ -2587,7 +2587,8 @@ export namespace SessionPrompt { // the next user message and gets one more turn to address it. const body = failures .map(({ validator, result: vRes }) => { - const head = `[altimate-validator: ${validator.name}] ${vRes.reason ?? "validation failed"}` + const reason = (validatorStrict ? vRes.strictReason : undefined) ?? vRes.reason + const head = `[altimate-validator: ${validator.name}] ${reason ?? "validation failed"}` const hint = (validatorStrict ? vRes.strictFixHint : undefined) ?? vRes.fixHint const tail = hint ? `\n${hint}` : "" return head + tail diff --git a/packages/opencode/src/session/validators/types.ts b/packages/opencode/src/session/validators/types.ts index ed465169e9..837377fa0e 100644 --- a/packages/opencode/src/session/validators/types.ts +++ b/packages/opencode/src/session/validators/types.ts @@ -86,6 +86,9 @@ export interface ValidatorResult { * message verbatim. */ fixHint?: string + /** Overrides the reason in strict mode when prior diagnostics must be preserved. */ + strictReason?: string + /** Overrides the fix hint in strict mode when prior diagnostics must be preserved. */ strictFixHint?: string diff --git a/packages/opencode/test/altimate/validators/dbt-deliverable-names.test.ts b/packages/opencode/test/altimate/validators/dbt-deliverable-names.test.ts index 17b3582c66..a3cdf69f58 100644 --- a/packages/opencode/test/altimate/validators/dbt-deliverable-names.test.ts +++ b/packages/opencode/test/altimate/validators/dbt-deliverable-names.test.ts @@ -132,6 +132,12 @@ describe("DbtDeliverableNamesValidator — check", () => { const result = await DbtDeliverableNamesValidator.check(ctx()) expect(result).toMatchObject({ ok: false, classification }) expect(result.advisory).toContain("manifest.json") + if (classification === "finding") { + expect(result.fixHint).not.toContain("fct_orders") + expect(result.strictReason).toContain("fct_orders") + expect(result.strictFixHint).toContain("fct_orders") + expect(result.strictFixHint).toContain("models/orders.sql") + } }) test("a seed satisfies a required name", async () => { diff --git a/packages/opencode/test/altimate/validators/dbt-schema-verify-stale.test.ts b/packages/opencode/test/altimate/validators/dbt-schema-verify-stale.test.ts index 2d4a0ad3d2..57b92882d0 100644 --- a/packages/opencode/test/altimate/validators/dbt-schema-verify-stale.test.ts +++ b/packages/opencode/test/altimate/validators/dbt-schema-verify-stale.test.ts @@ -148,5 +148,7 @@ describe("dbt-tests-pass result classification", () => { }, DbtTestsPassValidator) expect(result).toMatchObject({ ok: false, classification: "finding" }) expect(result.fixHint).toContain("not_null_orders_id") + expect(result.strictReason).toContain("could not be tested") + expect(result.strictFixHint).toContain("dbt exited with code 1") }) }) From e3cd7e7d375ed8e720bf0d0b89126ea92f792b57 Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Sat, 21 Nov 2026 11:01:36 -0800 Subject: [PATCH 10/12] test: cover advisory write recovery and strict retry messages --- .../src/session/validators/enforcement.ts | 5 +++ packages/opencode/test/session/prompt.test.ts | 12 ++++++ .../test/session/validator-advisories.test.ts | 37 +++++++++++++++++++ 3 files changed, 54 insertions(+) create mode 100644 packages/opencode/test/session/validator-advisories.test.ts diff --git a/packages/opencode/src/session/validators/enforcement.ts b/packages/opencode/src/session/validators/enforcement.ts index 3839f028f7..0f52c7c2c7 100644 --- a/packages/opencode/src/session/validators/enforcement.ts +++ b/packages/opencode/src/session/validators/enforcement.ts @@ -47,6 +47,7 @@ interface AdvisoryState { const log = Log.create({ service: "validator-advisories" }) // A transient write failure must not replay a delivered note on the next turn. const unsaved = new Map() +const MAX_UNSAVED_SESSIONS = 128 export function advisoryFingerprint(text: string): string { return createHash("sha256").update(text).digest("hex") @@ -80,7 +81,11 @@ export function saveAdvisories(sessionID: SessionID, state: AdvisoryState): void Filesystem.writeJsonAtomic(advisoryFile(sessionID), state, 0o600) unsaved.delete(sessionID) } catch (error) { + unsaved.delete(sessionID) unsaved.set(sessionID, state) + if (unsaved.size > MAX_UNSAVED_SESSIONS) { + unsaved.delete(unsaved.keys().next().value!) + } log.warn("failed to save validator advisories", { error }) } } diff --git a/packages/opencode/test/session/prompt.test.ts b/packages/opencode/test/session/prompt.test.ts index b869b1f45b..8a9c3d47c8 100644 --- a/packages/opencode/test/session/prompt.test.ts +++ b/packages/opencode/test/session/prompt.test.ts @@ -588,6 +588,8 @@ for (const mode of ["advisory", "advisory-after-error", "mixed", "finding", "str const requests: string[] = [] const note = "Could not verify orders because the schema tool failed." const retryNote = "Could not verify customers because the test tool failed." + const strictReason = "Previous failure summary." + const strictFixHint = "Previous fix guidance." let checkCount = 0 const runAll = spyOn(ValidatorRegistry, "runAll").mockImplementation(async () => { checkCount++ @@ -599,6 +601,10 @@ for (const mode of ["advisory", "advisory-after-error", "mixed", "finding", "str result: { ok: false, classification: "unverifiable" as const, reason: text, advisory: text }, }) if (mode === "finding") return [finding] + if (mode === "strict") { + const check = advisory(note) + return [{ ...check, result: { ...check.result, strictReason, strictFixHint } }] + } if (mode === "mixed") { if (checkCount === 1) return [finding, advisory(note)] if (checkCount === 2) return [advisory(retryNote)] @@ -668,6 +674,12 @@ for (const mode of ["advisory", "advisory-after-error", "mixed", "finding", "str expect(requests).toHaveLength(4) expect(retries).toEqual([0, 1, 2, 3]) expect(requests.every((request) => !request.includes(note))).toBe(true) + if (mode === "strict") { + const retryBody = JSON.stringify(stream.mock.calls[1]?.[0].messages) + expect(retryBody).toContain(strictReason) + expect(retryBody).toContain(strictFixHint) + expect(retryBody).not.toContain(note) + } } } finally { stream.mockRestore() diff --git a/packages/opencode/test/session/validator-advisories.test.ts b/packages/opencode/test/session/validator-advisories.test.ts new file mode 100644 index 0000000000..5754eb47d9 --- /dev/null +++ b/packages/opencode/test/session/validator-advisories.test.ts @@ -0,0 +1,37 @@ +import { expect, spyOn, test } from "bun:test" +import { SessionID } from "../../src/session/schema" +import { ValidatorEnforcement } from "../../src/session/validators/enforcement" +import { Filesystem } from "../../src/util/filesystem" +import { withTestStateHome } from "../fixture/fixture" + +test("a failed advisory write keeps delivered state until recovery or session deletion", async () => { + await withTestStateHome(async () => { + const sessionID = SessionID.make("ses_validator-advisories") + const note = "Schema verification was unavailable." + const pending = { pending: [note], shown: [] } + const delivered = { pending: [], shown: [ValidatorEnforcement.advisoryFingerprint(note)] } + ValidatorEnforcement.saveAdvisories(sessionID, pending) + + const write = spyOn(Filesystem, "writeJsonAtomic").mockImplementation(() => { + throw new Error("state directory is temporarily unwritable") + }) + try { + ValidatorEnforcement.saveAdvisories(sessionID, delivered) + expect(await ValidatorEnforcement.loadAdvisories(sessionID)).toEqual(delivered) + } finally { + write.mockRestore() + } + expect(await ValidatorEnforcement.loadAdvisories(sessionID)).toEqual(delivered) + + const failedWrite = spyOn(Filesystem, "writeJsonAtomic").mockImplementation(() => { + throw new Error("state directory is temporarily unwritable") + }) + try { + ValidatorEnforcement.saveAdvisories(sessionID, pending) + await ValidatorEnforcement.removeAdvisories(sessionID) + expect(await ValidatorEnforcement.loadAdvisories(sessionID)).toEqual({ pending: [], shown: [] }) + } finally { + failedWrite.mockRestore() + } + }) +}) From 4b648d28b506aa432205f556c3c63e24b6546188 Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Sat, 21 Nov 2026 11:06:07 -0800 Subject: [PATCH 11/12] fix: prevent advisory writes after session deletion --- packages/opencode/src/session/index.ts | 6 ++++-- packages/opencode/src/session/prompt.ts | 14 +++++++++++--- packages/opencode/src/session/session.ts | 6 ++++-- packages/opencode/test/session/prompt.test.ts | 13 +++++++++++-- .../test/session/validator-advisories.test.ts | 6 ++++++ 5 files changed, 36 insertions(+), 9 deletions(-) diff --git a/packages/opencode/src/session/index.ts b/packages/opencode/src/session/index.ts index 3c4d4989da..d96f69fe71 100644 --- a/packages/opencode/src/session/index.ts +++ b/packages/opencode/src/session/index.ts @@ -756,9 +756,8 @@ export namespace Session { await remove(child.id) } await unshare(sessionID).catch(() => {}) - // altimate_change start — release the deleted session's scratch and advisory state + // altimate_change start — release the deleted session's scratch store await closeScratch(sessionID) - await ValidatorEnforcement.removeAdvisories(sessionID) // altimate_change end // CASCADE delete handles messages and parts automatically Database.use((db) => { @@ -769,6 +768,9 @@ export namespace Session { }), ) }) + // altimate_change start — clean up only after the session row is gone + await ValidatorEnforcement.removeAdvisories(sessionID) + // altimate_change end } catch (e) { log.error(e) } diff --git a/packages/opencode/src/session/prompt.ts b/packages/opencode/src/session/prompt.ts index 8e648c941d..1c7fedf796 100644 --- a/packages/opencode/src/session/prompt.ts +++ b/packages/opencode/src/session/prompt.ts @@ -37,7 +37,7 @@ import { Bus } from "../bus" // altimate_change start - distinguish submitted requests from compaction-created user rows in the stream import { BusEvent } from "../bus/bus-event" import { Database, eq, sql } from "../storage/db" -import { MessageTable } from "@opencode-ai/core/session/sql" +import { MessageTable, SessionTable } from "@opencode-ai/core/session/sql" // altimate_change end import { ProviderTransform } from "../provider/transform" import { SystemPrompt } from "./system" @@ -906,6 +906,14 @@ export namespace SessionPrompt { process.env.ALTIMATE_VALIDATORS_ENABLED === "1" && !validatorStrict ? await ValidatorEnforcement.loadAdvisories(sessionID) : undefined + const saveValidatorAdvisories = () => { + if (!validatorAdvisories) return + // The validator may have awaited a subprocess while the session was deleted. + const exists = Database.use((db) => + db.select({ id: SessionTable.id }).from(SessionTable).where(eq(SessionTable.id, sessionID)).get(), + ) + if (exists) ValidatorEnforcement.saveAdvisories(sessionID, validatorAdvisories) + } // altimate_change end let sessionAgentName = "" let sessionHadError = false @@ -2303,7 +2311,7 @@ export namespace SessionPrompt { if (validatorAdvisories && advisoryContext && processor.message.finish && !processor.message.error && !abort.aborted) { validatorAdvisories.shown.push(...validatorAdvisories.pending.map(ValidatorEnforcement.advisoryFingerprint)) validatorAdvisories.pending = [] - ValidatorEnforcement.saveAdvisories(sessionID, validatorAdvisories) + saveValidatorAdvisories() } // altimate_change end @@ -2577,7 +2585,7 @@ export namespace SessionPrompt { ) if (additional.length > 0) { validatorAdvisories.pending.push(...new Set(additional)) - ValidatorEnforcement.saveAdvisories(sessionID, validatorAdvisories) + saveValidatorAdvisories() } } diff --git a/packages/opencode/src/session/session.ts b/packages/opencode/src/session/session.ts index 84a9306293..2cc468fc38 100644 --- a/packages/opencode/src/session/session.ts +++ b/packages/opencode/src/session/session.ts @@ -678,12 +678,14 @@ export const layer: Layer.Layer< yield* remove(child.id) } - // altimate_change start — release the deleted session's scratch and advisory state + // altimate_change start — release the deleted session's scratch store yield* Effect.promise(() => closeScratch(sessionID)) - yield* Effect.promise(() => ValidatorEnforcement.removeAdvisories(sessionID)) // altimate_change end yield* events.publish(SessionV1.Event.Deleted, { sessionID, info: session }) yield* events.remove(sessionID) + // altimate_change start — clean up only after the session row is gone + yield* Effect.promise(() => ValidatorEnforcement.removeAdvisories(sessionID)) + // altimate_change end } catch (error) { yield* Effect.logError("failed to remove session", { sessionID, error }) } diff --git a/packages/opencode/test/session/prompt.test.ts b/packages/opencode/test/session/prompt.test.ts index 8a9c3d47c8..d67f75899b 100644 --- a/packages/opencode/test/session/prompt.test.ts +++ b/packages/opencode/test/session/prompt.test.ts @@ -44,6 +44,7 @@ import { Instruction } from "../../src/session/instruction" import { SessionProcessor } from "../../src/session/processor" import { SessionPrompt } from "../../src/session/prompt" import { ValidatorRegistry } from "../../src/session/validators/registry" +import { ValidatorEnforcement } from "../../src/session/validators/enforcement" import { SessionRevert } from "../../src/session/revert" import { SessionRunState } from "../../src/session/run-state" import { MessageID, PartID, SessionID } from "../../src/session/schema" @@ -576,7 +577,7 @@ for (const downgrade of [true, false]) { // altimate_change end // altimate_change start — completion validator enforcement through the real prompt loop -for (const mode of ["advisory", "advisory-after-error", "mixed", "finding", "strict"] as const) { +for (const mode of ["advisory", "advisory-after-error", "deleted-session", "mixed", "finding", "strict"] as const) { noLLMServer.instance( `completion validators use the ${mode} policy`, () => Effect.gen(function* () { @@ -591,8 +592,9 @@ for (const mode of ["advisory", "advisory-after-error", "mixed", "finding", "str const strictReason = "Previous failure summary." const strictFixHint = "Previous fix guidance." let checkCount = 0 - const runAll = spyOn(ValidatorRegistry, "runAll").mockImplementation(async () => { + const runAll = spyOn(ValidatorRegistry, "runAll").mockImplementation(async (ctx) => { checkCount++ + if (mode === "deleted-session") await LegacySession.remove(SessionID.make(ctx.sessionID)) if (mode === "advisory" && checkCount === 2) return [] const validator = ValidatorRegistry.list()[0]! const finding = { validator, result: { ok: false, classification: "finding" as const, reason: "Confirmed failure." } } @@ -635,6 +637,13 @@ for (const mode of ["advisory", "advisory-after-error", "mixed", "finding", "str model: promptRef, parts: [{ type: "text", text }], }) + if (mode === "deleted-session") { + yield* Effect.exit(reply("Review invoice totals.")) + expect(runAll.mock.calls).toHaveLength(1) + const state = yield* Effect.promise(() => ValidatorEnforcement.loadAdvisories(chat.id)) + expect(state).toEqual({ pending: [], shown: [] }) + return + } yield* reply("Review invoice totals.") const retries = runAll.mock.calls.map(([ctx]) => ctx.retryCount) if (mode === "advisory") { diff --git a/packages/opencode/test/session/validator-advisories.test.ts b/packages/opencode/test/session/validator-advisories.test.ts index 5754eb47d9..910046cd24 100644 --- a/packages/opencode/test/session/validator-advisories.test.ts +++ b/packages/opencode/test/session/validator-advisories.test.ts @@ -1,4 +1,7 @@ import { expect, spyOn, test } from "bun:test" +import { readdir } from "node:fs/promises" +import path from "node:path" +import { Global } from "../../src/global" import { SessionID } from "../../src/session/schema" import { ValidatorEnforcement } from "../../src/session/validators/enforcement" import { Filesystem } from "../../src/util/filesystem" @@ -22,6 +25,9 @@ test("a failed advisory write keeps delivered state until recovery or session de write.mockRestore() } expect(await ValidatorEnforcement.loadAdvisories(sessionID)).toEqual(delivered) + const folder = path.join(Global.Path.state, "validator-advisories") + const [file] = await readdir(folder) + expect(await Filesystem.readJson(path.join(folder, file!))).toEqual(delivered) const failedWrite = spyOn(Filesystem, "writeJsonAtomic").mockImplementation(() => { throw new Error("state directory is temporarily unwritable") From 73d08bcfdd9f82ff060388da38060895f1f68dd4 Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Sat, 21 Nov 2026 11:07:47 -0800 Subject: [PATCH 12/12] test: type the persisted advisory state assertion --- packages/opencode/test/session/validator-advisories.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/opencode/test/session/validator-advisories.test.ts b/packages/opencode/test/session/validator-advisories.test.ts index 910046cd24..450d826abe 100644 --- a/packages/opencode/test/session/validator-advisories.test.ts +++ b/packages/opencode/test/session/validator-advisories.test.ts @@ -27,7 +27,7 @@ test("a failed advisory write keeps delivered state until recovery or session de expect(await ValidatorEnforcement.loadAdvisories(sessionID)).toEqual(delivered) const folder = path.join(Global.Path.state, "validator-advisories") const [file] = await readdir(folder) - expect(await Filesystem.readJson(path.join(folder, file!))).toEqual(delivered) + expect(await Filesystem.readJson(path.join(folder, file!))).toEqual(delivered) const failedWrite = spyOn(Filesystem, "writeJsonAtomic").mockImplementation(() => { throw new Error("state directory is temporarily unwritable")