Repository navigation
feat(review): opt-in AI verifier that confirms or refutes each AI comment from dbt evidence #1769
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: feat/review-posting-policy
Are you sure you want to change the base?
Changes from all commits
0815710
6f5d4f4
d21d5ee
7cb6b34
3a9e587
c4f7539
9eee7e6
c6fd21a
4f53698
2d7431a
a53f7fb
401e006
f5965c5
e3e6417
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -12,3 +12,42 @@ export async function readStreamText(stream: { | |
| } | ||
| return text | ||
| } | ||
|
|
||
| /** A tool call the model made in one step, as the AI SDK reports it. */ | ||
| export interface StepToolCall { | ||
| toolCallId: string | ||
| toolName: string | ||
| input: unknown | ||
| /** The SDK could not parse the call or the tool does not exist. */ | ||
| invalid?: boolean | ||
| } | ||
|
|
||
| /** | ||
| * Read one AI SDK step without discarding its tool calls: the text, why the | ||
| * step ended, the tool calls it made, and the response messages to append to | ||
| * the conversation for the next step. Tools without an `execute` are not run | ||
| * by the SDK; the caller runs them and appends the results. Like | ||
| * `readStreamText`, a stream error is thrown rather than lost. | ||
| */ | ||
| export async function readStreamStep(stream: { | ||
| fullStream: AsyncIterable<{ type: string; error?: unknown }> | ||
| readonly text: PromiseLike<string> | ||
| readonly finishReason: PromiseLike<string> | ||
| readonly response: PromiseLike<{ messages: unknown[] }> | ||
| }): Promise<{ text: string; finishReason: string; toolCalls: StepToolCall[]; messages: unknown[] }> { | ||
| const toolCalls: StepToolCall[] = [] | ||
| const text = await readStreamText({ | ||
| text: stream.text, | ||
| fullStream: (async function* () { | ||
| for await (const part of stream.fullStream) { | ||
| if (part.type === "tool-call") { | ||
| const p = part as unknown as StepToolCall | ||
| toolCalls.push({ toolCallId: p.toolCallId, toolName: p.toolName, input: p.input, invalid: p.invalid }) | ||
| } | ||
| yield part | ||
| } | ||
| })(), | ||
| }) | ||
| const [finishReason, response] = await Promise.all([stream.finishReason, stream.response]) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. SUGGESTION:
|
||
| return { text, finishReason, toolCalls, messages: response?.messages ?? [] } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,220 @@ | ||
| // altimate_change - new file: replay the AI lane's generator findings from an earlier envelope | ||
| // | ||
| // `ALTIMATE_REVIEW_AI_REPLAY=<path to a review --json envelope>` skips the | ||
| // generator model call and rebuilds its findings from that envelope: the AI | ||
| // findings it kept (with what the verifier added undone) plus the ones the | ||
| // verifier discarded (`ai.verifier.dropped`). The verifier then runs on them as | ||
| // usual, so verifier settings can be compared without paying for generation | ||
| // again. Ids are the stored ones; benchmark labels are keyed by them. | ||
| // Fidelity is approximate: discarded findings come back with bodies clamped to | ||
| // 600 characters, and kept ones after the orchestrator's anchoring/cleaning. | ||
| import { readFile } from "node:fs/promises" | ||
| import z from "zod" | ||
| import { Provider } from "@/provider/provider" | ||
| import { Log } from "@/altimate/util/log" | ||
| import { Finding, ReviewCategory, makeFinding, ruleId } from "./finding" | ||
| import type { AiLaneStatus, AiReviewInput, AiReviewResult } from "./ai-review" | ||
| import { aiVerifyEnabled, runAiVerify, type VerifierStatus } from "./ai-verify" | ||
| import { modelNameFromPath } from "./orchestrate" | ||
|
|
||
| const log = Log.create({ service: "ai-replay" }) | ||
|
|
||
| /** The path of the envelope to replay, when replay is on. */ | ||
| export function aiReplayPath(env: Record<string, string | undefined> = process.env): string | undefined { | ||
| return env.ALTIMATE_REVIEW_AI_REPLAY || undefined | ||
| } | ||
|
|
||
| const Tokens = z.object({ | ||
| input: z.number(), | ||
| inputTotal: z.number(), | ||
| output: z.number(), | ||
| reasoning: z.number(), | ||
| cacheRead: z.number(), | ||
| }) | ||
|
|
||
| /** The parts of an envelope replay reads; everything else is ignored. */ | ||
| const ReplayEnvelope = z.object({ | ||
| findings: z.array(z.unknown()), | ||
| ai: z | ||
| .object({ | ||
| state: z.enum(["ok", "skipped", "timeout", "error"]), | ||
| reason: z.string().optional(), | ||
| model: z.string().optional(), | ||
| tokens: Tokens.optional(), | ||
| cost: z.number().optional(), | ||
| calls: z.number().optional(), | ||
| verifier: z | ||
| .object({ | ||
| discarded: z.number().optional(), | ||
| dropped: z | ||
| .array( | ||
| z.object({ | ||
| id: z.string(), | ||
| file: z.string(), | ||
| startLine: z.number().optional(), | ||
| title: z.string(), | ||
| body: z.string(), | ||
| category: z.string(), | ||
| severity: z.string(), | ||
| }), | ||
| ) | ||
| .optional(), | ||
| decisions: z | ||
| .array(z.object({ id: z.string(), ts: z.object({ lowConfidence: z.boolean() }).partial().optional() })) | ||
| .optional(), | ||
| }) | ||
| .optional(), | ||
| }) | ||
| .optional(), | ||
| }) | ||
|
|
||
| export interface ReplayedLane { | ||
| findings: Finding[] | ||
| /** The stored generator status: state, reason, model, tokens, cost, calls. */ | ||
| status: Omit<AiLaneStatus, "durationMs" | "verifier"> | ||
| /** Problems found while rebuilding (findings that could not be recovered). */ | ||
| warnings: string[] | ||
| } | ||
|
|
||
| /** | ||
| * Rebuild the generator's findings from a parsed envelope (pure). Kept AI | ||
| * findings lose the verifier's record, its `Verified:` body suffix and the | ||
| * tiers set after generation (the generator sets none); discarded ones are | ||
| * rebuilt from `ai.verifier.dropped`. Order follows the verifier's | ||
| * `decisions` (the generator's order) when recorded. | ||
| */ | ||
| export function replayFindings(raw: unknown): ReplayedLane { | ||
| const env = ReplayEnvelope.parse(raw) | ||
| const ai = env.ai | ||
| if (!ai) throw new Error("the envelope has no AI lane status") | ||
| const status: ReplayedLane["status"] = { | ||
| state: ai.state, | ||
| ...(ai.reason ? { reason: ai.reason } : {}), | ||
| ...(ai.model ? { model: ai.model } : {}), | ||
| ...(ai.tokens ? { tokens: ai.tokens } : {}), | ||
| ...(ai.cost !== undefined ? { cost: ai.cost } : {}), | ||
| ...(ai.calls !== undefined ? { calls: ai.calls } : {}), | ||
| replayed: true, | ||
| } | ||
| const warnings: string[] = [] | ||
| const out: Finding[] = [] | ||
| for (const item of env.findings) { | ||
| const parsed = Finding.safeParse(item) | ||
| if (!parsed.success || parsed.data.evidence?.tool !== "ai-review") continue | ||
| out.push(unverify(parsed.data)) | ||
| } | ||
|
|
||
| const v = ai.verifier | ||
| const lowConfidence = new Map((v?.decisions ?? []).map((d) => [d.id, d.ts?.lowConfidence === true])) | ||
| for (const d of v?.dropped ?? []) { | ||
| const category = ReviewCategory.safeParse(d.category) | ||
| if (!category.success) { | ||
| warnings.push(`discarded finding ${d.id} has an unknown category "${d.category}"`) | ||
| continue | ||
| } | ||
| // The envelope does not keep a discarded finding's confidence; the verifier reads only whether it is low. | ||
| const confidence = lowConfidence.get(d.id) ? ("low" as const) : ("medium" as const) | ||
| out.push( | ||
| makeFinding({ | ||
| id: d.id, | ||
| severity: d.severity === "warning" ? "warning" : "suggestion", | ||
| category: category.data, | ||
| title: d.title, | ||
| body: d.body, | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P3: Discarded findings are rebuilt from the envelope's Prompt for AI agents
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Agreed, low priority (ai-replay.ts:121, clamped bodies not flagged). Leaving this for a follow-up PR rather than widening this one. |
||
| file: d.file, | ||
| model: modelNameFromPath(d.file), | ||
| ...(d.startLine ? { startLine: d.startLine, endLine: d.startLine } : {}), | ||
| confidence, | ||
| evidence: { tool: "ai-review", result: { confidence } }, | ||
| rule: ruleId("ai-review", d.category), | ||
| }), | ||
| ) | ||
| } | ||
|
|
||
| const listed = v?.dropped?.length ?? 0 | ||
| if ((v?.discarded ?? 0) > listed) | ||
| warnings.push(`the verifier discarded ${v!.discarded} findings but the envelope lists ${listed}`) | ||
| const have = new Set(out.map((f) => f.id)) | ||
| const missing = (v?.decisions ?? []).filter((d) => !have.has(d.id)).length | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. SUGGESTION: A merged repeat is reported as a missing decision on every replay
|
||
| if (missing) warnings.push(`${missing} findings the verifier decided are not in the envelope`) | ||
|
|
||
| if (v?.decisions?.length) { | ||
| const order = new Map(v.decisions.map((d, i) => [d.id, i])) | ||
| out.sort((a, b) => (order.get(a.id) ?? Infinity) - (order.get(b.id) ?? Infinity)) | ||
| } | ||
| return { findings: out, status: warnings.length ? { ...status, replayIncomplete: true } : status, warnings } | ||
| } | ||
|
|
||
| /** A kept finding close to how the generator returned it: no verification record, body suffix, | ||
| * tier, origin, or the orchestrator's `preExisting` / `evidenceLine` (re-derived on the next run). */ | ||
| function unverify(f: Finding): Finding { | ||
| const { tier: _tier, origin: _origin, preExisting: _pre, evidenceLine: _line, ...rest } = f | ||
| const prior = f.evidence?.result | ||
| if (!prior || typeof prior !== "object") return rest | ||
| const { verification, ...result } = prior as Record<string, unknown> & { | ||
| verification?: { rule?: string; evidence?: string } | ||
| } | ||
| let body = f.body | ||
| const suffix = verification?.rule === "verified" && verification.evidence ? `\n\nVerified: ${verification.evidence}` : "" | ||
| if (suffix && body.endsWith(suffix)) body = body.slice(0, -suffix.length) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. SUGGESTION:
Remove the last occurrence of the suffix instead of requiring a tail match, and strip the
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Agreed, low priority (ai-replay.ts:156, |
||
| return { ...rest, body, evidence: { tool: "ai-review", result } } | ||
| } | ||
|
|
||
| /** | ||
| * The AI lane in replay mode: no generator call. Fails loudly (stderr) and | ||
| * reports `state: "error"` when the envelope cannot be read; never falls back | ||
| * to calling the generator. | ||
| */ | ||
| export async function runAiReplay(input: AiReviewInput, path: string): Promise<AiReviewResult> { | ||
| const startedAt = Date.now() | ||
| const done = (findings: Finding[], status: Omit<AiLaneStatus, "durationMs">): AiReviewResult => ({ | ||
| findings, | ||
| status: { ...status, durationMs: Date.now() - startedAt }, | ||
| }) | ||
| let lane: ReplayedLane | ||
| try { | ||
| lane = replayFindings(JSON.parse(await readFile(path, "utf8"))) | ||
| } catch (err) { | ||
| const reason = `cannot replay ${path}: ${err instanceof Error ? err.message : String(err)}`.slice(0, 300) | ||
| process.stderr.write(`altimate review: ALTIMATE_REVIEW_AI_REPLAY: ${reason}\n`) | ||
| log.error("ai replay failed", { path, error: err }) | ||
| return done([], { state: "error", reason, replayed: true }) | ||
| } | ||
| for (const w of lane.warnings) { | ||
| process.stderr.write(`altimate review: ALTIMATE_REVIEW_AI_REPLAY: ${w}\n`) | ||
| log.warn("ai replay incomplete", { path, warning: w }) | ||
| } | ||
| log.info("ai replay", { path, findings: lane.findings.length }) | ||
| if (lane.status.state !== "ok") return done([], lane.status) | ||
|
|
||
| let findings = lane.findings | ||
| let verifier: VerifierStatus | undefined | ||
| if (input.evidence && aiVerifyEnabled()) { | ||
| let model: Awaited<ReturnType<typeof Provider.getModel>> | ||
| try { | ||
| const chosen = input.model ? Provider.parseModel(input.model) : await Provider.defaultModel() | ||
| model = await Provider.getModel(chosen.providerID, chosen.modelID) | ||
| } catch (err) { | ||
| // Fail open: the replayed generator findings stand, unverified. | ||
| const reason = `no usable model: ${err instanceof Error ? err.message : String(err)}`.slice(0, 200) | ||
| const n = findings.length | ||
| const skipped: VerifierStatus = { | ||
| state: "skipped", | ||
| reason, | ||
| durationMs: 0, | ||
| turns: 0, | ||
| toolCalls: 0, | ||
| reviewed: 0, | ||
| kept: n, | ||
| discarded: 0, | ||
| summary: 0, | ||
| unreviewed: n, | ||
| } | ||
| return done(findings, { ...lane.status, verifier: skipped }) | ||
| } | ||
| const v = await runAiVerify({ findings, grounding: input.grounding, evidence: input.evidence, model }) | ||
| findings = v.findings | ||
| verifier = v.status | ||
| } | ||
| return done(findings, { ...lane.status, ...(verifier ? { verifier } : {}) }) | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -9,6 +9,8 @@ import { redactSecrets } from "@/installation" | |
| import { Dispatcher } from "../native" | ||
| import { type Finding, type ReviewCategory, type Severity, makeFinding, ruleId } from "./finding" | ||
| import { readStreamText } from "../llm/stream-text" | ||
| import { type AiVerifyEvidence, type VerifierStatus, aiVerifyEnabled, runAiVerify } from "./ai-verify" | ||
| import { aiReplayPath, runAiReplay } from "./ai-replay" | ||
|
|
||
| const log = Log.create({ service: "ai-review" }) | ||
|
|
||
|
|
@@ -138,6 +140,8 @@ export interface AiReviewInput { | |
| prBody?: string | ||
| /** Explicit `provider/model`; omit to use the configured default model. */ | ||
| model?: string | ||
| /** What the verifier may read; it runs only when this is set and `ALTIMATE_REVIEW_AI_VERIFY=1`. */ | ||
| evidence?: AiVerifyEvidence | ||
| } | ||
|
|
||
| /** What happened in the advisory lane, reported on the envelope and in the comment. */ | ||
|
|
@@ -155,6 +159,13 @@ export interface AiLaneStatus { | |
| cost?: number | ||
| /** Model calls made (one per batch). */ | ||
| calls?: number | ||
| /** The dbt-grounded verifier's outcome, when it ran (opt-in). */ | ||
| verifier?: VerifierStatus | ||
| /** The generator's findings came from an earlier envelope (`ALTIMATE_REVIEW_AI_REPLAY`); | ||
| * tokens, cost and calls are that run's, so only `verifier` was spent now. */ | ||
| replayed?: boolean | ||
| /** The replayed envelope did not list every generator finding. */ | ||
| replayIncomplete?: boolean | ||
| } | ||
| /** Token counts as `Session.getUsage` reports them: `input` excludes cached | ||
| * tokens, `inputTotal` includes them. */ | ||
|
|
@@ -253,6 +264,8 @@ export function buildUserMessage(input: AiReviewInput, maxFileChars = MAX_DIFF_C | |
| * envelope and the PR comment. | ||
| */ | ||
| export async function runAiReview(input: AiReviewInput): Promise<AiReviewResult> { | ||
| const replay = aiReplayPath() | ||
| if (replay) return runAiReplay(input, replay) | ||
| const startedAt = Date.now() | ||
| const done = (findings: Finding[], status: Omit<AiLaneStatus, "durationMs">): AiReviewResult => ({ | ||
| findings, | ||
|
|
@@ -291,15 +304,22 @@ export async function runAiReview(input: AiReviewInput): Promise<AiReviewResult> | |
| const slice = batches.slice(i, i + BATCH_CONCURRENCY) | ||
| results.push(...(await Promise.all(slice.map((b) => reviewBatch(input, b, budget, system, model))))) | ||
| } | ||
| const findings = results.flatMap((r) => r.findings) | ||
| let findings = results.flatMap((r) => r.findings) | ||
| const failed = results.filter((r) => r.state !== "ok") | ||
| log.info("ai review complete", { findings: findings.length, batches: batches.length, failed: failed.length }) | ||
| // The lane is ok when any call answered; a partial failure is reported, not hidden. | ||
| const spent = laneUsage(results) | ||
| if (failed.length === results.length) | ||
| return done([], { state: failed[0].state, reason: failed[0].reason, model: modelLabel, ...spent }) | ||
| const reason = failed.length ? `${failed.length} of ${results.length} calls failed: ${failed[0].reason}` : undefined | ||
| return done(findings, { state: "ok", model: modelLabel, ...(reason ? { reason } : {}), ...spent }) | ||
| // Opt-in: try to refute each finding with dbt evidence. Never throws. | ||
| let verifier: VerifierStatus | undefined | ||
| if (input.evidence && aiVerifyEnabled()) { | ||
| const v = await runAiVerify({ findings, grounding: input.grounding, evidence: input.evidence, model }) | ||
| findings = v.findings | ||
| verifier = v.status | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. SUGGESTION: A fail-closed verifier failure is invisible in the report When |
||
| } | ||
| return done(findings, { state: "ok", model: modelLabel, ...(reason ? { reason } : {}), ...spent, ...(verifier ? { verifier } : {}) }) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P2: With the verifier on, lane-level Prompt for AI agents
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Not changing: the verifier is reported separately as |
||
| } catch (err) { | ||
| log.error("ai review failed", { error: err }) | ||
| return done([], { state: "error", reason: safeReason(err), model: modelLabel }) | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
P3:
readStreamStepadds a new error-propagation path and aninvalidpassthrough, but no test covers either. Add cases to stream-text.test.ts for a mid-streamerrorpart (must throw) and for a tool call withinvalid: true.Prompt for AI agents
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Not changing: nit. It is exercised through the verifier tests.