diff --git a/.opencode/plugins/sce-agent-trace.ts b/.opencode/plugins/sce-agent-trace.ts index 33e6ad7a..72e03cbb 100644 --- a/.opencode/plugins/sce-agent-trace.ts +++ b/.opencode/plugins/sce-agent-trace.ts @@ -1,10 +1,12 @@ -import type { Hooks, Plugin } from "@opencode-ai/plugin"; import { spawn } from "node:child_process"; +import type { Hooks, Plugin } from "@opencode-ai/plugin"; + type OpenCodeEvent = Parameters>[0]["event"]; const REQUIRED_EVENTS: Set = new Set([ "message.updated", + "message.part.updated", "session.created", "session.updated", ]); @@ -22,11 +24,54 @@ type DiffTracePayload = { model_id: string; }; +type ConversationTraceMessageUpdatedItem = { + session_id: string; + message_id: string; + role: EventMessageUpdated["properties"]["info"]["role"]; + generated_at_unix_ms: number; +}; + +type ConversationTraceMessagePartUpdatedItem = { + session_id: string; + message_id: string; + part_type: EventMessagePartUpdated["properties"]["part"]["type"]; + text: unknown; + generated_at_unix_ms: number; +}; + +type ConversationTraceMessageUpdatedPayload = { + type: "message.updated"; + payloads: ConversationTraceMessageUpdatedItem[]; +}; + +type ConversationTraceMessagePartUpdatedPayload = { + type: "message.part.updated"; + payloads: ConversationTraceMessagePartUpdatedItem[]; +}; + +type ConversationTracePayload = + | ConversationTraceMessageUpdatedPayload + | ConversationTraceMessagePartUpdatedPayload; + type EventMessageUpdated = Extract< NonNullable, { type: "message.updated" } >; +type EventMessagePartUpdated = Extract< + NonNullable, + { type: "message.part.updated" } +>; + +function extractDiffEntries( + eventInfo: EventMessageUpdated["properties"]["info"], +) { + if (typeof eventInfo.summary === "object") { + return eventInfo.summary.diffs; + } + return undefined; +} + function extractDiffTracePayload( event: EventMessageUpdated, ): DiffTracePayload | undefined { @@ -36,8 +81,7 @@ function extractDiffTracePayload( return undefined; } - // Access info.summary?.diffs via explicit checks - const diffEntries = eventInfo.summary?.diffs; + const diffEntries = extractDiffEntries(eventInfo); if (!diffEntries || diffEntries.length === 0) { return undefined; @@ -45,13 +89,9 @@ function extractDiffTracePayload( const patches: string[] = []; for (const entry of diffEntries) { - const entryObj = entry as { patch?: string }; - - if (!entryObj.patch) { - continue; + if ("patch" in entry && typeof entry.patch === "string") { + patches.push(entry.patch); } - - patches.push(entryObj.patch); } if (patches.length === 0) { @@ -70,6 +110,122 @@ function shouldCaptureEvent(eventType: OpenCodeEvent["type"]): boolean { return ALL_CAPTURED_EVENTS.has(eventType); } +function buildConversationTracePayload( + event: EventMessageUpdated, +): ConversationTraceMessageUpdatedPayload { + const eventInfo = event.properties.info; + + return { + type: "message.updated", + payloads: [ + { + session_id: eventInfo.sessionID, + message_id: eventInfo.id, + role: eventInfo.role, + generated_at_unix_ms: Date.now(), + }, + ], + }; +} + +export function buildMessagePartConversationTracePayload( + event: EventMessagePartUpdated, +): ConversationTraceMessagePartUpdatedPayload { + const eventPart = event.properties.part; + + return { + type: "message.part.updated", + payloads: [ + { + session_id: eventPart.sessionID, + message_id: eventPart.messageID, + part_type: eventPart.type, + text: "text" in eventPart ? eventPart.text : "", + generated_at_unix_ms: Date.now(), + }, + ], + }; +} + +function buildPatchConversationTracePayloads( + event: EventMessageUpdated, +): ConversationTracePayload[] | undefined { + const eventInfo = event.properties.info; + const diffEntries = extractDiffEntries(eventInfo); + + if (!diffEntries || diffEntries.length === 0) { + return undefined; + } + + const patchMessageId = `${eventInfo.id}-patch`; + const payloads: ConversationTracePayload[] = []; + + payloads.push({ + type: "message.updated", + payloads: [ + { + session_id: eventInfo.sessionID, + message_id: patchMessageId, + role: eventInfo.role, + generated_at_unix_ms: Date.now(), + }, + ], + }); + + for (const entry of diffEntries) { + if ("patch" in entry && typeof entry.patch === "string") { + payloads.push({ + type: "message.part.updated", + payloads: [ + { + session_id: eventInfo.sessionID, + message_id: patchMessageId, + part_type: "patch", + text: entry.patch, + generated_at_unix_ms: Date.now(), + }, + ], + }); + } + } + + return payloads; +} + +export async function recordConversationTrace( + repoRoot: string, + event: EventMessageUpdated | EventMessagePartUpdated, +): Promise { + if ( + event.type === "message.part.updated" && + (event.properties.part.type === "reasoning" || + event.properties.part.type === "text") && + event.properties.part.text + ) { + await runConversationTraceHook( + repoRoot, + buildMessagePartConversationTracePayload(event), + ); + return; + } + + if (event.type === "message.updated") { + const patchPayloads = buildPatchConversationTracePayloads(event); + + if (patchPayloads !== undefined) { + await Promise.all( + patchPayloads.map((p) => runConversationTraceHook(repoRoot, p)), + ); + return; + } + + await runConversationTraceHook( + repoRoot, + buildConversationTracePayload(event), + ); + } +} + async function buildTrace( repoRoot: string, event: EventMessageUpdated, @@ -120,9 +276,41 @@ async function runDiffTraceHook( }); } +async function runConversationTraceHook( + repoRoot: string, + payload: ConversationTracePayload, +): Promise { + await new Promise((resolve, reject) => { + const child = spawn("sce", ["hooks", "conversation-trace"], { + cwd: repoRoot, + stdio: ["pipe", "ignore", "inherit"], + }); + + child.on("error", reject); + + child.on("close", (code, signal) => { + if (code === 0) { + resolve(); + return; + } + + const reason = + signal === null ? `exit code ${String(code)}` : `signal ${signal}`; + reject( + new Error( + `Command 'sce hooks conversation-trace' failed with ${reason}.`, + ), + ); + }); + + child.stdin.end(`${JSON.stringify(payload)}\n`); + }); +} + export const SceAgentTracePlugin: Plugin = async ({ directory, worktree }) => { const repoRoot = worktree ?? directory ?? process.cwd(); const clientVersionsBySessionId: Map = new Map(); + const processedDiffsMessageIds: Set = new Set(); return { event: async (input) => { @@ -141,12 +329,30 @@ export const SceAgentTracePlugin: Plugin = async ({ directory, worktree }) => { } if (input.event.type === "message.updated") { + const eventInfo = input.event.properties.info; + const diffEntries = extractDiffEntries(eventInfo); + const hasDiffs = + diffEntries !== undefined && diffEntries.length > 0; + + if (hasDiffs) { + const dedupKey = `${eventInfo.sessionID}:${eventInfo.id}`; + if (processedDiffsMessageIds.has(dedupKey)) { + return; + } + processedDiffsMessageIds.add(dedupKey); + } + const clientVersion = clientVersionsBySessionId.get( input.event.properties.info.sessionID, ) || null; + await recordConversationTrace(repoRoot, input.event); await buildTrace(repoRoot, input.event, clientVersion); } + + if (input.event.type === "message.part.updated") { + await recordConversationTrace(repoRoot, input.event); + } }, }; }; diff --git a/cli/Cargo.lock b/cli/Cargo.lock index 6d89a53d..34accea5 100644 --- a/cli/Cargo.lock +++ b/cli/Cargo.lock @@ -76,6 +76,12 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "allocator-api2" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c880a97d28a3681c0267bd29cff89621202715b065127cd445fa0f0fe0aa2880" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -121,7 +127,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -132,7 +138,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -1034,7 +1040,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -1160,7 +1166,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -1513,7 +1519,7 @@ version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ - "allocator-api2", + "allocator-api2 0.2.21", "equivalent", "foldhash 0.2.0", ] @@ -1715,6 +1721,31 @@ dependencies = [ "cc", ] +[[package]] +name = "icu_collator" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b521b92a2666061ddda902769d8a4cf730b5c9529a845cc1b69770b12a6c9a71" +dependencies = [ + "icu_collator_data", + "icu_collections", + "icu_locale", + "icu_locale_core", + "icu_normalizer", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "zerovec", +] + +[[package]] +name = "icu_collator_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "038ed8e5817f2059c2f3efb0945ba78d060d3d25e8f1a1bea5139f821a21a2f0" + [[package]] name = "icu_collections" version = "2.2.0" @@ -1729,6 +1760,21 @@ dependencies = [ "zerovec", ] +[[package]] +name = "icu_locale" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5a396343c7208121dc86e35623d3dfe19814a7613cfd14964994cdc9c9a2e26" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_locale_data", + "icu_provider", + "potential_utf", + "tinystr", + "zerovec", +] + [[package]] name = "icu_locale_core" version = "2.2.0" @@ -1737,11 +1783,18 @@ checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" dependencies = [ "displaydoc", "litemap", + "serde", "tinystr", "writeable", "zerovec", ] +[[package]] +name = "icu_locale_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fdcc9ac77c6d74ff5cf6e65ef3181d6af32003b16fce3a77fb451d2f695993" + [[package]] name = "icu_normalizer" version = "2.2.0" @@ -1753,6 +1806,9 @@ dependencies = [ "icu_properties", "icu_provider", "smallvec", + "utf16_iter", + "utf8_iter", + "write16", "zerovec", ] @@ -1790,6 +1846,8 @@ checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" dependencies = [ "displaydoc", "icu_locale_core", + "serde", + "stable_deref_trait", "writeable", "yoke", "zerofrom", @@ -1909,7 +1967,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2339,7 +2397,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -2618,6 +2676,8 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" dependencies = [ + "serde_core", + "writeable", "zerovec", ] @@ -3069,7 +3129,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -3082,7 +3142,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3139,7 +3199,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3498,7 +3558,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -3767,7 +3827,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3837,6 +3897,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" dependencies = [ "displaydoc", + "serde_core", "zerovec", ] @@ -4051,9 +4112,8 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "turso" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f21b08119fb7fa81cdd441a0255cb26811ff5d9c0dc0ff65bb71a493b7472c86" +version = "0.7.0-pre.5" +source = "git+https://github.com/tursodatabase/turso?rev=fed0b50b8aa697f56ab015aa9e905e0f56371092#fed0b50b8aa697f56ab015aa9e905e0f56371092" dependencies = [ "mimalloc", "thiserror", @@ -4066,13 +4126,13 @@ dependencies = [ [[package]] name = "turso_core" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "584dcc247f472c45d4f369daf590487a609c16a7f8848f2fb2902d3d3f055d31" +version = "0.7.0-pre.5" +source = "git+https://github.com/tursodatabase/turso?rev=fed0b50b8aa697f56ab015aa9e905e0f56371092#fed0b50b8aa697f56ab015aa9e905e0f56371092" dependencies = [ "aegis", "aes", "aes-gcm", + "allocator-api2 0.4.0", "antithesis_sdk", "arc-swap", "bigdecimal", @@ -4089,6 +4149,8 @@ dependencies = [ "fallible-iterator", "fastbloom", "hex", + "icu_collator", + "icu_locale", "intrusive-collections", "io-uring", "libc", @@ -4132,9 +4194,8 @@ dependencies = [ [[package]] name = "turso_ext" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "870a80e0516ec000ee51aed89664d41e258e31c1b4a639f86e0ce7c2dd3bbc18" +version = "0.7.0-pre.5" +source = "git+https://github.com/tursodatabase/turso?rev=fed0b50b8aa697f56ab015aa9e905e0f56371092#fed0b50b8aa697f56ab015aa9e905e0f56371092" dependencies = [ "chrono", "getrandom 0.4.2", @@ -4143,9 +4204,8 @@ dependencies = [ [[package]] name = "turso_macros" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97578dbe06dd73634457b38b3546b868b4886aebefa57570b11a899679355761" +version = "0.7.0-pre.5" +source = "git+https://github.com/tursodatabase/turso?rev=fed0b50b8aa697f56ab015aa9e905e0f56371092#fed0b50b8aa697f56ab015aa9e905e0f56371092" dependencies = [ "proc-macro2", "quote", @@ -4154,9 +4214,8 @@ dependencies = [ [[package]] name = "turso_parser" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa8dd793f7e9c467568275b0acddfd527ec19b7d1057bed41364b4c77b1111b3" +version = "0.7.0-pre.5" +source = "git+https://github.com/tursodatabase/turso?rev=fed0b50b8aa697f56ab015aa9e905e0f56371092#fed0b50b8aa697f56ab015aa9e905e0f56371092" dependencies = [ "bitflags", "memchr", @@ -4169,9 +4228,8 @@ dependencies = [ [[package]] name = "turso_sdk_kit" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d00d981b511fd8838ed601b14e447fe4c407281cc2ddd1217762f5386cb4fc14" +version = "0.7.0-pre.5" +source = "git+https://github.com/tursodatabase/turso?rev=fed0b50b8aa697f56ab015aa9e905e0f56371092#fed0b50b8aa697f56ab015aa9e905e0f56371092" dependencies = [ "bindgen", "env_logger", @@ -4180,14 +4238,14 @@ dependencies = [ "tracing-appender", "tracing-subscriber", "turso_core", + "turso_ext", "turso_sdk_kit_macros", ] [[package]] name = "turso_sdk_kit_macros" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ece10adfe27b9ec4cbc8e22c4b34fb22b08c1f5027fbcdd5bfa69a8306a927f2" +version = "0.7.0-pre.5" +source = "git+https://github.com/tursodatabase/turso?rev=fed0b50b8aa697f56ab015aa9e905e0f56371092#fed0b50b8aa697f56ab015aa9e905e0f56371092" dependencies = [ "proc-macro2", "quote", @@ -4196,9 +4254,8 @@ dependencies = [ [[package]] name = "turso_sync_engine" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbe9e0f7293d024c1458d9bd8f3676590a39ea9aac7659442c41639cb35d1270" +version = "0.7.0-pre.5" +source = "git+https://github.com/tursodatabase/turso?rev=fed0b50b8aa697f56ab015aa9e905e0f56371092#fed0b50b8aa697f56ab015aa9e905e0f56371092" dependencies = [ "base64", "bytes", @@ -4218,9 +4275,8 @@ dependencies = [ [[package]] name = "turso_sync_sdk_kit" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "235fb4a05bc74c6f23db6b9391bd6d76befd987767485af5726b9d009d5cca1f" +version = "0.7.0-pre.5" +source = "git+https://github.com/tursodatabase/turso?rev=fed0b50b8aa697f56ab015aa9e905e0f56371092#fed0b50b8aa697f56ab015aa9e905e0f56371092" dependencies = [ "bindgen", "env_logger", @@ -4288,7 +4344,7 @@ checksum = "f2f6fb2847f6742cd76af783a2a2c49e9375d0a111c7bef6f71cd9e738c72d6e" dependencies = [ "memoffset", "tempfile", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -4364,6 +4420,12 @@ dependencies = [ "serde", ] +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + [[package]] name = "utf8-ranges" version = "1.0.5" @@ -4618,7 +4680,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -4967,6 +5029,12 @@ dependencies = [ "wasmparser", ] +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + [[package]] name = "writeable" version = "0.6.3" @@ -5134,6 +5202,7 @@ dependencies = [ "displaydoc", "yoke", "zerofrom", + "zerovec", ] [[package]] @@ -5142,6 +5211,7 @@ version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" dependencies = [ + "serde", "yoke", "zerofrom", "zerovec-derive", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 43e30545..7f1aacb9 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -30,7 +30,7 @@ categories = ["command-line-utilities", "development-tools"] anyhow = "1" chrono = "0.4" clap = { version = "4", features = ["derive"] } -turso = "0.6.0" +turso = { git = "https://github.com/tursodatabase/turso", rev = "fed0b50b8aa697f56ab015aa9e905e0f56371092" } clap_complete = "4" dirs = "6" hmac = "0.13" diff --git a/cli/migrations/agent-trace/008_create_messages.sql b/cli/migrations/agent-trace/008_create_messages.sql new file mode 100644 index 00000000..6eb86fd0 --- /dev/null +++ b/cli/migrations/agent-trace/008_create_messages.sql @@ -0,0 +1,9 @@ +CREATE TABLE IF NOT EXISTS messages ( + id INTEGER PRIMARY KEY, + session_id TEXT NOT NULL, + message_id TEXT NOT NULL, + role TEXT NOT NULL CHECK (role IN ('user', 'assistant')), + generated_at_unix_ms INTEGER NOT NULL CHECK (generated_at_unix_ms >= 0), + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) +); diff --git a/cli/migrations/agent-trace/009_create_parts.sql b/cli/migrations/agent-trace/009_create_parts.sql new file mode 100644 index 00000000..98f6d0ce --- /dev/null +++ b/cli/migrations/agent-trace/009_create_parts.sql @@ -0,0 +1,10 @@ +CREATE TABLE IF NOT EXISTS parts ( + id INTEGER PRIMARY KEY, + type TEXT NOT NULL CHECK (type IN ('text', 'reasoning', 'patch')), + text TEXT NOT NULL, + message_id TEXT NOT NULL, + session_id TEXT NOT NULL, + generated_at_unix_ms INTEGER NOT NULL CHECK (generated_at_unix_ms >= 0), + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) +); diff --git a/cli/migrations/agent-trace/010_create_messages_session_message_unique_index.sql b/cli/migrations/agent-trace/010_create_messages_session_message_unique_index.sql new file mode 100644 index 00000000..3035a5cb --- /dev/null +++ b/cli/migrations/agent-trace/010_create_messages_session_message_unique_index.sql @@ -0,0 +1,2 @@ +CREATE UNIQUE INDEX IF NOT EXISTS idx_messages_session_message +ON messages (session_id, message_id); diff --git a/cli/migrations/agent-trace/011_create_messages_session_order_index.sql b/cli/migrations/agent-trace/011_create_messages_session_order_index.sql new file mode 100644 index 00000000..dfd1d2ff --- /dev/null +++ b/cli/migrations/agent-trace/011_create_messages_session_order_index.sql @@ -0,0 +1,2 @@ +CREATE INDEX IF NOT EXISTS idx_messages_session_order +ON messages (session_id, generated_at_unix_ms, id); diff --git a/cli/migrations/agent-trace/012_create_parts_session_message_order_index.sql b/cli/migrations/agent-trace/012_create_parts_session_message_order_index.sql new file mode 100644 index 00000000..834c89f5 --- /dev/null +++ b/cli/migrations/agent-trace/012_create_parts_session_message_order_index.sql @@ -0,0 +1,2 @@ +CREATE INDEX IF NOT EXISTS idx_parts_session_message_order +ON parts (session_id, message_id, generated_at_unix_ms, id); diff --git a/cli/migrations/agent-trace/013_create_messages_updated_at_trigger.sql b/cli/migrations/agent-trace/013_create_messages_updated_at_trigger.sql new file mode 100644 index 00000000..de16cb10 --- /dev/null +++ b/cli/migrations/agent-trace/013_create_messages_updated_at_trigger.sql @@ -0,0 +1,12 @@ +CREATE TRIGGER IF NOT EXISTS trg_messages_updated_at +AFTER UPDATE ON messages +FOR EACH ROW +WHEN OLD.session_id IS NOT NEW.session_id + OR OLD.message_id IS NOT NEW.message_id + OR OLD.role IS NOT NEW.role + OR OLD.generated_at_unix_ms IS NOT NEW.generated_at_unix_ms +BEGIN + UPDATE messages + SET updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now') + WHERE id = NEW.id; +END; diff --git a/cli/migrations/agent-trace/014_create_parts_updated_at_trigger.sql b/cli/migrations/agent-trace/014_create_parts_updated_at_trigger.sql new file mode 100644 index 00000000..142e53ef --- /dev/null +++ b/cli/migrations/agent-trace/014_create_parts_updated_at_trigger.sql @@ -0,0 +1,13 @@ +CREATE TRIGGER IF NOT EXISTS trg_parts_updated_at +AFTER UPDATE ON parts +FOR EACH ROW +WHEN OLD.type IS NOT NEW.type + OR OLD.text IS NOT NEW.text + OR OLD.message_id IS NOT NEW.message_id + OR OLD.session_id IS NOT NEW.session_id + OR OLD.generated_at_unix_ms IS NOT NEW.generated_at_unix_ms +BEGIN + UPDATE parts + SET updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now') + WHERE id = NEW.id; +END; diff --git a/cli/src/app.rs b/cli/src/app.rs index 614d6457..017e760d 100644 --- a/cli/src/app.rs +++ b/cli/src/app.rs @@ -163,6 +163,7 @@ fn build_startup_context() -> Result { })?; let observability_config = services::config::resolve_observability_runtime_config(&cwd) .map_err(|error| app_support::classify_observability_configuration_error(&error))?; + services::config::init_database_retry_config_from_environment(&cwd); let startup_diagnostic = app_support::invalid_discovered_config_guidance(&observability_config); Ok(StartupContext { observability_config, diff --git a/cli/src/cli_schema.rs b/cli/src/cli_schema.rs index caa08b37..579b5729 100644 --- a/cli/src/cli_schema.rs +++ b/cli/src/cli_schema.rs @@ -279,6 +279,9 @@ pub enum HooksSubcommand { #[command(about = "Run diff-trace hook (reads JSON payload from STDIN)")] DiffTrace, + + #[command(about = "Run conversation-trace hook (reads JSON payload from STDIN)")] + ConversationTrace, } #[derive(ValueEnum, Clone, Copy, Debug, PartialEq, Eq)] diff --git a/cli/src/services/agent_trace.rs b/cli/src/services/agent_trace.rs index 6345036e..e5f306ea 100644 --- a/cli/src/services/agent_trace.rs +++ b/cli/src/services/agent_trace.rs @@ -12,7 +12,7 @@ //! - same hunk slot in `post_commit_patch` but not exact line-by-line match => `mixed` //! - hunk present in `post_commit_patch` but missing from `intersection_patch` => `unknown` -use std::{error::Error, fmt, io::Cursor, path::Path, sync::OnceLock}; +use std::{collections::BTreeSet, error::Error, fmt, io::Cursor, path::Path, sync::OnceLock}; use anyhow::{Context, Result}; use chrono::{DateTime, FixedOffset}; @@ -34,6 +34,7 @@ const RANGE_CONTENT_HASH_PREFIX: &str = "murmur3:"; const RANGE_CONTENT_HASH_INPUT_VERSION: &[u8] = b"sce-agent-trace-range-content-hash-v1\0"; const TOUCHED_LINE_ADDED_TAG: &[u8] = b"added\0"; const TOUCHED_LINE_REMOVED_TAG: &[u8] = b"removed\0"; +const SESSION_RELATED_URL_PREFIX: &str = "https://sce.crocoder.dev/sessions/"; fn default_agent_trace_version() -> String { AGENT_TRACE_VERSION.to_owned() @@ -218,6 +219,21 @@ pub struct Conversation { pub contributor: Contributor, /// Line ranges in the new file, derived from the `post_commit_patch` hunk metadata. pub ranges: Vec, + /// Optional related resources for this conversation entry. + #[serde(skip_serializing_if = "Option::is_none")] + pub related: Option>, +} + +/// A related resource for a conversation entry. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +#[allow(dead_code)] +pub struct ConversationRelated { + /// Free-form related resource type. + #[serde(rename = "type")] + pub kind: String, + /// Related resource URL. + pub url: String, } /// Nested contributor object for a conversation entry. @@ -383,7 +399,7 @@ fn parse_embedded_deleted_patch(file: &PatchFileChange) -> Option { .collect::>() .join("\n"); - let parsed_patch = parse_patch(&embedded_patch).ok()?; + let parsed_patch = parse_patch(&embedded_patch, None).ok()?; (!parsed_patch.files.is_empty()).then_some(parsed_patch) } @@ -404,29 +420,51 @@ fn build_trace_file( .hunks .iter() .map(|post_commit_hunk| { - let (contributor_kind, contributor_model_id) = match intersection_file { - Some(ifile) => { - let contributor_kind = classify_hunk(post_commit_hunk, &ifile.hunks); - let matched_intersection_hunk = ifile - .hunks - .iter() - .find(|h| h.old_start == post_commit_hunk.old_start); - let contributor_model_id = match contributor_kind { - HunkContributor::Ai | HunkContributor::Mixed => { - matched_intersection_hunk.and_then(|hunk| hunk.model_id.clone()) - } - HunkContributor::Unknown => None, - }; - (contributor_kind, contributor_model_id) - } - None => (HunkContributor::Unknown, None), - }; + let (contributor_kind, contributor_model_id, matched_intersection_hunk) = + match intersection_file { + Some(ifile) => { + let contributor_kind = classify_hunk(post_commit_hunk, &ifile.hunks); + let matched_intersection_hunk = ifile + .hunks + .iter() + .find(|h| h.old_start == post_commit_hunk.old_start); + let contributor_model_id = match contributor_kind { + HunkContributor::Ai | HunkContributor::Mixed => { + matched_intersection_hunk.and_then(|hunk| hunk.model_id.clone()) + } + HunkContributor::Unknown => None, + }; + ( + contributor_kind, + contributor_model_id, + matched_intersection_hunk, + ) + } + None => (HunkContributor::Unknown, None, None), + }; + let related_session_ids = matched_intersection_hunk + .into_iter() + .flat_map(|hunk| hunk.lines.iter()) + .filter_map(|line| line.session_id.as_deref()) + .filter(|session_id| !session_id.is_empty()) + .collect::>(); + let related = (!related_session_ids.is_empty()).then(|| { + related_session_ids + .into_iter() + .map(|session_id| ConversationRelated { + kind: String::from("session"), + url: format!("{SESSION_RELATED_URL_PREFIX}{session_id}"), + }) + .collect() + }); + Conversation { contributor: Contributor { kind: contributor_kind, model_id: contributor_model_id, }, ranges: vec![line_range_from_hunk(post_commit_file, post_commit_hunk)], + related, } }) .collect(); diff --git a/cli/src/services/agent_trace/tests.rs b/cli/src/services/agent_trace/tests.rs index d9bf9b7f..8cdd78b5 100644 --- a/cli/src/services/agent_trace/tests.rs +++ b/cli/src/services/agent_trace/tests.rs @@ -18,7 +18,7 @@ const TEST_COMMIT_REVISION: &str = "a0b1c2d3e4f5a6b7c8d9e0f11223344556677889"; fn parse_fixtures(fixtures: &[&str]) -> Vec { fixtures .iter() - .map(|fixture| parse_patch(fixture).expect("fixture patch should parse")) + .map(|fixture| parse_patch(fixture, None).expect("fixture patch should parse")) .collect() } @@ -53,7 +53,8 @@ const TEXT_FILE_LIFECYCLE_RECONSTRUCTION_INCREMENTALS: &[&str] = &[ fn assert_builds_expected_agent_trace(scenario: AgentTraceScenario) { let constructed_patch = combine_patches(&parse_fixtures(scenario.incremental)); - let post_commit_patch = parse_patch(scenario.post_commit).expect("fixture patch should parse"); + let post_commit_patch = + parse_patch(scenario.post_commit, None).expect("fixture patch should parse"); let golden: Value = serde_json::from_str(scenario.golden).expect("golden json should load"); validate_agent_trace_value(&golden).expect("golden json should validate against schema"); let actual = build_agent_trace( @@ -146,15 +147,20 @@ fn poem_edit_reconstruction_matches_golden_agent_trace() { #[test] fn poem_edit_reconstruction_maps_each_hunk_to_one_range() { - let constructed_patch = combine_patches(&parse_fixtures(&[ + let mut constructed_patch = combine_patches(&parse_fixtures(&[ include_str!("fixtures/poem_edit_reconstruction/incremental_01.patch"), include_str!("fixtures/poem_edit_reconstruction/incremental_02.patch"), ])); - let post_commit_patch = parse_patch(include_str!( - "fixtures/poem_edit_reconstruction/post_commit.patch" - )) + let post_commit_patch = parse_patch( + include_str!("fixtures/poem_edit_reconstruction/post_commit.patch"), + None, + ) .expect("fixture patch should parse"); + let first_hunk_lines = &mut constructed_patch.files[0].hunks[0].lines; + first_hunk_lines[0].session_id = Some(String::from("session-z")); + first_hunk_lines[1].session_id = Some(String::from("session-a")); + let agent_trace = build_agent_trace( &constructed_patch, &post_commit_patch, @@ -174,6 +180,42 @@ fn poem_edit_reconstruction_maps_each_hunk_to_one_range() { assert_eq!(agent_trace.files.len(), 1); assert_eq!(agent_trace.files[0].path, "poem.md"); assert_eq!(agent_trace.files[0].conversations.len(), 3); + assert_eq!( + agent_trace.files[0].conversations[0].related, + Some(vec![ + super::ConversationRelated { + kind: String::from("session"), + url: String::from("https://sce.crocoder.dev/sessions/session-a"), + }, + super::ConversationRelated { + kind: String::from("session"), + url: String::from("https://sce.crocoder.dev/sessions/session-z"), + }, + ]) + ); + assert_eq!(agent_trace.files[0].conversations[1].related, None); + assert_eq!(agent_trace.files[0].conversations[2].related, None); + assert_eq!( + actual_json["files"][0]["conversations"][0]["related"], + json!([ + { + "type": "session", + "url": "https://sce.crocoder.dev/sessions/session-a" + }, + { + "type": "session", + "url": "https://sce.crocoder.dev/sessions/session-z" + } + ]) + ); + assert!( + actual_json["files"][0]["conversations"][1]["related"].is_null(), + "conversations without session-backed lines should omit related" + ); + assert!( + actual_json["files"][0]["conversations"][2]["related"].is_null(), + "conversations without session-backed lines should omit related" + ); assert_eq!( agent_trace.files[0] .conversations diff --git a/cli/src/services/agent_trace_db/mod.rs b/cli/src/services/agent_trace_db/mod.rs index f3bc5324..ebf4c97d 100644 --- a/cli/src/services/agent_trace_db/mod.rs +++ b/cli/src/services/agent_trace_db/mod.rs @@ -3,6 +3,7 @@ use std::path::PathBuf; use anyhow::{Context, Result}; +use turso::Value as TursoValue; use crate::services::{ db::{DbSpec, TursoDb}, @@ -28,6 +29,22 @@ const ADD_AGENT_TRACES_REMOTE_URL_MIGRATION: &str = const CREATE_AGENT_TRACES_REMOTE_URL_INDEX_MIGRATION: &str = include_str!( "../../../migrations/agent-trace/007_create_agent_traces_vcs_remote_url_index.sql" ); +const CREATE_MESSAGES_MIGRATION: &str = + include_str!("../../../migrations/agent-trace/008_create_messages.sql"); +const CREATE_PARTS_MIGRATION: &str = + include_str!("../../../migrations/agent-trace/009_create_parts.sql"); +const CREATE_MESSAGES_SESSION_MESSAGE_UNIQUE_INDEX_MIGRATION: &str = include_str!( + "../../../migrations/agent-trace/010_create_messages_session_message_unique_index.sql" +); +const CREATE_MESSAGES_SESSION_ORDER_INDEX_MIGRATION: &str = + include_str!("../../../migrations/agent-trace/011_create_messages_session_order_index.sql"); +const CREATE_PARTS_SESSION_MESSAGE_ORDER_INDEX_MIGRATION: &str = include_str!( + "../../../migrations/agent-trace/012_create_parts_session_message_order_index.sql" +); +const CREATE_MESSAGES_UPDATED_AT_TRIGGER_MIGRATION: &str = + include_str!("../../../migrations/agent-trace/013_create_messages_updated_at_trigger.sql"); +const CREATE_PARTS_UPDATED_AT_TRIGGER_MIGRATION: &str = + include_str!("../../../migrations/agent-trace/014_create_parts_updated_at_trigger.sql"); const AGENT_TRACE_MIGRATIONS: &[(&str, &str)] = &[ ("001_create_diff_traces", CREATE_DIFF_TRACES_MIGRATION), @@ -52,8 +69,32 @@ const AGENT_TRACE_MIGRATIONS: &[(&str, &str)] = &[ "007_create_agent_traces_remote_url_index", CREATE_AGENT_TRACES_REMOTE_URL_INDEX_MIGRATION, ), + ("008_create_messages", CREATE_MESSAGES_MIGRATION), + ("009_create_parts", CREATE_PARTS_MIGRATION), + ( + "010_create_messages_session_message_unique_index", + CREATE_MESSAGES_SESSION_MESSAGE_UNIQUE_INDEX_MIGRATION, + ), + ( + "011_create_messages_session_order_index", + CREATE_MESSAGES_SESSION_ORDER_INDEX_MIGRATION, + ), + ( + "012_create_parts_session_message_order_index", + CREATE_PARTS_SESSION_MESSAGE_ORDER_INDEX_MIGRATION, + ), + ( + "013_create_messages_updated_at_trigger", + CREATE_MESSAGES_UPDATED_AT_TRIGGER_MIGRATION, + ), + ( + "014_create_parts_updated_at_trigger", + CREATE_PARTS_UPDATED_AT_TRIGGER_MIGRATION, + ), ]; +const AGENT_TRACE_SCHEMA_SETUP_GUIDANCE: &str = "Run 'sce setup'."; + /// Parameterized SQL for inserting a captured diff trace payload. pub const INSERT_DIFF_TRACE_SQL: &str = "INSERT INTO diff_traces (time_ms, session_id, patch, model_id, tool_name, tool_version) VALUES (?1, ?2, ?3, ?4, ?5, ?6)"; @@ -81,6 +122,20 @@ pub const INSERT_POST_COMMIT_PATCH_INTERSECTION_SQL: &str = pub const INSERT_AGENT_TRACE_SQL: &str = "INSERT INTO agent_traces (commit_id, commit_time_ms, trace_json, agent_trace_id, url, remote_url) VALUES (?1, ?2, ?3, ?4, ?5, ?6)"; +/// Parameterized SQL for inserting a message row. Duplicate +/// `(session_id, message_id)` writes are ignored. +#[allow(dead_code)] +pub const INSERT_MESSAGE_SQL: &str = + "INSERT INTO messages (session_id, message_id, role, generated_at_unix_ms) +VALUES (?1, ?2, ?3, ?4) +ON CONFLICT (session_id, message_id) DO NOTHING"; + +/// Parameterized SQL for inserting a part row (append-only, no upsert). +#[allow(dead_code)] +pub const INSERT_PART_SQL: &str = + "INSERT INTO parts (type, text, message_id, session_id, generated_at_unix_ms) +VALUES (?1, ?2, ?3, ?4, ?5)"; + /// Agent trace database configuration. pub struct AgentTraceDbSpec; @@ -96,6 +151,10 @@ impl DbSpec for AgentTraceDbSpec { fn migrations() -> &'static [(&'static str, &'static str)] { AGENT_TRACE_MIGRATIONS } + + fn db_config_key() -> &'static str { + "agent_trace_db" + } } /// Agent trace Turso database adapter. @@ -184,7 +243,84 @@ pub struct AgentTraceInsert<'a> { pub remote_url: &'a str, } +/// Message role constraint for the `messages` table. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[allow(dead_code)] +pub enum MessageRole { + User, + Assistant, +} + +impl std::fmt::Display for MessageRole { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::User => write!(f, "user"), + Self::Assistant => write!(f, "assistant"), + } + } +} + +/// Message insert payload for the `messages` table. +#[derive(Clone, Debug, Eq, PartialEq)] +#[allow(dead_code)] +pub struct InsertMessageInsert { + pub session_id: String, + pub message_id: String, + pub role: MessageRole, + pub generated_at_unix_ms: i64, +} + +/// Part type constraint for the `parts` table. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[allow(dead_code)] +pub enum PartType { + Text, + Reasoning, + Patch, +} + +impl std::fmt::Display for PartType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Text => write!(f, "text"), + Self::Reasoning => write!(f, "reasoning"), + Self::Patch => write!(f, "patch"), + } + } +} + +/// Part insert payload for the `parts` table (append-only, no upsert). +#[derive(Clone, Debug, Eq, PartialEq)] +#[allow(dead_code)] +pub struct InsertPartInsert { + pub part_type: PartType, + pub text: String, + pub session_id: String, + pub message_id: String, + pub generated_at_unix_ms: i64, +} + impl AgentTraceDb { + /// Open the Agent Trace DB for high-frequency hook runtime paths without + /// running embedded migrations. + /// + /// Setup/lifecycle initialization must continue to use [`AgentTraceDb::new`] + /// so schema migrations remain explicitly owned by setup flows. Hook callers + /// must verify schema readiness before reading or writing through this DB. + pub fn open_for_hooks_without_migrations() -> Result { + TursoDb::::open_without_migrations() + } + + /// Verify that the Agent Trace DB schema needed by hook runtime readers and + /// writers already exists. + /// + /// This check is intentionally non-mutating. Missing or incomplete schema is + /// reported with setup guidance instead of running migrations from a + /// high-frequency hook path. + pub fn ensure_schema_ready_for_hooks(&self) -> Result<()> { + self.ensure_schema_ready(AGENT_TRACE_SCHEMA_SETUP_GUIDANCE) + } + /// Insert a diff trace payload into the `diff_traces` table. pub fn insert_diff_trace(&self, input: DiffTraceInsert<'_>) -> Result { insert_diff_trace_with(self, input) @@ -212,6 +348,29 @@ impl AgentTraceDb { ) -> Result { recent_diff_trace_patches_with(self, cutoff_time_ms, end_time_ms) } + + /// Insert a message row, ignoring duplicate `(session_id, message_id)` rows. + #[allow(dead_code)] + pub fn insert_message(&self, input: InsertMessageInsert) -> Result { + insert_message_with(self, input) + } + + /// Insert message rows with one multi-row statement, ignoring duplicate + /// `(session_id, message_id)` rows. + pub fn insert_messages(&self, inputs: Vec) -> Result { + insert_messages_with(self, inputs) + } + + /// Append a part row (no upsert; multiple rows per message allowed). + #[allow(dead_code)] + pub fn insert_part(&self, input: InsertPartInsert) -> Result { + insert_part_with(self, input) + } + + /// Append part rows with one multi-row statement. + pub fn insert_parts(&self, inputs: Vec) -> Result { + insert_parts_with(self, inputs) + } } fn insert_diff_trace_with(db: &TursoDb, input: DiffTraceInsert<'_>) -> Result { @@ -260,6 +419,96 @@ fn insert_agent_trace_with(db: &TursoDb, input: AgentTraceInsert<' ) } +#[allow(dead_code)] +fn insert_message_with(db: &TursoDb, input: InsertMessageInsert) -> Result { + db.execute( + INSERT_MESSAGE_SQL, + ( + input.session_id, + input.message_id, + input.role.to_string(), + input.generated_at_unix_ms, + ), + ) +} + +fn insert_messages_with( + db: &TursoDb, + inputs: Vec, +) -> Result { + if inputs.is_empty() { + return Ok(0); + } + + let mut params = Vec::with_capacity(inputs.len() * 4); + let mut rows = Vec::with_capacity(inputs.len()); + + for (row_index, input) in inputs.into_iter().enumerate() { + let param_start = row_index * 4 + 1; + rows.push(numbered_placeholders(param_start, 4)); + params.push(TursoValue::Text(input.session_id)); + params.push(TursoValue::Text(input.message_id)); + params.push(TursoValue::Text(input.role.to_string())); + params.push(TursoValue::Integer(input.generated_at_unix_ms)); + } + + let sql = format!( + "INSERT INTO messages (session_id, message_id, role, generated_at_unix_ms)\nVALUES {}\nON CONFLICT (session_id, message_id) DO NOTHING", + rows.join(", ") + ); + + db.execute(&sql, params) +} + +#[allow(dead_code)] +fn insert_part_with(db: &TursoDb, input: InsertPartInsert) -> Result { + db.execute( + INSERT_PART_SQL, + ( + input.part_type.to_string(), + input.text, + input.message_id, + input.session_id, + input.generated_at_unix_ms, + ), + ) +} + +fn insert_parts_with(db: &TursoDb, inputs: Vec) -> Result { + if inputs.is_empty() { + return Ok(0); + } + + let mut params = Vec::with_capacity(inputs.len() * 5); + let mut rows = Vec::with_capacity(inputs.len()); + + for (row_index, input) in inputs.into_iter().enumerate() { + let param_start = row_index * 5 + 1; + rows.push(numbered_placeholders(param_start, 5)); + params.push(TursoValue::Text(input.part_type.to_string())); + params.push(TursoValue::Text(input.text)); + params.push(TursoValue::Text(input.message_id)); + params.push(TursoValue::Text(input.session_id)); + params.push(TursoValue::Integer(input.generated_at_unix_ms)); + } + + let sql = format!( + "INSERT INTO parts (type, text, message_id, session_id, generated_at_unix_ms)\nVALUES {}", + rows.join(", ") + ); + + db.execute(&sql, params) +} + +fn numbered_placeholders(start: usize, count: usize) -> String { + let placeholders = (start..start + count) + .map(|index| format!("?{index}")) + .collect::>() + .join(", "); + + format!("({placeholders})") +} + fn recent_diff_trace_patches_with( db: &TursoDb, cutoff_time_ms: i64, @@ -295,7 +544,7 @@ fn parse_recent_diff_trace_patch_rows(rows: Vec) -> RecentDif let mut skipped = Vec::new(); for row in rows { - match parse_patch(&row.patch) { + match parse_patch(&row.patch, Some(row.session_id.as_str())) { Ok(mut patch) => { for file in &mut patch.files { for hunk in &mut file.hunks { @@ -359,6 +608,10 @@ mod tests { fn migrations() -> &'static [(&'static str, &'static str)] { AGENT_TRACE_MIGRATIONS } + + fn db_config_key() -> &'static str { + "agent_trace_db" + } } struct BaselineAgentTraceDbSpec; @@ -378,6 +631,10 @@ mod tests { fn migrations() -> &'static [(&'static str, &'static str)] { AGENT_TRACE_MIGRATIONS } + + fn db_config_key() -> &'static str { + "agent_trace_db" + } } fn unique_test_db_path() -> PathBuf { @@ -557,6 +814,29 @@ mod tests { "index", "idx_agent_traces_remote_url" )); + assert!(sqlite_object_exists(&db, "table", "messages")); + assert!(sqlite_object_exists(&db, "table", "parts")); + assert!(sqlite_object_exists( + &db, + "index", + "idx_messages_session_message" + )); + assert!(sqlite_object_exists( + &db, + "index", + "idx_messages_session_order" + )); + assert!(sqlite_object_exists( + &db, + "index", + "idx_parts_session_message_order" + )); + assert!(sqlite_object_exists( + &db, + "trigger", + "trg_messages_updated_at" + )); + assert!(sqlite_object_exists(&db, "trigger", "trg_parts_updated_at")); assert_eq!( applied_migration_ids(&db), vec![ @@ -567,6 +847,13 @@ mod tests { "005_create_agent_traces_agent_trace_id_index", "006_add_agent_traces_remote_url", "007_create_agent_traces_remote_url_index", + "008_create_messages", + "009_create_parts", + "010_create_messages_session_message_unique_index", + "011_create_messages_session_order_index", + "012_create_parts_session_message_order_index", + "013_create_messages_updated_at_trigger", + "014_create_parts_updated_at_trigger", ] ); diff --git a/cli/src/services/auth_db/mod.rs b/cli/src/services/auth_db/mod.rs index 47f341e8..10a0a21e 100644 --- a/cli/src/services/auth_db/mod.rs +++ b/cli/src/services/auth_db/mod.rs @@ -40,6 +40,10 @@ impl DbSpec for AuthDbSpec { fn migrations() -> &'static [(&'static str, &'static str)] { AUTH_MIGRATIONS } + + fn db_config_key() -> &'static str { + "auth_db" + } } /// Encrypted auth Turso database adapter. @@ -83,6 +87,10 @@ mod tests { fn migrations() -> &'static [(&'static str, &'static str)] { AUTH_MIGRATIONS } + + fn db_config_key() -> &'static str { + "auth_db" + } } fn unique_test_db_path() -> PathBuf { diff --git a/cli/src/services/config/mod.rs b/cli/src/services/config/mod.rs index 5a6837cb..5da1a981 100644 --- a/cli/src/services/config/mod.rs +++ b/cli/src/services/config/mod.rs @@ -18,6 +18,8 @@ pub(crate) use resolver::{ }; pub(crate) use schema::validate_config_file; +pub(crate) use resolver::{get_database_retry_config, init_database_retry_config_from_environment}; + pub fn run_config_subcommand(subcommand: ConfigSubcommand) -> Result { match subcommand { ConfigSubcommand::Show(request) => { diff --git a/cli/src/services/config/render.rs b/cli/src/services/config/render.rs index 6959ba15..720907dd 100644 --- a/cli/src/services/config/render.rs +++ b/cli/src/services/config/render.rs @@ -6,6 +6,7 @@ use super::policy::{format_bash_policies_json, format_bash_policies_text}; use super::resolver::{ AuthConfigKeySpec, RuntimeConfig, PRECEDENCE_DESCRIPTION, WORKOS_CLIENT_ID_KEY, }; +use super::types::DatabaseRetryConfig; use super::{ConfigPathSource, ReportFormat, ResolvedOptionalValue, ValueSource}; pub(super) fn format_show_output(runtime: &RuntimeConfig, report_format: ReportFormat) -> String { @@ -34,6 +35,7 @@ pub(super) fn format_show_output(runtime: &RuntimeConfig, report_format: ReportF &runtime.workos_client_id, ), format_bash_policies_text(&runtime.bash_policies), + format_database_retry_text(&runtime.database_retry), format_validation_warnings_text(&warnings), ]; lines.splice(3..3, format_observability_text_lines(runtime)); @@ -68,6 +70,7 @@ pub(super) fn format_show_output(runtime: &RuntimeConfig, report_format: ReportF "workos_client_id": format_optional_auth_resolved_value_json(WORKOS_CLIENT_ID_KEY, &runtime.workos_client_id), "policies": { "bash": format_bash_policies_json(&runtime.bash_policies), + "database_retry": format_database_retry_json(&runtime.database_retry), } }, "warnings": warnings, @@ -365,3 +368,119 @@ fn abbreviate_text_value(value: &str) -> String { .collect(); format!("{prefix}...{suffix}") } + +fn retry_policy_display(policy: &crate::services::resilience::RetryPolicy) -> String { + format!( + "{} attempts, {}ms timeout, {}..{}ms backoff", + policy.max_attempts, policy.timeout_ms, policy.initial_backoff_ms, policy.max_backoff_ms + ) +} + +fn format_per_db_retry_text( + config: &super::types::PerDbRetryConfig, + db_label: &str, +) -> Vec { + let mut lines = Vec::new(); + if let Some(ref policy) = config.connection_open { + lines.push(format!( + " {}: {} (connection_open)", + style::label(db_label), + style::value(&retry_policy_display(policy)) + )); + } + if let Some(ref policy) = config.query { + lines.push(format!( + " {}: {} (query)", + style::label(db_label), + style::value(&retry_policy_display(policy)) + )); + } + lines +} + +fn format_database_retry_text(value: &ResolvedOptionalValue) -> String { + match (value.value.as_ref(), value.source) { + (Some(config), Some(source)) => { + let mut lines = vec![format!(" {}:", style::label("policies.database_retry"))]; + if let Some(ref per_db) = config.local_db { + lines.extend(format_per_db_retry_text(per_db, "local_db")); + } + if let Some(ref per_db) = config.agent_trace_db { + lines.extend(format_per_db_retry_text(per_db, "agent_trace_db")); + } + if let Some(ref per_db) = config.auth_db { + lines.extend(format_per_db_retry_text(per_db, "auth_db")); + } + match source.config_source() { + Some(config_source) => { + lines.push(format!( + " (source: {}, config_source: {})", + style::label(source.as_str()), + style::label(config_source.as_str()) + )); + } + None => { + lines.push(format!(" (source: {})", style::label(source.as_str()))); + } + } + lines.join("\n") + } + _ => format!( + " {}: {} (source: {})", + style::label("policies.database_retry"), + style::value("(unset)"), + style::label("none") + ), + } +} + +fn format_per_db_retry_json(config: &super::types::PerDbRetryConfig) -> Value { + let mut obj = serde_json::Map::new(); + if let Some(ref policy) = config.connection_open { + obj.insert( + "connection_open".to_string(), + json!({ + "max_attempts": policy.max_attempts, + "timeout_ms": policy.timeout_ms, + "initial_backoff_ms": policy.initial_backoff_ms, + "max_backoff_ms": policy.max_backoff_ms, + }), + ); + } + if let Some(ref policy) = config.query { + obj.insert( + "query".to_string(), + json!({ + "max_attempts": policy.max_attempts, + "timeout_ms": policy.timeout_ms, + "initial_backoff_ms": policy.initial_backoff_ms, + "max_backoff_ms": policy.max_backoff_ms, + }), + ); + } + Value::Object(obj) +} + +fn format_database_retry_json(value: &ResolvedOptionalValue) -> Value { + let config = value.value.as_ref(); + let mut resolved = serde_json::Map::new(); + if let Some(c) = config { + if let Some(ref per_db) = c.local_db { + resolved.insert("local_db".to_string(), format_per_db_retry_json(per_db)); + } + if let Some(ref per_db) = c.agent_trace_db { + resolved.insert( + "agent_trace_db".to_string(), + format_per_db_retry_json(per_db), + ); + } + if let Some(ref per_db) = c.auth_db { + resolved.insert("auth_db".to_string(), format_per_db_retry_json(per_db)); + } + } + json!({ + "resolved": if resolved.is_empty() { Value::Null } else { Value::Object(resolved) }, + "source": value.source.map(ValueSource::as_str), + "config_source": value.source.and_then(ValueSource::config_source).map(ConfigPathSource::as_str), + }) +} diff --git a/cli/src/services/config/resolver.rs b/cli/src/services/config/resolver.rs index 45241f76..63230d9f 100644 --- a/cli/src/services/config/resolver.rs +++ b/cli/src/services/config/resolver.rs @@ -13,10 +13,11 @@ use crate::services::default_paths::{resolve_sce_default_locations, RepoPaths}; use super::policy::{build_validation_warnings, resolve_bash_policy_config, BashPolicyConfig}; use super::schema; use super::types::{ - parse_bool_value_from, ConfigPathSource, ConfigRequest, LoadedConfigPath, LogFileMode, - LogFormat, LogLevel, ReportFormat, ResolvedAuthRuntimeConfig, ResolvedHookRuntimeConfig, - ResolvedObservabilityRuntimeConfig, ResolvedOptionalValue, ResolvedValue, ValueSource, - ENV_ATTRIBUTION_HOOKS_ENABLED, ENV_LOG_FILE, ENV_LOG_FILE_MODE, ENV_LOG_FORMAT, ENV_LOG_LEVEL, + parse_bool_value_from, ConfigPathSource, ConfigRequest, DatabaseRetryConfig, LoadedConfigPath, + LogFileMode, LogFormat, LogLevel, ReportFormat, ResolvedAuthRuntimeConfig, + ResolvedHookRuntimeConfig, ResolvedObservabilityRuntimeConfig, ResolvedOptionalValue, + ResolvedValue, ValueSource, ENV_ATTRIBUTION_HOOKS_ENABLED, ENV_LOG_FILE, ENV_LOG_FILE_MODE, + ENV_LOG_FORMAT, ENV_LOG_LEVEL, }; const DEFAULT_TIMEOUT_MS: u64 = 30000; @@ -63,6 +64,7 @@ pub(super) struct RuntimeConfig { pub(super) attribution_hooks_enabled: ResolvedValue, pub(super) workos_client_id: ResolvedOptionalValue, pub(super) bash_policies: ResolvedOptionalValue, + pub(super) database_retry: ResolvedOptionalValue, pub(super) validation_errors: Vec, pub(super) validation_warnings: Vec, } @@ -252,6 +254,7 @@ where workos_client_id: None, bash_policy_presets: None, bash_policy_custom: None, + database_retry: None, }; let mut validation_errors = Vec::new(); for loaded_path in &loaded_config_paths { @@ -291,6 +294,9 @@ where if let Some(bash_policy_custom) = layer.bash_policy_custom { file_config.bash_policy_custom = Some(bash_policy_custom); } + if let Some(database_retry) = layer.database_retry { + file_config.database_retry = Some(database_retry); + } } let mut resolved_log_level = ResolvedValue { @@ -431,6 +437,9 @@ where ); let validation_warnings = build_validation_warnings(&resolved_bash_policies); + let resolved_database_retry = + resolve_database_retry_config(file_config.database_retry.as_ref()); + Ok(RuntimeConfig { loaded_config_paths, log_level: resolved_log_level, @@ -441,6 +450,7 @@ where attribution_hooks_enabled: resolved_attribution_hooks_enabled, workos_client_id: resolved_workos_client_id, bash_policies: resolved_bash_policies, + database_retry: resolved_database_retry, validation_errors, validation_warnings, }) @@ -543,3 +553,51 @@ where fn resolve_default_global_config_path() -> Result { Ok(resolve_sce_default_locations()?.global_config_file()) } + +fn resolve_database_retry_config( + file_config: Option<&schema::FileConfigValue>, +) -> ResolvedOptionalValue { + match file_config { + Some(value) => ResolvedOptionalValue { + value: Some(value.value.clone()), + source: Some(ValueSource::ConfigFile(value.source)), + }, + None => ResolvedOptionalValue { + value: None, + source: None, + }, + } +} + +use std::sync::OnceLock; + +static DATABASE_RETRY_CONFIG: OnceLock = OnceLock::new(); + +pub(crate) fn init_database_retry_config(config: DatabaseRetryConfig) -> Result<()> { + DATABASE_RETRY_CONFIG + .set(config) + .map_err(|_| anyhow!("Database retry config has already been initialized.")) +} + +pub(crate) fn get_database_retry_config() -> Option<&'static DatabaseRetryConfig> { + DATABASE_RETRY_CONFIG.get() +} + +/// Resolve the full runtime config from the environment and initialize the +/// database retry `OnceLock`. Silently ignores errors — if the config cannot +/// be resolved, DB adapters fall back to hardcoded defaults. +pub(crate) fn init_database_retry_config_from_environment(cwd: &Path) { + if let Ok(runtime) = resolve_runtime_config( + &ConfigRequest { + report_format: ReportFormat::Text, + config_path: None, + log_level: None, + timeout_ms: None, + }, + cwd, + ) { + if let Some(config) = runtime.database_retry.value { + let _ = init_database_retry_config(config); + } + } +} diff --git a/cli/src/services/config/schema.rs b/cli/src/services/config/schema.rs index 5dbf4af5..7ff867a0 100644 --- a/cli/src/services/config/schema.rs +++ b/cli/src/services/config/schema.rs @@ -17,7 +17,8 @@ use serde::Deserialize; use serde_json::Value; use super::policy::{parse_bash_policy_presets, parse_custom_bash_policies, CustomBashPolicyEntry}; -use super::types::{ConfigPathSource, LogFileMode, LogFormat, LogLevel}; +use super::types::{ConfigPathSource, DatabaseRetryConfig, LogFileMode, LogFormat, LogLevel}; +use crate::services::resilience::RetryPolicy; pub(crate) const SCE_CONFIG_SCHEMA_JSON: &str = include_str!("../../../assets/generated/config/schema/sce-config.schema.json"); @@ -73,6 +74,7 @@ pub(crate) struct ParsedFileConfigDocument { pub(crate) struct ParsedPoliciesConfigDocument { pub(crate) bash: Option, pub(crate) attribution_hooks: Option, + pub(crate) database_retry: Option, } #[derive(Clone, Debug, Deserialize, Eq, PartialEq)] @@ -99,6 +101,28 @@ pub(crate) struct ParsedCustomBashPolicyMatchDocument { pub(crate) argv_prefix: Option>, } +#[allow(clippy::struct_field_names)] +#[derive(Clone, Debug, Deserialize, Eq, PartialEq)] +pub(crate) struct ParsedDatabaseRetryConfigDocument { + pub(crate) local_db: Option, + pub(crate) agent_trace_db: Option, + pub(crate) auth_db: Option, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq)] +pub(crate) struct ParsedPerDbRetryConfigDocument { + pub(crate) connection_open: Option, + pub(crate) query: Option, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq)] +pub(crate) struct ParsedRetryPolicyDocument { + pub(crate) max_attempts: Option, + pub(crate) timeout_ms: Option, + pub(crate) initial_backoff_ms: Option, + pub(crate) max_backoff_ms: Option, +} + #[derive(Clone, Debug, Eq, PartialEq)] pub(crate) struct FileConfigValue { pub(crate) value: T, @@ -116,6 +140,7 @@ pub(crate) struct FileConfig { pub(crate) workos_client_id: Option>, pub(crate) bash_policy_presets: Option>>, pub(crate) bash_policy_custom: Option>>, + pub(crate) database_retry: Option>, } pub(crate) type ParsedBashPolicyConfig = ( @@ -127,6 +152,7 @@ pub(crate) type ParsedFilePolicies = ( Option>, Option>>, Option>>, + Option>, ); pub(crate) fn validate_config_value_against_schema(value: &Value, path: &Path) -> Result<()> { @@ -257,7 +283,7 @@ pub(crate) fn parse_file_config( let workos_client_id = typed .workos_client_id .map(|value| FileConfigValue { value, source }); - let (attribution_hooks_enabled, bash_policy_presets, bash_policy_custom) = + let (attribution_hooks_enabled, bash_policy_presets, bash_policy_custom, database_retry) = map_policies_config(typed.policies.as_ref(), object, path, source)?; Ok(FileConfig { @@ -270,6 +296,7 @@ pub(crate) fn parse_file_config( workos_client_id, bash_policy_presets, bash_policy_custom, + database_retry, }) } @@ -280,7 +307,7 @@ pub(crate) fn map_policies_config( source: ConfigPathSource, ) -> Result { let Some(policies_value) = object.get("policies") else { - return Ok((None, None, None)); + return Ok((None, None, None, None)); }; let policies_object = policies_value.as_object().with_context(|| { @@ -294,8 +321,8 @@ pub(crate) fn map_policies_config( policies_object, path, Some("policies"), - &["bash", "attribution_hooks"], - "bash, attribution_hooks", + &["bash", "attribution_hooks", "database_retry"], + "bash, attribution_hooks, database_retry", )?; let bash = typed.and_then(|config| config.bash.as_ref()); @@ -307,11 +334,18 @@ pub(crate) fn map_policies_config( )?; let (bash_policy_presets, bash_policy_custom) = map_bash_policy_config(bash, policies_object, path, source)?; + let database_retry = map_database_retry_config( + typed.and_then(|config| config.database_retry.as_ref()), + policies_object, + path, + source, + )?; Ok(( attribution_hooks_enabled, bash_policy_presets, bash_policy_custom, + database_retry, )) } @@ -383,3 +417,155 @@ pub(crate) fn map_bash_policy_config( Ok((presets, custom)) } + +#[allow(clippy::too_many_lines)] +pub(crate) fn map_database_retry_config( + typed: Option<&ParsedDatabaseRetryConfigDocument>, + policies_object: &serde_json::Map, + path: &Path, + source: ConfigPathSource, +) -> Result>> { + let Some(database_retry_value) = policies_object.get("database_retry") else { + return Ok(None); + }; + + let database_retry_object = database_retry_value.as_object().with_context(|| { + format!( + "Config key 'policies.database_retry' in '{}' must be an object.", + path.display() + ) + })?; + + validate_object_keys( + database_retry_object, + path, + Some("policies.database_retry"), + &["local_db", "agent_trace_db", "auth_db"], + "local_db, agent_trace_db, auth_db", + )?; + + let build_retry_policy = + |parsed: &ParsedRetryPolicyDocument, context: &str| -> Result { + let max_attempts = parsed.max_attempts.with_context(|| { + format!( + "Config key '{context}.max_attempts' in '{}' must be present.", + path.display() + ) + })?; + let timeout_ms = parsed.timeout_ms.with_context(|| { + format!( + "Config key '{context}.timeout_ms' in '{}' must be present.", + path.display() + ) + })?; + let initial_backoff_ms = parsed.initial_backoff_ms.with_context(|| { + format!( + "Config key '{context}.initial_backoff_ms' in '{}' must be present.", + path.display() + ) + })?; + let max_backoff_ms = parsed.max_backoff_ms.with_context(|| { + format!( + "Config key '{context}.max_backoff_ms' in '{}' must be present.", + path.display() + ) + })?; + + if max_attempts == 0 { + bail!( + "Config key '{context}.max_attempts' in '{}' must be >= 1.", + path.display() + ); + } + if timeout_ms == 0 { + bail!( + "Config key '{context}.timeout_ms' in '{}' must be >= 1.", + path.display() + ); + } + if max_backoff_ms < initial_backoff_ms { + bail!( + "Config key '{context}.max_backoff_ms' in '{}' must be >= initial_backoff_ms.", + path.display() + ); + } + + Ok(RetryPolicy { + max_attempts, + timeout_ms, + initial_backoff_ms, + max_backoff_ms, + }) + }; + + let build_per_db = |db_key: &str| -> Result> { + let Some(db_value) = database_retry_object.get(db_key) else { + return Ok(None); + }; + + let db_object = db_value.as_object().with_context(|| { + format!( + "Config key 'policies.database_retry.{db_key}' in '{}' must be an object.", + path.display() + ) + })?; + + validate_object_keys( + db_object, + path, + Some(&format!("policies.database_retry.{db_key}")), + &["connection_open", "query"], + "connection_open, query", + )?; + + let typed_db = typed.and_then(|doc| match db_key { + "local_db" => doc.local_db.as_ref(), + "agent_trace_db" => doc.agent_trace_db.as_ref(), + "auth_db" => doc.auth_db.as_ref(), + _ => None, + }); + + let build_policy = |op_key: &str| -> Result> { + let Some(op_value) = db_object.get(op_key) else { + return Ok(None); + }; + + let _op_object = op_value.as_object().with_context(|| { + format!( + "Config key 'policies.database_retry.{db_key}.{op_key}' in '{}' must be an object.", + path.display() + ) + })?; + + let typed_policy = typed_db.and_then(|db| match op_key { + "connection_open" => db.connection_open.as_ref(), + "query" => db.query.as_ref(), + _ => None, + }); + + let parsed = typed_policy.with_context(|| { + format!( + "Config key 'policies.database_retry.{db_key}.{op_key}' in '{}' could not be parsed.", + path.display() + ) + })?; + + let context = format!("policies.database_retry.{db_key}.{op_key}"); + build_retry_policy(parsed, &context).map(Some) + }; + + Ok(Some(super::types::PerDbRetryConfig { + connection_open: build_policy("connection_open")?, + query: build_policy("query")?, + })) + }; + + Ok(Some(FileConfigValue { + value: DatabaseRetryConfig { + local_db: build_per_db("local_db")?, + agent_trace_db: build_per_db("agent_trace_db")?, + auth_db: build_per_db("auth_db")?, + }, + source, + })) +} diff --git a/cli/src/services/config/types.rs b/cli/src/services/config/types.rs index 84f43cf1..7686b82d 100644 --- a/cli/src/services/config/types.rs +++ b/cli/src/services/config/types.rs @@ -248,3 +248,19 @@ pub(crate) fn parse_bool_value_from(key: &str, raw: &str, source: &str) -> anyho _ => anyhow::bail!("Invalid {key} '{raw}' from {source}. Valid values: true, false, 1, 0."), } } + +use crate::services::resilience::RetryPolicy; + +#[allow(clippy::struct_field_names)] +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct DatabaseRetryConfig { + pub(crate) local_db: Option, + pub(crate) agent_trace_db: Option, + pub(crate) auth_db: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct PerDbRetryConfig { + pub(crate) connection_open: Option, + pub(crate) query: Option, +} diff --git a/cli/src/services/db/mod.rs b/cli/src/services/db/mod.rs index e332488b..92cc87a7 100644 --- a/cli/src/services/db/mod.rs +++ b/cli/src/services/db/mod.rs @@ -15,6 +15,7 @@ use anyhow::{Context, Result}; use crate::services::lifecycle::{ HealthCategory, HealthFixability, HealthProblem, HealthProblemKind, HealthSeverity, }; +use crate::services::resilience::{run_with_retry_sync, RetryPolicy}; const MIGRATIONS_TABLE_SQL: &str = "CREATE TABLE IF NOT EXISTS __sce_migrations ( id TEXT PRIMARY KEY, @@ -23,6 +24,20 @@ const MIGRATIONS_TABLE_SQL: &str = "CREATE TABLE IF NOT EXISTS __sce_migrations const SELECT_MIGRATION_SQL: &str = "SELECT id FROM __sce_migrations WHERE id = ?1 LIMIT 1"; const INSERT_MIGRATION_SQL: &str = "INSERT INTO __sce_migrations (id) VALUES (?1)"; const ENCRYPTION_CIPHER_AEGIS256: &str = "aegis256"; +const CONNECTION_OPEN_RETRY_POLICY: RetryPolicy = RetryPolicy { + max_attempts: 3, + timeout_ms: 1_000, + initial_backoff_ms: 25, + max_backoff_ms: 200, +}; +const CONNECTION_OPEN_RETRY_HINT: &str = "retry after the database lock clears; if the issue persists, stop other SCE processes using this database and rerun the command"; +const QUERY_RETRY_POLICY: RetryPolicy = RetryPolicy { + max_attempts: 5, + timeout_ms: 200, + initial_backoff_ms: 25, + max_backoff_ms: 100, +}; +const QUERY_RETRY_HINT: &str = "retry after the database lock clears; if the issue persists, stop other SCE processes using this database and rerun the command"; pub mod encryption_key; @@ -36,6 +51,10 @@ pub trait DbSpec { /// Ordered embedded migration SQL files as `(id, sql)` pairs. fn migrations() -> &'static [(&'static str, &'static str)]; + + /// Config-file lookup key under `policies.database_retry`. + /// One of `"local_db"`, `"agent_trace_db"`, `"auth_db"`. + fn db_config_key() -> &'static str; } /// Collect common filesystem health problems for a Turso database path. @@ -245,59 +264,43 @@ impl TursoConnectionCore { } } - fn execute(&self, sql: &str, params: impl turso::params::IntoParams) -> Result { - self.runtime.block_on(async { - self.conn - .execute(sql, params) - .await - .map_err(|e| anyhow::anyhow!("{} execute failed: {sql}: {e}", M::db_name())) - }) - } - - fn query(&self, sql: &str, params: impl turso::params::IntoParams) -> Result { - self.runtime.block_on(async { - self.conn - .query(sql, params) - .await - .map_err(|e| anyhow::anyhow!("{} query failed: {sql}: {e}", M::db_name())) - }) + fn run_migrations(&self) -> Result<()> { + run_embedded_migrations(&self.conn, &self.runtime, M::db_name(), M::migrations()) } +} - fn query_map( - &self, - sql: &str, - params: impl turso::params::IntoParams, - mut map_row: F, - ) -> Result> - where - F: FnMut(&turso::Row) -> Result, - { - self.runtime.block_on(async { - let mut rows = self - .conn - .query(sql, params) - .await - .map_err(|e| anyhow::anyhow!("{} query failed: {sql}: {e}", M::db_name()))?; - let mut results = Vec::new(); - - while let Some(row) = rows - .next() - .await - .map_err(|e| anyhow::anyhow!("{} row fetch failed: {sql}: {e}", M::db_name()))? - { - results.push( - map_row(&row) - .with_context(|| format!("{} row mapping failed: {sql}", M::db_name()))?, - ); +fn resolve_connection_open_retry_policy() -> RetryPolicy { + if let Some(config) = crate::services::config::get_database_retry_config() { + let per_db = match M::db_config_key() { + "local_db" => config.local_db.as_ref(), + "agent_trace_db" => config.agent_trace_db.as_ref(), + "auth_db" => config.auth_db.as_ref(), + _ => None, + }; + if let Some(per_db) = per_db { + if let Some(policy) = per_db.connection_open { + return policy; } - - Ok(results) - }) + } } + CONNECTION_OPEN_RETRY_POLICY +} - fn run_migrations(&self) -> Result<()> { - run_embedded_migrations(&self.conn, &self.runtime, M::db_name(), M::migrations()) +fn resolve_query_retry_policy() -> RetryPolicy { + if let Some(config) = crate::services::config::get_database_retry_config() { + let per_db = match M::db_config_key() { + "local_db" => config.local_db.as_ref(), + "agent_trace_db" => config.agent_trace_db.as_ref(), + "auth_db" => config.auth_db.as_ref(), + _ => None, + }; + if let Some(per_db) = per_db { + if let Some(policy) = per_db.query { + return policy; + } + } } + QUERY_RETRY_POLICY } /// Generic Turso database adapter. @@ -323,38 +326,59 @@ impl TursoDb { /// Parent directories are created automatically. Migrations are run after /// the database connection is established. pub fn new() -> Result { + let db = Self::open_without_migrations()?; + + db.run_migrations() + .with_context(|| format!("failed to run {} migrations", M::db_name()))?; + + Ok(db) + } + + /// Open or create the database at the spec-provided canonical path without + /// running embedded migrations. + /// + /// Parent directories are created automatically and the connection-open + /// retry policy is preserved. Runtime callers that use this path are + /// responsible for verifying schema readiness before query/write work. + pub fn open_without_migrations() -> Result { let db_name = M::db_name(); let db_path = M::db_path().with_context(|| format!("failed to resolve {db_name} path"))?; ensure_db_parent_dir(db_name, &db_path)?; let runtime = build_current_thread_runtime(db_name)?; - - let conn = runtime.block_on(async { - let path_str = db_path.to_str().ok_or_else(|| { - anyhow::anyhow!("invalid UTF-8 in database path: {}", db_path.display()) - })?; - let db = turso::Builder::new_local(path_str) - .build() - .await - .map_err(|e| { - anyhow::anyhow!( - "failed to open {db_name} database at {}: {e}", - db_path.display() - ) - })?; - db.connect() - .map_err(|e| anyhow::anyhow!("failed to connect to {db_name} database: {e}")) - })?; - - let db = Self { + let retry_policy = resolve_connection_open_retry_policy::(); + let operation_name = format!("open {db_name} database connection"); + + let conn = run_with_retry_sync( + retry_policy, + &operation_name, + CONNECTION_OPEN_RETRY_HINT, + |_| { + runtime.block_on(async { + let path_str = db_path.to_str().ok_or_else(|| { + anyhow::anyhow!("invalid UTF-8 in database path: {}", db_path.display()) + })?; + let db = turso::Builder::new_local(path_str) + .experimental_multiprocess_wal(true) + .build() + .await + .map_err(|e| { + anyhow::anyhow!( + "failed to open {db_name} database at {}: {e}", + db_path.display() + ) + })?; + db.connect().map_err(|e| { + anyhow::anyhow!("failed to connect to {db_name} database: {e}") + }) + }) + }, + )?; + + Ok(Self { core: TursoConnectionCore::new(conn, runtime), - }; - - db.run_migrations() - .with_context(|| format!("failed to run {db_name} migrations"))?; - - Ok(db) + }) } /// Execute a SQL statement that does not return rows. @@ -366,7 +390,25 @@ impl TursoDb { /// # Returns /// Number of rows affected. pub fn execute(&self, sql: &str, params: impl turso::params::IntoParams) -> Result { - self.core.execute(sql, params) + let params = turso::params::IntoParams::into_params(params).map_err(|e| { + anyhow::anyhow!("{} parameter conversion failed: {sql}: {e}", M::db_name()) + })?; + let operation_name = format!("execute {} database query", M::db_name()); + + run_with_retry_sync( + resolve_query_retry_policy::(), + &operation_name, + QUERY_RETRY_HINT, + |_| { + self.core.runtime.block_on(async { + self.core + .conn + .execute(sql, params.clone()) + .await + .map_err(|e| anyhow::anyhow!("{} execute failed: {sql}: {e}", M::db_name())) + }) + }, + ) } /// Execute a SQL query that returns rows. @@ -379,7 +421,25 @@ impl TursoDb { /// A `turso::Rows` iterator over the result set. #[allow(dead_code)] pub fn query(&self, sql: &str, params: impl turso::params::IntoParams) -> Result { - self.core.query(sql, params) + let params = turso::params::IntoParams::into_params(params).map_err(|e| { + anyhow::anyhow!("{} parameter conversion failed: {sql}: {e}", M::db_name()) + })?; + let operation_name = format!("query {} database", M::db_name()); + + run_with_retry_sync( + resolve_query_retry_policy::(), + &operation_name, + QUERY_RETRY_HINT, + |_| { + self.core.runtime.block_on(async { + self.core + .conn + .query(sql, params.clone()) + .await + .map_err(|e| anyhow::anyhow!("{} query failed: {sql}: {e}", M::db_name())) + }) + }, + ) } /// Execute a SQL query and synchronously map all returned rows. @@ -387,12 +447,53 @@ impl TursoDb { &self, sql: &str, params: impl turso::params::IntoParams, - map_row: F, + mut map_row: F, ) -> Result> where F: FnMut(&turso::Row) -> Result, { - self.core.query_map(sql, params, map_row) + let params = turso::params::IntoParams::into_params(params).map_err(|e| { + anyhow::anyhow!("{} parameter conversion failed: {sql}: {e}", M::db_name()) + })?; + let operation_name = format!("query and fetch {} database rows", M::db_name()); + + let rows = run_with_retry_sync( + resolve_query_retry_policy::(), + &operation_name, + QUERY_RETRY_HINT, + |_| { + self.core.runtime.block_on(async { + let mut rows = + self.core + .conn + .query(sql, params.clone()) + .await + .map_err(|e| { + anyhow::anyhow!("{} query failed: {sql}: {e}", M::db_name()) + })?; + let mut fetched_rows = Vec::new(); + + while let Some(row) = rows.next().await.map_err(|e| { + anyhow::anyhow!("{} row fetch failed: {sql}: {e}", M::db_name()) + })? { + fetched_rows.push(row); + } + + Ok(fetched_rows) + }) + }, + )?; + + let mut results = Vec::new(); + + for row in rows { + results.push( + map_row(&row) + .with_context(|| format!("{} row mapping failed: {sql}", M::db_name()))?, + ); + } + + Ok(results) } /// Run all embedded migrations in order. @@ -404,6 +505,86 @@ impl TursoDb { pub fn run_migrations(&self) -> Result<()> { self.core.run_migrations() } + + /// Check migration metadata for problems that would prevent safe hook + /// runtime access. + /// + /// Returns a list of problems: missing migration metadata table, + /// incomplete applied migrations, or unexpected extra migrations. + /// An empty list means the schema is ready. + pub fn migration_metadata_problems(&self) -> Result> { + let migration_table_exists = self.query_map( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = '__sce_migrations' LIMIT 1", + (), + |row| row.get::(0).map_err(Into::into), + )?; + + if migration_table_exists.is_empty() { + return Ok(vec![String::from("missing migration metadata table")]); + } + + let applied_ids = self.query_map( + "SELECT id FROM __sce_migrations ORDER BY id ASC", + (), + |row| row.get::(0).map_err(Into::into), + )?; + let expected_ids = M::migrations() + .iter() + .map(|(id, _)| *id) + .collect::>(); + let mut problems = Vec::new(); + + if applied_ids.len() != expected_ids.len() { + problems.push(format!( + "expected {} applied migrations, found {}", + expected_ids.len(), + applied_ids.len() + )); + } + + let missing_ids = expected_ids + .iter() + .copied() + .filter(|id| !applied_ids.iter().any(|applied_id| applied_id == id)) + .collect::>(); + if !missing_ids.is_empty() { + problems.push(format!("missing migrations {}", missing_ids.join(", "))); + } + + let unexpected_ids = applied_ids + .iter() + .filter(|applied_id| !expected_ids.iter().any(|id| id == &applied_id.as_str())) + .map(String::as_str) + .collect::>(); + if !unexpected_ids.is_empty() { + problems.push(format!( + "unexpected migrations {}", + unexpected_ids.join(", ") + )); + } + + Ok(problems) + } + + /// Verify that the database schema needed by hook runtime readers and + /// writers already exists. + /// + /// This check is intentionally non-mutating. Missing or incomplete schema + /// is reported with the provided setup guidance instead of running + /// migrations from a high-frequency hook path. + pub fn ensure_schema_ready(&self, setup_guidance: &str) -> Result<()> { + let problems = self.migration_metadata_problems()?; + + if problems.is_empty() { + return Ok(()); + } + + anyhow::bail!( + "{} schema is not initialized or is incomplete: {}. {setup_guidance}", + M::db_name(), + problems.join(", ") + ) + } } impl EncryptedTursoDb { @@ -420,33 +601,42 @@ impl EncryptedTursoDb { ensure_db_parent_dir(db_name, &db_path)?; let runtime = build_current_thread_runtime(db_name)?; - - let conn = runtime.block_on(async { - let path_str = db_path.to_str().ok_or_else(|| { - anyhow::anyhow!("invalid UTF-8 in database path: {}", db_path.display()) - })?; - - let encryption_opts = turso::EncryptionOpts { - hexkey: encryption_key, - cipher: ENCRYPTION_CIPHER_AEGIS256.to_string(), - }; - - let db = turso::Builder::new_local(path_str) - .experimental_encryption(true) - .with_encryption(encryption_opts) - .build() - .await - .map_err(|e| { - anyhow::anyhow!( - "failed to open encrypted {db_name} database at {} with cipher {ENCRYPTION_CIPHER_AEGIS256}. Try: verify the credential store encryption key is valid and that local Turso encryption support is available: {e}", - db_path.display() - ) - })?; - - db.connect().map_err(|e| { - anyhow::anyhow!("failed to connect to encrypted {db_name} database: {e}") - }) - })?; + let retry_policy = resolve_connection_open_retry_policy::(); + let operation_name = format!("open encrypted {db_name} database connection"); + + let conn = run_with_retry_sync( + retry_policy, + &operation_name, + CONNECTION_OPEN_RETRY_HINT, + |_| { + runtime.block_on(async { + let path_str = db_path.to_str().ok_or_else(|| { + anyhow::anyhow!("invalid UTF-8 in database path: {}", db_path.display()) + })?; + + let encryption_opts = turso::EncryptionOpts { + hexkey: encryption_key.clone(), + cipher: ENCRYPTION_CIPHER_AEGIS256.to_string(), + }; + + let db = turso::Builder::new_local(path_str) + .experimental_encryption(true) + .with_encryption(encryption_opts) + .build() + .await + .map_err(|e| { + anyhow::anyhow!( + "failed to open encrypted {db_name} database at {} with cipher {ENCRYPTION_CIPHER_AEGIS256}. Try: verify the credential store encryption key is valid and that local Turso encryption support is available: {e}", + db_path.display() + ) + })?; + + db.connect().map_err(|e| { + anyhow::anyhow!("failed to connect to encrypted {db_name} database: {e}") + }) + }) + }, + )?; let db = Self { core: TursoConnectionCore::new(conn, runtime), @@ -467,7 +657,25 @@ impl EncryptedTursoDb { /// # Returns /// Number of rows affected. pub fn execute(&self, sql: &str, params: impl turso::params::IntoParams) -> Result { - self.core.execute(sql, params) + let params = turso::params::IntoParams::into_params(params).map_err(|e| { + anyhow::anyhow!("{} parameter conversion failed: {sql}: {e}", M::db_name()) + })?; + let operation_name = format!("execute encrypted {} database query", M::db_name()); + + run_with_retry_sync( + resolve_query_retry_policy::(), + &operation_name, + QUERY_RETRY_HINT, + |_| { + self.core.runtime.block_on(async { + self.core + .conn + .execute(sql, params.clone()) + .await + .map_err(|e| anyhow::anyhow!("{} execute failed: {sql}: {e}", M::db_name())) + }) + }, + ) } /// Execute a SQL query that returns rows. @@ -480,7 +688,25 @@ impl EncryptedTursoDb { /// A `turso::Rows` iterator over the result set. #[allow(dead_code)] pub fn query(&self, sql: &str, params: impl turso::params::IntoParams) -> Result { - self.core.query(sql, params) + let params = turso::params::IntoParams::into_params(params).map_err(|e| { + anyhow::anyhow!("{} parameter conversion failed: {sql}: {e}", M::db_name()) + })?; + let operation_name = format!("query encrypted {} database", M::db_name()); + + run_with_retry_sync( + resolve_query_retry_policy::(), + &operation_name, + QUERY_RETRY_HINT, + |_| { + self.core.runtime.block_on(async { + self.core + .conn + .query(sql, params.clone()) + .await + .map_err(|e| anyhow::anyhow!("{} query failed: {sql}: {e}", M::db_name())) + }) + }, + ) } /// Execute a SQL query and synchronously map all returned rows. @@ -488,12 +714,53 @@ impl EncryptedTursoDb { &self, sql: &str, params: impl turso::params::IntoParams, - map_row: F, + mut map_row: F, ) -> Result> where F: FnMut(&turso::Row) -> Result, { - self.core.query_map(sql, params, map_row) + let params = turso::params::IntoParams::into_params(params).map_err(|e| { + anyhow::anyhow!("{} parameter conversion failed: {sql}: {e}", M::db_name()) + })?; + let operation_name = format!("query and fetch encrypted {} database rows", M::db_name()); + + let rows = run_with_retry_sync( + resolve_query_retry_policy::(), + &operation_name, + QUERY_RETRY_HINT, + |_| { + self.core.runtime.block_on(async { + let mut rows = + self.core + .conn + .query(sql, params.clone()) + .await + .map_err(|e| { + anyhow::anyhow!("{} query failed: {sql}: {e}", M::db_name()) + })?; + let mut fetched_rows = Vec::new(); + + while let Some(row) = rows.next().await.map_err(|e| { + anyhow::anyhow!("{} row fetch failed: {sql}: {e}", M::db_name()) + })? { + fetched_rows.push(row); + } + + Ok(fetched_rows) + }) + }, + )?; + + let mut results = Vec::new(); + + for row in rows { + results.push( + map_row(&row) + .with_context(|| format!("{} row mapping failed: {sql}", M::db_name()))?, + ); + } + + Ok(results) } /// Run all embedded migrations in order. diff --git a/cli/src/services/hooks/mod.rs b/cli/src/services/hooks/mod.rs index ae69d393..ad827b89 100644 --- a/cli/src/services/hooks/mod.rs +++ b/cli/src/services/hooks/mod.rs @@ -15,8 +15,8 @@ use crate::services::agent_trace::{ AgentTraceVcsType, }; use crate::services::agent_trace_db::{ - AgentTraceDb, AgentTraceInsert, DiffTraceInsert, PostCommitPatchIntersectionInsert, - RecentDiffTracePatches, + AgentTraceDb, AgentTraceInsert, DiffTraceInsert, InsertMessageInsert, InsertPartInsert, + MessageRole, PartType, PostCommitPatchIntersectionInsert, RecentDiffTracePatches, }; use crate::services::config; use crate::services::observability::traits::Logger; @@ -34,6 +34,8 @@ const AGENT_TRACE_URL_PREFIX: &str = "sce.crocoder.dev/trace/"; const MAX_TRACE_FILE_CREATE_ATTEMPTS: u64 = 1_000_000; +type PayloadValidationError = fn(&str) -> String; + #[derive(Clone, Debug, Eq, PartialEq)] pub enum HookSubcommand { PreCommit, @@ -48,6 +50,7 @@ pub enum HookSubcommand { rewrite_method: String, }, DiffTrace, + ConversationTrace, } #[derive(Clone, Debug, Eq, PartialEq, Serialize)] @@ -61,6 +64,47 @@ struct DiffTracePayload { tool_version: Option, } +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum ConversationTracePayload { + MessageUpdated(ConversationTraceMessageBatch), + MessagePartUpdated(ConversationTracePartBatch), +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ConversationTraceMessageBatch { + pub inserts: Vec, + pub skipped: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ConversationTracePartBatch { + pub inserts: Vec, + pub skipped: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct SkippedConversationTracePayload { + pub index: usize, + pub reason: String, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct ConversationTracePersistenceSummary { + event_type: &'static str, + attempted_count: usize, + persisted_count: usize, + skipped_count: usize, +} + +impl ConversationTracePersistenceSummary { + fn render(&self) -> String { + format!( + "conversation-trace hook persisted {} payload batch to AgentTraceDb: attempted={}, persisted={}, skipped={}.", + self.event_type, self.attempted_count, self.persisted_count, self.skipped_count + ) + } +} + /// Required `sce hooks diff-trace` STDIN payload shape: /// `{ sessionID, diff, time, model_id, tool_name, tool_version }`. /// @@ -100,9 +144,367 @@ fn run_hooks_subcommand_in_repo( run_post_rewrite_subcommand_with_trace(repository_root, subcommand, rewrite_method) } HookSubcommand::DiffTrace => run_diff_trace_subcommand(repository_root, logger), + HookSubcommand::ConversationTrace => run_conversation_trace_subcommand(logger), + } +} + +fn run_conversation_trace_subcommand(logger: Option<&dyn Logger>) -> Result { + let stdin_payload = read_hook_stdin()?; + let result = run_conversation_trace_subcommand_from_payload(&stdin_payload, logger); + if let Err(ref error) = result { + if let Some(log) = logger { + log.error( + "sce.hooks.conversation_trace.error", + &error.to_string(), + &[], + ); + } + } + result +} + +fn run_conversation_trace_subcommand_from_payload( + stdin_payload: &str, + logger: Option<&dyn Logger>, +) -> Result { + let payload = parse_conversation_trace_payload(stdin_payload)?; + persist_conversation_trace_payload_to_agent_trace_db(payload, logger) +} + +fn persist_conversation_trace_payload_to_agent_trace_db( + payload: ConversationTracePayload, + logger: Option<&dyn Logger>, +) -> Result { + let db = open_agent_trace_db_for_hook_runtime( + "Failed to open Agent Trace DB for conversation-trace persistence.", + )?; + + let summary = match payload { + ConversationTracePayload::MessageUpdated(batch) => { + persist_message_updated_batch_to_agent_trace_db(&db, batch, logger) + } + ConversationTracePayload::MessagePartUpdated(batch) => { + persist_message_part_updated_batch_to_agent_trace_db(&db, batch, logger) + } + }; + + Ok(summary.render()) +} + +fn open_agent_trace_db_for_hook_runtime(context_message: &'static str) -> Result { + prepare_agent_trace_db_for_hook_runtime_with( + AgentTraceDb::open_for_hooks_without_migrations, + AgentTraceDb::ensure_schema_ready_for_hooks, + context_message, + ) +} + +fn prepare_agent_trace_db_for_hook_runtime_with( + open_db: O, + ensure_schema_ready: R, + context_message: &'static str, +) -> Result +where + O: FnOnce() -> Result, + R: FnOnce(&D) -> Result<()>, +{ + let db = open_db().context(context_message)?; + ensure_schema_ready(&db).context(context_message)?; + + Ok(db) +} + +fn persist_message_updated_batch_to_agent_trace_db( + db: &AgentTraceDb, + batch: ConversationTraceMessageBatch, + logger: Option<&dyn Logger>, +) -> ConversationTracePersistenceSummary { + const EVENT_TYPE: &str = "message.updated"; + + let attempted_count = batch.inserts.len() + batch.skipped.len(); + let mut skipped_count = batch.skipped.len(); + + log_skipped_conversation_trace_payloads(logger, EVENT_TYPE, &batch.skipped); + + let valid_count = batch.inserts.len(); + let persisted_count = if valid_count == 0 { + 0 + } else { + match db.insert_messages(batch.inserts) { + Ok(affected_rows) => usize::try_from(affected_rows) + .unwrap_or(usize::MAX) + .min(valid_count), + Err(error) => { + skipped_count += valid_count; + log_conversation_trace_batch_insert_failure( + logger, + EVENT_TYPE, + valid_count, + &error, + ); + 0 + } + } + }; + + ConversationTracePersistenceSummary { + event_type: EVENT_TYPE, + attempted_count, + persisted_count, + skipped_count, + } +} + +fn persist_message_part_updated_batch_to_agent_trace_db( + db: &AgentTraceDb, + batch: ConversationTracePartBatch, + logger: Option<&dyn Logger>, +) -> ConversationTracePersistenceSummary { + const EVENT_TYPE: &str = "message.part.updated"; + + let attempted_count = batch.inserts.len() + batch.skipped.len(); + let mut skipped_count = batch.skipped.len(); + + log_skipped_conversation_trace_payloads(logger, EVENT_TYPE, &batch.skipped); + + let valid_count = batch.inserts.len(); + let persisted_count = if valid_count == 0 { + 0 + } else { + match db.insert_parts(batch.inserts) { + Ok(affected_rows) => usize::try_from(affected_rows) + .unwrap_or(usize::MAX) + .min(valid_count), + Err(error) => { + skipped_count += valid_count; + log_conversation_trace_batch_insert_failure( + logger, + EVENT_TYPE, + valid_count, + &error, + ); + 0 + } + } + }; + + ConversationTracePersistenceSummary { + event_type: EVENT_TYPE, + attempted_count, + persisted_count, + skipped_count, + } +} + +fn log_skipped_conversation_trace_payloads( + logger: Option<&dyn Logger>, + event_type: &str, + skipped_payloads: &[SkippedConversationTracePayload], +) { + let Some(log) = logger else { + return; + }; + + for skipped in skipped_payloads { + let index = skipped.index.to_string(); + log.warn( + "sce.hooks.conversation_trace.payload_skipped", + &skipped.reason, + &[ + ("event_type", event_type), + ("payload_index", index.as_str()), + ], + ); } } +fn log_conversation_trace_batch_insert_failure( + logger: Option<&dyn Logger>, + event_type: &str, + valid_count: usize, + error: &anyhow::Error, +) { + if let Some(log) = logger { + let count = valid_count.to_string(); + log.warn( + "sce.hooks.conversation_trace.agent_trace_db_batch_failed", + &error.to_string(), + &[("event_type", event_type), ("valid_count", count.as_str())], + ); + } +} + +pub fn parse_conversation_trace_payload(stdin_payload: &str) -> Result { + let parsed: Value = serde_json::from_str(stdin_payload) + .context("Invalid conversation-trace payload from STDIN: expected valid JSON.")?; + let payload = parsed.as_object().ok_or_else(|| { + anyhow!(conversation_trace_validation_error( + "expected a JSON object" + )) + })?; + let event_type = required_string_field(payload, "type", conversation_trace_validation_error)?; + let payloads = required_payloads_array(payload)?; + + match event_type.as_str() { + "message.updated" => parse_message_updated_payloads(payloads), + "message.part.updated" => parse_message_part_updated_payloads(payloads), + _ => bail!(conversation_trace_validation_error( + "field 'type' must be one of 'message.updated' or 'message.part.updated'" + )), + } +} + +fn required_payloads_array(payload: &serde_json::Map) -> Result<&Vec> { + required_field(payload, "payloads", conversation_trace_validation_error)? + .as_array() + .ok_or_else(|| { + anyhow!(conversation_trace_validation_error( + "field 'payloads' must be an array" + )) + }) +} + +fn parse_message_updated_payloads(payloads: &[Value]) -> Result { + let mut inserts = Vec::new(); + let mut skipped = Vec::new(); + + for (index, item) in payloads.iter().enumerate() { + let Some(item) = conversation_trace_payload_item(item, index, &mut skipped)? else { + continue; + }; + match parse_message_updated_item(item) { + Ok(input) => inserts.push(input), + Err(error) => skipped.push(SkippedConversationTracePayload { + index, + reason: error.to_string(), + }), + } + } + + Ok(ConversationTracePayload::MessageUpdated( + ConversationTraceMessageBatch { inserts, skipped }, + )) +} + +fn parse_message_part_updated_payloads(payloads: &[Value]) -> Result { + let mut inserts = Vec::new(); + let mut skipped = Vec::new(); + + for (index, item) in payloads.iter().enumerate() { + let Some(item) = conversation_trace_payload_item(item, index, &mut skipped)? else { + continue; + }; + match parse_message_part_updated_item(item) { + Ok(input) => inserts.push(input), + Err(error) => skipped.push(SkippedConversationTracePayload { + index, + reason: error.to_string(), + }), + } + } + + Ok(ConversationTracePayload::MessagePartUpdated( + ConversationTracePartBatch { inserts, skipped }, + )) +} + +fn conversation_trace_payload_item<'a>( + item: &'a Value, + index: usize, + skipped: &mut Vec, +) -> Result>> { + let Some(payload) = item.as_object() else { + skipped.push(SkippedConversationTracePayload { + index, + reason: conversation_trace_validation_error(&format!( + "payloads[{index}] must be an object" + )), + }); + return Ok(None); + }; + + if payload.contains_key("type") { + bail!(conversation_trace_validation_error(&format!( + "payloads[{index}] must not declare its own 'type'; use the top-level 'type' for homogeneous batches" + ))); + } + + Ok(Some(payload)) +} + +fn parse_message_updated_item( + payload: &serde_json::Map, +) -> Result { + Ok(InsertMessageInsert { + session_id: required_non_empty_string_field( + payload, + "session_id", + conversation_trace_validation_error, + )?, + message_id: required_non_empty_string_field( + payload, + "message_id", + conversation_trace_validation_error, + )?, + role: parse_message_role(payload)?, + generated_at_unix_ms: required_i64_millisecond_field( + payload, + "generated_at_unix_ms", + conversation_trace_validation_error, + )?, + }) +} + +fn parse_message_part_updated_item( + payload: &serde_json::Map, +) -> Result { + Ok(InsertPartInsert { + session_id: required_non_empty_string_field( + payload, + "session_id", + conversation_trace_validation_error, + )?, + message_id: required_non_empty_string_field( + payload, + "message_id", + conversation_trace_validation_error, + )?, + part_type: parse_part_type(payload)?, + text: required_string_field(payload, "text", conversation_trace_validation_error)?, + generated_at_unix_ms: required_i64_millisecond_field( + payload, + "generated_at_unix_ms", + conversation_trace_validation_error, + )?, + }) +} + +fn parse_message_role(payload: &serde_json::Map) -> Result { + match required_string_field(payload, "role", conversation_trace_validation_error)?.as_str() { + "user" => Ok(MessageRole::User), + "assistant" => Ok(MessageRole::Assistant), + _ => bail!(conversation_trace_validation_error( + "field 'role' must be one of 'user' or 'assistant'" + )), + } +} + +fn parse_part_type(payload: &serde_json::Map) -> Result { + match required_string_field(payload, "part_type", conversation_trace_validation_error)?.as_str() + { + "text" => Ok(PartType::Text), + "reasoning" => Ok(PartType::Reasoning), + "patch" => Ok(PartType::Patch), + _ => bail!(conversation_trace_validation_error( + "field 'part_type' must be one of 'text', 'reasoning' or 'patch'" + )), + } +} + +fn conversation_trace_validation_error(detail: &str) -> String { + format!("Invalid conversation-trace payload from STDIN: {detail}.") +} + fn run_diff_trace_subcommand( repository_root: &Path, logger: Option<&dyn Logger>, @@ -166,12 +568,19 @@ fn parse_diff_trace_payload(stdin_payload: &str) -> Result { .as_object() .ok_or_else(|| anyhow!(diff_trace_validation_error("expected a JSON object")))?; - let session_id = required_non_empty_string_field(payload, "sessionID")?; - let diff = required_non_empty_string_field(payload, "diff")?; - let time = required_u64_millisecond_field(payload, "time")?; - let model_id = required_non_empty_string_field(payload, "model_id")?; - let tool_name = required_non_empty_string_field(payload, "tool_name")?; - let tool_version = required_nullable_or_non_empty_string_field(payload, "tool_version")?; + let session_id = + required_non_empty_string_field(payload, "sessionID", diff_trace_validation_error)?; + let diff = required_non_empty_string_field(payload, "diff", diff_trace_validation_error)?; + let time = required_u64_millisecond_field(payload, "time", diff_trace_validation_error)?; + let model_id = + required_non_empty_string_field(payload, "model_id", diff_trace_validation_error)?; + let tool_name = + required_non_empty_string_field(payload, "tool_name", diff_trace_validation_error)?; + let tool_version = required_nullable_or_non_empty_string_field( + payload, + "tool_version", + diff_trace_validation_error, + )?; Ok(DiffTracePayload { session_id, @@ -186,21 +595,22 @@ fn parse_diff_trace_payload(stdin_payload: &str) -> Result { fn required_nullable_or_non_empty_string_field( payload: &serde_json::Map, field_name: &str, + validation_error: PayloadValidationError, ) -> Result> { - let raw = required_field(payload, field_name)?; + let raw = required_field(payload, field_name, validation_error)?; if raw.is_null() { return Ok(None); } let value = raw.as_str().ok_or_else(|| { - anyhow!(diff_trace_validation_error(&format!( + anyhow!(validation_error(&format!( "field '{field_name}' must be null or a non-empty string" ))) })?; if value.trim().is_empty() { - bail!(diff_trace_validation_error(&format!( + bail!(validation_error(&format!( "field '{field_name}' must be null or a non-empty string" ))); } @@ -211,22 +621,31 @@ fn required_nullable_or_non_empty_string_field( fn required_non_empty_string_field( payload: &serde_json::Map, field_name: &str, + validation_error: PayloadValidationError, ) -> Result { - let raw = required_field(payload, field_name)?; - - let value = raw.as_str().ok_or_else(|| { - anyhow!(diff_trace_validation_error(&format!( - "field '{field_name}' must be a non-empty string" - ))) - })?; + let value = required_string_field(payload, field_name, validation_error)?; if value.trim().is_empty() { - bail!(diff_trace_validation_error(&format!( + bail!(validation_error(&format!( "field '{field_name}' must be a non-empty string" ))); } - Ok(value.to_string()) + Ok(value) +} + +fn required_string_field( + payload: &serde_json::Map, + field_name: &str, + validation_error: PayloadValidationError, +) -> Result { + let raw = required_field(payload, field_name, validation_error)?; + + raw.as_str().map(ToString::to_string).ok_or_else(|| { + anyhow!(validation_error(&format!( + "field '{field_name}' must be a string" + ))) + }) } #[allow( @@ -237,8 +656,9 @@ fn required_non_empty_string_field( fn required_u64_millisecond_field( payload: &serde_json::Map, field_name: &str, + validation_error: PayloadValidationError, ) -> Result { - let raw = required_field(payload, field_name)?; + let raw = required_field(payload, field_name, validation_error)?; if let Some(value) = raw.as_u64() { return Ok(value); @@ -246,7 +666,7 @@ fn required_u64_millisecond_field( if let Some(value) = raw.as_i64() { if value < 0 { - bail!(diff_trace_validation_error(&format!( + bail!(validation_error(&format!( "field '{field_name}' must be a u64 Unix epoch millisecond value, got a negative number" ))); } @@ -255,34 +675,70 @@ fn required_u64_millisecond_field( if let Some(value) = raw.as_f64() { if value.fract() != 0.0 { - bail!(diff_trace_validation_error(&format!( + bail!(validation_error(&format!( "field '{field_name}' must be a u64 Unix epoch millisecond value, got a fractional number" ))); } if value < 0.0 { - bail!(diff_trace_validation_error(&format!( + bail!(validation_error(&format!( "field '{field_name}' must be a u64 Unix epoch millisecond value, got a negative number" ))); } if value > u64::MAX as f64 { - bail!(diff_trace_validation_error(&format!( + bail!(validation_error(&format!( "field '{field_name}' must be a u64 Unix epoch millisecond value" ))); } return Ok(value as u64); } - bail!(diff_trace_validation_error(&format!( + bail!(validation_error(&format!( "field '{field_name}' must be a u64 Unix epoch millisecond value" ))) } +fn required_i64_millisecond_field( + payload: &serde_json::Map, + field_name: &str, + validation_error: PayloadValidationError, +) -> Result { + let raw = required_field(payload, field_name, validation_error)?; + + if let Some(value) = raw.as_i64() { + if value < 0 { + bail!(validation_error(&format!( + "field '{field_name}' must be a non-negative signed 64-bit Unix epoch millisecond value" + ))); + } + return Ok(value); + } + + if let Some(value) = raw.as_u64() { + return i64::try_from(value).map_err(|_| { + anyhow!(validation_error(&format!( + "field '{field_name}' must fit in a signed 64-bit Unix epoch millisecond value for Agent Trace DB storage" + ))) + }); + } + + if raw.as_f64().is_some_and(|value| value.fract() != 0.0) { + bail!(validation_error(&format!( + "field '{field_name}' must be a non-negative signed 64-bit Unix epoch millisecond value, got a fractional number" + ))); + } + + bail!(validation_error(&format!( + "field '{field_name}' must be a non-negative signed 64-bit Unix epoch millisecond value" + ))) +} + fn required_field<'a>( payload: &'a serde_json::Map, field_name: &str, + validation_error: PayloadValidationError, ) -> Result<&'a Value> { payload.get(field_name).ok_or_else(|| { - anyhow!(diff_trace_validation_error(&format!( + anyhow!(validation_error(&format!( "missing required field '{field_name}'" ))) }) @@ -313,8 +769,9 @@ fn persist_diff_trace_payload( fn persist_diff_trace_payload_to_agent_trace_db(payload: &DiffTracePayload) -> Result<()> { persist_diff_trace_payload_to_agent_trace_db_with(payload, |input| { - let db = AgentTraceDb::new() - .context("Failed to open Agent Trace DB for diff-trace persistence.")?; + let db = open_agent_trace_db_for_hook_runtime( + "Failed to open Agent Trace DB for diff-trace persistence.", + )?; db.insert_diff_trace(input) .context("Failed to persist diff-trace payload to Agent Trace DB.")?; @@ -537,7 +994,9 @@ fn run_post_commit_agent_trace_flow( vcs_type: Option, remote_url: &str, ) -> Result { - let db = AgentTraceDb::new().context("Failed to open Agent Trace DB for post-commit trace.")?; + let db = open_agent_trace_db_for_hook_runtime( + "Failed to open Agent Trace DB for post-commit trace.", + )?; run_post_commit_agent_trace_flow_with( flow_result, @@ -604,7 +1063,7 @@ where .context("Failed to serialize post-commit Agent Trace payload for persistence.")? ); - let constructed_url = format!("{}{}", AGENT_TRACE_URL_PREFIX, agent_trace.id); + let constructed_url = format!("{AGENT_TRACE_URL_PREFIX}{}", agent_trace.id); let insert_input = AgentTraceInsert { commit_id: &flow_result.post_commit_data.commit_oid, @@ -625,8 +1084,9 @@ const RECENT_DAYS_MILLIS: i64 = 7 * 24 * 60 * 60 * 1000; fn run_post_commit_intersection_flow( repository_root: &Path, ) -> Result { - let db = AgentTraceDb::new() - .context("Failed to open Agent Trace DB for post-commit intersection.")?; + let db = open_agent_trace_db_for_hook_runtime( + "Failed to open Agent Trace DB for post-commit intersection.", + )?; run_post_commit_intersection_flow_with( repository_root, @@ -757,6 +1217,7 @@ fn hook_runtime_invocation_name(subcommand: &HookSubcommand) -> &'static str { HookSubcommand::PostCommit { .. } => "post-commit runtime invocation", HookSubcommand::PostRewrite { .. } => "post-rewrite runtime invocation", HookSubcommand::DiffTrace => "diff-trace runtime invocation", + HookSubcommand::ConversationTrace => "conversation-trace runtime invocation", } } @@ -799,6 +1260,7 @@ fn hook_trace_name(subcommand: &HookSubcommand) -> &'static str { HookSubcommand::PostCommit { .. } => "post-commit", HookSubcommand::PostRewrite { .. } => "post-rewrite", HookSubcommand::DiffTrace => "diff-trace", + HookSubcommand::ConversationTrace => "conversation-trace", } } @@ -1003,7 +1465,7 @@ pub fn capture_post_commit_patch_from_git(repository_root: &Path) -> Result &'static [(&'static str, &'static str)] { &[] } + + fn db_config_key() -> &'static str { + "local_db" + } } /// Local Turso database adapter. diff --git a/cli/src/services/parse/command_runtime.rs b/cli/src/services/parse/command_runtime.rs index 3b6df8d0..0fabea8b 100644 --- a/cli/src/services/parse/command_runtime.rs +++ b/cli/src/services/parse/command_runtime.rs @@ -378,6 +378,11 @@ fn convert_hooks_subcommand( subcommand: services::hooks::HookSubcommand::DiffTrace, })) } + cli_schema::HooksSubcommand::ConversationTrace => { + Ok(Box::new(services::hooks::command::HooksCommand { + subcommand: services::hooks::HookSubcommand::ConversationTrace, + })) + } } } diff --git a/cli/src/services/patch.rs b/cli/src/services/patch.rs index 6b63f23a..9cb31544 100644 --- a/cli/src/services/patch.rs +++ b/cli/src/services/patch.rs @@ -80,6 +80,9 @@ pub struct TouchedLine { pub line_number: u64, /// Content of the line (without the leading `+`/`-` prefix). pub content: String, + /// Optional session identifier associated with this touched line. + #[serde(default)] + pub session_id: Option, } /// Kind of touched line. @@ -190,8 +193,8 @@ pub fn load_patch_from_json_bytes(input: &[u8]) -> Result = post_commit_hunk .lines .iter() - .filter(|line| { + .filter_map(|line| { if let Some(index) = find_available_line_match( &available_lines, &used_lines, @@ -242,7 +245,11 @@ pub fn intersect_patches( matched_model_id = available_lines[index].model_id.map(str::to_string); } used_lines[index] = true; - return true; + let mut overlapping_line = line.clone(); + overlapping_line + .session_id + .clone_from(&available_lines[index].line.session_id); + return Some(overlapping_line); } if let Some(index) = find_available_line_match( @@ -255,12 +262,15 @@ pub fn intersect_patches( matched_model_id = available_lines[index].model_id.map(str::to_string); } used_lines[index] = true; - return true; + let mut overlapping_line = line.clone(); + overlapping_line + .session_id + .clone_from(&available_lines[index].line.session_id); + return Some(overlapping_line); } - false + None }) - .cloned() .collect(); if overlapping_lines.is_empty() { @@ -506,7 +516,7 @@ pub fn combine_patches(patches: &[ParsedPatch]) -> ParsedPatch { /// Returns `ParseError` with an actionable message when the input is malformed, /// such as an invalid hunk header or a `---`/`+++` line that cannot be parsed. #[allow(dead_code)] -pub fn parse_patch(input: &str) -> Result { +pub fn parse_patch(input: &str, session_id: Option<&str>) -> Result { let mut files: Vec = Vec::new(); let mut current_file: Option = None; @@ -591,7 +601,7 @@ pub fn parse_patch(input: &str) -> Result { // Parse hunk header: @@ -old_start[,old_count] +new_start[,new_count] @@ if let Some(rest) = line.strip_prefix("@@ ") { if let Some(fb) = current_file.as_mut() { - let hunk = parse_hunk_header_and_body(rest, &mut lines)?; + let hunk = parse_hunk_header_and_body(rest, &mut lines, session_id)?; fb.add_hunk(hunk); } } @@ -773,6 +783,7 @@ fn parse_diff_path(rest: &str) -> String { fn parse_hunk_header_and_body<'a, I>( rest: &str, lines: &mut std::iter::Peekable, + session_id: Option<&str>, ) -> Result where I: Iterator, @@ -837,6 +848,7 @@ where kind: TouchedLineKind::Added, line_number: new_line_num, content: content.to_string(), + session_id: session_id.map(str::to_string), }); new_line_num += 1; } else if let Some(content) = line.strip_prefix('-') { @@ -845,6 +857,7 @@ where kind: TouchedLineKind::Removed, line_number: old_line_num, content: content.to_string(), + session_id: session_id.map(str::to_string), }); old_line_num += 1; } else if line.starts_with(' ') || line.starts_with('\t') { diff --git a/cli/src/services/patch/tests.rs b/cli/src/services/patch/tests.rs index 57271a49..14c43f19 100644 --- a/cli/src/services/patch/tests.rs +++ b/cli/src/services/patch/tests.rs @@ -10,7 +10,7 @@ struct PatchScenario { fn parse_fixtures(fixtures: &[&str]) -> Vec { fixtures .iter() - .map(|fixture| parse_patch(fixture).expect("fixture patch should parse")) + .map(|fixture| parse_patch(fixture, None).expect("fixture patch should parse")) .collect() } @@ -45,7 +45,7 @@ const TEXT_FILE_LIFECYCLE_RECONSTRUCTION_INCREMENTALS: &[&str] = &[ fn assert_reconstructs_post_commit(scenario: PatchScenario) { let combined = combine_patches(&parse_fixtures(scenario.incremental)); - let post_commit = parse_patch(scenario.post_commit).expect("fixture patch should parse"); + let post_commit = parse_patch(scenario.post_commit, None).expect("fixture patch should parse"); let golden: ParsedPatch = serde_json::from_str(scenario.golden).expect("golden json should load"); diff --git a/cli/src/services/resilience.rs b/cli/src/services/resilience.rs index 3e948364..fb1de8c9 100644 --- a/cli/src/services/resilience.rs +++ b/cli/src/services/resilience.rs @@ -1,5 +1,6 @@ use std::future::Future; -use std::time::Duration; +use std::thread; +use std::time::{Duration, Instant}; use anyhow::{anyhow, ensure, Result}; @@ -96,3 +97,169 @@ where retry_hint )) } + +#[allow(dead_code)] +pub fn run_with_retry_sync( + policy: RetryPolicy, + operation_name: &str, + retry_hint: &str, + mut operation: Op, +) -> Result +where + Op: FnMut(u32) -> Result, +{ + ensure!( + policy.max_attempts > 0, + "Retry policy requires max_attempts >= 1" + ); + ensure!( + policy.timeout_ms > 0, + "Retry policy requires timeout_ms >= 1" + ); + ensure!( + policy.max_backoff_ms >= policy.initial_backoff_ms, + "Retry policy requires max_backoff_ms >= initial_backoff_ms" + ); + + let mut last_error = String::new(); + + for attempt in 1..=policy.max_attempts { + let started_at = Instant::now(); + let outcome = operation(attempt); + let timed_out = started_at.elapsed() >= policy.timeout(); + + match (timed_out, outcome) { + (false, Ok(value)) => return Ok(value), + (true, _) => { + last_error = format!("attempt {attempt} timed out after {}ms", policy.timeout_ms); + } + (false, Err(error)) => { + last_error = error.to_string(); + } + } + + if attempt == policy.max_attempts { + break; + } + + let backoff = policy.backoff_for_attempt(attempt + 1); + tracing::warn!( + event_id = "sce.resilience.retry", + operation = operation_name, + attempt, + max_attempts = policy.max_attempts, + timeout_ms = policy.timeout_ms, + backoff_ms = u64::try_from(backoff.as_millis()).unwrap_or(u64::MAX), + error = %last_error, + "Retrying operation after transient failure" + ); + thread::sleep(backoff); + } + + Err(anyhow!( + "Operation '{operation_name}' failed after {} attempt(s) (timeout={}ms, backoff={}..{}ms). Last error: {}. Try: {}", + policy.max_attempts, + policy.timeout_ms, + policy.initial_backoff_ms, + policy.max_backoff_ms, + last_error, + retry_hint + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sync_retry_succeeds_after_transient_failures() { + let policy = RetryPolicy { + max_attempts: 3, + timeout_ms: 1_000, + initial_backoff_ms: 0, + max_backoff_ms: 0, + }; + let mut attempts = 0; + + let result = + run_with_retry_sync(policy, "sync success", "retry the operation", |attempt| { + attempts = attempt; + if attempt < 3 { + return Err(anyhow!("transient failure {attempt}")); + } + + Ok("ok") + }); + + assert_eq!(result.unwrap(), "ok"); + assert_eq!(attempts, 3); + } + + #[test] + fn sync_retry_reports_exhausted_failures_with_guidance() { + let policy = RetryPolicy { + max_attempts: 2, + timeout_ms: 1_000, + initial_backoff_ms: 0, + max_backoff_ms: 0, + }; + let mut attempts = 0; + + let error = + run_with_retry_sync::<(), _>(policy, "sync failure", "retry later", |attempt| { + attempts = attempt; + Err(anyhow!("nope {attempt}")) + }) + .unwrap_err(); + + assert_eq!(attempts, 2); + let message = error.to_string(); + assert!(message.contains("Operation 'sync failure' failed after 2 attempt(s)")); + assert!(message.contains("Last error: nope 2")); + assert!(message.contains("Try: retry later")); + } + + #[test] + fn sync_retry_treats_slow_attempt_as_timeout() { + let policy = RetryPolicy { + max_attempts: 1, + timeout_ms: 5, + initial_backoff_ms: 0, + max_backoff_ms: 0, + }; + let mut attempts = 0; + + let error = run_with_retry_sync( + policy, + "sync timeout", + "try again when the resource is available", + |attempt| { + attempts = attempt; + thread::sleep(Duration::from_millis(20)); + Ok("late success") + }, + ) + .unwrap_err(); + + assert_eq!(attempts, 1); + assert!(error + .to_string() + .contains("Last error: attempt 1 timed out after 5ms")); + } + + #[test] + fn retry_policy_backoff_is_exponential_and_capped() { + let policy = RetryPolicy { + max_attempts: 5, + timeout_ms: 1_000, + initial_backoff_ms: 5, + max_backoff_ms: 12, + }; + + assert_eq!(policy.backoff_for_attempt(1), Duration::from_millis(0)); + assert_eq!(policy.backoff_for_attempt(2), Duration::from_millis(5)); + assert_eq!(policy.backoff_for_attempt(3), Duration::from_millis(10)); + assert_eq!(policy.backoff_for_attempt(4), Duration::from_millis(12)); + assert_eq!(policy.backoff_for_attempt(5), Duration::from_millis(12)); + } +} diff --git a/config/.opencode/plugins/sce-agent-trace.ts b/config/.opencode/plugins/sce-agent-trace.ts index 33e6ad7a..72e03cbb 100644 --- a/config/.opencode/plugins/sce-agent-trace.ts +++ b/config/.opencode/plugins/sce-agent-trace.ts @@ -1,10 +1,12 @@ -import type { Hooks, Plugin } from "@opencode-ai/plugin"; import { spawn } from "node:child_process"; +import type { Hooks, Plugin } from "@opencode-ai/plugin"; + type OpenCodeEvent = Parameters>[0]["event"]; const REQUIRED_EVENTS: Set = new Set([ "message.updated", + "message.part.updated", "session.created", "session.updated", ]); @@ -22,11 +24,54 @@ type DiffTracePayload = { model_id: string; }; +type ConversationTraceMessageUpdatedItem = { + session_id: string; + message_id: string; + role: EventMessageUpdated["properties"]["info"]["role"]; + generated_at_unix_ms: number; +}; + +type ConversationTraceMessagePartUpdatedItem = { + session_id: string; + message_id: string; + part_type: EventMessagePartUpdated["properties"]["part"]["type"]; + text: unknown; + generated_at_unix_ms: number; +}; + +type ConversationTraceMessageUpdatedPayload = { + type: "message.updated"; + payloads: ConversationTraceMessageUpdatedItem[]; +}; + +type ConversationTraceMessagePartUpdatedPayload = { + type: "message.part.updated"; + payloads: ConversationTraceMessagePartUpdatedItem[]; +}; + +type ConversationTracePayload = + | ConversationTraceMessageUpdatedPayload + | ConversationTraceMessagePartUpdatedPayload; + type EventMessageUpdated = Extract< NonNullable, { type: "message.updated" } >; +type EventMessagePartUpdated = Extract< + NonNullable, + { type: "message.part.updated" } +>; + +function extractDiffEntries( + eventInfo: EventMessageUpdated["properties"]["info"], +) { + if (typeof eventInfo.summary === "object") { + return eventInfo.summary.diffs; + } + return undefined; +} + function extractDiffTracePayload( event: EventMessageUpdated, ): DiffTracePayload | undefined { @@ -36,8 +81,7 @@ function extractDiffTracePayload( return undefined; } - // Access info.summary?.diffs via explicit checks - const diffEntries = eventInfo.summary?.diffs; + const diffEntries = extractDiffEntries(eventInfo); if (!diffEntries || diffEntries.length === 0) { return undefined; @@ -45,13 +89,9 @@ function extractDiffTracePayload( const patches: string[] = []; for (const entry of diffEntries) { - const entryObj = entry as { patch?: string }; - - if (!entryObj.patch) { - continue; + if ("patch" in entry && typeof entry.patch === "string") { + patches.push(entry.patch); } - - patches.push(entryObj.patch); } if (patches.length === 0) { @@ -70,6 +110,122 @@ function shouldCaptureEvent(eventType: OpenCodeEvent["type"]): boolean { return ALL_CAPTURED_EVENTS.has(eventType); } +function buildConversationTracePayload( + event: EventMessageUpdated, +): ConversationTraceMessageUpdatedPayload { + const eventInfo = event.properties.info; + + return { + type: "message.updated", + payloads: [ + { + session_id: eventInfo.sessionID, + message_id: eventInfo.id, + role: eventInfo.role, + generated_at_unix_ms: Date.now(), + }, + ], + }; +} + +export function buildMessagePartConversationTracePayload( + event: EventMessagePartUpdated, +): ConversationTraceMessagePartUpdatedPayload { + const eventPart = event.properties.part; + + return { + type: "message.part.updated", + payloads: [ + { + session_id: eventPart.sessionID, + message_id: eventPart.messageID, + part_type: eventPart.type, + text: "text" in eventPart ? eventPart.text : "", + generated_at_unix_ms: Date.now(), + }, + ], + }; +} + +function buildPatchConversationTracePayloads( + event: EventMessageUpdated, +): ConversationTracePayload[] | undefined { + const eventInfo = event.properties.info; + const diffEntries = extractDiffEntries(eventInfo); + + if (!diffEntries || diffEntries.length === 0) { + return undefined; + } + + const patchMessageId = `${eventInfo.id}-patch`; + const payloads: ConversationTracePayload[] = []; + + payloads.push({ + type: "message.updated", + payloads: [ + { + session_id: eventInfo.sessionID, + message_id: patchMessageId, + role: eventInfo.role, + generated_at_unix_ms: Date.now(), + }, + ], + }); + + for (const entry of diffEntries) { + if ("patch" in entry && typeof entry.patch === "string") { + payloads.push({ + type: "message.part.updated", + payloads: [ + { + session_id: eventInfo.sessionID, + message_id: patchMessageId, + part_type: "patch", + text: entry.patch, + generated_at_unix_ms: Date.now(), + }, + ], + }); + } + } + + return payloads; +} + +export async function recordConversationTrace( + repoRoot: string, + event: EventMessageUpdated | EventMessagePartUpdated, +): Promise { + if ( + event.type === "message.part.updated" && + (event.properties.part.type === "reasoning" || + event.properties.part.type === "text") && + event.properties.part.text + ) { + await runConversationTraceHook( + repoRoot, + buildMessagePartConversationTracePayload(event), + ); + return; + } + + if (event.type === "message.updated") { + const patchPayloads = buildPatchConversationTracePayloads(event); + + if (patchPayloads !== undefined) { + await Promise.all( + patchPayloads.map((p) => runConversationTraceHook(repoRoot, p)), + ); + return; + } + + await runConversationTraceHook( + repoRoot, + buildConversationTracePayload(event), + ); + } +} + async function buildTrace( repoRoot: string, event: EventMessageUpdated, @@ -120,9 +276,41 @@ async function runDiffTraceHook( }); } +async function runConversationTraceHook( + repoRoot: string, + payload: ConversationTracePayload, +): Promise { + await new Promise((resolve, reject) => { + const child = spawn("sce", ["hooks", "conversation-trace"], { + cwd: repoRoot, + stdio: ["pipe", "ignore", "inherit"], + }); + + child.on("error", reject); + + child.on("close", (code, signal) => { + if (code === 0) { + resolve(); + return; + } + + const reason = + signal === null ? `exit code ${String(code)}` : `signal ${signal}`; + reject( + new Error( + `Command 'sce hooks conversation-trace' failed with ${reason}.`, + ), + ); + }); + + child.stdin.end(`${JSON.stringify(payload)}\n`); + }); +} + export const SceAgentTracePlugin: Plugin = async ({ directory, worktree }) => { const repoRoot = worktree ?? directory ?? process.cwd(); const clientVersionsBySessionId: Map = new Map(); + const processedDiffsMessageIds: Set = new Set(); return { event: async (input) => { @@ -141,12 +329,30 @@ export const SceAgentTracePlugin: Plugin = async ({ directory, worktree }) => { } if (input.event.type === "message.updated") { + const eventInfo = input.event.properties.info; + const diffEntries = extractDiffEntries(eventInfo); + const hasDiffs = + diffEntries !== undefined && diffEntries.length > 0; + + if (hasDiffs) { + const dedupKey = `${eventInfo.sessionID}:${eventInfo.id}`; + if (processedDiffsMessageIds.has(dedupKey)) { + return; + } + processedDiffsMessageIds.add(dedupKey); + } + const clientVersion = clientVersionsBySessionId.get( input.event.properties.info.sessionID, ) || null; + await recordConversationTrace(repoRoot, input.event); await buildTrace(repoRoot, input.event, clientVersion); } + + if (input.event.type === "message.part.updated") { + await recordConversationTrace(repoRoot, input.event); + } }, }; }; diff --git a/config/automated/.opencode/plugins/sce-agent-trace.ts b/config/automated/.opencode/plugins/sce-agent-trace.ts index 33e6ad7a..72e03cbb 100644 --- a/config/automated/.opencode/plugins/sce-agent-trace.ts +++ b/config/automated/.opencode/plugins/sce-agent-trace.ts @@ -1,10 +1,12 @@ -import type { Hooks, Plugin } from "@opencode-ai/plugin"; import { spawn } from "node:child_process"; +import type { Hooks, Plugin } from "@opencode-ai/plugin"; + type OpenCodeEvent = Parameters>[0]["event"]; const REQUIRED_EVENTS: Set = new Set([ "message.updated", + "message.part.updated", "session.created", "session.updated", ]); @@ -22,11 +24,54 @@ type DiffTracePayload = { model_id: string; }; +type ConversationTraceMessageUpdatedItem = { + session_id: string; + message_id: string; + role: EventMessageUpdated["properties"]["info"]["role"]; + generated_at_unix_ms: number; +}; + +type ConversationTraceMessagePartUpdatedItem = { + session_id: string; + message_id: string; + part_type: EventMessagePartUpdated["properties"]["part"]["type"]; + text: unknown; + generated_at_unix_ms: number; +}; + +type ConversationTraceMessageUpdatedPayload = { + type: "message.updated"; + payloads: ConversationTraceMessageUpdatedItem[]; +}; + +type ConversationTraceMessagePartUpdatedPayload = { + type: "message.part.updated"; + payloads: ConversationTraceMessagePartUpdatedItem[]; +}; + +type ConversationTracePayload = + | ConversationTraceMessageUpdatedPayload + | ConversationTraceMessagePartUpdatedPayload; + type EventMessageUpdated = Extract< NonNullable, { type: "message.updated" } >; +type EventMessagePartUpdated = Extract< + NonNullable, + { type: "message.part.updated" } +>; + +function extractDiffEntries( + eventInfo: EventMessageUpdated["properties"]["info"], +) { + if (typeof eventInfo.summary === "object") { + return eventInfo.summary.diffs; + } + return undefined; +} + function extractDiffTracePayload( event: EventMessageUpdated, ): DiffTracePayload | undefined { @@ -36,8 +81,7 @@ function extractDiffTracePayload( return undefined; } - // Access info.summary?.diffs via explicit checks - const diffEntries = eventInfo.summary?.diffs; + const diffEntries = extractDiffEntries(eventInfo); if (!diffEntries || diffEntries.length === 0) { return undefined; @@ -45,13 +89,9 @@ function extractDiffTracePayload( const patches: string[] = []; for (const entry of diffEntries) { - const entryObj = entry as { patch?: string }; - - if (!entryObj.patch) { - continue; + if ("patch" in entry && typeof entry.patch === "string") { + patches.push(entry.patch); } - - patches.push(entryObj.patch); } if (patches.length === 0) { @@ -70,6 +110,122 @@ function shouldCaptureEvent(eventType: OpenCodeEvent["type"]): boolean { return ALL_CAPTURED_EVENTS.has(eventType); } +function buildConversationTracePayload( + event: EventMessageUpdated, +): ConversationTraceMessageUpdatedPayload { + const eventInfo = event.properties.info; + + return { + type: "message.updated", + payloads: [ + { + session_id: eventInfo.sessionID, + message_id: eventInfo.id, + role: eventInfo.role, + generated_at_unix_ms: Date.now(), + }, + ], + }; +} + +export function buildMessagePartConversationTracePayload( + event: EventMessagePartUpdated, +): ConversationTraceMessagePartUpdatedPayload { + const eventPart = event.properties.part; + + return { + type: "message.part.updated", + payloads: [ + { + session_id: eventPart.sessionID, + message_id: eventPart.messageID, + part_type: eventPart.type, + text: "text" in eventPart ? eventPart.text : "", + generated_at_unix_ms: Date.now(), + }, + ], + }; +} + +function buildPatchConversationTracePayloads( + event: EventMessageUpdated, +): ConversationTracePayload[] | undefined { + const eventInfo = event.properties.info; + const diffEntries = extractDiffEntries(eventInfo); + + if (!diffEntries || diffEntries.length === 0) { + return undefined; + } + + const patchMessageId = `${eventInfo.id}-patch`; + const payloads: ConversationTracePayload[] = []; + + payloads.push({ + type: "message.updated", + payloads: [ + { + session_id: eventInfo.sessionID, + message_id: patchMessageId, + role: eventInfo.role, + generated_at_unix_ms: Date.now(), + }, + ], + }); + + for (const entry of diffEntries) { + if ("patch" in entry && typeof entry.patch === "string") { + payloads.push({ + type: "message.part.updated", + payloads: [ + { + session_id: eventInfo.sessionID, + message_id: patchMessageId, + part_type: "patch", + text: entry.patch, + generated_at_unix_ms: Date.now(), + }, + ], + }); + } + } + + return payloads; +} + +export async function recordConversationTrace( + repoRoot: string, + event: EventMessageUpdated | EventMessagePartUpdated, +): Promise { + if ( + event.type === "message.part.updated" && + (event.properties.part.type === "reasoning" || + event.properties.part.type === "text") && + event.properties.part.text + ) { + await runConversationTraceHook( + repoRoot, + buildMessagePartConversationTracePayload(event), + ); + return; + } + + if (event.type === "message.updated") { + const patchPayloads = buildPatchConversationTracePayloads(event); + + if (patchPayloads !== undefined) { + await Promise.all( + patchPayloads.map((p) => runConversationTraceHook(repoRoot, p)), + ); + return; + } + + await runConversationTraceHook( + repoRoot, + buildConversationTracePayload(event), + ); + } +} + async function buildTrace( repoRoot: string, event: EventMessageUpdated, @@ -120,9 +276,41 @@ async function runDiffTraceHook( }); } +async function runConversationTraceHook( + repoRoot: string, + payload: ConversationTracePayload, +): Promise { + await new Promise((resolve, reject) => { + const child = spawn("sce", ["hooks", "conversation-trace"], { + cwd: repoRoot, + stdio: ["pipe", "ignore", "inherit"], + }); + + child.on("error", reject); + + child.on("close", (code, signal) => { + if (code === 0) { + resolve(); + return; + } + + const reason = + signal === null ? `exit code ${String(code)}` : `signal ${signal}`; + reject( + new Error( + `Command 'sce hooks conversation-trace' failed with ${reason}.`, + ), + ); + }); + + child.stdin.end(`${JSON.stringify(payload)}\n`); + }); +} + export const SceAgentTracePlugin: Plugin = async ({ directory, worktree }) => { const repoRoot = worktree ?? directory ?? process.cwd(); const clientVersionsBySessionId: Map = new Map(); + const processedDiffsMessageIds: Set = new Set(); return { event: async (input) => { @@ -141,12 +329,30 @@ export const SceAgentTracePlugin: Plugin = async ({ directory, worktree }) => { } if (input.event.type === "message.updated") { + const eventInfo = input.event.properties.info; + const diffEntries = extractDiffEntries(eventInfo); + const hasDiffs = + diffEntries !== undefined && diffEntries.length > 0; + + if (hasDiffs) { + const dedupKey = `${eventInfo.sessionID}:${eventInfo.id}`; + if (processedDiffsMessageIds.has(dedupKey)) { + return; + } + processedDiffsMessageIds.add(dedupKey); + } + const clientVersion = clientVersionsBySessionId.get( input.event.properties.info.sessionID, ) || null; + await recordConversationTrace(repoRoot, input.event); await buildTrace(repoRoot, input.event, clientVersion); } + + if (input.event.type === "message.part.updated") { + await recordConversationTrace(repoRoot, input.event); + } }, }; }; diff --git a/config/lib/agent-trace-plugin/opencode-sce-agent-trace-plugin.ts b/config/lib/agent-trace-plugin/opencode-sce-agent-trace-plugin.ts index 33e6ad7a..72e03cbb 100644 --- a/config/lib/agent-trace-plugin/opencode-sce-agent-trace-plugin.ts +++ b/config/lib/agent-trace-plugin/opencode-sce-agent-trace-plugin.ts @@ -1,10 +1,12 @@ -import type { Hooks, Plugin } from "@opencode-ai/plugin"; import { spawn } from "node:child_process"; +import type { Hooks, Plugin } from "@opencode-ai/plugin"; + type OpenCodeEvent = Parameters>[0]["event"]; const REQUIRED_EVENTS: Set = new Set([ "message.updated", + "message.part.updated", "session.created", "session.updated", ]); @@ -22,11 +24,54 @@ type DiffTracePayload = { model_id: string; }; +type ConversationTraceMessageUpdatedItem = { + session_id: string; + message_id: string; + role: EventMessageUpdated["properties"]["info"]["role"]; + generated_at_unix_ms: number; +}; + +type ConversationTraceMessagePartUpdatedItem = { + session_id: string; + message_id: string; + part_type: EventMessagePartUpdated["properties"]["part"]["type"]; + text: unknown; + generated_at_unix_ms: number; +}; + +type ConversationTraceMessageUpdatedPayload = { + type: "message.updated"; + payloads: ConversationTraceMessageUpdatedItem[]; +}; + +type ConversationTraceMessagePartUpdatedPayload = { + type: "message.part.updated"; + payloads: ConversationTraceMessagePartUpdatedItem[]; +}; + +type ConversationTracePayload = + | ConversationTraceMessageUpdatedPayload + | ConversationTraceMessagePartUpdatedPayload; + type EventMessageUpdated = Extract< NonNullable, { type: "message.updated" } >; +type EventMessagePartUpdated = Extract< + NonNullable, + { type: "message.part.updated" } +>; + +function extractDiffEntries( + eventInfo: EventMessageUpdated["properties"]["info"], +) { + if (typeof eventInfo.summary === "object") { + return eventInfo.summary.diffs; + } + return undefined; +} + function extractDiffTracePayload( event: EventMessageUpdated, ): DiffTracePayload | undefined { @@ -36,8 +81,7 @@ function extractDiffTracePayload( return undefined; } - // Access info.summary?.diffs via explicit checks - const diffEntries = eventInfo.summary?.diffs; + const diffEntries = extractDiffEntries(eventInfo); if (!diffEntries || diffEntries.length === 0) { return undefined; @@ -45,13 +89,9 @@ function extractDiffTracePayload( const patches: string[] = []; for (const entry of diffEntries) { - const entryObj = entry as { patch?: string }; - - if (!entryObj.patch) { - continue; + if ("patch" in entry && typeof entry.patch === "string") { + patches.push(entry.patch); } - - patches.push(entryObj.patch); } if (patches.length === 0) { @@ -70,6 +110,122 @@ function shouldCaptureEvent(eventType: OpenCodeEvent["type"]): boolean { return ALL_CAPTURED_EVENTS.has(eventType); } +function buildConversationTracePayload( + event: EventMessageUpdated, +): ConversationTraceMessageUpdatedPayload { + const eventInfo = event.properties.info; + + return { + type: "message.updated", + payloads: [ + { + session_id: eventInfo.sessionID, + message_id: eventInfo.id, + role: eventInfo.role, + generated_at_unix_ms: Date.now(), + }, + ], + }; +} + +export function buildMessagePartConversationTracePayload( + event: EventMessagePartUpdated, +): ConversationTraceMessagePartUpdatedPayload { + const eventPart = event.properties.part; + + return { + type: "message.part.updated", + payloads: [ + { + session_id: eventPart.sessionID, + message_id: eventPart.messageID, + part_type: eventPart.type, + text: "text" in eventPart ? eventPart.text : "", + generated_at_unix_ms: Date.now(), + }, + ], + }; +} + +function buildPatchConversationTracePayloads( + event: EventMessageUpdated, +): ConversationTracePayload[] | undefined { + const eventInfo = event.properties.info; + const diffEntries = extractDiffEntries(eventInfo); + + if (!diffEntries || diffEntries.length === 0) { + return undefined; + } + + const patchMessageId = `${eventInfo.id}-patch`; + const payloads: ConversationTracePayload[] = []; + + payloads.push({ + type: "message.updated", + payloads: [ + { + session_id: eventInfo.sessionID, + message_id: patchMessageId, + role: eventInfo.role, + generated_at_unix_ms: Date.now(), + }, + ], + }); + + for (const entry of diffEntries) { + if ("patch" in entry && typeof entry.patch === "string") { + payloads.push({ + type: "message.part.updated", + payloads: [ + { + session_id: eventInfo.sessionID, + message_id: patchMessageId, + part_type: "patch", + text: entry.patch, + generated_at_unix_ms: Date.now(), + }, + ], + }); + } + } + + return payloads; +} + +export async function recordConversationTrace( + repoRoot: string, + event: EventMessageUpdated | EventMessagePartUpdated, +): Promise { + if ( + event.type === "message.part.updated" && + (event.properties.part.type === "reasoning" || + event.properties.part.type === "text") && + event.properties.part.text + ) { + await runConversationTraceHook( + repoRoot, + buildMessagePartConversationTracePayload(event), + ); + return; + } + + if (event.type === "message.updated") { + const patchPayloads = buildPatchConversationTracePayloads(event); + + if (patchPayloads !== undefined) { + await Promise.all( + patchPayloads.map((p) => runConversationTraceHook(repoRoot, p)), + ); + return; + } + + await runConversationTraceHook( + repoRoot, + buildConversationTracePayload(event), + ); + } +} + async function buildTrace( repoRoot: string, event: EventMessageUpdated, @@ -120,9 +276,41 @@ async function runDiffTraceHook( }); } +async function runConversationTraceHook( + repoRoot: string, + payload: ConversationTracePayload, +): Promise { + await new Promise((resolve, reject) => { + const child = spawn("sce", ["hooks", "conversation-trace"], { + cwd: repoRoot, + stdio: ["pipe", "ignore", "inherit"], + }); + + child.on("error", reject); + + child.on("close", (code, signal) => { + if (code === 0) { + resolve(); + return; + } + + const reason = + signal === null ? `exit code ${String(code)}` : `signal ${signal}`; + reject( + new Error( + `Command 'sce hooks conversation-trace' failed with ${reason}.`, + ), + ); + }); + + child.stdin.end(`${JSON.stringify(payload)}\n`); + }); +} + export const SceAgentTracePlugin: Plugin = async ({ directory, worktree }) => { const repoRoot = worktree ?? directory ?? process.cwd(); const clientVersionsBySessionId: Map = new Map(); + const processedDiffsMessageIds: Set = new Set(); return { event: async (input) => { @@ -141,12 +329,30 @@ export const SceAgentTracePlugin: Plugin = async ({ directory, worktree }) => { } if (input.event.type === "message.updated") { + const eventInfo = input.event.properties.info; + const diffEntries = extractDiffEntries(eventInfo); + const hasDiffs = + diffEntries !== undefined && diffEntries.length > 0; + + if (hasDiffs) { + const dedupKey = `${eventInfo.sessionID}:${eventInfo.id}`; + if (processedDiffsMessageIds.has(dedupKey)) { + return; + } + processedDiffsMessageIds.add(dedupKey); + } + const clientVersion = clientVersionsBySessionId.get( input.event.properties.info.sessionID, ) || null; + await recordConversationTrace(repoRoot, input.event); await buildTrace(repoRoot, input.event, clientVersion); } + + if (input.event.type === "message.part.updated") { + await recordConversationTrace(repoRoot, input.event); + } }, }; }; diff --git a/config/pkl/base/sce-config-schema.pkl b/config/pkl/base/sce-config-schema.pkl index d2ff424e..80f8c73f 100644 --- a/config/pkl/base/sce-config-schema.pkl +++ b/config/pkl/base/sce-config-schema.pkl @@ -17,6 +17,39 @@ local mutuallyExclusiveConstraints = bash_policy_presets.mutually_exclusive.toLi } ).toListing() +local retryPolicyFieldsSchema = new JsonSchema { + type = "object" + additionalProperties = false + required = new { "max_attempts"; "timeout_ms"; "initial_backoff_ms"; "max_backoff_ms" } + properties { + ["max_attempts"] = new JsonSchema { + type = "integer" + minimum = 1 + } + ["timeout_ms"] = new JsonSchema { + type = "integer" + minimum = 1 + } + ["initial_backoff_ms"] = new JsonSchema { + type = "integer" + minimum = 0 + } + ["max_backoff_ms"] = new JsonSchema { + type = "integer" + minimum = 0 + } + } +} + +local perDbRetrySchema = new JsonSchema { + type = "object" + additionalProperties = false + properties { + ["connection_open"] = retryPolicyFieldsSchema + ["query"] = retryPolicyFieldsSchema + } +} + local sceConfigSchema = new JsonSchema { $schema = "https://json-schema.org/draft/2020-12/schema" $id = "https://sce.crocoder.dev/config.json" @@ -65,6 +98,15 @@ local sceConfigSchema = new JsonSchema { } } } + ["database_retry"] = new JsonSchema { + type = "object" + additionalProperties = false + properties { + ["local_db"] = perDbRetrySchema + ["agent_trace_db"] = perDbRetrySchema + ["auth_db"] = perDbRetrySchema + } + } ["bash"] = new JsonSchema { type = "object" additionalProperties = false diff --git a/config/pkl/rendered b/config/pkl/rendered new file mode 100644 index 00000000..06e1e1a8 --- /dev/null +++ b/config/pkl/rendered @@ -0,0 +1,3 @@ +{ + "rendered": "{\n \"$id\": \"https://sce.crocoder.dev/config.json\",\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"title\": \"SCE Config\",\n \"description\": \"Canonical JSON Schema for global and repo-local sce/config.json files.\",\n \"type\": \"object\",\n \"properties\": {\n \"$schema\": {\n \"type\": \"string\",\n \"const\": \"https://sce.crocoder.dev/config.json\"\n },\n \"log_level\": {\n \"type\": \"string\",\n \"enum\": [\n \"error\",\n \"warn\",\n \"info\",\n \"debug\"\n ]\n },\n \"log_format\": {\n \"type\": \"string\",\n \"enum\": [\n \"text\",\n \"json\"\n ]\n },\n \"log_file\": {\n \"type\": \"string\",\n \"minLength\": 1\n },\n \"log_file_mode\": {\n \"type\": \"string\",\n \"enum\": [\n \"truncate\",\n \"append\"\n ]\n },\n \"timeout_ms\": {\n \"type\": \"integer\",\n \"minimum\": 0\n },\n \"workos_client_id\": {\n \"type\": \"string\"\n },\n \"policies\": {\n \"type\": \"object\",\n \"properties\": {\n \"attribution_hooks\": {\n \"type\": \"object\",\n \"properties\": {\n \"enabled\": {\n \"type\": \"boolean\"\n }\n },\n \"additionalProperties\": false\n },\n \"database_retry\": {\n \"type\": \"object\",\n \"properties\": {\n \"local_db\": {\n \"type\": \"object\",\n \"properties\": {\n \"connection_open\": {\n \"type\": \"object\",\n \"properties\": {\n \"max_attempts\": {\n \"type\": \"integer\",\n \"minimum\": 1\n },\n \"timeout_ms\": {\n \"type\": \"integer\",\n \"minimum\": 1\n },\n \"initial_backoff_ms\": {\n \"type\": \"integer\",\n \"minimum\": 0\n },\n \"max_backoff_ms\": {\n \"type\": \"integer\",\n \"minimum\": 0\n }\n },\n \"additionalProperties\": false,\n \"required\": [\n \"max_attempts\",\n \"timeout_ms\",\n \"initial_backoff_ms\",\n \"max_backoff_ms\"\n ]\n },\n \"query\": {\n \"type\": \"object\",\n \"properties\": {\n \"max_attempts\": {\n \"type\": \"integer\",\n \"minimum\": 1\n },\n \"timeout_ms\": {\n \"type\": \"integer\",\n \"minimum\": 1\n },\n \"initial_backoff_ms\": {\n \"type\": \"integer\",\n \"minimum\": 0\n },\n \"max_backoff_ms\": {\n \"type\": \"integer\",\n \"minimum\": 0\n }\n },\n \"additionalProperties\": false,\n \"required\": [\n \"max_attempts\",\n \"timeout_ms\",\n \"initial_backoff_ms\",\n \"max_backoff_ms\"\n ]\n }\n },\n \"additionalProperties\": false\n },\n \"agent_trace_db\": {\n \"type\": \"object\",\n \"properties\": {\n \"connection_open\": {\n \"type\": \"object\",\n \"properties\": {\n \"max_attempts\": {\n \"type\": \"integer\",\n \"minimum\": 1\n },\n \"timeout_ms\": {\n \"type\": \"integer\",\n \"minimum\": 1\n },\n \"initial_backoff_ms\": {\n \"type\": \"integer\",\n \"minimum\": 0\n },\n \"max_backoff_ms\": {\n \"type\": \"integer\",\n \"minimum\": 0\n }\n },\n \"additionalProperties\": false,\n \"required\": [\n \"max_attempts\",\n \"timeout_ms\",\n \"initial_backoff_ms\",\n \"max_backoff_ms\"\n ]\n },\n \"query\": {\n \"type\": \"object\",\n \"properties\": {\n \"max_attempts\": {\n \"type\": \"integer\",\n \"minimum\": 1\n },\n \"timeout_ms\": {\n \"type\": \"integer\",\n \"minimum\": 1\n },\n \"initial_backoff_ms\": {\n \"type\": \"integer\",\n \"minimum\": 0\n },\n \"max_backoff_ms\": {\n \"type\": \"integer\",\n \"minimum\": 0\n }\n },\n \"additionalProperties\": false,\n \"required\": [\n \"max_attempts\",\n \"timeout_ms\",\n \"initial_backoff_ms\",\n \"max_backoff_ms\"\n ]\n }\n },\n \"additionalProperties\": false\n },\n \"auth_db\": {\n \"type\": \"object\",\n \"properties\": {\n \"connection_open\": {\n \"type\": \"object\",\n \"properties\": {\n \"max_attempts\": {\n \"type\": \"integer\",\n \"minimum\": 1\n },\n \"timeout_ms\": {\n \"type\": \"integer\",\n \"minimum\": 1\n },\n \"initial_backoff_ms\": {\n \"type\": \"integer\",\n \"minimum\": 0\n },\n \"max_backoff_ms\": {\n \"type\": \"integer\",\n \"minimum\": 0\n }\n },\n \"additionalProperties\": false,\n \"required\": [\n \"max_attempts\",\n \"timeout_ms\",\n \"initial_backoff_ms\",\n \"max_backoff_ms\"\n ]\n },\n \"query\": {\n \"type\": \"object\",\n \"properties\": {\n \"max_attempts\": {\n \"type\": \"integer\",\n \"minimum\": 1\n },\n \"timeout_ms\": {\n \"type\": \"integer\",\n \"minimum\": 1\n },\n \"initial_backoff_ms\": {\n \"type\": \"integer\",\n \"minimum\": 0\n },\n \"max_backoff_ms\": {\n \"type\": \"integer\",\n \"minimum\": 0\n }\n },\n \"additionalProperties\": false,\n \"required\": [\n \"max_attempts\",\n \"timeout_ms\",\n \"initial_backoff_ms\",\n \"max_backoff_ms\"\n ]\n }\n },\n \"additionalProperties\": false\n }\n },\n \"additionalProperties\": false\n },\n \"bash\": {\n \"type\": \"object\",\n \"properties\": {\n \"presets\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\",\n \"enum\": [\n \"forbid-git-all\",\n \"forbid-git-commit\",\n \"use-pnpm-over-npm\",\n \"use-bun-over-npm\",\n \"use-nix-flake-over-cargo\"\n ]\n },\n \"uniqueItems\": true,\n \"allOf\": [\n {\n \"not\": {\n \"allOf\": [\n {\n \"contains\": {\n \"const\": \"use-pnpm-over-npm\"\n }\n },\n {\n \"contains\": {\n \"const\": \"use-bun-over-npm\"\n }\n }\n ]\n }\n }\n ]\n },\n \"custom\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"minLength\": 1,\n \"not\": {\n \"enum\": [\n \"forbid-git-all\",\n \"forbid-git-commit\",\n \"use-pnpm-over-npm\",\n \"use-bun-over-npm\",\n \"use-nix-flake-over-cargo\"\n ]\n }\n },\n \"match\": {\n \"type\": \"object\",\n \"properties\": {\n \"argv_prefix\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\",\n \"minLength\": 1\n },\n \"minItems\": 1\n }\n },\n \"additionalProperties\": false,\n \"required\": [\n \"argv_prefix\"\n ]\n },\n \"message\": {\n \"type\": \"string\",\n \"minLength\": 1\n }\n },\n \"additionalProperties\": false,\n \"required\": [\n \"id\",\n \"match\",\n \"message\"\n ]\n }\n }\n },\n \"additionalProperties\": false\n }\n },\n \"additionalProperties\": false\n }\n },\n \"additionalProperties\": false,\n \"dependentRequired\": {\n \"log_file_mode\": [\n \"log_file\"\n ]\n }\n}\n" +} diff --git a/config/schema/sce-config.schema.json b/config/schema/sce-config.schema.json index 10fb446d..a68e9bd2 100644 --- a/config/schema/sce-config.schema.json +++ b/config/schema/sce-config.schema.json @@ -55,6 +55,198 @@ }, "additionalProperties": false }, + "database_retry": { + "type": "object", + "properties": { + "local_db": { + "type": "object", + "properties": { + "connection_open": { + "type": "object", + "properties": { + "max_attempts": { + "type": "integer", + "minimum": 1 + }, + "timeout_ms": { + "type": "integer", + "minimum": 1 + }, + "initial_backoff_ms": { + "type": "integer", + "minimum": 0 + }, + "max_backoff_ms": { + "type": "integer", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "max_attempts", + "timeout_ms", + "initial_backoff_ms", + "max_backoff_ms" + ] + }, + "query": { + "type": "object", + "properties": { + "max_attempts": { + "type": "integer", + "minimum": 1 + }, + "timeout_ms": { + "type": "integer", + "minimum": 1 + }, + "initial_backoff_ms": { + "type": "integer", + "minimum": 0 + }, + "max_backoff_ms": { + "type": "integer", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "max_attempts", + "timeout_ms", + "initial_backoff_ms", + "max_backoff_ms" + ] + } + }, + "additionalProperties": false + }, + "agent_trace_db": { + "type": "object", + "properties": { + "connection_open": { + "type": "object", + "properties": { + "max_attempts": { + "type": "integer", + "minimum": 1 + }, + "timeout_ms": { + "type": "integer", + "minimum": 1 + }, + "initial_backoff_ms": { + "type": "integer", + "minimum": 0 + }, + "max_backoff_ms": { + "type": "integer", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "max_attempts", + "timeout_ms", + "initial_backoff_ms", + "max_backoff_ms" + ] + }, + "query": { + "type": "object", + "properties": { + "max_attempts": { + "type": "integer", + "minimum": 1 + }, + "timeout_ms": { + "type": "integer", + "minimum": 1 + }, + "initial_backoff_ms": { + "type": "integer", + "minimum": 0 + }, + "max_backoff_ms": { + "type": "integer", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "max_attempts", + "timeout_ms", + "initial_backoff_ms", + "max_backoff_ms" + ] + } + }, + "additionalProperties": false + }, + "auth_db": { + "type": "object", + "properties": { + "connection_open": { + "type": "object", + "properties": { + "max_attempts": { + "type": "integer", + "minimum": 1 + }, + "timeout_ms": { + "type": "integer", + "minimum": 1 + }, + "initial_backoff_ms": { + "type": "integer", + "minimum": 0 + }, + "max_backoff_ms": { + "type": "integer", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "max_attempts", + "timeout_ms", + "initial_backoff_ms", + "max_backoff_ms" + ] + }, + "query": { + "type": "object", + "properties": { + "max_attempts": { + "type": "integer", + "minimum": 1 + }, + "timeout_ms": { + "type": "integer", + "minimum": 1 + }, + "initial_backoff_ms": { + "type": "integer", + "minimum": 0 + }, + "max_backoff_ms": { + "type": "integer", + "minimum": 0 + } + }, + "additionalProperties": false, + "required": [ + "max_attempts", + "timeout_ms", + "initial_backoff_ms", + "max_backoff_ms" + ] + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, "bash": { "type": "object", "properties": { diff --git a/context/architecture.md b/context/architecture.md index c7641aaa..e1cdf5a5 100644 --- a/context/architecture.md +++ b/context/architecture.md @@ -98,16 +98,15 @@ The repository includes a new placeholder Rust binary crate at `cli/`. - `cli/src/services/observability.rs` no longer owns duplicate log enums or parsing helpers; it consumes the canonical primitive seam from `cli/src/services/config/mod.rs` and stays focused on logger and telemetry runtime behavior. - `cli/src/cli_schema.rs` is now the canonical owner for top-level command metadata for the real clap-backed command set (`auth`, `config`, `setup`, `doctor`, `hooks`, `version`, `completion`), including the slim top-level help purpose text and per-command visibility on `sce`, `sce help`, and `sce --help`; `cli/src/command_surface.rs` remains the custom top-level help renderer and known-command classifier, adding the synthetic `help` row plus the ASCII banner while consuming that shared metadata instead of maintaining a parallel command catalog. - `cli/src/services/default_paths.rs` is the canonical production path catalog for the CLI: it resolves config/state/cache roots with platform-aware XDG or `dirs` fallbacks through an internal `roots` seam, exposes named default paths for current persisted artifacts and database files (global config, auth tokens, auth DB, local DB, agent trace DB), and owns canonical repo-relative, embedded-asset, install/runtime, hook, and context-path accessors so non-test production path definitions have one shared owner. Current production consumers such as config discovery, doctor reporting, setup/install flows, database adapters, and local hook runtime path resolution consume this shared catalog rather than defining owned path literals in their own modules. -- `cli/src/services/config/mod.rs` is the config service facade and `sce config` orchestration surface (`show`, `validate`, `--help`), with bare `sce config` routed by `cli/src/app.rs` to the same help payload as `sce config --help`. Focused submodules own the implementation slices: `types.rs` owns shared config/runtime primitives, `schema.rs` owns generated schema embedding plus typed file parsing, `policy.rs` owns bash-policy semantic validation plus policy-specific formatting, `resolver.rs` owns deterministic config-file discovery, file-layer merging, explicit value precedence (`flags > env > config file > defaults` where flag-backed), shared auth-key resolution, observability-runtime resolution, attribution-hooks runtime gate resolution, default-discovered invalid-file degradation, and explicit-path fatal errors for `--config` / `SCE_CONFIG_FILE`, and private `render.rs` owns `sce config show` / `sce config validate` text and JSON output construction plus rendering-specific display-value helpers. The facade preserves existing `services::config` imports for startup/auth/hooks callers while delegating command execution to resolution plus rendering submodules. +- `cli/src/services/config/mod.rs` is the config service facade and `sce config` orchestration surface (`show`, `validate`, `--help`), with bare `sce config` routed by `cli/src/app.rs` to the same help payload as `sce config --help`. Focused submodules own the implementation slices: `types.rs` owns shared config/runtime primitives, `schema.rs` owns generated schema embedding plus typed file parsing, `policy.rs` owns bash-policy semantic validation plus policy-specific formatting, `resolver.rs` owns deterministic config-file discovery, file-layer merging, explicit value precedence (`flags > env > config file > defaults` where flag-backed), shared auth-key resolution, observability-runtime resolution, attribution-hooks runtime gate resolution, database-retry config resolution and `DATABASE_RETRY_CONFIG` `OnceLock` initialization, default-discovered invalid-file degradation, and explicit-path fatal errors for `--config` / `SCE_CONFIG_FILE`, and private `render.rs` owns `sce config show` / `sce config validate` text and JSON output construction plus rendering-specific display-value helpers. The facade preserves existing `services::config` imports for startup/auth/hooks callers while delegating command execution to resolution plus rendering submodules. - `cli/src/services/output_format.rs` defines the canonical shared CLI output-format contract (`OutputFormat`) for supporting commands, with deterministic `text|json` parsing and command-scoped actionable invalid-value guidance. - `cli/src/services/config/types.rs` is the canonical owner for the shared runtime/config primitive seam used by the CLI: `LogLevel`, `LogFormat`, `LogFileMode`, the observability env-key constants, and the shared bool parsing helpers used by both config resolution and observability bootstrap; `cli/src/services/config/mod.rs` re-exports those primitives through the facade. - `cli/src/services/capabilities.rs` defines the current broad CLI dependency-injection capability traits consumed by `AppContext`: `FsOps` with `StdFsOps` for filesystem operations and `GitOps` with `ProcessGitOps` for git command execution plus repository-root/hooks-directory resolution. Existing services do not consume these traits internally yet; doctor/setup/hooks/config migration is deferred to later lifecycle/AppContext tasks. - `cli/src/services/lifecycle.rs` defines the current compile-safe `ServiceLifecycle` trait seam. It has default no-op `diagnose(&AppContext)`, `fix(&AppContext, &[HealthProblem])`, and `setup(&AppContext)` methods, with lifecycle-owned health, fix, and setup result types so the trait contract is not publicly anchored to doctor/setup module types. The same module owns the shared lifecycle provider catalog/factory, returning providers in deterministic order (config → local_db → auth_db → agent_trace_db → hooks when requested). Hooks exposes a `HooksLifecycle` provider in `cli/src/services/hooks/lifecycle.rs` for hook rollout diagnosis/fix/setup using lifecycle-owned health records plus the canonical required-hook installer. Config exposes a `ConfigLifecycle` provider in `cli/src/services/config/lifecycle.rs` for global/repo-local config validation and repo-local `.sce/config.json` bootstrap. local_db exposes a `LocalDbLifecycle` provider in `cli/src/services/local_db/lifecycle.rs` for canonical local DB path health, parent-directory readiness/bootstrap, and `LocalDb::new()` setup. auth_db exposes an `AuthDbLifecycle` provider in `cli/src/services/auth_db/lifecycle.rs` for canonical auth DB path health, parent-directory readiness/bootstrap, and `AuthDb::new()` setup. agent_trace_db exposes an `AgentTraceDbLifecycle` provider in `cli/src/services/agent_trace_db/lifecycle.rs` for canonical Agent Trace DB path health, parent-directory readiness/bootstrap, and `AgentTraceDb::new()` setup. Doctor runtime aggregates the full provider catalog for `diagnose` and `fix` and adapts lifecycle records into doctor report/fix records at the orchestration boundary; setup command aggregates the shared catalog for `setup` with hooks included only when requested and adapts hook setup outcomes before rendering setup-owned messages. - `cli/src/services/auth_command/mod.rs` defines the implemented auth command surface for `sce auth login|renew|logout|status`, including device-flow login, stored-token renewal (`--force` supported for renew), logout, and status rendering in text/JSON formats; `cli/src/services/auth_command/command.rs` owns the `AuthCommand` struct and its `RuntimeCommand` impl. -- `cli/src/services/db/mod.rs` provides the shared generic Turso infrastructure seam: `DbSpec` supplies a service-specific name, path, and ordered embedded migrations, while `TursoDb` owns parent-directory creation, `Builder::new_local(...)` initialization, Turso connection setup, and tokio current-thread runtime bridging. The same module also provides `EncryptedTursoDb`, a structurally parallel encrypted adapter that resolves the encryption key via `encryption_key::get_or_create_encryption_key()` and enables Turso local encryption with strict `aegis256` cipher selection. Both public adapters delegate their synchronous `execute`/`query`/`query_map` wrappers and generic migration execution to the shared internal `TursoConnectionCore`, preserving one operation path and per-database `__sce_migrations` metadata. Existing DB files without migration metadata are upgraded by re-applying the current idempotent migration set and recording each migration ID, so setup/lifecycle initialization applies later migrations to already-created databases. The same module owns shared DB lifecycle helpers for path-health problem collection and DB parent-directory bootstrap. `cli/src/services/db/encryption_key.rs` first derives a Turso-compatible 64-character hex key from non-empty `SCE_AUTH_DB_ENCRYPTION_KEY` env-secret text when present, otherwise falls back to keyring-backed credential-store get-or-create behavior; no plaintext auth DB fallback exists. -- `cli/src/services/local_db/mod.rs` provides the concrete local DB spec and `LocalDb` type alias over the shared generic `TursoDb` adapter. `LocalDbSpec` resolves the deterministic persistent runtime DB target through the shared default-path seam and declares no local migrations; `TursoDb` supplies blocking `execute`/`query`, parent-directory creation, Turso connection setup, tokio current-thread runtime bridging, and generic migration execution. -- `cli/src/services/auth_db/mod.rs` provides the encrypted auth DB spec and `AuthDb` type alias over `EncryptedTursoDb`. `AuthDbSpec` resolves `/sce/auth.db` through the shared default-path seam and embeds ordered auth migrations where baseline SQL creates `auth_credentials` without `user_id`, with `updated_at`, and a trigger that auto-refreshes `updated_at` on row updates. Auth DB lifecycle setup/doctor integration is wired through `AuthDbLifecycle`; auth command/token-storage reads/writes are directed through `token_storage.rs`, which now persists tokens via the `auth_credentials` table instead of a JSON file. -- `cli/src/services/agent_trace_db/mod.rs` provides the Agent Trace DB spec and `AgentTraceDb` type alias over `TursoDb`. `AgentTraceDbSpec` resolves `/sce/agent-trace.db` through the shared default-path seam and embeds an ordered split fresh-start baseline migration set (`001_create_diff_traces`, `002_create_post_commit_patch_intersections`, `003_create_agent_traces`, `004_create_diff_traces_time_ms_id_index`, `005_create_agent_traces_agent_trace_id_index`, `006_add_agent_traces_remote_url`, `007_create_agent_traces_remote_url_index`) without `AUTOINCREMENT`; `agent_traces.agent_trace_id` is `NOT NULL UNIQUE`, `agent_traces.remote_url` is nullable, and indexes include `idx_agent_traces_agent_trace_id` plus `idx_agent_traces_remote_url`. The module adds `DiffTraceInsert<'_>`/`insert_diff_trace()` (including `model_id`, `tool_name`, and nullable `tool_version` writes), `PostCommitPatchIntersectionInsert<'_>`/`insert_post_commit_patch_intersection()`, and `AgentTraceInsert<'_>`/`insert_agent_trace()` for parameterized writes plus `recent_diff_trace_patches(cutoff_time_ms, end_time_ms)` for inclusive chronological `diff_traces` reads that parse valid raw patch text and return skipped malformed-row reports. `cli/src/services/agent_trace_db/lifecycle.rs` registers Agent Trace DB setup/doctor lifecycle behavior; runtime writes come from `sce hooks diff-trace` (`diff_traces`) and `sce hooks post-commit` (`post_commit_patch_intersections` + built `agent_traces`). +- `cli/src/services/db/mod.rs` provides the shared generic Turso infrastructure seam: `DbSpec` supplies a service-specific name, path, ordered embedded migrations, and config-file lookup key (`db_config_key()`), while `TursoDb` owns parent-directory creation, `Builder::new_local(...)` initialization (with `experimental_multiprocess_wal(true)` for safe concurrent access), Turso connection setup, tokio current-thread runtime bridging, retry-backed blocking `execute`/`query`/`query_map` wrappers, and generic migration execution with per-database `__sce_migrations` metadata. `TursoDb::new()` and `EncryptedTursoDb::new()` wrap only their local open/connect block in `run_with_retry_sync` using a config-driven connection-open policy resolved from the `DATABASE_RETRY_CONFIG` `OnceLock` (initialized at app startup from `policies.database_retry`) with fallback to the hardcoded defaults (`3` attempts, `1s` timeout, `25ms..200ms` backoff), while both adapters' `execute()`, `query()`, and `query_map()` methods use a config-driven operation policy from the same source with fallback to hardcoded defaults (`3` attempts, `500ms` timeout, `25ms..100ms` backoff). Both adapters' `query_map()` methods retry the initial query and row-fetch loop, then apply caller row mapping after retry completion so mapping failures are not retried. Migration execution remains after connection construction and is not retried. The same module also provides `EncryptedTursoDb`, a structurally parallel encrypted adapter that resolves the encryption key through `encryption_key::get_or_create_encryption_key()` (first checking non-empty `SCE_AUTH_DB_ENCRYPTION_KEY` env-secret before keyring-backed credential-store get-or-create with no plaintext fallback), enables Turso local encryption with strict `aegis256` cipher selection, and exposes retry-backed synchronous `execute`/`query`/`query_map` wrappers plus migration execution. Existing DB files without migration metadata are upgraded by re-applying the current idempotent migration set and recording each migration ID, so setup/lifecycle initialization applies later migrations to already-created databases. The same module owns shared DB lifecycle helpers for path-health problem collection and DB parent-directory bootstrap. `cli/src/services/db/encryption_key.rs` provides stable `OnceLock` plus atomic retry guard for credential-store default registration without mutex poisoning, platform-specific credential-store remediation mentioning the env fallback, and first derives a Turso-compatible 64-character hex key from the env secret when present before falling back to keyring-backed behavior. +- `cli/src/services/local_db/mod.rs` provides the concrete local DB spec and `LocalDb` type alias over the shared generic `TursoDb` adapter. `LocalDbSpec` resolves the deterministic persistent runtime DB target through the shared default-path seam and declares no local migrations; `TursoDb` supplies retry-backed blocking `execute`/`query`, parent-directory creation, Turso connection setup, tokio current-thread runtime bridging, and generic migration execution. +- `cli/src/services/agent_trace_db/mod.rs` provides the Agent Trace DB spec and `AgentTraceDb` type alias over `TursoDb`. `AgentTraceDbSpec` resolves `/sce/agent-trace.db` through the shared default-path seam and embeds ordered fresh-start migrations `001..014` for `diff_traces`, `post_commit_patch_intersections`, `agent_traces`, parent `messages` without `text`, `agent`, or `summary_diffs` columns, append-only `parts` with required `text`, indexes, and `updated_at` triggers; `agent_traces.agent_trace_id` is `NOT NULL UNIQUE` and indexed by `idx_agent_traces_agent_trace_id`. The module adds `DiffTraceInsert<'_>`/`insert_diff_trace()` (including `model_id`, `tool_name`, and nullable `tool_version` writes), `PostCommitPatchIntersectionInsert<'_>`/`insert_post_commit_patch_intersection()`, `AgentTraceInsert<'_>`/`insert_agent_trace()`, `InsertMessageInsert`/`insert_message()` plus `insert_messages()` for parent message inserts with duplicate `(session_id, message_id)` writes ignored, and `InsertPartInsert`/`insert_part()` plus `insert_parts()` for append-only part text writes plus `recent_diff_trace_patches(cutoff_time_ms, end_time_ms)` for inclusive chronological `diff_traces` reads that parse valid raw patch text and return skipped malformed-row reports. `cli/src/services/agent_trace_db/lifecycle.rs` registers Agent Trace DB setup/doctor lifecycle behavior through migration-running `AgentTraceDb::new()`; active hook runtime writes/read paths use `AgentTraceDb::open_for_hooks_without_migrations()` followed by `ensure_schema_ready_for_hooks()` and fail with setup guidance when migration metadata is missing or incomplete. - `cli/src/test_support.rs` provides a shared test-only temp-directory helper (`TestTempDir`) used by service tests that need filesystem fixtures. - `cli/src/services/setup/mod.rs` defines the setup command contract (`SetupMode`, `SetupTarget`, `SetupRequest`, CLI flag parser/validator), an `inquire`-backed interactive target prompter (`InquireSetupTargetPrompter`), setup dispatch outcomes (proceed/cancelled), compile-time embedded asset access (`EmbeddedAsset`, target-scoped iterators, required-hook asset iterators/lookups) generated by `cli/build.rs` from the ephemeral crate-local `cli/assets/generated/config/{opencode,claude}/**` mirror plus `cli/assets/hooks/**`, and focused internal support seams for install-flow vs prompt-flow logic; `cli/src/services/setup/command.rs` owns `SetupCommand` and its `RuntimeCommand` impl. Its install engine/orchestrator stages embedded files and uses a unified remove-and-replace policy (removing existing targets before swapping staged content, with deterministic recovery guidance on swap failure and no backup artifact creation), and formats deterministic completion messaging; required-hook install orchestration (`install_required_git_hooks`) follows the same remove-and-replace policy (removing existing hooks before swapping staged content, with deterministic recovery guidance on swap failure). The setup command derives a repo-root-scoped context from the runtime `AppContext` before aggregating `ServiceLifecycle::setup` calls across lifecycle providers (config → local_db → auth_db → agent_trace_db → hooks when requested), so setup providers receive the runtime logger, telemetry, and capability objects instead of a setup-local replacement context. - `cli/src/services/setup/mod.rs` keeps those responsibilities inside one file for now, but the current ownership split is explicit: the inline `install` module owns repository-path normalization, staging/swap install behavior, required-hook installation, and filesystem safety guards, while the inline `prompt` module owns interactive target selection and prompt styling. @@ -115,8 +114,10 @@ The repository includes a new placeholder Rust binary crate at `cli/`. - `cli/src/services/doctor/mod.rs` owns the current doctor request/report surface while focused submodules (`doctor/inspect.rs`, `doctor/render.rs`, `doctor/fixes.rs`, `doctor/types.rs`) split report fact collection, rendering, manual fix reporting, and doctor-owned domain types into smaller seams; `cli/src/services/doctor/command.rs` owns `DoctorCommand` and its `RuntimeCommand` impl. Runtime doctor execution receives `AppContext`, requests the shared lifecycle provider catalog with hooks included for service-owned `diagnose` and `fix` behavior, adapts lifecycle-owned health/fix records into doctor-owned problem/fix records, and then renders stable text/JSON problem records with category/severity/fixability/remediation fields plus deterministic fix-result reporting in fix mode. Report fact collection still preserves current environment/repository/hook/integration display data, while service-owned lifecycle providers now own config validation, local DB and Agent Trace DB readiness/bootstrap, and hook rollout diagnosis/repair. - `cli/src/services/version/mod.rs` defines the version command parser/rendering contract (`parse_version_request`, `render_version`) with deterministic text output and stable JSON runtime-identification fields; `cli/src/services/version/command.rs` owns the `VersionCommand` struct and its `RuntimeCommand` impl. - `cli/src/services/completion/mod.rs` defines completion parser/rendering contract (`parse_completion_request`, `render_completion`) with deterministic Bash/Zsh/Fish script output aligned to current parser-valid command/flag surfaces; `cli/src/services/completion/command.rs` owns the `CompletionCommand` struct and its `RuntimeCommand` impl. -- `cli/src/services/hooks/mod.rs` defines the current local hook runtime parsing/dispatch (`HookSubcommand`, `run_hooks_subcommand`) plus a commit-msg co-author policy seam (`apply_commit_msg_coauthor_policy`) that injects one canonical SCE trailer only when the disabled-default attribution-hooks config/env control is enabled and `SCE_DISABLED` is false; `cli/src/services/hooks/command.rs` owns `HooksCommand` and its `RuntimeCommand` impl. In the current attribution-only baseline, `pre-commit` and `post-rewrite` are deterministic no-op surfaces; `post-commit` requires validated `--remote-url`, threads that URL through the Agent Trace flow, prints it to stderr, and remains an active intersection + Agent Trace persistence entrypoint (captures current commit patch, queries recent `diff_traces` from the bounded past-7-days window, combines valid patches via `patch::combine_patches`, intersects with post-commit patch via `patch::intersect_patches`, persists result to `post_commit_patch_intersections`, then persists built Agent Trace payloads with range-level `content_hash` values to `agent_traces` in AgentTraceDb without post-commit file artifacts); while `diff-trace` performs STDIN JSON intake, validates required non-empty `sessionID`/`diff`/`model_id`/`tool_name` and required `tool_version` (present and either `null` or non-empty string) plus required `u64` `time` (Unix epoch milliseconds), rejects values that cannot fit AgentTraceDb signed `time_ms` storage, writes one collision-safe parsed-payload `context/tmp/-000000-diff-trace.json` artifact, and inserts the parsed payload fields into AgentTraceDb. Success requires both persistence paths to succeed. -- `cli/src/services/resilience.rs` defines bounded retry/timeout/backoff execution policy (`RetryPolicy`, `run_with_retry`) for transient operation hardening with deterministic failure messaging and retry observability. +- `cli/src/services/hooks/mod.rs` defines the current local hook runtime parsing/dispatch (`HookSubcommand`, `run_hooks_subcommand`) plus a commit-msg co-author policy seam (`apply_commit_msg_coauthor_policy`) that injects one canonical SCE trailer only when the disabled-default attribution-hooks config/env control is enabled and `SCE_DISABLED` is false; `cli/src/services/hooks/command.rs` owns `HooksCommand` and its `RuntimeCommand` impl. In the current local-hook baseline, `pre-commit` and `post-rewrite` are deterministic no-op surfaces; `post-commit` is an active intersection + Agent Trace persistence entrypoint (captures current commit patch, queries recent `diff_traces` from the bounded past-7-days window, combines valid patches via `patch::combine_patches`, intersects with post-commit patch via `patch::intersect_patches`, persists result to `post_commit_patch_intersections`, then persists built Agent Trace payloads with range-level `content_hash` values to `agent_traces` in AgentTraceDb without post-commit file artifacts); `diff-trace` performs STDIN JSON intake, validates required non-empty `sessionID`/`diff`/`model_id`/`tool_name` and required `tool_version` (present and either `null` or non-empty string) plus required `u64` `time` (Unix epoch milliseconds), rejects values that cannot fit AgentTraceDb signed `time_ms` storage, writes one collision-safe parsed-payload `context/tmp/-000000-diff-trace.json` artifact, and inserts the parsed payload fields into AgentTraceDb with both persistence paths required to succeed; and `conversation-trace` validates typed batch `{ type, payloads }` STDIN envelopes for homogeneous normalized snake_case `message.updated` / `message.part.updated` item arrays, records skipped-item parser diagnostics for invalid items, then writes valid textless parent `message.updated` rows through one multi-row `AgentTraceDb::insert_messages(...)` call with duplicate `(session_id, message_id)` writes ignored and valid `message.part.updated` rows with required part `text` through one multi-row `AgentTraceDb::insert_parts(...)` call with DB open failures propagated as command failures, valid-item multi-row insert failures counted as whole-batch skipped without row-by-row fallback, and no `context/tmp` artifact persistence. +- `conversation-trace` persistence accounting is best-effort after readiness-gated DB access: one no-migration `AgentTraceDb` is opened per invocation and schema readiness is checked before insertion, parser-skipped rows are logged/counted as skipped before DB insertion, valid-item multi-row insert failures are logged once and count the whole valid-item batch as skipped without row-by-row fallback, DB open/readiness failures remain command-failing, and success output reports deterministic attempted/persisted/skipped counts. +- `cli/src/services/resilience.rs` defines bounded retry/timeout/backoff execution policy (`RetryPolicy`, async `run_with_retry`, and sync `run_with_retry_sync`) for transient operation hardening with deterministic failure messaging and retry observability. The sync helper uses blocking sleep and elapsed-attempt timeout classification for pure synchronous call sites; the shared database adapters currently use it for constructor open/connect retry, and both `TursoDb` and `EncryptedTursoDb` use it for `execute()`/`query()`/`query_map()` operation retry. +- `TursoDb::open_without_migrations()` is the shared no-migration open seam for hot runtime paths: it uses the same parent-directory creation and connection-open retry behavior as `new()` but intentionally skips `run_migrations()`, so it does not create `__sce_migrations` or apply schema SQL. `AgentTraceDb::open_for_hooks_without_migrations()` exposes this as the Agent Trace-specific runtime-open API, and `AgentTraceDb::ensure_schema_ready_for_hooks()` checks Agent Trace migration metadata parity before hook persistence; `AgentTraceDb::new()` remains the migration-running setup/lifecycle initializer. `cli/src/services/hooks/mod.rs` centralizes active Agent Trace hook DB construction through this no-migration readiness gate for `conversation-trace`, `diff-trace`, and both `post-commit` Agent Trace DB flows. - No user-invocable `sce sync` command is wired in the current runtime; local DB and Agent Trace DB bootstrap flows through lifecycle providers aggregated by setup, and DB health/repair flows through the doctor surface. - `cli/src/services/patch.rs` defines the standalone patch domain model (`ParsedPatch`, `PatchFileChange`, `FileChangeKind`, `PatchHunk`, `TouchedLine`, `TouchedLineKind`) for in-memory parsed unified-diff representation, capturing only touched lines (added/removed) plus minimal per-file/per-hunk metadata while excluding non-hunk headers and unchanged context lines. All types are `serde`-serializable/deserializable with `snake_case` JSON field naming. The module also provides `parse_patch`, a public parser function that converts raw unified-diff text (both `Index:` SVN-style and `diff --git` git-style formats) into `ParsedPatch` structs, with `ParseError` for actionable malformed-input diagnostics. Storage-agnostic JSON load helpers (`load_patch_from_json` for string input, `load_patch_from_json_bytes` for byte input) reconstruct `ParsedPatch` from serialized JSON content with `PatchLoadError` for actionable deserialization diagnostics. Its patch-set operations now include deterministic ordered combination plus target-shaped intersection that prefers exact touched-line matches and falls back to historical `kind`+`content` matching when incremental diffs and canonical post-commit diffs have drifted line numbers; `parse_patch`, `combine_patches`, and `intersect_patches` are consumed by the active post-commit hook runtime. - `cli/src/services/` contains module boundaries for command_registry, lifecycle, auth_command, config, setup, doctor, hooks, version, completion, help, patch, shared database infrastructure, local DB adapters, encrypted auth DB adapters, and Agent Trace DB adapters with explicit trait seams for future implementations. `cli/src/services/command_registry.rs` defines the `RuntimeCommand` trait, `RuntimeCommandHandle` type alias, `CommandRegistry` struct, and `build_default_registry()` function for the command dispatch registry. Service-owned command modules now own the migrated runtime command structs and `RuntimeCommand` impls for help/help-text, version, completion, auth, config, setup, doctor, and hooks. diff --git a/context/cli/cli-command-surface.md b/context/cli/cli-command-surface.md index ce2f4d75..c77a4f52 100644 --- a/context/cli/cli-command-surface.md +++ b/context/cli/cli-command-surface.md @@ -91,8 +91,8 @@ A user-invocable `sync` command is not wired in the current CLI surface; local D - `cli/src/services/doctor/mod.rs` defines the implemented doctor request/report contract (`DoctorRequest`, `DoctorMode`, `run_doctor`) while focused submodules under `cli/src/services/doctor/` handle runtime command dispatch (`command.rs`), diagnosis (`inspect.rs`), rendering (`render.rs`), fix execution (`fixes.rs`), and doctor-owned domain types (`types.rs`). Together they preserve explicit fix-mode parsing, stable text/JSON problem and database-record rendering, deterministic fix-result reporting, and aggregation of `ServiceLifecycle::diagnose`/`ServiceLifecycle::fix` across registered providers (`config`, `local_db`, `auth_db`, `agent_trace_db`, `hooks`). The doctor module coordinates state-root/config/database reporting and validation, an empty default repo-scoped database inventory, path-source detection plus required-hook presence/executable/content checks when a repository target is detected, repo-root installed OpenCode integration presence inventory for `plugins`, `agents`, `commands`, and `skills` derived from the embedded OpenCode setup asset catalog, shared-style bracketed human status token rendering (`[PASS]`, `[FAIL]`, `[MISS]`) with simplified `label (path)` text rows, and repair-mode delegation to service-owned fix implementations. - `cli/src/services/version/mod.rs` defines the version parser/output contract (`parse_version_request`, `render_version`) with deterministic text/JSON output modes; `cli/src/services/version/command.rs` owns the version runtime command handler. - `cli/src/services/completion/mod.rs` defines the completion output contract (`render_completion`) using clap_complete to generate deterministic shell scripts for Bash, Zsh, and Fish; `cli/src/services/completion/command.rs` owns the completion runtime command handler. -- `cli/src/services/hooks/mod.rs` defines production local hook runtime parsing/dispatch (`HookSubcommand`, `run_hooks_subcommand`) for `pre-commit`, `commit-msg`, `post-commit`, `post-rewrite`, and `diff-trace`; `cli/src/services/hooks/command.rs` owns the hook runtime command handler. Current runtime behavior is commit-msg-only attribution behind the disabled-default attribution gate; `pre-commit` and `post-rewrite` are deterministic no-ops; `post-commit` requires validated `--remote-url`, threads that value through Agent Trace flow, prints it to stderr, and remains an active intersection + Agent Trace DB persistence path (captures current commit patch, combines/intersects recent `diff_traces`, persists intersection metadata to `post_commit_patch_intersections`, then persists built Agent Trace payload with range-level `content_hash` values to `agent_traces`); and `diff-trace` performs STDIN JSON intake, required non-empty `sessionID`/`diff`/`model_id`/`tool_name`, required nullable/non-empty `tool_version`, plus required `u64` `time` (Unix epoch milliseconds) validation, non-lossy AgentTraceDb `time_ms` conversion, collision-safe parsed-payload `context/tmp/-000000-diff-trace.json` persistence, and command-failing AgentTraceDb insertion. `cli/src/services/hooks/lifecycle.rs` implements `ServiceLifecycle` for hook health checks, fix, and setup (hook rollout integrity and required-hook installation). -- `cli/src/services/resilience.rs` defines shared bounded retry/timeout/backoff execution policy (`RetryPolicy`, `run_with_retry`) with deterministic failure messaging and retry observability hooks. +- `cli/src/services/hooks/mod.rs` defines production local hook runtime parsing/dispatch (`HookSubcommand`, `run_hooks_subcommand`) for `pre-commit`, `commit-msg`, `post-commit`, `post-rewrite`, `diff-trace`, and `conversation-trace`; `cli/src/services/hooks/command.rs` owns the hook runtime command handler. Current runtime behavior is commit-msg-only attribution behind the disabled-default attribution gate; `pre-commit` and `post-rewrite` are deterministic no-ops; `post-commit` is an active intersection + Agent Trace DB persistence path (captures current commit patch, combines/intersects recent `diff_traces`, persists intersection metadata to `post_commit_patch_intersections`, then persists built Agent Trace payload with range-level `content_hash` values to `agent_traces`); `diff-trace` performs STDIN JSON intake, required non-empty `sessionID`/`diff`/`model_id`/`tool_name`, required nullable/non-empty `tool_version`, plus required `u64` `time` (Unix epoch milliseconds) validation, non-lossy AgentTraceDb `time_ms` conversion, collision-safe parsed-payload `context/tmp/-000000-diff-trace.json` persistence, and command-failing AgentTraceDb insertion; and `conversation-trace` validates typed batch `{ type, payloads }` STDIN envelopes for homogeneous normalized snake_case `message.updated` / `message.part.updated` item arrays, opens one Agent Trace DB per invocation, logs/counts parser-skipped rows before DB insertion, writes valid Agent Trace DB `messages` through one multi-row `AgentTraceDb::insert_messages(...)` call with duplicate `(session_id, message_id)` writes ignored and valid `parts` through one multi-row `AgentTraceDb::insert_parts(...)` call, logs/counts valid-item multi-row insert failures as whole-batch skipped without row-by-row fallback, reports deterministic attempted/persisted/skipped success counts, and writes no `context/tmp` artifacts. `cli/src/services/hooks/lifecycle.rs` implements `ServiceLifecycle` for hook health checks, fix, and setup (hook rollout integrity and required-hook installation). +- `cli/src/services/resilience.rs` defines shared bounded retry/timeout/backoff execution policy (`RetryPolicy`, async `run_with_retry`, sync `run_with_retry_sync`) with deterministic failure messaging and retry observability hooks; the sync helper is used by shared DB constructors for local open/connect retry and by `TursoDb` operation methods for `execute()`/`query()`/`query_map()` retry. - No `cli/src/services/sync.rs` module exists in the current codebase; `sce sync` command wiring is deferred, while local DB initialization and health ownership are split between setup and doctor. - `cli/src/services/default_paths.rs` defines the canonical per-user persisted-location seam for config/state/cache roots plus named default file paths for current persisted artifacts (`global config`, `auth tokens`, `local DB`, `agent trace DB`) used by config discovery, token storage, database adapters, and doctor diagnostics; its internal `roots` seam now owns the platform-aware root-directory resolution so non-test production modules consume shared path accessors instead of resolving owned roots directly. - `cli/src/services/token_storage.rs` defines WorkOS token persistence (`save_tokens`, `load_tokens`, `delete_tokens`) via the encrypted `AuthDb` `auth_credentials` table using a `OnceLock` lazy singleton with constant integer row ID `1`. `token_file_path()` returns the auth DB path. `TokenStorageError` exposes `PathResolution` and `Database` variants. No JSON file I/O remains. @@ -101,7 +101,7 @@ A user-invocable `sync` command is not wired in the current CLI surface; local D ## Local and Agent Trace Turso adapter behavior -- `cli/src/services/local_db/mod.rs` provides `LocalDb = TursoDb` with `new()`, `execute()`, and `query()` inherited from the shared Turso adapter. +- `cli/src/services/local_db/mod.rs` provides `LocalDb = TursoDb` with retry-backed `new()`, `execute()`, `query()`, and `query_map()` inherited from the shared Turso adapter. - `LocalDb::new()` resolves the canonical per-user DB path through `default_paths::local_db_path()`, creates parent directories, opens the local Turso database, and currently runs zero local migrations. - `cli/src/services/agent_trace_db/mod.rs` provides `AgentTraceDb = TursoDb` plus `DiffTraceInsert<'_>` and `insert_diff_trace()` for parameterized writes to `diff_traces`, including nullable `model_id` storage. - `AgentTraceDb::new()` resolves `/sce/agent-trace.db` through `default_paths::agent_trace_db_path()`, creates parent directories through `TursoDb`, opens the Turso database, and runs the ordered embedded `cli/migrations/agent-trace/*.sql` migration set. @@ -127,7 +127,7 @@ A user-invocable `sync` command is not wired in the current CLI surface; local D - `cli/src/app.rs` unit tests cover default-help behavior, auth/config/setup/hooks routing, auth bare/help/nested-help routing, command-local `--help` routing for `doctor`/`hooks`, and failure paths for unknown commands/options and extra arguments. - `cli/src/app.rs` additionally validates setup contract routing for interactive default, explicit target flags, and mutually-exclusive setup flag failures. - `cli/src/services/local_db/mod.rs` tests cover in-memory and file-backed local Turso initialization plus execute/query smoke checks. -- `cli/src/services/resilience.rs` tests lock deterministic retry behavior for transient failures, timeout exhaustion, and actionable terminal error messaging. +- `cli/src/services/resilience.rs` tests lock deterministic sync retry behavior for transient failures, timeout classification, capped backoff, and actionable terminal error messaging. - `cli/src/services/setup/mod.rs` and `cli/src/services/hooks/mod.rs` include contract-focused tests for setup flag parsing/validation, interactive selection/cancellation dispatch, setup run messaging, and hook runtime argument/IO/finalization behavior. - `cli/src/services/token_storage.rs` tests cover token save/load round-trips, missing-file handling, token deletion outcomes, invalid JSON corruption handling, and Unix `0600` file-permission enforcement. - `cli/src/services/auth.rs` tests cover WorkOS device/token payload shape parsing, RFC 8628 device and refresh grant constant wiring, terminal OAuth error mapping with `Try:` guidance, polling decision handling for `authorization_pending`/`slow_down`/terminal outcomes, token-expiry evaluation, and refresh-token re-login guidance for terminal refresh errors. diff --git a/context/cli/patch-service.md b/context/cli/patch-service.md index 7714d532..0f8bccdb 100644 --- a/context/cli/patch-service.md +++ b/context/cli/patch-service.md @@ -8,14 +8,14 @@ Standalone patch domain model and parser in `cli/src/services/patch.rs` for in-m - `PatchFileChange` — per-file change with `old_path`, `new_path`, `FileChangeKind`, and hunks - `FileChangeKind` — enum: `Added`, `Modified`, `Deleted`, `Renamed` (serialized as `snake_case`) - `PatchHunk` — hunk with `old_start`/`old_count`/`new_start`/`new_count` and touched lines -- `TouchedLine` — a single added or removed line with `kind`, `line_number`, and `content` +- `TouchedLine` — a single added or removed line with `kind`, `line_number`, `content`, and optional `session_id` - `TouchedLineKind` — enum: `Added`, `Removed` (serialized as `snake_case`) All types derive `Clone, Debug, Deserialize, Eq, PartialEq, Serialize` and support JSON round-trip fidelity via `serde` with `snake_case` field naming. `TouchedLineKind` additionally derives `Hash` to support set-based intersection operations. ## Parser -`parse_patch(input: &str) -> Result` converts raw unified-diff text into `ParsedPatch` structs. +`parse_patch(input: &str, session_id: Option<&str>) -> Result` converts raw unified-diff text into `ParsedPatch` structs and propagates `session_id` onto each produced touched line (`Some` when provided, `None` otherwise). ### Supported formats @@ -50,7 +50,7 @@ Both functions wrap `serde_json::from_str`/`serde_json::from_slice` and map serd - **File matching**: files are matched by post-change path identity — exact `new_path` equality, or absolute-vs-relative path variants whose normalized path segments share the same relative suffix - **Touched-line matching**: matching prefers exact identity (`kind`, `line_number`, and `content`); when no exact match exists, it falls back to historical reconstruction matching by `kind` and `content` only so canonical post-commit patches can still intersect with earlier incremental diffs whose line numbers drifted -- **Result structure**: only files with at least one overlapping touched line appear in the result; hunks with no overlapping lines are excluded; hunk range metadata (`old_start`, `old_count`, `new_start`, `new_count`) is preserved from the second patch (`b`) so the result keeps the target patch shape, while hunk `model_id` provenance is inherited from the matched hunk in the first patch (`a`) when available (and remains `None` when matched constructed provenance is absent) +- **Result structure**: only files with at least one overlapping touched line appear in the result; hunks with no overlapping lines are excluded; hunk range metadata (`old_start`, `old_count`, `new_start`, `new_count`) is preserved from the second patch (`b`) so the result keeps the target patch shape, while matched-line `session_id` and hunk `model_id` provenance are inherited from the first patch (`a`) when available (and remain `None` when matched constructed provenance is absent) - **Determinism**: the same inputs always produce the same output - **Equivalent-hunk behavior**: semantically identical hunks still intersect when they differ only in surrounding context windows, hunk header ranges, or absolute-vs-relative `Index:` path spelling, as long as their touched-line identities match exactly - **Consumed by**: the post-commit hook runtime combines recent DB diff-trace patches and then intersects with the current commit patch (see `agent-trace-hooks-command-routing.md`). Previously listed as "not yet wired" before T04. diff --git a/context/context-map.md b/context/context-map.md index db2a3ce2..31b4522c 100644 --- a/context/context-map.md +++ b/context/context-map.md @@ -10,9 +10,8 @@ Primary context files: Feature/domain context: - `context/cli/cli-command-surface.md` (CLI command surface including top-level help with ASCII art banner and gradient rendering, setup install flow, WorkOS device authorization flow + token storage behavior, attribution-only hook routing with validated post-commit `--remote-url` plumbing plus DB-backed `diff-trace` dual persistence and post-commit Agent Trace payload persistence including range `content_hash`, setup-owned local DB + Agent Trace DB bootstrap plus doctor DB health coverage, nested flake release package/app installability, and Cargo local install + crates.io readiness policy; `sce sync` command wiring is deferred to `0.4.0`; migrated runtime command structs for help/version/completion/auth/config/setup/doctor/hooks are owned by their respective `services/{name}/command.rs` files, while clap-to-runtime conversion lives in `services/parse/command_runtime.rs`) -- `context/cli/default-path-catalog.md` (canonical production CLI path-ownership contract centered on `cli/src/services/default_paths.rs`, including persisted, repo-relative, embedded-asset, install/runtime, hook, and context-path families plus the regression guard that keeps production path ownership centralized) - `context/cli/default-path-catalog.md` (canonical production CLI path-ownership contract centered on `cli/src/services/default_paths.rs`, including persisted auth/config files, named DB paths for auth/local/Agent Trace databases, repo-relative, embedded-asset, install/runtime, hook, and context-path families plus the regression guard that keeps production path ownership centralized) -- `context/cli/patch-service.md` (standalone patch domain model, parser, JSON load helpers, and set operations in `cli/src/services/patch.rs` for in-memory parsed unified-diff representation, capturing only touched lines plus minimal per-file/per-hunk metadata, supporting both `Index:` SVN-style and `diff --git` git-style formats, with `ParseError` for actionable malformed-input diagnostics, `PatchLoadError`/`load_patch_from_json`/`load_patch_from_json_bytes` for storage-agnostic JSON reconstruction, `intersect_patches` for target-shaped overlap with exact-match-first and historical `kind`+`content` fallback semantics plus matched-constructed-hunk `model_id` provenance inheritance, and `combine_patches` for ordered patch combination with later-wins conflict resolution plus winning-hunk `model_id` provenance inheritance; `parse_patch`, `intersect_patches`, and `combine_patches` are consumed by the active post-commit hook runtime) +- `context/cli/patch-service.md` (standalone patch domain model, parser, JSON load helpers, and set operations in `cli/src/services/patch.rs` for in-memory parsed unified-diff representation, capturing only touched lines plus minimal per-file/per-hunk metadata, supporting both `Index:` SVN-style and `diff --git` git-style formats, with `ParseError` for actionable malformed-input diagnostics, `PatchLoadError`/`load_patch_from_json`/`load_patch_from_json_bytes` for storage-agnostic JSON reconstruction, `intersect_patches` for target-shaped overlap with exact-match-first and historical `kind`+`content` fallback semantics plus matched-constructed-line `session_id` and matched-constructed-hunk `model_id` provenance inheritance, and `combine_patches` for ordered patch combination with later-wins conflict resolution plus winning-hunk `model_id` provenance inheritance; `parse_patch`, `intersect_patches`, and `combine_patches` are consumed by the active post-commit hook runtime) - `context/cli/styling-service.md` (CLI text-mode output styling with `owo-colors` and `comfy-table`, TTY/`NO_COLOR` policy, shared helper API for human-facing surfaces, and per-column right-to-left RGB gradient banner rendering) - `context/cli/config-precedence-contract.md` (implemented `sce config` show/validate command contract, deterministic `flags > env > config file > defaults` resolution order, focused `config/resolver.rs` ownership for config discovery/merge/runtime precedence plus default-discovered invalid-file degradation, focused `config/render.rs` ownership for `show`/`validate` text+JSON output construction, canonical `$schema` acceptance for startup-loaded `sce/config.json` files, shared auth-key env/config/optional baked-default support starting with `workos_client_id`, shared runtime resolution for flat logging observability keys, canonical Pkl-generated `sce/config.json` schema ownership plus CLI embedding/reuse contract, config-file selection order, `show` provenance output, and trimmed `validate` output contract) - `context/cli/capability-traits.md` (current broad CLI dependency-injection capability seam in `cli/src/services/capabilities.rs`, including `FsOps`/`StdFsOps`, `GitOps`/`ProcessGitOps`, git root/hooks resolution behavior, AppContext wiring with capability accessors plus repo-root-scoped context derivation, and test-only unimplemented stubs; current service internals do not consume these traits until later lifecycle migration tasks) @@ -41,19 +40,18 @@ Feature/domain context: - `context/sce/cli-security-hardening-contract.md` (T06 CLI redaction contract, setup `--repo` canonicalization/validation, and setup write-permission probe behavior) - `context/sce/agent-trace-post-rewrite-local-remap-ingestion.md` (current post-rewrite no-op baseline plus historical remap-ingestion reference) - `context/sce/agent-trace-rewrite-trace-transformation.md` (current post-rewrite no-op baseline plus historical rewrite-transformation reference) -- `context/sce/local-db.md` (implemented `cli/src/services/local_db/mod.rs` local database spec with `LocalDb = TursoDb`, canonical local DB path resolution, zero local migrations, and inherited blocking `execute`/`query` methods using the shared Turso adapter) -- `context/sce/shared-turso-db.md` (current shared `cli/src/services/db/mod.rs` Turso database infrastructure seam, including `DbSpec`, generic `TursoDb`, `EncryptedTursoDb` encrypted constructor path with `SCE_AUTH_DB_ENCRYPTION_KEY` env-secret precedence before keyring-backed encryption key resolution via `encryption_key::get_or_create_encryption_key()`, strict `aegis256` selection via Turso `EncryptionOpts`, shared internal `TursoConnectionCore` operation/migration path for both public adapters, stable `OnceLock` plus atomic retry guard for credential-store default registration without mutex poisoning, platform-specific credential-store remediation mentioning the env fallback, per-database `__sce_migrations` tracking, generic embedded migration execution, and current concrete wrappers for `LocalDb`, `AgentTraceDb`, and encrypted `AuthDb`) -- `context/sce/agent-trace-db.md` (implemented `cli/src/services/agent_trace_db/mod.rs` Agent Trace database wrapper with canonical `/sce/agent-trace.db` path, ordered `diff_traces`, `post_commit_patch_intersections`, `diff_traces(time_ms, id)` index, `agent_traces`, nullable `diff_traces.model_id`, nullable `diff_traces.tool_name`, nullable `diff_traces.tool_version`, and nullable `agent_traces.agent_trace_id` migrations applied through shared migration metadata, typed parameterized insert helpers for diff traces including `model_id` + tool metadata, post-commit intersection rows, and built `agent_traces` rows with `agent_trace_id` plus schema-validated trace JSON containing range `content_hash`, inclusive bounded chronological recent `diff_traces` query/parse support with malformed-row skip accounting, registered setup/doctor lifecycle provider, and active hook writers for `diff_traces` intake plus post-commit intersection/agent-trace persistence) -- `context/sce/auth-db.md` (current encrypted auth DB foundation: canonical `/sce/auth.db` path resolver, `AuthDb = EncryptedTursoDb` wrapper, encryption key resolution from non-empty `SCE_AUTH_DB_ENCRYPTION_KEY` env-secret text before OS keyring fallback with no plaintext mode, platform-specific missing/unavailable credential-store remediation that points headless/CI users to the env fallback, baseline migration 001 creating `auth_credentials` without `user_id`, with `updated_at`, and 002 creating the `updated_at` auto-refresh trigger instead of a `user_id` index, and `AuthDbLifecycle` provider registered in the shared lifecycle catalog) +- `context/sce/local-db.md` (implemented `cli/src/services/local_db/mod.rs` local database spec with `LocalDb = TursoDb`, canonical local DB path resolution, zero local migrations, and inherited retry-backed blocking `execute`/`query`/`query_map` methods using the shared Turso adapter) +- `context/sce/shared-turso-db.md` (current shared `cli/src/services/db/mod.rs` Turso database infrastructure seam, including `DbSpec`, generic `TursoDb`, encrypted `EncryptedTursoDb`, config-driven constructor/open-connect retry via `run_with_retry_sync`, no-migration `TursoDb::open_without_migrations()` for hot runtime paths, migration-running `new()`/`run_migrations()` with per-database `__sce_migrations` tracking, config-driven operation retry for `execute`/`query`/`query_map`, row-mapping excluded from retry, generic embedded migration execution, non-mutating `migration_metadata_problems()` and `ensure_schema_ready(setup_guidance)` readiness methods on `TursoDb`, and concrete wrappers for `LocalDb`, `AuthDb`, plus `AgentTraceDb`) +- `context/sce/agent-trace-db.md` (implemented `cli/src/services/agent_trace_db/mod.rs` Agent Trace database wrapper with canonical `/sce/agent-trace.db` path, migration-running `AgentTraceDb::new()` setup/lifecycle initialization, no-migration `AgentTraceDb::open_for_hooks_without_migrations()` runtime-open API, non-mutating `ensure_schema_ready_for_hooks()` delegation to `TursoDb::ensure_schema_ready()` with `Run 'sce setup'.` guidance, ordered `diff_traces`, `post_commit_patch_intersections`, `agent_traces`, parent `messages`, append-only `parts`, indexes/triggers, typed parameterized insert helpers, bounded chronological recent `diff_traces` query/parse support with malformed-row skip accounting, registered setup/doctor lifecycle provider, and active hook writers for `diff_traces`, post-commit intersection/agent-trace persistence, `messages`, and `parts`) - `context/sce/agent-trace-core-schema-migrations.md` (historical reference for removed local DB schema bootstrap behavior; T03 now implements the actual local DB with migrations) - `context/sce/agent-trace-retry-queue-observability.md` (inactive local-hook retry path plus historical retry/metrics reference) - `context/sce/agent-trace-local-hooks-mvp-contract-gap-matrix.md` (T01 Local Hooks MVP production contract freeze and deterministic gap matrix for `agent-trace-local-hooks-production-mvp`) - `context/sce/agent-trace-minimal-generator.md` (implemented a library minimal Agent Trace generator seam at `cli/src/services/agent_trace.rs`, used by the active post-commit hook flow to produce strict `0.1.0` JSON payloads with top-level `version`, UUIDv7 `id` derived from commit-time metadata, caller-provided commit-time `timestamp`, optional top-level `vcs` metadata emitted when present (`type` from enum `git|jj|hg|svn`, `revision` from metadata input; current post-commit flow provides `git`), optional top-level `tool` metadata (`name`/`version`) sourced from builder metadata inputs when overlapping AI content exists, and always-emitted `metadata.sce.version` sourced from the compiled `sce` CLI package version, plus per-file trace data from patch inputs via `intersect_patches(constructed_patch, post_commit_patch)` then `post_commit_patch`-anchored hunk classification into `ai`/`mixed`/`unknown` contributor categories, serialized per conversation as nested `contributor.type` with optional `contributor.model_id` omitted when provenance is missing, one derived `ranges[{start_line,end_line,content_hash}]` entry per post-commit or embedded-patch hunk, and range `content_hash` values that hash touched-line kind/content independent of positions and metadata) -- `context/sce/agent-trace-hooks-command-routing.md` (implemented `sce hooks` command routing plus current runtime behavior: disabled-default commit-msg attribution, no-op `pre-commit`/`post-rewrite` entrypoints, active `post-commit` intersection entrypoint requiring validated `--remote-url`, threading that URL to the Agent Trace flow, printing it to stderr, capturing current commit patch, querying recent `diff_traces` from past 7 days, combining/intersecting patches via `patch::combine_patches` / `patch::intersect_patches`, persisting results to `post_commit_patch_intersections`, building/schema-validating post-commit Agent Trace payloads enriched with optional top-level `tool` metadata, `metadata.sce.version`, and range `content_hash`, and persisting validated payloads to AgentTraceDb `agent_traces` (DB-only), plus `diff-trace` STDIN intake with required non-empty `sessionID`/`diff`/`model_id`/`tool_name`, required nullable/non-empty `tool_version`, required `u64` `time` validation, dual persistence to AgentTraceDb, and collision-safe `context/tmp/-000000-diff-trace.json` artifacts) +- `context/sce/agent-trace-hooks-command-routing.md` (implemented `sce hooks` command routing plus current runtime behavior: disabled-default commit-msg attribution, no-op `pre-commit`/`post-rewrite` entrypoints, active Agent Trace hook DB paths using no-migration readiness-gated AgentTraceDb access, active `post-commit` intersection entrypoint capturing current commit patch, querying recent `diff_traces` from past 7 days, combining valid patches via `patch::combine_patches`, intersecting via `patch::intersect_patches`, persisting results to `post_commit_patch_intersections`, building post-commit Agent Trace payloads enriched with optional top-level `tool` metadata, `metadata.sce.version`, and range `content_hash`, schema-validating them, and persisting validated payloads to AgentTraceDb `agent_traces` (DB-only), plus `diff-trace` STDIN intake with required non-empty `sessionID`/`diff`/`model_id`/`tool_name`, required nullable/non-empty `tool_version`, required `u64` `time` validation, dual persistence to AgentTraceDb, collision-safe `context/tmp/-000000-diff-trace.json` artifacts, and `conversation-trace` typed batch `{ type, payloads }` STDIN validation/persistence for homogeneous normalized textless parent `message.updated` / part-text `message.part.updated` item arrays into AgentTraceDb `messages`/`parts` without `context/tmp` artifacts, parser-skipped-item logging, one-DB-per-invocation multi-row writes, whole-valid-batch insert failure logging/skip accounting without row-by-row fallback, deterministic attempted/persisted/skipped success output, and generated OpenCode plugin one-element typed batch handoff for captured messages/parts) - `context/sce/automated-profile-contract.md` (deterministic gate policy for automated OpenCode profile, including 10 gate categories, permission mappings, automated `/commit` single-commit execution behavior, and automated profile constraints) - `context/sce/bash-tool-policy-enforcement-contract.md` (approved bash-tool blocking contract plus the implementation target for generated OpenCode enforcement, including config schema, argv-prefix matching, fixed preset catalog/messages, and precedence rules) - `context/sce/generated-opencode-plugin-registration.md` (current generated OpenCode plugin-registration contract, canonical Pkl ownership, generated manifest/plugin paths including `sce-bash-policy` + `sce-agent-trace`, and TypeScript source ownership; Claude bash-policy enforcement has been removed from generated outputs) -- `context/sce/opencode-agent-trace-plugin-runtime.md` (current OpenCode agent-trace plugin runtime behavior, including `message.updated` capture filtered to user messages with diffs, `{ sessionID, diff, time, model_id }` extraction from message info, session-scoped OpenCode client version capture from `session.created`/`session.updated`, and CLI handoff to `sce hooks diff-trace` over STDIN JSON with required `tool_name="opencode"` plus required nullable `tool_version`; Rust hook parsing and AgentTraceDb insertion persist required payload fields including `model_id`; `session.diff` event capture has been removed) +- `context/sce/opencode-agent-trace-plugin-runtime.md` (current OpenCode agent-trace plugin runtime behavior, including captured `message.updated` handoff with `summary.diffs` branching: when diffs exist sends `-patch` variant `message.updated` + per-diff `message.part.updated` with `part_type: "patch"` concurrently via `Promise.all`, when no diffs sends original `message.updated` payload; in-memory dedup `Set` keyed by `"${sessionID}:${messageID}"`; captured `message.part.updated` handoff to `sce hooks conversation-trace` as `{ type: "message.part.updated", payloads: [{ session_id, message_id, part_type, text, generated_at_unix_ms }] }` with `text`/`reasoning` only; existing user-message diff extraction for `{ sessionID, diff, time, model_id }`; session-scoped OpenCode client version capture from `session.created`/`session.updated`; and CLI handoff to `sce hooks diff-trace` over STDIN JSON with required `tool_name="opencode"` plus required nullable `tool_version`; Rust hook parsing and AgentTraceDb insertion persist required payload fields including `model_id`) - `context/sce/cli-first-install-channels-contract.md` (current first-wave `sce` install/distribution contract covering supported channels, canonical naming, `.version` release authority, and Nix-owned build policy) - `context/sce/optional-install-channel-integration-test-entrypoint.md` (current opt-in flake app contract for install-channel integration coverage, including thin flake delegation to the Rust runner, shared harness ownership, real npm+Bun+Cargo install flows, channel selector semantics, and the explicit non-default execution boundary) - `context/sce/cli-release-artifact-contract.md` (shared `sce` release artifact naming, checksum/manifest outputs, GitHub Releases as the canonical artifact publication surface, and the current three-target Linux/macOS release workflow topology) diff --git a/context/glossary.md b/context/glossary.md index 7d9ef029..b2d0833b 100644 --- a/context/glossary.md +++ b/context/glossary.md @@ -29,21 +29,27 @@ - `command loop`: The `clap` derive-based parser + dispatcher in `cli/src/cli_schema.rs`, `cli/src/services/parse/command_runtime.rs`, and `cli/src/app.rs` that routes `help`, `config`, `setup`, `doctor`, `auth`, `hooks`, `version`, and `completion`, executes implemented command flows, emits command-local help payloads for supported subcommand trees, and returns deterministic actionable errors for invalid invocation. - `RuntimeCommand seam`: Internal command-execution abstraction where clap-parsed commands are converted into boxed command objects with `name()` and `execute(&AppContext)` methods, allowing app lifecycle orchestration to log and run commands without a single central dispatch `match` covering every command; the `RuntimeCommand` trait and `RuntimeCommandHandle` type alias are defined in `cli/src/services/command_registry.rs`, and the `CommandRegistry` struct maps command names to zero-arg constructor functions for dispatch. Migrated commands (`HelpCommand`, `HelpTextCommand`, `VersionCommand`, `CompletionCommand`, `AuthCommand`, `ConfigCommand`, `SetupCommand`, `DoctorCommand`, `HooksCommand`) live in service-owned `command.rs` files; parsed request construction lives in `cli/src/services/parse/command_runtime.rs` when user-provided options or subcommands are required. - `sce dependency baseline`: Current crate dependency set declared in `cli/Cargo.toml` (`anyhow`, `clap`, `clap_complete`, `dirs`, `hmac`, `inquire`, `reqwest`, `serde`, `serde_json`, `sha2`, `tokio`, `tracing`, `tracing-subscriber`, `turso`) and validated through normal compile/test coverage. -- `local Turso adapter`: Module in `cli/src/services/local_db/mod.rs` that defines `LocalDbSpec` and exposes `LocalDb` as a `TursoDb` alias. It resolves the canonical local DB path with `local_db_path()`, currently declares zero migrations, and inherits `new()`, `execute()`, and `query()` from the shared generic adapter. -- `encrypted Turso adapter`: Generic adapter seam in `cli/src/services/db/mod.rs` exposed as `EncryptedTursoDb`, structurally parallel to `TursoDb` (connection, tokio runtime bridge, spec typing). Its constructor resolves the encryption key via `encryption_key::get_or_create_encryption_key(&db_path, db_name)`, which derives a Turso-compatible 64-character hex key from non-empty `SCE_AUTH_DB_ENCRYPTION_KEY` env-secret text before falling back to OS credential-store keyring get-or-create behavior; credential-store default registration is guarded by stable `OnceLock` plus an atomic in-progress flag so errors or panics leave initialization retryable without mutex poisoning. The adapter enables Turso local encryption with strict `aegis256` cipher selection through `turso::EncryptionOpts`, runs embedded migrations after connect, and exposes synchronous `execute`, `query`, `query_map`, and `run_migrations` helpers with `__sce_migrations` tracking parity. +- `local Turso adapter`: Module in `cli/src/services/local_db/mod.rs` that defines `LocalDbSpec` and exposes `LocalDb` as a `TursoDb` alias. It resolves the canonical local DB path with `local_db_path()`, currently declares zero migrations, and inherits retry-backed `new()`, `execute()`, `query()`, and `query_map()` behavior from the shared generic adapter. +- `encrypted Turso adapter`: Generic adapter seam in `cli/src/services/db/mod.rs` exposed as `EncryptedTursoDb`, structurally parallel to `TursoDb` (connection, tokio runtime bridge, spec typing). Its constructor resolves the encryption key via `encryption_key::get_or_create_encryption_key(&db_path, db_name)`, which derives a Turso-compatible 64-character hex key from non-empty `SCE_AUTH_DB_ENCRYPTION_KEY` env-secret text before falling back to OS credential-store keyring get-or-create behavior; credential-store default registration is guarded by stable `OnceLock` plus an atomic in-progress flag so errors or panics leave initialization retryable without mutex poisoning. The adapter enables Turso local encryption with strict `aegis256` cipher selection through `turso::EncryptionOpts`, wraps encrypted local open/connect in the default DB connection-open retry policy, and runs embedded migrations after retry has produced a connection; the adapter also exposes retry-backed synchronous `execute`, `query`, `query_map`, and `run_migrations` helpers with `__sce_migrations` tracking parity. - `auth DB adapter`: Module in `cli/src/services/auth_db/mod.rs` that defines `AuthDbSpec` and exposes `AuthDb` as an `EncryptedTursoDb` alias. It resolves the canonical `/sce/auth.db` path with `auth_db_path()`, keeps encryption mandatory with `SCE_AUTH_DB_ENCRYPTION_KEY` env-secret precedence before OS keyring fallback and no plaintext mode, and embeds ordered auth migrations where baseline SQL creates `auth_credentials` without `user_id`, with `updated_at`, and a trigger that auto-refreshes `updated_at` on row updates. Auth runtime token-storage is now wired through `cli/src/services/token_storage.rs`, which persists tokens via the `auth_credentials` table in the encrypted auth DB instead of a JSON file. - `AuthDbLifecycle`: Lifecycle provider in `cli/src/services/auth_db/lifecycle.rs` that implements `ServiceLifecycle` for encrypted auth DB setup/doctor integration. `diagnose` collects auth DB path health problems, `fix` bootstraps missing auth DB parent directory, and `setup` calls `AuthDb::new()`. Registered as `LifecycleProviderId::AuthDb` in the shared lifecycle catalog. -- `agent trace DB adapter`: Module in `cli/src/services/agent_trace_db/mod.rs` that defines `AgentTraceDbSpec`, exposes `AgentTraceDb` as a `TursoDb` alias, resolves `/sce/agent-trace.db` through `agent_trace_db_path()`, embeds an ordered split fresh-start migration set (`001..005`) that creates `diff_traces`, `post_commit_patch_intersections`, and `agent_traces` plus `idx_diff_traces_time_ms_id` and `idx_agent_traces_agent_trace_id`, with `agent_traces.agent_trace_id` enforced as `NOT NULL UNIQUE`; provides typed parameterized insert helpers for diff traces including `model_id` + tool metadata, post-commit intersection rows, and built agent-trace rows (including `agent_trace_id`); exposes chronological recent `diff_traces` query/parse support with malformed-row skip accounting; has `AgentTraceDbLifecycle` for setup/doctor integration; and is written by `sce hooks diff-trace` (`diff_traces`) plus `sce hooks post-commit` (`post_commit_patch_intersections` and built `agent_traces`). - - `agent trace DB adapter`: Module in `cli/src/services/agent_trace_db/mod.rs` that defines `AgentTraceDbSpec`, exposes `AgentTraceDb` as a `TursoDb` alias, resolves `/sce/agent-trace.db` through `agent_trace_db_path()`, embeds an ordered split fresh-start migration set (`001..007`) that creates `diff_traces`, `post_commit_patch_intersections`, and `agent_traces`, adds nullable `agent_traces.remote_url`, and creates `idx_diff_traces_time_ms_id`, `idx_agent_traces_agent_trace_id`, and `idx_agent_traces_remote_url`, with `agent_traces.agent_trace_id` enforced as `NOT NULL UNIQUE`; provides typed parameterized insert helpers for diff traces including `model_id` + tool metadata, post-commit intersection rows, and built agent-trace rows (including `agent_trace_id`); exposes chronological recent `diff_traces` query/parse support with malformed-row skip accounting; has `AgentTraceDbLifecycle` for setup/doctor integration; and is written by `sce hooks diff-trace` (`diff_traces`) plus `sce hooks post-commit` (`post_commit_patch_intersections` and built `agent_traces`). +- `agent trace DB adapter`: Module in `cli/src/services/agent_trace_db/mod.rs` that defines `AgentTraceDbSpec`, exposes `AgentTraceDb` as a `TursoDb` alias, resolves `/sce/agent-trace.db` through `agent_trace_db_path()`, embeds an ordered split fresh-start baseline migration set (`001..014`) covering `diff_traces`, `post_commit_patch_intersections`, `agent_traces`, `messages`, and `parts` plus indexes (`idx_diff_traces_time_ms_id`, `idx_agent_traces_agent_trace_id`, `idx_agent_traces_remote_url`, `idx_messages_session_message` (unique), `idx_messages_session_order`, `idx_parts_session_message_order`) and `updated_at` triggers (`trg_messages_updated_at`, `trg_parts_updated_at`), with `agent_traces.agent_trace_id` enforced as `NOT NULL UNIQUE` and no foreign keys between `messages` and `parts`; provides typed parameterized insert helpers for diff traces including `model_id` + tool metadata, post-commit intersection rows, built agent-trace rows (including `agent_trace_id`), duplicate-ignore parent message inserts via `INSERT_MESSAGE_SQL` (`MessageRole`, `InsertMessageInsert`), and append-only part insert via `INSERT_PART_SQL` (`PartType`, `InsertPartInsert`); exposes chronological recent `diff_traces` query/parse support with malformed-row skip accounting; has `AgentTraceDbLifecycle` for setup/doctor migration-running initialization; and is accessed by active Agent Trace hooks through the no-migration readiness gate before `diff_traces`, post-commit intersection/agent-trace, `messages`, or `parts` reads/writes. - `Agent Trace SCE metadata`: Implementation-owned top-level metadata emitted by `build_agent_trace(...)` as `metadata.sce.version`; the value is sourced from the compiled `sce` CLI package version via `env!("CARGO_PKG_VERSION")`, is schema-validated with the rest of the payload, and is persisted in AgentTraceDb `agent_traces.trace_json` without changing the top-level Agent Trace payload/schema `version`. - `Agent Trace range content_hash`: Per-range `content_hash` emitted by `build_agent_trace(...)` inside every `ranges[]` entry as `murmur3:`, computed from the touched-line kind/content of the `post_commit_patch` or embedded-patch hunk used to emit that range while excluding positions, paths, metadata, and database IDs. - `DiffTraceInsert`: Insert payload in `cli/src/services/agent_trace_db/mod.rs` carrying `time_ms`, `session_id`, `patch`, `model_id`, `tool_name`, and nullable `tool_version` for parameterized writes to the `diff_traces` table. -- `DbSpec`: Service-specific database metadata trait in `cli/src/services/db/mod.rs` that supplies a diagnostic database name, canonical path resolver, and ordered embedded migration list for `TursoDb`. -- `TursoDb`: Generic unencrypted Turso database adapter in `cli/src/services/db/mod.rs`; owns parent-directory creation and Turso local open/connect flow, then delegates synchronous `execute()`/`query()`/`query_map()` wrappers and migration execution to the shared internal `TursoConnectionCore` for a `DbSpec` implementation. -- `TursoConnectionCore`: Internal shared operation core in `cli/src/services/db/mod.rs` used by both `TursoDb` and `EncryptedTursoDb`; owns tokio current-thread runtime bridging, synchronous Turso operation wrappers, per-database `__sce_migrations` metadata, and generic embedded migration execution. +- `DbSpec`: Service-specific database metadata trait in `cli/src/services/db/mod.rs` that supplies a diagnostic database name, canonical path resolver, ordered embedded migration list, and config-file lookup key (`db_config_key()`) for `TursoDb`. +- `TursoDb`: Generic unencrypted Turso database adapter in `cli/src/services/db/mod.rs`; owns parent-directory creation and Turso local open/connect flow wrapped in config-driven connection-open retry, then delegates synchronous `execute()`/`query()`/`query_map()` wrappers with config-driven query retry and migration execution through the shared internal `TursoConnectionCore` for a `DbSpec` implementation. +- `TursoConnectionCore`: Internal shared operation core in `cli/src/services/db/mod.rs` used by both `TursoDb` and `EncryptedTursoDb`; owns the Turso connection and tokio current-thread runtime bridging used by the public adapter methods; generic embedded migration execution with per-database `__sce_migrations` metadata is delegated to `run_embedded_migrations` helpers. +- `no-migration DB open path`: `TursoDb::open_without_migrations()` plus the Agent Trace-specific `AgentTraceDb::open_for_hooks_without_migrations()` seam; opens/connects a local Turso database with parent-directory creation and configured connection-open retry but does not create `__sce_migrations` or run embedded schema migrations. Active Agent Trace hook callers route through this path and must pass `AgentTraceDb::ensure_schema_ready_for_hooks()` before persistence or reads. +- `TursoDb migration readiness check`: Public methods on `TursoDb` in `cli/src/services/db/mod.rs` for non-mutating schema-readiness verification: `migration_metadata_problems(&self) -> Result>` queries `__sce_migrations` metadata and compares applied IDs against `M::migrations()`, returning problems (missing table, incomplete migrations, unexpected migrations) or an empty list when ready; `ensure_schema_ready(&self, setup_guidance: &str) -> Result<()>` calls `migration_metadata_problems()` and bails with a formatted error including `M::db_name()` and the caller-provided guidance string when problems are found. `AgentTraceDb::ensure_schema_ready_for_hooks()` delegates to `TursoDb::ensure_schema_ready()` with the Agent Trace–specific `AGENT_TRACE_SCHEMA_SETUP_GUIDANCE` constant. +- `database_retry config namespace`: Nested config namespace under `policies.database_retry` in `sce/config.json`, authored in `config/pkl/base/sce-config-schema.pkl` and parsed/resolved in `cli/src/services/config/mod.rs`. Supports per-database overrides (`local_db`, `agent_trace_db`, `auth_db`) each with optional `connection_open` and `query` objects containing `max_attempts`, `timeout_ms`, `initial_backoff_ms`, `max_backoff_ms`. Validated against JSON Schema at config load and surfaced in `sce config show`/`validate`. Wired into DB adapter constructors and operation methods via config-aware retry resolution with fallback to hardcoded defaults. +- `DatabaseRetryConfig`: Rust type in `cli/src/services/config/mod.rs` holding parsed and validated per-database retry policy overrides (`local_db`/`agent_trace_db`/`auth_db`, each `Option`) from the `policies.database_retry` config namespace. Initialized at app startup via `DATABASE_RETRY_CONFIG` `OnceLock` and consumed by config-aware retry resolution in DB adapters. +- `PerDbRetryConfig`: Rust type in `cli/src/services/config/mod.rs` holding optional `connection_open` and `query` retry policies (`Option`) for one database in the `database_retry` config namespace. +- `DB connection-open retry policy`: Retry policy used by `TursoDb::new()` and `EncryptedTursoDb::new()` for local Turso open/connect, resolved at app startup from `policies.database_retry..connection_open` via the `DATABASE_RETRY_CONFIG` `OnceLock` with fallback to hardcoded defaults (`3` attempts, `1s` elapsed-attempt timeout, `25ms` initial backoff, `200ms` max backoff) through `run_with_retry_sync`; embedded migrations are not covered by this policy. +- `DB query retry policy`: Retry policy used by `TursoDb::execute()`, `TursoDb::query()`, `TursoDb::query_map()`, `EncryptedTursoDb::execute()`, `EncryptedTursoDb::query()`, and `EncryptedTursoDb::query_map()` for local Turso operation retry, resolved from `policies.database_retry..query` via the `DATABASE_RETRY_CONFIG` `OnceLock` with fallback to hardcoded defaults (`3` attempts, `500ms` elapsed-attempt timeout, `25ms` initial backoff, `100ms` max backoff) through `run_with_retry_sync`. `query_map()` retries the initial query and row-fetch loop, then runs caller row mapping outside retry. - `__sce_migrations`: Per-database migration metadata table created by the shared `TursoConnectionCore` migration path behind public adapter `run_migrations()` methods; records applied migration IDs after successful execution so later setup/lifecycle initialization applies only migrations not yet recorded, while existing metadata-less DBs are brought forward by re-applying the current idempotent migration set and recording each ID. - `sync command deferral`: Current plan/state note that a user-invocable `sce sync` command is not wired yet and is deferred to `0.4.0`; local DB and Agent Trace DB bootstrap now flow through lifecycle providers aggregated by the setup command, and DB health/repair flows through the doctor surface. -- `CLI bounded resilience wrapper`: Shared policy in `cli/src/services/resilience.rs` (`RetryPolicy`, `run_with_retry`) that applies deterministic retries/timeouts/capped backoff to transient operations, emits retry observability events, and returns actionable terminal failure guidance. +- `CLI bounded resilience wrapper`: Shared policy in `cli/src/services/resilience.rs` (`RetryPolicy`, async `run_with_retry`, sync `run_with_retry_sync`) that applies deterministic retries/timeouts/capped backoff to transient operations, emits retry observability events, and returns actionable terminal failure guidance. The sync helper is currently wired into shared database constructors for local open/connect retry and into `TursoDb`/`EncryptedTursoDb` operation retry for `execute()`/`query()`/`query_map()`. - `setup service orchestration`: Setup execution logic in `cli/src/services/setup/command.rs` that resolves the repository root, derives a repo-root-scoped `AppContext` from the runtime command context, aggregates `ServiceLifecycle::setup` calls across lifecycle providers (config → local_db → auth_db → agent_trace_db → hooks when requested), handles interactive target selection for config asset installation, and emits deterministic success messaging per target. - `setup target flags`: Mutually-exclusive `sce setup` target selectors (`--opencode`, `--claude`, `--both`) that force non-interactive mode for automation. - `setup mode contract`: `cli/src/services/setup/mod.rs` model where `SetupMode::Interactive` is the default and `SetupMode::NonInteractive(SetupTarget)` is selected only when exactly one target flag is provided. @@ -101,7 +107,7 @@ - `auth config baked default`: Optional key-declared fallback in `cli/src/services/config/mod.rs` (with schema/parsing in `schema.rs`) used only after env and config-file inputs are absent; the first implemented case is `workos_client_id`, which currently falls back to `client_sce_default`. - `setup install engine`: Installer in `cli/src/services/setup/mod.rs` (`install_embedded_setup_assets`) that writes embedded setup assets into per-target staging directories and swaps them into repository-root `.opencode/`/`.claude/` destinations, using a unified remove-and-replace policy that removes existing targets before swapping staged content. - `setup remove-and-replace`: Replacement choreography in `cli/src/services/setup/mod.rs` where existing install targets are removed before staged content is promoted; on swap failure, the engine cleans temporary staging paths and returns deterministic recovery guidance (recover from version control). No backup artifacts are created. -- `hooks command routing contract`: Current hook command parser/dispatcher plus runtime wiring in `cli/src/services/hooks/mod.rs` (`HookSubcommand`, `run_hooks_subcommand`) that supports `pre-commit`, `commit-msg `, `post-commit`, `post-rewrite `, and `diff-trace` with deterministic invocation validation/usage errors; `commit-msg` is the only active attribution path behind the attribution hooks gate, `pre-commit`/`post-rewrite` are deterministic no-op entrypoints, `post-commit` requires validated `--remote-url`, threads that value through the Agent Trace flow, prints it to stderr, captures the current commit patch, queries recent `diff_traces` from the past 7 days, combines valid patches via `patch::combine_patches`, intersects with the post-commit patch via `patch::intersect_patches`, persists the intersection result to `post_commit_patch_intersections`, and persists built Agent Trace payloads to AgentTraceDb `agent_traces` (DB-only, no post-commit Agent Trace file artifact), and `diff-trace` performs STDIN JSON intake with required non-empty `sessionID`/`diff`/`model_id`/`tool_name`, required `tool_version` (present and either `null` or non-empty string), required `u64` `time` validation, non-lossy AgentTraceDb `time_ms` conversion, collision-safe per-invocation artifact persistence at `context/tmp/-000000-diff-trace.json`, and AgentTraceDb insertion. +- `hooks command routing contract`: Current hook command parser/dispatcher plus runtime wiring in `cli/src/services/hooks/mod.rs` (`HookSubcommand`, `run_hooks_subcommand`) that supports `pre-commit`, `commit-msg `, `post-commit`, `post-rewrite `, `diff-trace`, and `conversation-trace` with deterministic invocation validation/usage errors; `commit-msg` is the only active attribution path behind the attribution hooks gate, `pre-commit`/`post-rewrite` are deterministic no-op entrypoints, active Agent Trace hook DB paths use no-migration readiness-gated AgentTraceDb access, `post-commit` actively captures the current commit patch, queries recent `diff_traces` from the past 7 days, combines valid patches via `patch::combine_patches`, intersects with the post-commit patch via `patch::intersect_patches`, persists the intersection result to `post_commit_patch_intersections`, and persists built Agent Trace payloads to AgentTraceDb `agent_traces` (DB-only, no post-commit Agent Trace file artifact), `diff-trace` performs STDIN JSON intake with required non-empty `sessionID`/`diff`/`model_id`/`tool_name`, required `tool_version`, required `u64` `time` validation, collision-safe `context/tmp` artifact persistence, and AgentTraceDb insertion, and `conversation-trace` validates typed batch `{ type, payloads }` STDIN envelopes for homogeneous normalized `message.updated` / `message.part.updated` item arrays before writing valid Agent Trace DB `messages` via one multi-row `insert_messages` call or `parts` via one multi-row `insert_parts` call, with parser-skipped item accounting and whole-valid-batch insert failure skip accounting. - `cloud sync gateway placeholder`: Abstraction in `cli/src/services/sync.rs` (`CloudSyncGateway`) that returns deferred cloud-sync checkpoints while `sync` remains non-production. - `sce CLI onboarding guide`: Crate-local documentation at `cli/README.md` that defines runnable placeholder commands, non-goals/safety limits, and roadmap mapping to service modules. - `plan/code overlap map`: Context artifact at `context/sce/plan-code-overlap-map.md` that classifies Shared Context Plan/Code, `/change-to-plan`, `/next-task`, `/commit`, and core skills into role-specific vs shared-reusable instruction blocks with explicit dedup targets. @@ -119,7 +125,9 @@ - `agent trace historical reference docs`: Retained `context/sce/agent-trace-*.md` artifacts that describe the removed pre-v0.3 Agent Trace design and task slices; they are reference-only and do not describe the active local-hook runtime. - `agent trace commit-msg co-author policy`: Current contract in `cli/src/services/hooks/mod.rs` (`apply_commit_msg_coauthor_policy`) that applies exactly one canonical trailer (`Co-authored-by: SCE `) only when attribution hooks are enabled and SCE is not disabled; duplicate canonical trailers are deduped idempotently. - `local DB migration contract`: `cli/src/services/local_db/mod.rs` delegates migration execution to `TursoDb` through the `DbSpec::migrations()` contract. The current `LocalDbSpec` migration list is empty, so `LocalDb::new()` opens/creates the canonical local DB without creating local tables. -- `hook no-op baseline`: Current `cli/src/services/hooks/mod.rs` runtime posture where `pre-commit` and `post-rewrite` return deterministic no-op status text, `commit-msg` is a gated mutating path behind the disabled-default attribution-hooks control, `post-commit` requires validated `--remote-url`, threads that value through the Agent Trace flow, prints it to stderr, captures current commit patch, queries recent `diff_traces` from past 7 days, combines/intersects patches, persists to `post_commit_patch_intersections`, and persists built Agent Trace payloads to `agent_traces` without post-commit file artifacts, and `diff-trace` is an active intake path (validates required STDIN payload fields including `model_id`, `tool_name`, and required nullable/non-empty `tool_version`, writes collision-safe parsed-payload `context/tmp/-000000-diff-trace.json` artifacts, and inserts parsed payload fields into AgentTraceDb). +- `hook no-op baseline`: Current `cli/src/services/hooks/mod.rs` runtime posture where `pre-commit` and `post-rewrite` return deterministic no-op status text, `commit-msg` is a gated mutating path behind the disabled-default attribution-hooks control, `post-commit` is an active intersection + Agent Trace DB path (captures current commit patch, queries recent `diff_traces` from past 7 days, combines/intersects patches, persists to `post_commit_patch_intersections`, and persists built Agent Trace payloads to `agent_traces` without post-commit file artifacts), `diff-trace` is an active intake path (validates required STDIN payload fields including `model_id`, `tool_name`, and required nullable/non-empty `tool_version`, writes collision-safe parsed-payload `context/tmp/-000000-diff-trace.json` artifacts, and inserts parsed payload fields into AgentTraceDb), and `conversation-trace` is an active typed batch normalized message/part intake path that records parser skipped-item diagnostics and writes valid AgentTraceDb `messages` via one multi-row `insert_messages` call and `parts` via one multi-row `insert_parts` call without `context/tmp` artifacts. +- `conversation-trace typed batch envelope`: Rust hook STDIN shape for `sce hooks conversation-trace`: a top-level JSON object with `type` (`message.updated` or `message.part.updated`) and `payloads` (an array of same-kind normalized snake_case item objects). Item-level `type` fields are rejected to keep batches homogeneous. +- `conversation-trace best-effort accounting`: Current `sce hooks conversation-trace` persistence behavior where one `AgentTraceDb` is opened per hook invocation, parser-skipped items are logged/counted as skipped before DB insertion, valid-item multi-row insert failures are logged once and count the whole valid-item batch as skipped without row-by-row fallback, DB open failure remains command-failing, and success output reports deterministic attempted/persisted/skipped counts. - `sce doctor` operator-health contract: `cli/src/services/doctor/mod.rs` is the stable doctor entrypoint, with focused `doctor/{inspect,render,fixes,types}.rs` submodules implementing the current approved operator-health surface in `context/sce/agent-trace-hook-doctor.md`: `sce doctor --fix` selects repair intent, help/output expose deterministic doctor mode, JSON includes stable problem taxonomy/fixability fields plus database records and fix-result records, the runtime validates state-root resolution, global and repo-local `sce/config.json` readability/schema health, local DB and Agent Trace DB path/health, DB-parent readiness barriers, git availability, non-repo vs bare-repo targeting failures, effective hook-path source resolution, required hook presence/executable/content drift against canonical embedded hook assets, and repo-root installed OpenCode integration presence for `OpenCode plugins`, `OpenCode agents`, `OpenCode commands`, and `OpenCode skills`. Human text mode now uses the approved sectioned layout (`Environment`, `Configuration` (includes Agent Trace DB row), `Repository`, `Git Hooks`, `Integrations`), `SCE doctor diagnose` / `SCE doctor fix` headers, bracketed `[PASS]`/`[FAIL]`/`[MISS]` status tokens with shared-style green/red colorization when enabled, simplified `label (path)` row formatting, top-level-only hook rows, and presence-only integration parent/child rows where missing required files surface as `[MISS]` children and `[FAIL]` parent groups. Fix mode still reuses canonical setup hook installation for missing/stale/non-executable required hooks and missing hooks directories and can bootstrap canonical missing SCE-owned DB parent directories. - `cli warnings-denied lint policy`: `cli/Cargo.toml` sets `warnings = "deny"`, so plain `cargo clippy --manifest-path cli/Cargo.toml` already fails on warnings without needing an extra `-- -D warnings` tail. - `agent trace local DB schema migration contract`: Retired `apply_core_schema_migrations` behavior removed from the current runtime during `agent-trace-removal-and-hook-noop-reset` T01; the local DB baseline is now file open/create only. @@ -146,7 +154,7 @@ - `banner gradient rendering`: Per-column right-to-left RGB color gradient applied to ASCII art banner lines in `cli/src/services/style.rs`; uses `owo-colors` truecolor support with cyan (0, 255, 255) on the right fading to magenta (255, 0, 255) on the left when color is enabled, and plain uncolored ASCII when color is disabled (non-TTY or `NO_COLOR`); spaces in the banner are left unstyled to avoid trailing-space ANSI artifacts - `SCE_BANNER_LINES`: ASCII art "SCE" logo constant in `cli/src/command_surface.rs` rendered at the top of `sce`, `sce help`, and `sce --help` output via `banner_with_gradient`; plain ASCII when color is disabled - `patch domain model`: Standalone `serde`-serializable domain types in `cli/src/services/patch.rs` (`ParsedPatch`, `PatchFileChange`, `FileChangeKind`, `PatchHunk`, `TouchedLine`, `TouchedLineKind`) for in-memory parsed unified-diff representation; captures only touched lines (added/removed) plus minimal per-file/per-hunk metadata, excludes non-hunk headers and unchanged context lines, and supports JSON round-trip fidelity with `snake_case` field naming -- `patch parser`: Public function `parse_patch` in `cli/src/services/patch.rs` that converts raw unified-diff text into `ParsedPatch` structs; supports both `Index:` (SVN-style) and `diff --git` (git-style) formats, handles `/dev/null` paths for new/deleted files, strips `a/`/`b/` prefixes, skips context lines and `\ No newline at end of file` markers, returns actionable `ParseError` for malformed input, and is consumed by post-commit patch capture plus Agent Trace DB recent-row parsing +- `patch parser`: Public function `parse_patch` in `cli/src/services/patch.rs` (`parse_patch(input, session_id)`) that converts raw unified-diff text into `ParsedPatch` structs, propagating the optional session ID into each produced `TouchedLine`; supports both `Index:` (SVN-style) and `diff --git` (git-style) formats, handles `/dev/null` paths for new/deleted files, strips `a/`/`b/` prefixes, skips context lines and `\ No newline at end of file` markers, returns actionable `ParseError` for malformed input, and is consumed by post-commit patch capture plus Agent Trace DB recent-row parsing - `ParseError`: Error type in `cli/src/services/patch.rs` produced when `parse_patch` encounters malformed input (missing file headers, invalid hunk headers, missing `@@` delimiters); carries an actionable `message` field - `FileChangeKind`: Enum in `cli/src/services/patch.rs` classifying a file change as `Added`, `Modified`, `Deleted`, or `Renamed`; serialized as `snake_case` JSON strings - `TouchedLineKind`: Enum in `cli/src/services/patch.rs` classifying a touched line as `Added` or `Removed`; serialized as `snake_case` JSON strings; derives `Hash` to support set-based intersection operations @@ -164,8 +172,10 @@ - `AgentTraceMetadataInput`: Metadata input struct in `cli/src/services/agent_trace.rs` that carries `commit_timestamp` (RFC 3339 commit-time value used as `AgentTrace.timestamp`), `commit_revision` (mapped to `AgentTrace.vcs.revision` when VCS metadata is emitted), and optional `vcs_type` (`Option`, mapped to `AgentTrace.vcs.type` and controlling whether top-level `vcs` is emitted). - `AgentTraceVcsType`: Schema-aligned VCS enum in `cli/src/services/agent_trace.rs` (`Git`, `Jj`, `Hg`, `Svn`) serialized as `snake_case` JSON values (`git`, `jj`, `hg`, `svn`) for `AgentTrace.vcs.type`. - `build_agent_trace`: Public function in `cli/src/services/agent_trace.rs` that computes `intersection_patch = intersect_patches(constructed_patch, post_commit_patch)`, iterates over `post_commit_patch` files and hunks, classifies each hunk against `intersection_patch`, validates `AgentTraceMetadataInput.commit_timestamp` as RFC 3339, derives UUIDv7 `AgentTrace.id` from that same commit-time moment, and returns `Result` with top-level metadata fields plus one `Conversation` per `post_commit_patch` hunk; consumed by the active post-commit hook flow, with no standalone `sce agent-trace` command surface. -- `agent-trace plugin diff extraction seam`: Exported helper `extractDiffTracePayload` in `config/lib/agent-trace-plugin/opencode-sce-agent-trace-plugin.ts` that reads `input.event` and returns `{ sessionID, diff, time, model_id }` only when the event is `message.updated` with `properties.info.role === "user"`; extracts `sessionID` from `info.sessionID` (falling back to `"unknown"` when missing/empty), joins object-entry `patch` fields from `info.summary?.diffs[]` with `\n` while preserving empty patch strings for Rust-side validation, uses `Date.now()` for `time`, and builds `model_id` as `info.model.providerID/info.model.modelID`; returns `undefined` for non-`message.updated` events, non-user messages, messages without a non-empty `summary.diffs` array, or diffs arrays without object entries. - `agent-trace plugin diff extraction seam`: Helper `extractDiffTracePayload` in `config/lib/agent-trace-plugin/opencode-sce-agent-trace-plugin.ts` that accepts a typed `message.updated` event and returns `{ sessionID, diff, time, model_id }` only for user-role messages with non-empty `summary?.diffs`; it joins present object-entry `patch` fields with `\n`, skips entries without `patch`, returns `undefined` when no usable patches remain, uses `Date.now()` for `time`, and builds `model_id` as `providerID/modelID` from `event.properties.info.model`. - `get_or_create_encryption_key`: Public keyring-backed helper in `cli/src/services/db/encryption_key.rs` that retrieves or generates a 64-character hex encryption key from the OS credential store (macOS Keychain, Linux Secret Service via zbus, Windows Credential Store); uses `keyring_core::Entry` with service name `"sce"` and the database name as username. Actively consumed by `EncryptedTursoDb::new()` via the shared adapter constructor. +- `agent-trace plugin conversation-trace handoff seam`: Helper path in `config/lib/agent-trace-plugin/opencode-sce-agent-trace-plugin.ts` where `recordConversationTrace(repoRoot, event)` builds normalized snake_case one-element typed batch envelopes for `message.updated` (`{ type: "message.updated", payloads: [{ session_id, message_id, role, generated_at_unix_ms }] }`) and `message.part.updated` (`{ type: "message.part.updated", payloads: [{ session_id, message_id, part_type, text, generated_at_unix_ms }] }`) and invokes `sce hooks conversation-trace` over STDIN JSON; `message.updated` handoff runs before the plugin's existing diff-trace flow, while part events do not invoke diff-trace. Rust owns validation and AgentTraceDb persistence. - `agent-trace plugin diff-trace hook handoff seam`: Internal helper `runDiffTraceHook` in `config/lib/agent-trace-plugin/opencode-sce-agent-trace-plugin.ts` that invokes `sce hooks diff-trace`, streams `{ sessionID, diff, time, model_id, tool_name, tool_version }` to STDIN JSON (`tool_name` fixed to `opencode`, `tool_version` session-derived when available), and surfaces deterministic invocation failures. - `agent-trace plugin secondary diff artifact ownership`: Current runtime contract where `buildTrace` no longer writes diff-trace artifacts or database rows directly; extracted diff payloads are forwarded to CLI `diff-trace` intake and the Rust hook runtime owns AgentTraceDb insertion plus collision-safe per-invocation artifact persistence. +- `messages table (Agent Trace DB)`: Agent Trace DB table created by migration `008_create_messages.sql`; stores session-scoped parent messages with columns `session_id`, `message_id`, `role` (`user`/`assistant` via CHECK constraint), `generated_at_unix_ms`, `created_at`, and `updated_at`. Message body text belongs to `parts.text`, not the parent `messages` row. Has a unique index on `(session_id, message_id)` for duplicate-ignore parent message inserts and a compound index on `(session_id, generated_at_unix_ms, id)` for chronological session message retrieval. No foreign keys to any other table. +- `parts table (Agent Trace DB)`: Agent Trace DB table created by migration `007_create_parts.sql`; stores append-only message parts with columns `type` (`text`/`reasoning` via CHECK constraint), `text`, `message_id`, `session_id`, `generated_at_unix_ms`, `created_at`, `updated_at`. Uses only the internal `id` for row identity (no upsert/dedup). Multiple parts can exist for the same `(session_id, message_id)`. A compound index on `(session_id, message_id, generated_at_unix_ms, id)` enables ordered joins. No foreign keys to `messages` or any other table, so parts may be inserted before their parent message exists. diff --git a/context/overview.md b/context/overview.md index 77b19d25..061fd95b 100644 --- a/context/overview.md +++ b/context/overview.md @@ -18,13 +18,13 @@ The `setup` command includes an `inquire`-backed target-selection flow: default The CLI now compiles an embedded setup asset manifest from `config/.opencode/**`, `config/.claude/**`, and `cli/assets/hooks/**` via `cli/build.rs`; `cli/src/services/setup/mod.rs` exposes deterministic normalized relative paths plus file bytes and target-scoped iteration without runtime reads from `config/`. The setup service also provides repository-root install orchestration: it resolves the repository root, derives a repo-root-scoped `AppContext` from the runtime command context, aggregates `ServiceLifecycle::setup` calls across lifecycle providers (config → local_db → auth_db → agent_trace_db → hooks when requested), handles interactive or flag-based target selection for config asset installation, and reports deterministic completion details (selected target(s) and installed file counts). Setup uses a unified remove-and-replace policy for all write flows — it removes existing targets before swapping staged content and returns deterministic recovery guidance (recover from version control) on swap failure, without creating backup artifacts. The setup command gates all modes on an existing git repository before any writes. Internally, `cli/src/services/setup/mod.rs` now separates install-flow logic from interactive prompt logic through focused support seams. The CLI now also applies baseline security hardening for reliability-driven automation: diagnostics/logging paths use deterministic secret redaction, `sce setup --hooks --repo ` canonicalizes and validates repository paths before execution, and setup write flows run explicit directory write-permission probes before staging/swap operations. -The config service now provides deterministic runtime config resolution with explicit precedence (`flags > env > config file > defaults`), strict config-file validation (`$schema`, `log_level`, `log_format`, `log_file`, `log_file_mode`, `timeout_ms`, `workos_client_id`, and nested `policies.bash`), deterministic default discovery/merge of global+local config files (`${config_root}/sce/config.json` then `.sce/config.json` with local override, where `config_root` comes from the shared default-path seam with XDG/`dirs::config_dir()` config-root resolution), defaults for the resolved observability value set (`log_level=error`, `log_format=text`, `log_file_mode=truncate`), shared auth-key resolution with optional baked defaults starting at `workos_client_id`, first-class bash-policy preset/custom parsing with deterministic conflict and duplicate-prefix validation, and a canonical Pkl-authored `sce/config.json` JSON Schema generated to `config/schema/sce-config.schema.json` and embedded by `cli/src/services/config/mod.rs` for both `sce config validate` and doctor-time config checks. Runtime startup config loading now keeps parity with that schema by accepting the canonical `"$schema": "https://sce.crocoder.dev/config.json"` declaration in repo-local and global config files, so startup commands such as `sce version` no longer fail before dispatch on that field. App-runtime observability now consumes flat logging keys through the shared resolver, so env values still override config-file values while config files provide deterministic fallback for file logging; `sce config show` reports resolved observability/auth/policy values with provenance, while `sce config validate` is now a trimmed validation surface that reports only pass/fail plus validation errors or warnings in text and JSON modes. The canonical preset catalog and matching contract live in `config/pkl/data/bash-policy-presets.json` and `context/sce/bash-tool-policy-enforcement-contract.md`. +The config service now provides deterministic runtime config resolution with explicit precedence (`flags > env > config file > defaults`), strict config-file validation (`$schema`, `log_level`, `log_format`, `log_file`, `log_file_mode`, `timeout_ms`, `workos_client_id`, and nested `policies.bash`, `policies.attribution_hooks.enabled`, plus `policies.database_retry` with per-DB `connection_open`/`query` retry policy specs), deterministic default discovery/merge of global+local config files (`${config_root}/sce/config.json` then `.sce/config.json` with local override, where `config_root` comes from the shared default-path seam with XDG/`dirs::config_dir()` config-root resolution), defaults for the resolved observability value set (`log_level=error`, `log_format=text`, `log_file_mode=truncate`), shared auth-key resolution with optional baked defaults starting at `workos_client_id`, first-class bash-policy preset/custom parsing with deterministic conflict and duplicate-prefix validation, and a canonical Pkl-authored `sce/config.json` JSON Schema generated to `config/schema/sce-config.schema.json` and embedded by `cli/src/services/config/mod.rs` for both `sce config validate` and doctor-time config checks. Runtime startup config loading now keeps parity with that schema by accepting the canonical `"$schema": "https://sce.crocoder.dev/config.json"` declaration in repo-local and global config files, so startup commands such as `sce version` no longer fail before dispatch on that field. App-runtime observability now consumes flat logging keys through the shared resolver, so env values still override config-file values while config files provide deterministic fallback for file logging; `sce config show` reports resolved observability/auth/policy values with provenance, while `sce config validate` is now a trimmed validation surface that reports only pass/fail plus validation errors or warnings in text and JSON modes. The canonical preset catalog and matching contract live in `config/pkl/data/bash-policy-presets.json` and `context/sce/bash-tool-policy-enforcement-contract.md`. Invalid default-discovered config files now also degrade gracefully at startup: `sce` keeps running with degraded observability defaults, logs `sce.config.invalid_config` warnings, and reserves hard failures for explicit `--config` / `SCE_CONFIG_FILE` targets or other truly invalid runtime observability inputs. `cli/src/services/config/mod.rs` is now a module facade that declares focused config submodules (`types`, `schema`, `policy`, `resolver`, private `render`, `command`, and `lifecycle`), re-exporting `pub use types::*` and `pub(crate) use schema::validate_config_file`. Shared config primitive ownership is delegated to `cli/src/services/config/types.rs`; schema loading and file parsing to `cli/src/services/config/schema.rs`; bash-policy semantic validation and policy-specific formatting to `cli/src/services/config/policy.rs`; runtime discovery/precedence to `cli/src/services/config/resolver.rs`; and `sce config show` / `sce config validate` text+JSON output construction to `cli/src/services/config/render.rs`. Downstream modules continue importing through `services::config` unchanged. The CLI now has a minimal `AppContext` dependency-injection container in `cli/src/app.rs` holding `Arc`, `Arc`, `Arc`, `Arc`, and an optional `repo_root: Option`; it can derive repo-root-scoped contexts with `with_repo_root(...)` while preserving runtime dependencies. The broad capability seam lives in `cli/src/services/capabilities.rs`, where `FsOps`/`StdFsOps` wrap filesystem operations and `GitOps`/`ProcessGitOps` wrap git process execution plus repository-root/hooks-directory resolution. The shared default path service in `cli/src/services/default_paths.rs` is now the canonical owner for production CLI path definitions. It resolves per-user config/state/cache roots through a dedicated internal `roots` seam, exposes the current persisted-artifact inventory (global config and auth tokens), and also defines named DB paths (auth DB, local DB, Agent Trace DB) plus the repo-relative, embedded-asset, install/runtime, hook, and context-path accessors consumed across current CLI production code. Non-test production modules should consume this shared catalog instead of hardcoding owned path literals. No default cache-backed persisted artifact currently exists, so cache-root resolution remains available without speculative cache-path features and no legacy default-path fallback is supported. The same config resolver now also owns the attribution-hooks gate used by local hook runtime: `SCE_ATTRIBUTION_HOOKS_ENABLED` overrides `policies.attribution_hooks.enabled`, and the gate defaults to disabled. The config service split now includes `cli/src/services/config/resolver.rs` as the focused owner for config-file discovery, file-layer merging, env/flag/default precedence, auth-key resolution, observability resolution, attribution-hooks resolution, and default-discovered invalid-file degradation; `cli/src/services/config/mod.rs` remains the facade/rendering orchestration surface while preserving existing `services::config` imports. -Generated config now includes repo-local OpenCode plugin assets for both profiles: `sce-bash-policy.ts` plus `sce-agent-trace.ts` are emitted under `config/.opencode/plugins/` and `config/automated/.opencode/plugins/`; the agent-trace plugin extracts `{ sessionID, diff, time, model_id }` from user `message.updated` events with diffs, tracks per-session OpenCode client version from `session.created`/`session.updated`, and sends payloads to `sce hooks diff-trace` with `tool_name="opencode"` plus optional `tool_version`; the Rust hook continues to validate required fields and persists `model_id`, `tool_name`, and nullable `tool_version` into `diff_traces` through AgentTraceDb. Bash-policy also emits shared runtime logic and preset data under `config/.opencode/lib/` (also emitted for `config/automated/.opencode/**`). Claude bash-policy enforcement has been removed from generated outputs. +Generated config now includes repo-local OpenCode plugin assets for both profiles: `sce-bash-policy.ts` plus `sce-agent-trace.ts` are emitted under `config/.opencode/plugins/` and `config/automated/.opencode/plugins/`; the agent-trace plugin sends every captured `message.updated` event to `sce hooks conversation-trace` first as a typed batch envelope `{ type: "message.updated", payloads: [item] }`, sends every captured `message.part.updated` event to the same hook as `{ type: "message.part.updated", payloads: [item] }`, then preserves the existing user-message diff extraction path that sends `{ sessionID, diff, time, model_id }` to `sce hooks diff-trace` with `tool_name="opencode"` plus optional `tool_version` from tracked session lifecycle events. Part events do not invoke diff-trace. The plugin does not currently wrap its event handler in a try/catch; subprocess failures in `runConversationTraceHook` or `buildTrace` will result in unhandled promise rejections. The Rust hooks own validation/persistence, including typed batch conversation-trace `messages`/`parts` writes and `model_id`, `tool_name`, and nullable `tool_version` writes into `diff_traces`. Bash-policy also emits shared runtime logic and preset data under `config/.opencode/lib/` (also emitted for `config/automated/.opencode/**`). Claude bash-policy enforcement has been removed from generated outputs. The `doctor` command now exposes explicit inspection mode (`sce doctor`) and repair-intent mode (`sce doctor --fix`) at the CLI/help/schema level while keeping diagnosis mode read-only. It now validates both current global operator health and the current repo/hook-integrity slice: state-root resolution, global config path resolution, global and repo-local `sce/config.json` readability/schema validity, local DB and Agent Trace DB path + health, DB parent-directory readiness, git availability, non-repo vs bare-repo targeting failures, effective git hook-path source (default, per-repo `core.hooksPath`, or global `core.hooksPath`), hooks-directory health, required hook presence/executable permissions/content drift against canonical embedded SCE-managed hook assets, and repo-root OpenCode integration presence across the installed `plugins`, `agents`, `commands`, and `skills` inventories with embedded SHA-256 content verification for OpenCode assets. Text mode now renders the approved human-only layout with ordered `Environment` / `Configuration` / `Repository` / `Git Hooks` / `Integrations` sections, `SCE doctor diagnose` / `SCE doctor fix` headers, bracketed `[PASS]`/`[FAIL]`/`[MISS]` status tokens, shared-style green pass plus red fail/miss coloring when color output is enabled, simplified `label (path)` row formatting, top-level-only hook rows, and integration parent/child rows that reflect missing vs content-mismatch states; JSON output now reports Agent Trace DB health under `agent_trace_db` (as a row within the Configuration section in text mode). Repo-scoped database reporting is empty by default because no repo-owned SCE database currently exists. Fix mode reuses the canonical setup hook install flow to repair missing/stale/non-executable required hooks and can also bootstrap missing canonical DB parent directories while preserving manual-only guidance for unsupported issues. Local database bootstrap is now owned by `LocalDbLifecycle::setup` and `AgentTraceDbLifecycle::setup` aggregated by the setup command, while doctor validates both DB paths/health and can bootstrap missing parent directories. Wiring a user-invocable `sce sync` command is deferred to `0.4.0`. The repository-root flake (`flake.nix`) now applies a Rust overlay-backed stable toolchain pinned to `1.93.1` (with `rustfmt` and `clippy`), reads package/check version from the repo-root `.version` file, builds `packages.sce` through a Crane `buildDepsOnly` + `buildPackage` pipeline with filtered package sources for the Cargo tree plus required embedded config/assets, and runs `cli-tests`, `cli-clippy`, and `cli-fmt` through Crane-backed check derivations (`cargoTest`, `cargoClippy`, `cargoFmt`) that reuse the same filtered source/toolchain setup. @@ -45,10 +45,10 @@ Context sync now uses an important-change gate: cross-cutting/policy/architectur The `/change-to-plan` command body is also intentionally thin orchestration: it delegates clarification and plan-shape contracts to `sce-plan-authoring` (including one-task/one-atomic-commit task slicing) while keeping wrapper-level plan output and handoff obligations explicit. The generated OpenCode command doc now also emits `entry-skill: sce-plan-authoring` plus an ordered `skills` list. The targeted support commands (`handover`, `commit`, `validate`) keep their thin-wrapper behavior and now also emit machine-readable OpenCode command frontmatter describing their entry skill and ordered skill chain. `/commit` is now split by profile: manual generated commands remain proposal-only and allow split guidance when staged changes mix unrelated goals, while the automated OpenCode `/commit` command generates exactly one commit message and runs `git commit` against the staged diff. The shared `sce-atomic-commit` contract also requires commit bodies to cite affected plan slug(s) and updated task ID(s) when staged changes include `context/plans/*.md`, and to stop for clarification instead of inventing those references when the staged plan diff is ambiguous. The prior no-git-wrapper Agent Trace design artifacts under `context/sce/agent-trace-*.md` are retained only as historical reference; the current CLI runtime no longer wires the removed Agent Trace schema adaptation, payload building, retry replay, or rewrite handling paths into local hook execution. -The hooks service now uses a minimal attribution-only runtime: `commit-msg` is the only hook that mutates behavior, conditionally injecting exactly one canonical SCE trailer when the attribution-hooks gate is enabled and `SCE_DISABLED` is false; `pre-commit` and `post-rewrite` remain deterministic no-op entrypoints; `post-commit` requires validated `--remote-url`, threads that URL through the Agent Trace flow, prints it to stderr, captures current commit patch, queries recent `diff_traces` from past 7 days, combines/intersects patches, persists intersection metadata to `post_commit_patch_intersections`, and persists the schema-validated built Agent Trace payload, including optional top-level `tool` metadata from recent diff-trace rows, top-level `metadata.sce.version` from the compiled `sce` CLI package version, and range-level `content_hash` values, to AgentTraceDb `agent_traces` (DB-only, no post-commit Agent Trace file artifact); and `diff-trace` currently validates/persists required non-empty `sessionID`/`diff`/`model_id`/`tool_name`, required `tool_version` (must be present and either `null` or a non-empty string), plus required `u64` millisecond `time`, with non-lossy AgentTraceDb `time_ms` conversion and collision-safe timestamp+attempt artifact filenames. +The hooks service now uses a local hook runtime where `commit-msg` is the only attribution mutator, conditionally injecting exactly one canonical SCE trailer when the attribution-hooks gate is enabled and `SCE_DISABLED` is false; `pre-commit` and `post-rewrite` remain deterministic no-op entrypoints; active Agent Trace hook paths open AgentTraceDb through the no-migration runtime API and require `ensure_schema_ready_for_hooks()` before reads/writes, so missing or incomplete schema fails with setup guidance instead of running migrations on the hot path; `post-commit` is an active intersection entrypoint that captures current commit patch, queries recent `diff_traces` from past 7 days, combines/intersects patches, persists intersection metadata to `post_commit_patch_intersections`, and persists the schema-validated built Agent Trace payload, including optional top-level `tool` metadata from recent diff-trace rows, top-level `metadata.sce.version` from the compiled `sce` CLI package version, and range-level `content_hash` values, to AgentTraceDb `agent_traces` (DB-only, no post-commit Agent Trace file artifact); `diff-trace` currently validates/persists required non-empty `sessionID`/`diff`/`model_id`/`tool_name`, required `tool_version` (must be present and either `null` or a non-empty string), plus required `u64` millisecond `time`, with non-lossy AgentTraceDb `time_ms` conversion and collision-safe timestamp+attempt artifact filenames; and `conversation-trace` validates typed batch `{ type, payloads }` STDIN envelopes for homogeneous `message.updated` / `message.part.updated` item arrays, uses one readiness-gated AgentTraceDb per invocation, logs/counts parser-skipped items before DB insertion, writes valid messages through one multi-row `AgentTraceDb::insert_messages(...)` call with duplicate `(session_id, message_id)` writes ignored, writes valid parts through one multi-row `AgentTraceDb::insert_parts(...)` call, logs/counts valid-item multi-row insert failures as whole-batch skipped without row-by-row fallback, emits attempted/persisted/skipped success counts, and writes no `context/tmp` artifacts. The CLI now also includes an approved operator-environment doctor contract documented in `context/sce/agent-trace-hook-doctor.md`; the runtime now matches the implemented T06 slice for `sce doctor --fix` parsing/help, stable problem/fix-result reporting, canonical hook-repair reuse, and bounded doctor-owned local-DB directory bootstrap for the missing SCE-owned DB parent path. -The local DB service now provides `LocalDb` as a thin `TursoDb` alias in `cli/src/services/local_db/mod.rs`; `LocalDbSpec` resolves the canonical local DB path from the shared default-path catalog and currently declares zero migrations. Shared Turso infrastructure lives in `cli/src/services/db/mod.rs`, where `DbSpec` and generic `TursoDb` support dual-mode operation — local mode via `turso::Builder::new_local()` when `SCE_SYNC_URL`+`SCE_SYNC_TOKEN` are absent, or sync (Turso Cloud) mode via `turso::sync::Builder::new_remote()` when both are set. It owns parent-directory creation, connection setup, tokio current-thread runtime bridging, synchronous `execute`/`query`/`query_map`, generic migration execution, sync operations (`push`/`pull`/`checkpoint`/`stats`) that are no-ops in local mode (sync is never triggered automatically from `execute()`), and shared DB lifecycle helpers for service-specific database wrappers. Auth DB persistence now has a thin encrypted wrapper in `cli/src/services/auth_db/mod.rs`: `AuthDb = EncryptedTursoDb` resolves `/sce/auth.db` and embeds ordered `auth_tokens` table/index migrations, with lifecycle registration wired through `AuthDbLifecycle` in `cli/src/services/auth_db/lifecycle.rs`; auth runtime token-storage is now wired through `token_storage.rs`, which persists tokens via the `auth_credentials` table instead of a JSON file. Agent Trace persistence now has its own `cli/src/services/agent_trace_db/mod.rs` wrapper, canonical `/sce/agent-trace.db` path, a split fresh-start baseline migration set (`001..007`) covering `diff_traces`, `post_commit_patch_intersections`, `agent_traces`, nullable `agent_traces.remote_url`, and indexes (`idx_diff_traces_time_ms_id`, `idx_agent_traces_agent_trace_id`, `idx_agent_traces_remote_url`) without `AUTOINCREMENT`, plus `agent_traces.agent_trace_id` as `NOT NULL UNIQUE`; it also provides typed parameterized insert helpers for diff traces, post-commit intersection rows, and built agent-trace rows, chronological recent `diff_traces` query/parse support with malformed-row skip accounting, registered setup/doctor lifecycle provider, active `sce hooks diff-trace` writes for `diff_traces`, and active `sce hooks post-commit` writes for built `agent_traces` payloads. -The hooks command surface now also supports concrete runtime subcommand routing (`pre-commit`, `commit-msg`, `post-commit`, `post-rewrite`, `diff-trace`) with deterministic argument/STDIN validation. Current runtime behavior keeps attribution disabled by default: the attribution gate enables canonical trailer insertion in `commit-msg`, `pre-commit`/`post-rewrite` remain deterministic no-ops, `post-commit` requires validated `--remote-url`, threads that URL into the Agent Trace flow, prints it to stderr, and remains the active bounded recent-diff-trace intersection path, while `diff-trace` is the active intake path for parsed STDIN `{ sessionID, diff, time, model_id, tool_name, tool_version }` payload persistence with required non-empty `tool_name`, required nullable/non-empty `tool_version`, required `u64` millisecond `time`, non-lossy AgentTraceDb `time_ms` conversion, and collision-safe timestamp+attempt artifact filenames. This behavior is documented in `context/sce/agent-trace-hooks-command-routing.md`. +The local DB service now provides `LocalDb` as a thin `TursoDb` alias in `cli/src/services/local_db/mod.rs`; `LocalDbSpec` resolves the canonical local DB path from the shared default-path catalog and currently declares zero migrations. Shared Turso infrastructure lives in `cli/src/services/db/mod.rs`, where `DbSpec` and generic `TursoDb` own parent-directory creation, local open/connect via `turso::Builder::new_local()` with `experimental_multiprocess_wal(true)`, config-driven constructor open/connect retry through `run_with_retry_sync` (resolved from `DATABASE_RETRY_CONFIG` `OnceLock` with fallback to hardcoded defaults: `3` attempts, `1s` timeout, `25ms..200ms` backoff), config-driven operation retry for `execute`/`query`/`query_map` on both `TursoDb` and `EncryptedTursoDb` (resolved from the same source with fallback to hardcoded defaults: `3` attempts, `500ms` timeout, `25ms..100ms` backoff, with `query_map` row mapping outside retry), tokio current-thread runtime bridging, generic migration execution outside retry, and shared DB lifecycle helpers for service-specific database wrappers. `EncryptedTursoDb` applies the same constructor retry treatment to encrypted local open/connect after resolving its OS credential-store encryption key. Agent Trace persistence has its own `cli/src/services/agent_trace_db/mod.rs` wrapper, canonical `/sce/agent-trace.db` path, ordered migrations for `diff_traces`, `post_commit_patch_intersections`, `agent_traces`, `messages`, and `parts` plus indexes/triggers, typed parameterized insert helpers, chronological recent `diff_traces` query/parse support, and lifecycle provider coverage. +The hooks command surface now also supports concrete runtime subcommand routing (`pre-commit`, `commit-msg`, `post-commit`, `post-rewrite`, `diff-trace`, `conversation-trace`) with deterministic argument/STDIN validation where implemented. Current runtime behavior keeps attribution disabled by default: the attribution gate enables canonical trailer insertion in `commit-msg`, `pre-commit`/`post-rewrite` remain deterministic no-ops, `post-commit` is the active bounded recent-diff-trace intersection path, `diff-trace` is the active intake path for parsed STDIN `{ sessionID, diff, time, model_id, tool_name, tool_version }` payload persistence with required non-empty `tool_name`, required nullable/non-empty `tool_version`, required `u64` millisecond `time`, non-lossy AgentTraceDb `time_ms` conversion, and collision-safe timestamp+attempt artifact filenames, and `conversation-trace` is the active typed batch message/part intake path for Agent Trace DB `messages`/`parts` writes. Active Agent Trace hook DB access is no-migration and readiness-gated; setup/lifecycle remains responsible for applying migrations. This behavior is documented in `context/sce/agent-trace-hooks-command-routing.md`. The setup service now also exposes deterministic required-hook embedded asset accessors (`iter_required_hook_assets`, `get_required_hook_asset`) backed by canonical templates in `cli/assets/hooks/` for `pre-commit`, `commit-msg`, and `post-commit`; this behavior is documented in `context/sce/setup-githooks-hook-asset-packaging.md`. The setup service now also includes required-hook install orchestration (`install_required_git_hooks`) that resolves repository root and effective hooks path from git truth, enforces deterministic per-hook outcomes (`Installed`/`Updated`/`Skipped`), and uses a unified remove-and-replace policy that removes existing hooks before swapping staged content with deterministic recovery guidance on swap failures; this behavior is documented in `context/sce/setup-githooks-install-flow.md`. The setup command parser/dispatch now also supports composable setup+hooks runs (`sce setup --opencode|--claude|--both --hooks`) plus hooks-only mode (`sce setup --hooks` with optional `--repo `), enforces deterministic compatibility validation (`--repo` requires `--hooks`; target flags remain mutually exclusive), and emits deterministic setup/hook outcome messaging (`installed`/`updated`/`skipped`); this behavior is documented in `context/sce/setup-githooks-cli-ux.md`. diff --git a/context/patterns.md b/context/patterns.md index 16c1e157..bc9f06aa 100644 --- a/context/patterns.md +++ b/context/patterns.md @@ -125,7 +125,7 @@ - Route local Turso access through service adapters so command handlers do not expose low-level `turso` API details. New Turso-backed services should build on `cli/src/services/db/mod.rs` (`DbSpec` + `TursoDb`, or `EncryptedTursoDb` when at-rest encryption is required) for runtime, connection, per-database `__sce_migrations` tracking, and migration infrastructure, then expose domain-specific methods from their own service modules. - For current local DB flows, route initialization through the dedicated adapter (`cli/src/services/local_db/mod.rs`) and invoke it from approved orchestration surfaces such as setup or doctor rather than exposing a partial user command before its contract is approved. - For Turso-backed services with setup/doctor ownership, add service-owned lifecycle providers that reuse shared DB path-health and parent-bootstrap helpers, then register them through `lifecycle_providers()` instead of adding command-local database checks. -- For transient local IO/database hotspots, apply bounded resilience wrappers with explicit retry count, timeout, and capped backoff (`cli/src/services/resilience.rs`) and surface terminal failures with deterministic `Try:` remediation guidance. +- For transient local IO/database hotspots, apply bounded resilience wrappers with explicit retry count, timeout, and capped backoff (`cli/src/services/resilience.rs`) and surface terminal failures with deterministic `Try:` remediation guidance. Use async `run_with_retry` for async operations and sync `run_with_retry_sync` for pure blocking contexts where a Tokio sleep/timeout future cannot be awaited. For Turso database constructors/openers, wrap only the local open/connect operation in retry; keep migration execution outside retry because schema changes must not be replayed. Use `TursoDb::new()` for setup/lifecycle-owned schema initialization and `TursoDb::open_without_migrations()` only for hot runtime paths that verify required schema separately before query/write work. For `TursoDb` and `EncryptedTursoDb` operation retry, convert params to owned cloneable Turso params before retrying `execute()`/`query()`, retry `query_map()` query plus row-fetch failures, and keep caller row-mapping outside retry. Retry policies for both connection-open and query operations can now be configured per database via `policies.database_retry` in `sce/config.json`, parsed and resolved in `cli/src/services/config/mod.rs`, with fallback to hardcoded defaults when the config key is absent. - For SCE operator-health commands, prefer deterministic local diagnostics over implicit pass/fail behavior: report the inspected environment scope, stable problem categories, severity/fixability classes, actionable remediation text, and any path/location facts needed to repair the issue; when repair mode exists, keep outcome vocabulary deterministic and idempotent (`cli/src/services/doctor/mod.rs`, with focused diagnosis/render/fix helpers under `cli/src/services/doctor/`). - For service-owned operator health, keep command modules as thin aggregators over `ServiceLifecycle` providers once a lifecycle slice is wired: providers own diagnosis/fix problem production, while command-specific report builders preserve existing output facts and rendering contracts. - Keep lifecycle provider-list construction centralized in the lifecycle service layer so doctor/setup choose provider inclusion without maintaining parallel concrete provider lists. @@ -134,11 +134,11 @@ - For cross-service CLI dependencies that will be injected through `AppContext`, prefer broad capability traits in `cli/src/services/capabilities.rs` over one-off per-service abstractions; keep production wrappers thin over `std::fs` and `git` process execution until call-site migration tasks approve deeper service refactors. - For future CLI domains, define trait-first service contracts with request/plan models in `cli/src/services/*` and keep placeholder implementations explicitly non-runnable until production behavior is approved. - Model deferred integration boundaries with concrete event/capability data structures (for example hook-runtime attribution snapshots/policies and cloud-sync checkpoints) so later tasks can implement behavior without reshaping public seams. -- For the current local-hook baseline, keep `pre-commit` and `post-rewrite` as deterministic no-op entrypoints; keep `post-commit` as the active bounded recent-diff-trace intersection entrypoint with validated `--remote-url` plumbed through Agent Trace flow and any direct diagnostics printed to stderr; keep `diff-trace` as an explicit STDIN intake path with deterministic required-field validation for `sessionID`, `diff`, `time`, `model_id`, `tool_name`, and `tool_version` (present and either `null` or non-empty string), non-lossy AgentTraceDb `time_ms` conversion, collision-safe `context/tmp/-000000-diff-trace.json` persistence using atomic create-new retry semantics, and command-failing AgentTraceDb insertion through the existing database adapter. +- For the current local-hook baseline, keep `pre-commit` and `post-rewrite` as deterministic no-op entrypoints; keep active Agent Trace hook DB access on `AgentTraceDb::open_for_hooks_without_migrations()` plus `ensure_schema_ready_for_hooks()` so setup/lifecycle owns migrations and missing schema fails with setup guidance; keep `post-commit` as the active bounded recent-diff-trace intersection entrypoint; keep `diff-trace` as an explicit STDIN intake path with deterministic required-field validation for `sessionID`, `diff`, `time`, `model_id`, `tool_name`, and `tool_version` (present and either `null` or non-empty string), non-lossy AgentTraceDb `time_ms` conversion, collision-safe `context/tmp/-000000-diff-trace.json` persistence using atomic create-new retry semantics, and command-failing readiness-gated AgentTraceDb insertion through the existing database adapter; keep `conversation-trace` as a typed batch `{ type, payloads }` STDIN intake path for homogeneous normalized snake_case `message.updated` / `message.part.updated` item arrays; parser-invalid items are logged and counted as skipped before DB insertion, one readiness-gated `AgentTraceDb` is opened per invocation, valid message rows write without message-level `text` through one multi-row `AgentTraceDb::insert_messages(...)` call with duplicate `(session_id, message_id)` writes ignored, valid part rows write with required part `text` through one multi-row `AgentTraceDb::insert_parts(...)` call, multi-row insert failures are logged/counted as the whole valid-item batch skipped without row-by-row fallback or command failure, success output reports attempted/persisted/skipped counts, and no `context/tmp` artifacts are written. - For commit-msg co-author policy seams, gate canonical trailer insertion on runtime controls (`SCE_DISABLED` plus the shared attribution-hooks enablement gate), and enforce idempotent dedupe so allowed cases end with exactly one `Co-authored-by: SCE ` trailer. - For local hook attribution flows, resolve the top-level enablement gate through the shared config precedence model (`SCE_ATTRIBUTION_HOOKS_ENABLED` over `policies.attribution_hooks.enabled`, default `false`) so commit-msg attribution stays disabled by default without adding hook-specific config parsing. -- Do not assume post-commit persistence, retry replay, remap ingestion, or rewrite trace transformation are active in the current local-hook runtime; those paths are removed from the current baseline. -- For the current local DB baseline, resolve one deterministic per-user persistent DB target (Linux: `${XDG_STATE_HOME:-~/.local/state}/sce/local.db`; platform-equivalent state roots elsewhere), keep the path neutral rather than Agent Trace-branded, create parent directories before first use, and route initialization through `LocalDb::new()`. As database services split, keep path/migration ownership in each `DbSpec`: `LocalDbSpec` owns the neutral local DB path with zero migrations, `AuthDbSpec` owns encrypted `/sce/auth.db` plus ordered auth migrations, `AgentTraceDbSpec` owns `/sce/agent-trace.db` plus ordered Agent Trace migrations, and shared Turso mechanics plus migration metadata stay in `TursoDb` / `EncryptedTursoDb`. +- Do not assume conversation-trace retry/backfill/artifact persistence, retry replay, remap ingestion, or rewrite trace transformation are active in the current local-hook runtime; those paths are removed from or deferred beyond the current baseline. +- For the current local DB baseline, resolve one deterministic per-user persistent DB target (Linux: `${XDG_STATE_HOME:-~/.local/state}/sce/local.db`; platform-equivalent state roots elsewhere), keep the path neutral rather than Agent Trace-branded, create parent directories before first use, and route initialization through `LocalDb::new()`. As database services split, keep path/migration ownership in each `DbSpec`: `LocalDbSpec` owns the neutral local DB path with zero migrations, `AgentTraceDbSpec` owns `/sce/agent-trace.db` plus ordered Agent Trace migrations, and shared Turso mechanics plus migration metadata stay in `TursoDb`. - For hosted event intake seams, verify provider signatures before payload parsing (GitHub `sha256=` HMAC over body, GitLab token-equality secret check), resolve old/new heads from provider payload fields, and derive deterministic reconciliation run idempotency keys from provider+event+repo+head tuple material. - For hosted rewrite mapping seams, resolve candidates deterministically in strict precedence order (patch-id exact, then range-diff score, then fuzzy score), classify top-score ties as `ambiguous`, enforce low-confidence unresolved behavior below `0.60`, and preserve stable outcome ordering via canonical candidate SHA sorting. - For hosted reconciliation observability, publish run-level mapped/unmapped counts, confidence histogram buckets, runtime timing, and normalized error-class labels so retry/quality drift can be monitored without requiring a full dashboard surface. diff --git a/context/plans/agent_trace_hook_no_hot_path_migrations.md b/context/plans/agent_trace_hook_no_hot_path_migrations.md new file mode 100644 index 00000000..c0fe8065 --- /dev/null +++ b/context/plans/agent_trace_hook_no_hot_path_migrations.md @@ -0,0 +1,138 @@ +# Agent Trace Hook No-Hot-Path Migrations Plan + +## Change summary + +Stop high-frequency Agent Trace hook invocations from running Agent Trace DB migrations during normal trace persistence. `sce setup` and lifecycle/doctor-owned setup remain responsible for schema initialization and upgrades. All Agent Trace hook paths must open the database for runtime reads/writes without migration execution, verify that the required schema is already present, and fail with clear setup/doctor guidance when the schema is missing. + +This targets transient `database is locked` failures caused by hook processes racing through migration metadata setup (`__sce_migrations`) while preserving explicit schema initialization through setup/doctor flows. + +## Success criteria + +- `sce hooks conversation-trace`, `sce hooks diff-trace`, and `sce hooks post-commit` no longer run Agent Trace DB migrations as part of their high-frequency runtime path. +- Agent Trace DB migrations still run through setup/lifecycle initialization, including `sce setup` and existing `AgentTraceDbLifecycle::setup` behavior. +- Hook runtime paths fail with actionable guidance when required Agent Trace DB schema is absent, directing the operator to run `sce setup` or `sce doctor --fix`. +- Runtime query/write retry behavior remains available for hook database operations. +- Existing hook behavior and output contracts remain unchanged except for clearer missing-schema failures and reduced migration-lock exposure. +- Tests cover no-migration hook DB opening/schema readiness behavior and the missing-schema guidance path. +- Context files are synced to describe the resulting current-state architecture. + +## Constraints and non-goals + +- Do not add new database libraries or external services. +- Do not remove migrations from setup, lifecycle, or explicit database initialization flows. +- Do not opportunistically run migrations from Agent Trace hook paths when schema is missing. +- Do not add retry/backfill queues or `context/tmp` artifacts for `conversation-trace`. +- Do not change Agent Trace DB schema shape as part of this plan unless strictly required for readiness checks. +- Do not broaden hook command surface or top-level CLI help visibility. + +## Assumptions + +- Scope includes all high-frequency Agent Trace hook paths: `conversation-trace`, `diff-trace`, and `post-commit`. +- Missing hook schema should fail fast with clear guidance, not run migrations as a fallback. +- `sce doctor --fix` may remain limited to safe parent-directory repair; if it cannot apply migrations today, guidance may include `sce setup` as the canonical schema initialization command. + +## Task stack + +- [x] T01: `Add no-migration Agent Trace DB open path` (status:done) + - Task ID: T01 + - Goal: Introduce an explicit Agent Trace DB construction/open path for runtime hooks that opens/connects the database without running embedded migrations while preserving existing retry-backed query/write methods. + - Boundaries (in/out of scope): In - adapter/API seam needed by Agent Trace hook code, tests proving migration execution is bypassed for this new path, preservation of existing `AgentTraceDb::new()` migration behavior for setup/lifecycle. Out - changing schema definitions, changing auth/local DB behavior, changing setup/doctor command output. + - Done when: A named no-migration Agent Trace DB open API exists; existing setup/lifecycle code still uses migration-running initialization; tests distinguish migration-running initialization from no-migration hook open behavior. + - Verification notes (commands or checks): Run the narrow Rust tests for the DB adapter/Agent Trace DB module through Nix, then include them in final `nix flake check`. + - Completed: 2026-06-09 + - Files changed: `cli/src/services/db/mod.rs`, `cli/src/services/agent_trace_db/mod.rs` + - Evidence: `nix develop -c sh -c 'cd cli && cargo fmt' && nix flake check` passed. Initial narrow `cargo test agent_trace_db` command was blocked by repo bash policy in favor of `nix flake check`. + - Notes: Added shared `TursoDb::::open_without_migrations()` plus Agent Trace-specific `AgentTraceDb::open_for_hooks_without_migrations()` for later hook routing. `TursoDb::::new()` still runs migrations. Focused Agent Trace DB tests distinguish migration-running initialization from no-migration opening. + +- [x] T02: `Add Agent Trace schema readiness checks` (status:done) + - Task ID: T02 + - Goal: Add a deterministic schema readiness check for the Agent Trace DB tables/indexes required by all active Agent Trace hook writers/readers. + - Boundaries (in/out of scope): In - readiness helper(s) for required objects such as `diff_traces`, `post_commit_patch_intersections`, `agent_traces`, `messages`, `parts`, and required indexes/columns where needed; actionable error type/message for missing schema. Out - running migrations, repairing schema, backfilling legacy DBs. + - Done when: The readiness check can identify an uninitialized or incomplete Agent Trace DB before hook persistence proceeds and returns a stable error with `Run 'sce setup'` guidance. + - Verification notes (commands or checks): Unit tests cover ready schema, empty DB, and at least one partially missing required object case. + - Completed: 2026-06-09 + - Files changed: `cli/src/services/agent_trace_db/mod.rs`, `context/sce/agent-trace-db.md`, `context/architecture.md`, `context/context-map.md` + - Evidence: `nix develop -c sh -c 'cd cli && cargo fmt'` passed; `nix flake check` passed; `nix run .#pkl-check-generated` passed; `git diff --check` passed. A direct focused `cargo test agent_trace_db` attempt was blocked by repo bash policy in favor of `nix flake check`. + - Notes: Added non-mutating `AgentTraceDb::ensure_schema_ready_for_hooks()` readiness checking against the canonical Agent Trace migration metadata (`__sce_migrations`) and `AGENT_TRACE_MIGRATIONS`. Missing or incomplete metadata fails with stable setup guidance (`Run 'sce setup'.`) without inspecting table/index/column objects or running migrations. Context sync classified the change as Agent Trace DB domain-local with small root architecture discoverability update. + +- [x] T03: `Route all Agent Trace hooks through no-migration readiness-gated DB access` (status:done) + - Task ID: T03 + - Goal: Update `conversation-trace`, `diff-trace`, and `post-commit` Agent Trace DB access to use the no-migration open path plus schema readiness checks before runtime reads/writes. + - Boundaries (in/out of scope): In - hook DB construction call sites, preserving existing parser/accounting/output behavior, ensuring missing-schema DB failures are command-failing where current DB open failures are command-failing. Out - changing generated OpenCode plugin behavior, changing attribution-only `commit-msg`, changing no-op `pre-commit`/`post-rewrite`. + - Done when: No active Agent Trace hook path calls the migration-running constructor during runtime persistence; missing schema produces clear runtime guidance; normal ready-schema paths retain existing persisted data behavior. + - Verification notes (commands or checks): Focused hook tests for `conversation-trace`, `diff-trace`, and `post-commit` cover ready-schema behavior and missing-schema failure guidance where feasible. + - Completed: 2026-06-09 + - Files changed: `cli/src/services/hooks/mod.rs`, `cli/src/services/agent_trace_db/mod.rs` + - Evidence: `nix develop -c sh -c 'cd cli && cargo fmt' && nix flake check` passed. + - Notes: Added a shared hook-runtime Agent Trace DB opener that uses `AgentTraceDb::open_for_hooks_without_migrations()` followed by `AgentTraceDb::ensure_schema_ready_for_hooks()` before returning the DB. Routed `conversation-trace`, `diff-trace`, post-commit intersection, and post-commit Agent Trace persistence through that readiness gate. User requested removal of generated unit tests; no new tests remain from this task. + +- [x] T04: `Keep setup and doctor lifecycle as schema initialization owners` (status:done) + - Task ID: T04 + - Goal: Verify and, if needed, adjust setup/lifecycle/doctor documentation and tests so schema initialization remains owned by setup/lifecycle flows, not hook runtime flows. + - Boundaries (in/out of scope): In - tests or assertions around `AgentTraceDbLifecycle::setup` using migration-running initialization; operator guidance consistency between hook missing-schema errors, `sce setup`, and doctor reporting. Out - broad doctor repair expansion beyond existing safe behavior unless required to make guidance truthful. + - Done when: Setup/lifecycle initialization remains the tested path for applying Agent Trace migrations; hook missing-schema guidance aligns with available commands. + - Verification notes (commands or checks): Run focused lifecycle/setup/doctor tests relevant to Agent Trace DB initialization and health reporting. + - Completed: 2026-06-09 + - Files changed: None (verification-only task) + - Evidence: `nix flake check` passed. Code review confirmed: (1) `AgentTraceDbLifecycle::setup()` calls `AgentTraceDb::new()` (migration-running path); (2) hook runtime uses `open_for_hooks_without_migrations()` + `ensure_schema_ready_for_hooks()` with `"Run 'sce setup'."` guidance; (3) existing test `new_applies_baseline_agent_trace_migration_and_indexes` covers migration-running initialization; (4) existing tests `schema_readiness_rejects_empty_agent_trace_schema_with_setup_guidance` and `schema_readiness_rejects_partial_agent_trace_schema` cover missing-schema guidance; (5) doctor diagnoses path/parent-dir health and can bootstrap parent dirs via `--fix`, consistent with setup owning schema initialization. + - Notes: No code changes needed — all acceptance criteria were already satisfied by existing code and tests. T04 was a verification-only task confirming that setup/lifecycle owns schema initialization and hook missing-schema guidance aligns with available commands. + +- [x] T05: `Sync current-state context for Agent Trace DB runtime migration policy` (status:done) + - Task ID: T05 + - Goal: Update durable context to describe the new split between migration-running setup/lifecycle initialization and no-migration hook runtime access. + - Boundaries (in/out of scope): In - current-state updates to `context/sce/shared-turso-db.md`, `context/sce/agent-trace-db.md`, `context/sce/agent-trace-hooks-command-routing.md`, and root context files/glossary if the architecture terminology changes. Out - completed-work summaries, historical narration, unrelated context churn. + - Done when: Context accurately states that high-frequency Agent Trace hook paths do not run migrations and instead require pre-initialized schema with clear setup/doctor guidance on failure. + - Verification notes (commands or checks): Review context against code truth; run generated-output parity if generated docs/config are touched. + - Completed: 2026-06-09 + - Files changed: `context/sce/shared-turso-db.md` + - Evidence: Context review verified all in-scope files against code truth. `agent-trace-db.md` and `agent-trace-hooks-command-routing.md` were already current from T02/T03 context syncs. Root context files (`context-map.md`, `overview.md`, `architecture.md`, `glossary.md`) already accurately describe the no-migration/readiness-gate split. Only `shared-turso-db.md` had a stale Agent Trace DB integration description that was updated to reflect the full migration set (`001..014`) and the setup/lifecycle vs hook runtime split. + - Notes: This was a verify-heavy context-sync task. The only edit was updating the Agent Trace DB integration description in `shared-turso-db.md` to reflect the complete migration set and the no-migration hook runtime split. No generated docs/config were touched, so `pkl-check-generated` was not required. + +- [x] T06: `Validation and cleanup` (status:done) + - Task ID: T06 + - Goal: Run full validation, remove temporary scaffolding, and confirm all success criteria are met. + - Boundaries (in/out of scope): In - full repo validation, targeted manual/automated command checks as appropriate, final plan evidence capture. Out - new behavior changes beyond fixes required by validation failures. + - Done when: `nix flake check` passes; `nix run .#pkl-check-generated` passes; targeted hook/DB tests pass; no temporary test files or debug instrumentation remain; plan status/evidence is updated. + - Verification notes (commands or checks): `nix run .#pkl-check-generated`; `nix flake check`; any focused Rust test commands used during earlier tasks. + - Completed: 2026-06-09 + - Files changed: None (validation-only task) + - Evidence: `nix flake check` passed (all checks including cli-tests, cli-clippy, cli-fmt, pkl-parity, npm-bun-tests, config-lib-bun-tests, biome checks); `nix run .#pkl-check-generated` passed ("Generated outputs are up to date."); no `dbg!` or `todo!` macros in production code; no temporary test files or debug instrumentation found; `context/tmp/` contains only `.gitignore`. + - Notes: All success criteria met. No code changes needed. No scaffolding to remove. Plan is complete. + +## Open questions + +- None. User confirmed all Agent Trace hook paths are in scope and missing schema should fail with clear guidance instead of running migrations. + +## Validation Report + +### Commands run +- `nix flake check` -> exit 0 (all checks passed: cli-tests, cli-clippy, cli-fmt, pkl-parity, npm-bun-tests, npm-biome-check, npm-biome-format, config-lib-bun-tests, config-lib-biome-check, config-lib-biome-format, integrations-install-tests, integrations-install-clippy, integrations-install-fmt) +- `nix run .#pkl-check-generated` -> exit 0 ("Generated outputs are up to date.") + +### Scaffolding check +- No `dbg!` macros in production code +- No `todo!` macros in production code +- No temporary test files or debug instrumentation found +- `context/tmp/` contains only `.gitignore` + +### Context verification +- `context/overview.md` — verified, accurately describes no-migration/readiness-gate split +- `context/architecture.md` — verified, accurately describes `open_without_migrations()` and `ensure_schema_ready_for_hooks()` seams +- `context/glossary.md` — verified, includes `no-migration DB open path` and `AgentTraceDb::open_for_hooks_without_migrations()` entries +- `context/patterns.md` — verified, accurately describes hook runtime DB access patterns +- `context/context-map.md` — verified, all domain file links are current +- `context/sce/shared-turso-db.md` — verified, updated in T05 to reflect full migration set and no-migration hook split +- `context/sce/agent-trace-db.md` — verified, current from T02 context sync +- `context/sce/agent-trace-hooks-command-routing.md` — verified, current from T03 context sync + +### Success-criteria verification +- [x] `sce hooks conversation-trace`, `sce hooks diff-trace`, and `sce hooks post-commit` no longer run Agent Trace DB migrations as part of their high-frequency runtime path — confirmed via T01/T03 code changes and T04 verification +- [x] Agent Trace DB migrations still run through setup/lifecycle initialization, including `sce setup` and existing `AgentTraceDbLifecycle::setup` behavior — confirmed via T04 verification of `AgentTraceDb::new()` migration-running path +- [x] Hook runtime paths fail with actionable guidance when required Agent Trace DB schema is absent, directing the operator to run `sce setup` or `sce doctor --fix` — confirmed via T02 `ensure_schema_ready_for_hooks()` with `"Run 'sce setup'."` guidance +- [x] Runtime query/write retry behavior remains available for hook database operations — confirmed via T01/T03 preserving retry-backed `execute`/`query`/`query_map` methods +- [x] Existing hook behavior and output contracts remain unchanged except for clearer missing-schema failures and reduced migration-lock exposure — confirmed via T03 preserving parser/accounting/output behavior +- [x] Tests cover no-migration hook DB opening/schema readiness behavior and the missing-schema guidance path — confirmed via T01/T02 focused tests plus `nix flake check` passing +- [x] Context files are synced to describe the resulting current-state architecture — confirmed via T05 context sync and T06 verify-only context review + +### Residual risks +- None identified. diff --git a/context/plans/move-migration-readiness-to-core-db.md b/context/plans/move-migration-readiness-to-core-db.md new file mode 100644 index 00000000..6357eb0a --- /dev/null +++ b/context/plans/move-migration-readiness-to-core-db.md @@ -0,0 +1,96 @@ +# Plan: Move migration readiness check to core DB + +## Change summary + +Move the generic migration readiness functions (`ensure_schema_ready_for_hooks_with` and `schema_migration_metadata_problems`) from `agent_trace_db/mod.rs` to the core `db/mod.rs` module, generalize the API as a public method on `TursoDb`, add a unit test for migration readiness checking in `db/mod.rs`, and update `AgentTraceDb::ensure_schema_ready_for_hooks()` to delegate to the moved core method. + +## Success criteria + +- `TursoDb` exposes a public `migration_metadata_problems()` method and a public `ensure_schema_ready(setup_guidance)` method that any `DbSpec` consumer can call. +- `AgentTraceDb::ensure_schema_ready_for_hooks()` delegates to `TursoDb::ensure_schema_ready()` with the Agent Trace–specific setup guidance string. +- `agent_trace_db/mod.rs` no longer contains the `ensure_schema_ready_for_hooks_with` or `schema_migration_metadata_problems` free functions. +- A new unit test in `db/mod.rs` exercises the readiness check with scenarios: missing migration metadata table, incomplete migrations, unexpected migrations, and complete/ready schema. +- `nix flake check` passes. +- Context files (`context/sce/shared-turso-db.md`, `context/sce/agent-trace-db.md`, `context/context-map.md`) reflect the moved ownership. + +## Constraints and non-goals + +- No behavior change to the readiness check logic itself — this is a pure refactor + test addition. +- `LocalDb` and `AuthDb` are not wired to call `ensure_schema_ready()` in this plan; the API is available for future use but no new callers are added. +- The `AGENT_TRACE_SCHEMA_SETUP_GUIDANCE` constant stays in `agent_trace_db/mod.rs` as a domain-specific string. +- No changes to `EncryptedTursoDb` in this plan; the readiness check is only on `TursoDb` since no encrypted-DB consumer currently needs it. + +## Task stack + +- [x] T01: `Move migration_metadata_problems and ensure_schema_ready to TursoDb` (status:done) + - Task ID: T01 + - Goal: Move the two generic free functions from `agent_trace_db/mod.rs` into `db/mod.rs` as public methods on `TursoDb`, update `AgentTraceDb::ensure_schema_ready_for_hooks()` to delegate, and remove the old free functions. + - Boundaries (in/out of scope): In — moving `schema_migration_metadata_problems` and `ensure_schema_ready_for_hooks_with` logic to `TursoDb` as `migration_metadata_problems(&self)` and `ensure_schema_ready(&self, setup_guidance: &str)`, updating `AgentTraceDb::ensure_schema_ready_for_hooks()` to call `self.ensure_schema_ready(AGENT_TRACE_SCHEMA_SETUP_GUIDANCE)`, removing the old free functions from `agent_trace_db/mod.rs`. Out — no changes to `EncryptedTursoDb`, no new callers for `LocalDb`/`AuthDb`, no behavior changes to the readiness logic. + - Done when: `agent_trace_db/mod.rs` no longer contains `ensure_schema_ready_for_hooks_with` or `schema_migration_metadata_problems`; `db/mod.rs` contains both methods on `TursoDb`; `AgentTraceDb::ensure_schema_ready_for_hooks()` delegates to the core method; `nix flake check` passes. + - Verification notes: `nix flake check`; `grep -r 'ensure_schema_ready_for_hooks_with\|schema_migration_metadata_problems' cli/src/services/agent_trace_db/` returns no matches; `grep 'fn ensure_schema_ready\|fn migration_metadata_problems' cli/src/services/db/mod.rs` returns both methods. + - **Status:** done + - **Completed:** 2026-06-09 + - **Files changed:** `cli/src/services/db/mod.rs`, `cli/src/services/agent_trace_db/mod.rs` + - **Evidence:** `nix flake check` all checks passed; `nix run .#pkl-check-generated` up to date; grep confirms old free functions removed from `agent_trace_db/mod.rs` and new methods present on `TursoDb` in `db/mod.rs`; `AgentTraceDb::ensure_schema_ready_for_hooks()` now delegates to `self.ensure_schema_ready(AGENT_TRACE_SCHEMA_SETUP_GUIDANCE)`. + - **Notes:** Pure refactor with no behavior change. The `ensure_schema_ready` error message now uses `M::db_name()` for the database name prefix instead of the hardcoded "Agent Trace DB" string, making it generic for any `DbSpec` consumer. + +- [x] T02: `Add migration readiness unit test in db/mod.rs` (status:done) + - Task ID: T02 + - Goal: Add a `#[cfg(test)] mod tests` block in `db/mod.rs` with a test that exercises `TursoDb::migration_metadata_problems()` and `TursoDb::ensure_schema_ready()` using a lightweight `TestDbSpec` with a small embedded migration, covering: (1) missing `__sce_migrations` table, (2) incomplete applied migrations, (3) unexpected extra migrations, and (4) complete/ready schema returns `Ok(())`. + - Boundaries (in/out of scope): In — test module in `db/mod.rs`, `TestDbSpec` with one or two minimal migrations, test cases for the four readiness scenarios. Out — no changes to production code, no changes to existing tests in other modules. + - Done when: `cargo test` (via `nix flake check`) includes the new test and all four scenarios pass; the test creates and cleans up a temporary database. + - Verification notes: `nix develop -c sh -c 'cd cli && cargo test migration_metadata_problems -- --exact'`; `nix flake check`. + - **Status:** done + - **Completed:** 2026-06-09 + - **Files changed:** none (test addition deferred per operator instruction) + - **Evidence:** Operator instructed to skip test implementation; T01 production code changes are verified and the readiness methods on `TursoDb` are in place. + - **Notes:** T02 was marked done without adding unit tests per operator instruction. The `migration_metadata_problems()` and `ensure_schema_ready()` methods on `TursoDb` are exercised indirectly through existing `AgentTraceDb` integration tests. + +- [x] T03: `Update context files to reflect moved ownership` (status:done) + - Task ID: T03 + - Goal: Update `context/sce/shared-turso-db.md`, `context/sce/agent-trace-db.md`, and `context/context-map.md` to reflect that `migration_metadata_problems()` and `ensure_schema_ready()` are now on `TursoDb` in the core DB module, and `AgentTraceDb::ensure_schema_ready_for_hooks()` delegates to the core method. + - Boundaries (in/out of scope): In — updating the three context files. Out — no code changes. + - Done when: Context files accurately describe the current code ownership; `shared-turso-db.md` documents the two new public methods on `TursoDb`; `agent-trace-db.md` describes delegation to the core method; `context-map.md` references are current. + - Verification notes: Manual review of updated context files against the code changes from T01. + - **Status:** done + - **Completed:** 2026-06-09 + - **Files changed:** none (context files were already current from prior sync) + - **Evidence:** Verified all three context files (`shared-turso-db.md`, `agent-trace-db.md`, `context-map.md`) plus the glossary entry (`TursoDb migration readiness check`) already reflect the moved ownership: `migration_metadata_problems()` and `ensure_schema_ready()` documented on `TursoDb` in `shared-turso-db.md`; `ensure_schema_ready_for_hooks()` delegation documented in `agent-trace-db.md`; `context-map.md` references current. No edits needed. + - **Notes:** Verify-only pass — context was already aligned with code truth from T01. + +- [x] T04: `Validate and clean up` (status:done) + - Task ID: T04 + - Goal: Run full validation suite and verify no stale references remain. + - Boundaries (in/out of scope): In — `nix flake check`, `nix run .#pkl-check-generated`, grep for stale references. Out — no code changes. + - Done when: `nix flake check` passes; `nix run .#pkl-check-generated` passes; no references to removed free functions remain in production code. + - Verification notes: `nix flake check`; `nix run .#pkl-check-generated`; `grep -r 'ensure_schema_ready_for_hooks_with\|schema_migration_metadata_problems' cli/src/` returns no matches. + - **Status:** done + - **Completed:** 2026-06-09 + - **Files changed:** none + - **Evidence:** `nix flake check` all checks passed; `nix run .#pkl-check-generated` reports "Generated outputs are up to date"; `grep -r 'ensure_schema_ready_for_hooks_with\|schema_migration_metadata_problems' cli/src/` returns no matches (exit code 1). + - **Notes:** Validation-only task. All checks pass. No stale references remain. + +## Validation Report + +### Commands run +- `nix flake check` → exit 0 (all checks passed: cli-tests, cli-clippy, cli-fmt, pkl-parity, npm-bun-tests, npm-biome-check, npm-biome-format, config-lib-bun-tests, config-lib-biome-check, config-lib-biome-format, integrations-install-tests, integrations-install-clippy, integrations-install-fmt) +- `nix run .#pkl-check-generated` → exit 0 ("Generated outputs are up to date") +- `grep -r 'ensure_schema_ready_for_hooks_with\|schema_migration_metadata_problems' cli/src/` → exit 1 (no matches found) + +### Success-criteria verification +- [x] `TursoDb` exposes public `migration_metadata_problems()` and `ensure_schema_ready(setup_guidance)` → confirmed in `db/mod.rs` lines 515-587 +- [x] `AgentTraceDb::ensure_schema_ready_for_hooks()` delegates to `TursoDb::ensure_schema_ready()` → confirmed in `agent_trace_db/mod.rs` line 321 +- [x] `agent_trace_db/mod.rs` no longer contains `ensure_schema_ready_for_hooks_with` or `schema_migration_metadata_problems` → confirmed by grep returning no matches +- [x] Unit test in `db/mod.rs` for readiness check → deferred per operator instruction (T02); methods exercised indirectly through existing AgentTraceDb integration tests +- [x] `nix flake check` passes → confirmed (exit 0, all checks passed) +- [x] Context files reflect moved ownership → confirmed: `shared-turso-db.md`, `agent-trace-db.md`, `context-map.md`, and `glossary.md` all current + +### Temporary scaffolding removed +- None introduced during this plan. + +### Residual risks +- None identified. + +## Open questions + +None — the scope is clear and the API design is straightforward. \ No newline at end of file diff --git a/context/plans/tighten-local-db-retry-defaults.md b/context/plans/tighten-local-db-retry-defaults.md new file mode 100644 index 00000000..0667685c --- /dev/null +++ b/context/plans/tighten-local-db-retry-defaults.md @@ -0,0 +1,92 @@ +# Plan: Tighten Local DB Retry/Backoff Defaults + +## Change summary + +Reduce hardcoded database retry/backoff defaults in `cli/src/services/db/mod.rs` to values appropriate for a local embedded Turso database with multiprocess WAL, where operations complete in microseconds rather than seconds. The current defaults (5s/3s timeouts, 100ms–1s backoff) are calibrated for remote/network databases and produce worst-case latencies of ~15s for connection open and ~9s for queries — unacceptable for a CLI tool operating on local files. + +## Success criteria + +- Hardcoded `CONNECTION_OPEN_RETRY_POLICY` and `QUERY_RETRY_POLICY` constants in `cli/src/services/db/mod.rs` are updated to the new tighter defaults. +- All existing tests pass (`nix flake check`). +- Context documentation reflects the new defaults. +- No behavioral change to the config-driven override mechanism — users who need longer timeouts can still set `policies.database_retry` in `sce/config.json`. + +## Constraints and non-goals + +- **In scope**: Update hardcoded default constants, update context docs, verify tests pass. +- **Out of scope**: Changing the `RetryPolicy` struct, the config override mechanism, the Pkl schema, or the resilience module logic. The config-driven override path remains the escape hatch for unusual environments. +- **Out of scope**: Changing `max_attempts` (3 is appropriate for local DBs). +- **Out of scope**: Adding new per-database differentiated defaults (all three DBs share the same hardcoded fallback today; this plan keeps that pattern). + +## New default values + +| Constant | Field | Current | New | Rationale | +|---|---|---|---|---| +| `CONNECTION_OPEN_RETRY_POLICY` | `timeout_ms` | 5,000 | 1,000 | Local file open completes in <10ms; 1s is generous | +| `CONNECTION_OPEN_RETRY_POLICY` | `initial_backoff_ms` | 100 | 25 | Local lock contention resolves in µs; 25ms is a brief pause | +| `CONNECTION_OPEN_RETRY_POLICY` | `max_backoff_ms` | 1,000 | 200 | Multiprocess WAL writer lock held for ms at most | +| `QUERY_RETRY_POLICY` | `timeout_ms` | 3,000 | 500 | Local queries complete in <1ms; 500ms is generous | +| `QUERY_RETRY_POLICY` | `initial_backoff_ms` | 100 | 25 | Same rationale as connection open | +| `QUERY_RETRY_POLICY` | `max_backoff_ms` | 500 | 100 | Same rationale as connection open | + +New worst-case timings: +- Connection open: ~3.025s (down from ~15.1s) +- Query: ~1.525s (down from ~9.1s) + +## Task stack + +- [x] T01: `Update hardcoded retry policy constants to local-DB-appropriate defaults` (status:done) + - Task ID: T01 + - Goal: Change the `CONNECTION_OPEN_RETRY_POLICY` and `QUERY_RETRY_POLICY` constant values in `cli/src/services/db/mod.rs` to the new tighter defaults. + - Boundaries (in/out of scope): In — the two `const` blocks at lines 27–39. Out — `RetryPolicy` struct, config override resolution, Pkl schema, resilience module. + - Done when: `CONNECTION_OPEN_RETRY_POLICY` has `timeout_ms: 1_000, initial_backoff_ms: 25, max_backoff_ms: 200` and `QUERY_RETRY_POLICY` has `timeout_ms: 500, initial_backoff_ms: 25, max_backoff_ms: 100`. `max_attempts` stays 3 for both. + - Verification notes: `nix develop -c sh -c 'cd cli && cargo test'` or `nix flake check`. + - **Status:** done + - **Completed:** 2026-06-09 + - **Files changed:** `cli/src/services/db/mod.rs` + - **Evidence:** `nix flake check` — all checks passed (cli-tests, cli-clippy, cli-fmt, pkl-parity, plus JS checks) + - **Notes:** Six numeric literal changes across two const blocks; no structural changes; config-driven override path untouched + +- [x] T02: `Update context documentation to reflect new retry defaults` (status:done) + - Task ID: T02 + - Goal: Update all context files that reference the old hardcoded default values to reflect the new values. + - Boundaries (in/out of scope): In — `context/glossary.md` (entries for `DB connection-open retry policy` and `DB query retry policy`), `context/sce/shared-turso-db.md` (default values in contract description), `context/overview.md` (if it mentions specific default values). Out — code changes, Pkl schema changes. + - Done when: All context files that previously referenced `5s`/`3s`/`100ms`/`1000ms`/`500ms` defaults now reference `1s`/`500ms`/`25ms`/`200ms`/`100ms` as appropriate. + - Verification notes: Grep context files for old values to confirm no stale references remain. + - **Status:** done + - **Completed:** 2026-06-09 + - **Files changed:** `context/glossary.md`, `context/sce/shared-turso-db.md`, `context/overview.md`, `context/architecture.md` + - **Evidence:** `nix flake check` — all checks passed (cli-tests, cli-clippy, cli-fmt, pkl-parity, plus JS checks); grep confirms no stale old default references remain in context documentation files (only the plan file retains historical rationale) + - **Notes:** Four context files updated with new default values; connection-open: `5s→1s`, `100ms→25ms`, `1000ms→200ms`; query: `3s→500ms`, `100ms→25ms`, `500ms→100ms`; no code changes + +- [x] T03: `Validate and clean up` (status:done) + - Task ID: T03 + - Goal: Run full validation suite and confirm no stale references to old defaults remain anywhere in the repo. + - Boundaries (in/out of scope): In — `nix flake check`, `nix run .#pkl-check-generated`, grep for old default values in code and docs. Out — any code or doc changes beyond what T01 and T02 already made. + - Done when: `nix flake check` passes, `nix run .#pkl-check-generated` passes, no stale references to old default values (`5000`/`1000`/`3000`/`500` as retry constants) remain in code or context files. + - Verification notes: `nix flake check && nix run .#pkl-check-generated`; `grep -r '5_000\|5000\|3_000\|3000' cli/src/services/db/mod.rs` should show no matches for old timeout values; `grep -r '5s.*timeout\|3s.*timeout\|100ms.*backoff\|1000ms.*backoff\|500ms.*backoff' context/` should show no stale references. + - **Status:** done + - **Completed:** 2026-06-09 + - **Files changed:** None (validation-only task) + - **Evidence:** `nix flake check` — all checks passed; `nix run .#pkl-check-generated` — generated outputs up to date; grep confirms no stale old default references in code (`5_000`/`3_000` absent from `db/mod.rs`) or context `.md` files (all reflect new values; only the plan file references old values as historical rationale) + - **Notes:** Pure validation task — no code or doc changes needed; all three tasks in the plan are now complete + +## Validation Report + +### Commands run +- `nix flake check` → exit 0 (all checks passed: cli-tests, cli-clippy, cli-fmt, pkl-parity, integrations-install-tests/clippy/fmt, npm-bun-tests, npm-biome-check/format, config-lib-bun-tests, config-lib-biome-check/format) +- `nix run .#pkl-check-generated` → exit 0 (generated outputs up to date) +- `grep -n '5_000\|5000\|3_000\|3000' cli/src/services/db/mod.rs` → no matches (old timeout constants removed) +- `grep -rn '5s.*timeout\|3s.*timeout\|100ms.*backoff\|1000ms.*backoff\|500ms.*backoff' context/ --include='*.md'` → only matches in plan file (historical rationale) and new-value references (`500ms` timeout, `25ms..100ms` backoff); no stale old defaults found + +### Success-criteria verification +- [x] Hardcoded `CONNECTION_OPEN_RETRY_POLICY` and `QUERY_RETRY_POLICY` constants updated → confirmed: `timeout_ms: 1_000, initial_backoff_ms: 25, max_backoff_ms: 200` and `timeout_ms: 500, initial_backoff_ms: 25, max_backoff_ms: 100` +- [x] All existing tests pass (`nix flake check`) → confirmed: all checks passed +- [x] Context documentation reflects the new defaults → confirmed: `context/glossary.md`, `context/overview.md`, `context/architecture.md`, `context/sce/shared-turso-db.md` all reflect new values +- [x] No behavioral change to config-driven override mechanism → confirmed: only const values changed; config resolution path untouched + +### Temporary scaffolding +- None introduced by this plan. + +### Residual risks +- None identified. \ No newline at end of file diff --git a/context/sce/agent-trace-db.md b/context/sce/agent-trace-db.md index 556c9dfa..5a4f7dbc 100644 --- a/context/sce/agent-trace-db.md +++ b/context/sce/agent-trace-db.md @@ -9,7 +9,9 @@ pub type AgentTraceDb = TursoDb; ## Module structure - `AgentTraceDbSpec`: `DbSpec` implementation for Agent Trace persistence. -- `AgentTraceDb`: type alias for `TursoDb`. +- `AgentTraceDb`: type alias for `TursoDb`, inheriting shared constructor and operation retry behavior. +- `open_for_hooks_without_migrations()`: Agent Trace-specific runtime-open API for high-frequency hook paths; opens/connects via `TursoDb::open_without_migrations()` and does not run embedded migrations. +- `ensure_schema_ready_for_hooks()`: non-mutating hook-readiness check that delegates to the shared `TursoDb::ensure_schema_ready()` method with the Agent Trace–specific `AGENT_TRACE_SCHEMA_SETUP_GUIDANCE` constant (`"Run 'sce setup'."`); verifies the Agent Trace DB has the expected applied migration metadata in `__sce_migrations` for every ID in `AGENT_TRACE_MIGRATIONS`; missing/incomplete metadata fails with `Run 'sce setup'.` guidance instead of running migrations. - `DiffTraceInsert<'a>`: insert payload with `time_ms: i64`, `session_id: &'a str`, `patch: &'a str`, `model_id: &'a str`, `tool_name: &'a str`, and nullable `tool_version: Option<&'a str>`. - `insert_diff_trace()`: domain-specific insert helper using parameterized SQL. - `RecentDiffTracePatches`: parsed recent `diff_traces` query result containing valid parsed patches plus skipped-row reports. @@ -18,8 +20,23 @@ pub type AgentTraceDb = TursoDb; - `insert_post_commit_patch_intersection()`: domain-specific insert helper using parameterized SQL. - `AgentTraceInsert<'a>`: insert payload for built Agent Trace rows with `commit_id`, `commit_time_ms`, serialized `trace_json`, `agent_trace_id`, non-null `url`, and required `remote_url: &'a str` (Rust-API-only; DB column stays nullable). - `insert_agent_trace()`: domain-specific insert helper for `agent_traces` using parameterized SQL. +- `MessageRole` enum: `User` / `Assistant` — maps to `messages.role` DB constraint. +- `InsertMessageInsert`: owned payload struct with insertable parent `messages` columns (`session_id`, `message_id`, `role`, `generated_at_unix_ms`); message body text belongs to `parts.text`, not the parent message row. +- `INSERT_MESSAGE_SQL`: parameterized single-row SQL using `INSERT ... ON CONFLICT (session_id, message_id) DO NOTHING` — leverages the unique index `idx_messages_session_message` so duplicate parent-message events remain non-failing without mutating the existing row. +- `insert_message(input)`: typed single-row helper that executes the duplicate-ignore parent-message insert; retained as part of the adapter surface. +- `insert_messages(inputs)`: typed batch helper that generates and executes one parameterized multi-row `messages` insert for valid conversation-trace `message.updated` batches while preserving duplicate-ignore semantics. +- `PartType` enum: `Text` / `Reasoning` / `Patch` — maps to `parts.type` DB constraint. +- `InsertPartInsert`: owned payload struct with `part_type`, `text`, `session_id`, `message_id`, and `generated_at_unix_ms`. +- `INSERT_PART_SQL`: parameterized single-row append-only INSERT into `parts` (no upsert; multiple rows per `(session_id, message_id)` allowed). +- `insert_part(input)`: typed single-row helper that inserts a part row without requiring a matching `messages` row (supports out-of-order writes); retained as part of the adapter surface. +- `insert_parts(inputs)`: typed batch helper that generates and executes one parameterized multi-row append-only `parts` insert for valid conversation-trace `message.part.updated` batches. - `lifecycle.rs`: service lifecycle provider for setup/doctor integration. +## Non-goals + +- No read/query helper for loading messages with their joined parts exists in the current runtime; the typed write helpers (`insert_message`, `insert_messages`, `insert_part`, `insert_parts`) are the only exposed message/part API surface. Message/part query helpers are deferred to a future task. +- No part upsert/deduplication; `parts` uses only the internal integer `id` for row identity (append-only per the `INSERT_PART_SQL` contract). + ## Database path The Agent Trace DB path is resolved from the shared default-path catalog: @@ -40,9 +57,18 @@ The Agent Trace DB path is resolved from the shared default-path catalog: - `005_create_agent_traces_agent_trace_id_index.sql` - `006_add_agent_traces_vcs_remote_url.sql` (historical filename; migration ID `006_add_agent_traces_remote_url` adds the `remote_url` column) - `007_create_agent_traces_vcs_remote_url_index.sql` (historical filename; migration ID `007_create_agent_traces_remote_url_index` creates `idx_agent_traces_remote_url`) +- `008_create_messages.sql` +- `009_create_parts.sql` +- `010_create_messages_session_message_unique_index.sql` +- `011_create_messages_session_order_index.sql` +- `012_create_parts_session_message_order_index.sql` +- `013_create_messages_updated_at_trigger.sql` +- `014_create_parts_updated_at_trigger.sql` The shared `TursoDb` runner records applied IDs in the database-local `__sce_migrations` table. Existing Agent Trace DB files without metadata are brought forward by re-applying the idempotent migration set and recording each ID, so rerunning `sce setup` / `AgentTraceDb::new()` applies later Agent Trace migrations to an already-created `~/.local/state/sce/agent-trace.db`. +`AgentTraceDb::open_for_hooks_without_migrations()` is the named no-migration Agent Trace open path for hook runtime code. It preserves Turso open/connect retry behavior from the shared adapter but intentionally skips `run_migrations()`, so it neither creates `__sce_migrations` nor applies Agent Trace schema SQL. Active hook callers (`conversation-trace`, `diff-trace`, and both post-commit Agent Trace DB flows) use this path and call `ensure_schema_ready_for_hooks()` before reads/writes; readiness is based on exact migration metadata parity with `AGENT_TRACE_MIGRATIONS`, not table/index/column introspection. + The `diff_traces` baseline migration creates: - `id INTEGER PRIMARY KEY` @@ -77,11 +103,44 @@ The `agent_traces` baseline migration creates: - `agent_trace_id TEXT NOT NULL UNIQUE` - `created_at TEXT NOT NULL DEFAULT (...)` +The `messages` migration creates: + +- `id INTEGER PRIMARY KEY` +- `session_id TEXT NOT NULL` +- `message_id TEXT NOT NULL` +- `role TEXT NOT NULL CHECK (role IN ('user', 'assistant'))` +- `generated_at_unix_ms INTEGER NOT NULL CHECK (generated_at_unix_ms >= 0)` +- `created_at TEXT NOT NULL DEFAULT (...)` +- `updated_at TEXT NOT NULL DEFAULT (...)` + +The `parts` migration creates: + +- `id INTEGER PRIMARY KEY` +- `type TEXT NOT NULL CHECK (type IN ('text', 'reasoning', 'patch'))` +- `text TEXT NOT NULL` +- `message_id TEXT NOT NULL` +- `session_id TEXT NOT NULL` +- `generated_at_unix_ms INTEGER NOT NULL CHECK (generated_at_unix_ms >= 0)` +- `created_at TEXT NOT NULL DEFAULT (...)` +- `updated_at TEXT NOT NULL DEFAULT (...)` + +No foreign keys exist between `messages` and `parts`; rows may be written out of order. The data model uses natural identifiers (`session_id`, `message_id`) for joins rather than DB-level referential integrity. + Lookup indexes created by the baseline migration set: - `idx_diff_traces_time_ms_id` on `(time_ms, id)` - `idx_agent_traces_agent_trace_id` on `(agent_trace_id)` - `idx_agent_traces_remote_url` on `(remote_url)` +- `idx_messages_session_message` unique index on `(session_id, message_id)` — enables duplicate-ignore parent message inserts by natural key +- `idx_messages_session_order` on `(session_id, generated_at_unix_ms, id)` — enables chronological session-scoped message retrieval +- `idx_parts_session_message_order` on `(session_id, message_id, generated_at_unix_ms, id)` — enables ordered part joins per message + +`updated_at` triggers defined by the migration set: + +- `trg_messages_updated_at`: fires on `UPDATE` for non-`updated_at` column changes on `messages` +- `trg_parts_updated_at`: fires on `UPDATE` for non-`updated_at` column changes on `parts` + +Both triggers compare `OLD.*` vs `NEW.*` for all mutable columns (excluding `updated_at` itself) and refresh the timestamp only when a real change occurred. ## Lifecycle integration @@ -98,11 +157,22 @@ Lookup indexes created by the baseline migration set: - The hook path validates required STDIN `{ sessionID, diff, time, model_id, tool_name, tool_version }` before persistence (`tool_name` non-empty; `tool_version` present and either `null` or non-empty string) and passes parsed `model_id`, `tool_name`, and nullable `tool_version` into `DiffTraceInsert`. - `time` is accepted as a `u64` Unix epoch millisecond input and must fit the signed `i64` `time_ms` column before any persistence starts. -- The hook writes the existing collision-safe `context/tmp/-000000-diff-trace.json` parsed-payload artifact and inserts the parsed payload fields through `AgentTraceDb::insert_diff_trace()`. +- The hook writes the existing collision-safe `context/tmp/-000000-diff-trace.json` parsed-payload artifact and inserts the parsed payload fields through readiness-gated `AgentTraceDb::insert_diff_trace()`. - Command success requires both artifact and database persistence to succeed. - Existing artifact files are not backfilled into the database. -Post-commit intersection rows are written by the active `post-commit` hook flow, and the same flow now also inserts built Agent Trace payloads into `agent_traces` via `AgentTraceDb::insert_agent_trace()` (see [agent-trace-hooks-command-routing.md](agent-trace-hooks-command-routing.md)). The persisted `trace_json` is the schema-validated `build_agent_trace(...)` output and includes top-level `metadata.sce.version` from the compiled `sce` CLI package version plus `content_hash` on every emitted range. Range `content_hash` values are computed from the touched-line kind/content of the post-commit hunk that produced the persisted range, not from DB IDs, paths, line positions, or runtime metadata. +Post-commit intersection rows are written by the active `post-commit` hook flow through readiness-gated AgentTraceDb access, and the same flow now also inserts built Agent Trace payloads into `agent_traces` via `AgentTraceDb::insert_agent_trace()` (see [agent-trace-hooks-command-routing.md](agent-trace-hooks-command-routing.md)). The persisted `trace_json` is the schema-validated `build_agent_trace(...)` output and includes top-level `metadata.sce.version` from the compiled `sce` CLI package version plus `content_hash` on every emitted range. Range `content_hash` values are computed from the touched-line kind/content of the post-commit hunk that produced the persisted range, not from DB IDs, paths, line positions, or runtime metadata. + +`sce hooks conversation-trace` is the current runtime writer for `messages` and `parts`. + +- The hook accepts only normalized snake_case typed batch STDIN envelopes: top-level `type` is `message.updated` or `message.part.updated`, and top-level `payloads` is an array of same-kind item objects. +- `message.updated` batch items validate and map payloads without message-level `text`, `agent`, or `summary_diffs` to `InsertMessageInsert`; valid rows are inserted through one multi-row `AgentTraceDb::insert_messages(...)` call so repeated `(session_id, message_id)` events are ignored without failing. +- `message.part.updated` batch items validate and map payloads with required part `text` to `InsertPartInsert`; valid rows are inserted through one multi-row `AgentTraceDb::insert_parts(...)` call so parts remain append-only and do not require a pre-existing message row. +- Per-item parser validation failures are retained as skipped-item diagnostics, logged, and counted as skipped while valid sibling items remain eligible for persistence. +- The hook opens one no-migration `AgentTraceDb` per invocation and checks schema readiness before insertion; DB open or readiness failures remain command-failing because no rows can be attempted. +- Multi-row insert failures are logged once and count the whole valid-item batch as skipped without failing the command; the hook does not fall back to row-by-row insertion after a batch failure. Successful inserts contribute to deterministic success output counts (`attempted`, `persisted`, `skipped`). Duplicate parent message inserts preserve the existing `ON CONFLICT DO NOTHING` affected-row semantics. +- No `context/tmp` artifact is written for conversation traces. +- The generated OpenCode agent-trace plugin is a runtime caller for both conversation event variants and currently sends one-element typed batch envelopes for captured `message.updated` and `message.part.updated` events. ## Recent patch reads diff --git a/context/sce/agent-trace-hooks-command-routing.md b/context/sce/agent-trace-hooks-command-routing.md index 33322d16..09ff2291 100644 --- a/context/sce/agent-trace-hooks-command-routing.md +++ b/context/sce/agent-trace-hooks-command-routing.md @@ -12,6 +12,7 @@ - `sce hooks post-commit [--vcs ] --remote-url ` - `sce hooks post-rewrite ` - `sce hooks diff-trace` +- `sce hooks conversation-trace` ## Parser and dispatch behavior @@ -35,6 +36,8 @@ - Writes back only when the attribution gate is enabled, `SCE_DISABLED` is false, and the transformed content differs. - `pre-commit` is a deterministic no-op entrypoint. - **`post-commit` is an active intersection entrypoint** (see [agent-trace-db.md](agent-trace-db.md)): + - Agent Trace DB access uses `AgentTraceDb::open_for_hooks_without_migrations()` followed by `ensure_schema_ready_for_hooks()` before both recent-patch reads/intersection writes and built Agent Trace persistence. + - Missing or incomplete Agent Trace DB schema is a command-failing runtime error with `Run 'sce setup'.` guidance; hook runtime does not run migrations as a fallback. - Captures the current commit's patch from git using `capture_post_commit_patch_from_git()`. - Queries recent `diff_traces` patches from the past 7 days via `AgentTraceDb::recent_diff_trace_patches()`. - Recent-row patch parsing carries nullable row `model_id` into each produced `PatchHunk`, so combined/intersection patch inputs retain per-hunk model provenance for downstream Agent Trace attribution building. @@ -55,8 +58,17 @@ - Post-commit Agent Trace success requires both schema validation and Agent Trace DB `agent_traces` persistence to succeed. - Current command-surface success output is: `post-commit hook processed intersection: commit=, intersection_files=`. - `post-rewrite` is a deterministic no-op entrypoint. -- `diff-trace` reads STDIN JSON, validates required non-empty `sessionID`/`diff`/`model_id`/`tool_name`, validates required `tool_version` (must be present and either `null` or a non-empty string), validates required `u64` `time` (Unix epoch milliseconds), rejects `time` values that cannot fit the Agent Trace DB signed `time_ms` column, writes one parsed-payload artifact per invocation to `context/tmp/-000000-diff-trace.json` with atomic create-new retry semantics, and inserts the parsed payload fields into AgentTraceDb via `DiffTraceInsert` + `insert_diff_trace()` including `model_id`. +- `diff-trace` reads STDIN JSON, validates required non-empty `sessionID`/`diff`/`model_id`/`tool_name`, validates required `tool_version` (must be present and either `null` or a non-empty string), validates required `u64` `time` (Unix epoch milliseconds), rejects `time` values that cannot fit the Agent Trace DB signed `time_ms` column, writes one parsed-payload artifact per invocation to `context/tmp/-000000-diff-trace.json` with atomic create-new retry semantics, opens AgentTraceDb through the no-migration readiness gate, and inserts the parsed payload fields via `DiffTraceInsert` + `insert_diff_trace()` including `model_id`. - `diff-trace` success requires both persistence paths to succeed; artifact write failures and AgentTraceDb open/insert failures are command-failing runtime errors logged through `sce.hooks.diff_trace.error`. +- `conversation-trace` is a recognized hidden hook subcommand routed through `HookSubcommand::ConversationTrace`. Rust intake now accepts only typed batch STDIN JSON with a top-level `type` discriminator and `payloads` array; the previous single-event envelope is no longer accepted. + - `type: "message.updated"` parses each item in `payloads` into `InsertMessageInsert` with required non-empty `session_id`, `message_id`, valid `role` (`user|assistant`), and non-negative signed-64-bit `generated_at_unix_ms`; message-level `text`, `agent`, and `summary_diffs` are not required or mapped because body text belongs to `message.part.updated` / `parts.text` and the parent `messages` row no longer stores the obsolete fields. + - `type: "message.part.updated"` parses each item in `payloads` into `InsertPartInsert` with required non-empty `session_id`, `message_id`, valid `part_type` (`text|reasoning|patch`), string `text`, and non-negative signed-64-bit `generated_at_unix_ms`. + - Item objects must not declare their own `type`; homogeneous batch type is owned by the top-level discriminator. + - Per-item validation failures are recorded as skipped-item diagnostics (`index`, `reason`) while valid sibling items remain eligible for persistence; skipped validation items are logged through `sce.hooks.conversation_trace.payload_skipped`. Top-level JSON/type/`payloads` shape failures still fail deterministically with `Invalid conversation-trace payload from STDIN: ...` diagnostics. + - Current persistence opens one no-migration `AgentTraceDb` per hook invocation, checks schema readiness, then inserts each valid `message.updated` batch through one multi-row `AgentTraceDb::insert_messages(...)` call or each valid `message.part.updated` batch through one multi-row `AgentTraceDb::insert_parts(...)` call. + - DB open or schema-readiness failures are command-failing runtime errors logged through `sce.hooks.conversation_trace.error`; valid-item multi-row insert failures are logged once through `sce.hooks.conversation_trace.agent_trace_db_batch_failed`, count the whole valid-item batch as skipped, and do not fail the command. The hook does not fall back to row-by-row insertion after a multi-row insert failure. + - Current success output reports deterministic batch accounting: `conversation-trace hook persisted payload batch to AgentTraceDb: attempted=, persisted=, skipped=.` The hook does not persist `context/tmp` artifacts. + - OpenCode's generated agent-trace plugin calls this hook with one-element typed batch envelopes for every captured `message.updated` event before its existing diff-trace flow and for every captured `message.part.updated` event without invoking diff-trace. ## Explicit non-goals in the current baseline @@ -65,3 +77,4 @@ - No backfill/import of existing `context/tmp/*-diff-trace.json` artifacts into AgentTraceDb - No retry queue replay - No rewrite remap ingestion +- No `conversation-trace` retry/backfill path or `context/tmp` artifact persistence diff --git a/context/sce/agent-trace-minimal-generator.md b/context/sce/agent-trace-minimal-generator.md index b99f51f8..2a53e659 100644 --- a/context/sce/agent-trace-minimal-generator.md +++ b/context/sce/agent-trace-minimal-generator.md @@ -13,7 +13,8 @@ Given a `constructed_patch` (AI candidate) and a `post_commit_patch` (canonical - **`mixed`** — `intersection_patch` hunk exists at the same slot but content differs. - **`unknown`** — no `intersection_patch` hunk at the same `old_start` slot. 4. Map `Conversation.contributor.model_id` from the matched `intersection_patch` hunk when contributor type is `ai` or `mixed`; omit `model_id` when provenance is missing (`None`). -5. Emit one `Conversation` per `post_commit_patch` hunk, one `TraceFile` per `post_commit_patch` file, and one range per hunk with a deterministic `content_hash` computed from that hunk's touched-line kind/content. +5. For each emitted conversation, derive optional `conversation.related` entries from non-empty `session_id` values on touched lines in the matched `intersection_patch` hunk; emit related entries as `{ "type": "session", "url": "https://sce.crocoder.dev/sessions/" }`, deduplicated by session ID with deterministic ordering, and omit `related` when no included lines provide `session_id`. +6. Emit one `Conversation` per `post_commit_patch` hunk, one `TraceFile` per `post_commit_patch` file, and one range per hunk with a deterministic `content_hash` computed from that hunk's touched-line kind/content. ## Domain types @@ -21,8 +22,9 @@ Given a `constructed_patch` (AI candidate) and a `post_commit_patch` (canonical | ----------------------- | ------------------------------------------------------------------------------------------------------------ | | `HunkContributor` | Enum: `Ai`, `Mixed`, `Unknown` | | `Contributor` | Nested per-conversation object carrying `type: HunkContributor` and optional `model_id` omitted when absent | +| `ConversationRelated` | Schema-aligned related-link entry shape (`type` as free-form string + `url`) for optional `conversation.related` | | `LineRange` | New-file line span with `start_line` + `end_line` + `content_hash` | -| `Conversation` | Per-hunk entry: nested contributor + `ranges` (currently exactly one range derived from `post_commit_patch`) | +| `Conversation` | Per-hunk entry: nested contributor + `ranges` (currently exactly one range derived from `post_commit_patch`) + optional `related` omitted when `None` | | `TraceFile` | Per-file entry: path + conversations | | `AgentTraceVcs` | Optional top-level VCS metadata object carrying `type` + `revision` when present | | `AgentTraceTool` | Optional top-level tool metadata object carrying optional `name` + optional `version` | @@ -30,7 +32,7 @@ Given a `constructed_patch` (AI candidate) and a `post_commit_patch` (canonical | `AgentTraceSceMetadata` | Nested `metadata.sce` object carrying the compiled SCE CLI package `version` | | `AgentTrace` | Top-level payload: `version`, `id`, `timestamp`, optional `vcs`, optional `tool`, `metadata`, `files` | -All types are `serde`-serializable with `snake_case` field naming. `Conversation.contributor` serializes as a nested object with a JSON field named `type`; `model_id` is present only when a concrete value exists. +All types are `serde`-serializable with `snake_case` field naming. `Conversation.contributor` serializes as a nested object with a JSON field named `type`; `model_id` is present only when a concrete value exists. `Conversation.related` is optional and omitted when `None` (`skip_serializing_if = "Option::is_none"`) and is now populated from matched intersection-line `session_id` provenance as session links. ## Payload shape diff --git a/context/sce/auth-db.md b/context/sce/auth-db.md index 89e5d3fa..d48f4d0d 100644 --- a/context/sce/auth-db.md +++ b/context/sce/auth-db.md @@ -8,7 +8,7 @@ The encrypted auth DB foundation provides a thin Rust wrapper, path and migratio - Database file path: `/sce/auth.db`. - Service wrapper: `cli/src/services/auth_db/mod.rs`. - `AuthDbSpec` implements `DbSpec` with diagnostic name `auth DB`, `auth_db_path()`, and ordered embedded auth migrations. -- `AuthDb` is a type alias for `EncryptedTursoDb`, consumed by the lifecycle provider at `cli/src/services/auth_db/lifecycle.rs`. +- `AuthDb` is a type alias for `EncryptedTursoDb`, inheriting retry-backed constructor and operation behavior and consumed by the lifecycle provider at `cli/src/services/auth_db/lifecycle.rs`. - Migration directory: `cli/migrations/auth/`. - Ordered migrations: - `001_create_auth_tokens.sql` diff --git a/context/sce/local-db.md b/context/sce/local-db.md index 04394d11..8a487584 100644 --- a/context/sce/local-db.md +++ b/context/sce/local-db.md @@ -10,7 +10,7 @@ pub type LocalDb = TursoDb; - `LocalDbSpec`: `DbSpec` implementation for the canonical per-user local database. - `LocalDb`: type alias for `TursoDb`. -- `LocalDb::new()`, `execute()`, and `query()`: inherited from `TursoDb`. +- `LocalDb::new()`, `execute()`, `query()`, and `query_map()`: inherited from `TursoDb` with current hardcoded constructor and operation retry policies. - `local_db/lifecycle.rs`: owns local DB health, parent-directory bootstrap, and setup initialization through `LocalDb::new()`. ## Database path @@ -40,6 +40,6 @@ let mut rows = db.query("PRAGMA database_list", ())?; ## Error handling -The shared `TursoDb` adapter returns `anyhow::Result` with service-name-qualified diagnostics for path resolution, parent-directory creation, runtime creation, database open/connect, migration execution, and SQL execution/query failures. +The shared `TursoDb` adapter returns `anyhow::Result` with service-name-qualified diagnostics for path resolution, parent-directory creation, runtime creation, database open/connect, migration execution, and SQL execution/query failures. Open/connect and `execute`/`query`/`query_map` transient failures are retry-wrapped by the shared adapter; migration execution and row-mapping failures are not retried. See also: [shared-turso-db.md](shared-turso-db.md), [agent-trace-db.md](agent-trace-db.md), [overview.md](../overview.md), [glossary.md](../glossary.md), [context-map.md](../context-map.md) diff --git a/context/sce/opencode-agent-trace-plugin-runtime.md b/context/sce/opencode-agent-trace-plugin-runtime.md index 8891a142..598b32b5 100644 --- a/context/sce/opencode-agent-trace-plugin-runtime.md +++ b/context/sce/opencode-agent-trace-plugin-runtime.md @@ -4,10 +4,20 @@ Current runtime source: `config/lib/agent-trace-plugin/opencode-sce-agent-trace- ## Event capture baseline -- The plugin captures `message.updated` events, filtered to user messages with diffs. -- When diff extraction succeeds, the plugin invokes `sce hooks diff-trace` and sends `{ sessionID, diff, time, model_id, tool_name, tool_version }` over STDIN JSON (`tool_name` is always `"opencode"`; `tool_version` is captured from session lifecycle events when available). +- The plugin registers for `message.updated`, `message.part.updated`, `session.created`, and `session.updated` events. +- For every captured `message.updated` event, the plugin checks for `summary.diffs` via `buildPatchConversationTracePayloads`: + - **When diffs exist**: builds a `-patch` conversation trace payload set (one `message.updated` with `message_id = "${id}-patch"` + per-diff `message.part.updated` payloads with `part_type: "patch"`) and dispatches all payloads concurrently via `Promise.all` to `sce hooks conversation-trace`. The original `message.updated` event is replaced — no original `message.updated` payload is sent. + - **When no diffs exist**: builds a single `message.updated` typed batch envelope `{ type: "message.updated", payloads: [item] }` via `buildConversationTracePayload` and invokes `sce hooks conversation-trace` over STDIN JSON (original behavior preserved). +- For every captured `message.part.updated` event, the plugin builds a typed batch envelope `{ type: "message.part.updated", payloads: [item] }` via `buildMessagePartConversationTracePayload` and invokes `sce hooks conversation-trace` over STDIN JSON; only `text` and `reasoning` part types with non-empty `text` are dispatched. +- Existing diff-trace capture remains filtered to user messages with usable diffs. +- When diff extraction succeeds, the plugin invokes `sce hooks diff-trace` after conversation-trace handoff and sends `{ sessionID, diff, time, model_id, tool_name, tool_version }` over STDIN JSON (`tool_name` is always `"opencode"`; `tool_version` is captured from session lifecycle events when available). - The plugin no longer writes diff-trace artifacts or database rows directly; the Rust `diff-trace` hook path owns AgentTraceDb insertion plus collision-safe timestamp+attempt artifact writes. -- `session.diff` event capture has been removed. + +## In-memory dedup cache + +The plugin maintains a `Set` (`processedDiffsMessageIds`) in the `SceAgentTracePlugin` closure, keyed by `"${sessionID}:${messageID}"`. Only `message.updated` events that carry `summary.diffs` are checked against and added to the set. An event without diffs does not interact with the set at all — it is processed normally and does not block subsequent events for the same `(sessionID, messageID)` pair. + +This prevents duplicate processing of diff-bearing `message.updated` events while allowing a non-diff event (e.g., initial `message.updated` without `summary.diffs`) to be followed by a later diff-bearing event for the same message. The set lives for the lifetime of the plugin instance and is not time-bounded — once a diff-bearing `(sessionID, messageID)` pair is processed, it is never re-processed. ## Diff extraction seam @@ -28,9 +38,42 @@ Returns `{ sessionID, diff, time, model_id }` only when all checks pass: Otherwise, the helper returns `undefined`. +## Patch conversation trace + +The `buildPatchConversationTracePayloads(event)` helper processes `message.updated` events that carry `summary.diffs`. When the event has diff items with usable `patch` fields, it returns an array of `ConversationTracePayload` envelopes; when no diff items have usable patches, it returns `undefined`. + +### Payload shape when diffs exist + +1. One `message.updated` payload with: + - `session_id` from `event.properties.info.sessionID` + - `role` from `event.properties.info.role` + - `message_id` = `${event.properties.info.id}-patch` + - `generated_at_unix_ms = Date.now()` +2. For each diff item with a non-empty `patch` field, a `message.part.updated` payload with: + - Same `session_id` as the parent `message.updated` payload + - `message_id = "${id}-patch"` (same as the parent) + - `part_type: "patch"` + - `text: entryObj.patch` + - `generated_at_unix_ms = Date.now()` + +### Concurrent dispatch + +`recordConversationTrace` dispatches all payloads (the parent `message.updated` and all per-diff `message.part.updated` payloads) **concurrently** using `await Promise.all(...)`. + +### No-diff fallback + +When `buildPatchConversationTracePayloads` returns `undefined` (no diff items have usable patches), `recordConversationTrace` falls back to sending the original `message.updated` payload unchanged via `buildConversationTracePayload`. + ## Current usage boundary -- The extraction seam is internal to the source module and is used by `buildTrace` at runtime. -- `buildTrace` is now called only for captured event types and exits early unless the event is `message.updated`; if extraction returns `undefined` (non-user role, empty diffs array, or no usable patch entries), no hook invocation occurs. +- `recordConversationTrace(repoRoot, event)` branches on event type: + - For `message.updated` events: calls `buildPatchConversationTracePayloads` first. + - If patch payloads are returned (diffs exist), dispatches them concurrently via `Promise.all` — the original `message.updated` payload is not sent. + - If `undefined` (no diffs), sends the original `message.updated` payload unchanged via `buildConversationTracePayload`. + - For `message.part.updated` events (only `text` and `reasoning` with non-empty `text`): uses `buildMessagePartConversationTracePayload` (unchanged). +- The `message.updated` conversation-trace batch (no-diff fallback) maps OpenCode event fields mechanically into a `payloads[0]` item with `session_id`, `message_id`, `role`, and `generated_at_unix_ms`; it does not emit message-level `agent` or `summary_diffs` fields and does not duplicate Rust hook validation. +- `buildMessagePartConversationTracePayload(event)` maps `event.properties.part.sessionID`, `messageID`, `type`, and `text` into a `payloads[0]` item with `session_id`, `message_id`, `part_type`, and `text`, and uses `Date.now()` for `generated_at_unix_ms`. +- The diff extraction seam is internal to the source module and is used by `buildTrace` at runtime. +- `buildTrace` exits early when extraction returns `undefined` (non-user role, empty diffs array, or no usable patch entries), so no diff-trace hook invocation occurs for those events. - The plugin tracks OpenCode client version per session ID from `session.created` / `session.updated` events and forwards it as `tool_version` when available. - When extraction succeeds, `buildTrace` forwards the extracted payload with required `tool_name="opencode"` and required `tool_version` (nullable when session version is unavailable) to `sce hooks diff-trace` via STDIN JSON; the Rust hook runtime validates required non-empty `sessionID`/`diff`/`model_id`/`tool_name`, required nullable/non-empty `tool_version`, plus required `time`, and persists the DB-backed diff-trace fields through AgentTraceDb `diff_traces` insertion. diff --git a/context/sce/shared-turso-db.md b/context/sce/shared-turso-db.md index 95394426..04be9d4c 100644 --- a/context/sce/shared-turso-db.md +++ b/context/sce/shared-turso-db.md @@ -10,12 +10,18 @@ - `migrations()` returns ordered embedded migration `(id, sql)` pairs. - `TursoDb`: generic unencrypted adapter that owns: - tokio current-thread runtime creation - - Turso local database open/connect flow + - Turso local database open/connect flow using `turso::Builder::new_local()` with `experimental_multiprocess_wal(true)` so concurrent `sce` processes can safely access the same local database without WAL lock contention + - config-driven connection-open retry around only the `build().await.connect()` block using `run_with_retry_sync` (resolved from `policies.database_retry..connection_open` via `DATABASE_RETRY_CONFIG` `OnceLock` with fallback to hardcoded defaults `3` attempts, `1s` timeout, `25ms..200ms` backoff) + - config-driven operation retry for `execute()`, `query()`, and `query_map()` using `run_with_retry_sync` (resolved from `policies.database_retry..query` via the same `OnceLock` with fallback to hardcoded defaults `5` attempts, `200ms` timeout, `25ms..100ms` backoff) - parent-directory creation - - delegation of synchronous `execute()`, `query()`, row-mapping `query_map()`, and `run_migrations()` to the shared internal `TursoConnectionCore` -- `EncryptedTursoDb`: encrypted-adapter seam parallel to `TursoDb` with the same structural shape (connection, runtime bridge, and spec marker). `EncryptedTursoDb::new()` resolves the encryption key via `encryption_key::get_or_create_encryption_key()`, enables Turso experimental local encryption, applies strict `aegis256` cipher selection through `turso::EncryptionOpts` during local DB open/connect, and runs embedded migrations after connect. -- `EncryptedTursoDb` exposes the same public synchronous `execute()`, `query()`, `query_map()`, and `run_migrations()` methods by delegating to the same `TursoConnectionCore` operation path used by `TursoDb`. -- `TursoConnectionCore` is internal to `cli/src/services/db/mod.rs` and owns the shared blocking Turso operation wrappers plus embedded migration execution with per-database `__sce_migrations` metadata; encryption vs unencrypted behavior remains constructor-only at the public adapter layer. + - retry-backed synchronous `execute()`, `query()`, and row-mapping `query_map()` wrappers via the public adapter methods, with config-driven query retry resolved from `policies.database_retry..query` + - migration-running initialization through `new()` and generic embedded migration execution through `run_migrations()` delegated to the shared internal `TursoConnectionCore` with per-database `__sce_migrations` metadata + - no-migration opening through `open_without_migrations()`, which preserves parent-directory creation and connection-open retry but does not create `__sce_migrations` or apply embedded migrations + - `migration_metadata_problems(&self) -> Result>`: non-mutating readiness check that queries `__sce_migrations` metadata and compares applied migration IDs against `M::migrations()`; returns a list of problems (missing metadata table, incomplete applied migrations, unexpected extra migrations) or an empty list when the schema is ready + - `ensure_schema_ready(&self, setup_guidance: &str) -> Result<()>`: non-mutating hook-readiness gate that calls `migration_metadata_problems()` and bails with a formatted error including `M::db_name()` and the caller-provided guidance string when problems are found; returns `Ok(())` when the schema is ready +- `EncryptedTursoDb`: encrypted-adapter seam parallel to `TursoDb` with the same structural shape (connection, runtime bridge, and spec marker). `EncryptedTursoDb::new()` resolves the encryption key via `encryption_key::get_or_create_encryption_key()` (environment variable `SCE_AUTH_DB_ENCRYPTION_KEY` with OS credential-store fallback), enables Turso experimental local encryption, applies strict `aegis256` cipher selection through `turso::EncryptionOpts` during local DB open/connect, wraps that open/connect block in the same connection-open retry policy resolved from `policies.database_retry..connection_open`, and runs embedded migrations after connect. +- `EncryptedTursoDb` exposes the same public synchronous `execute()`, `query()`, `query_map()`, and `run_migrations()` methods; operation methods use the same config-driven query retry policy as `TursoDb`. +- `TursoConnectionCore` is internal to `cli/src/services/db/mod.rs` and owns the shared Turso connection plus tokio current-thread runtime bridging used by the public adapter methods; generic embedded migration execution with per-database `__sce_migrations` metadata is delegated to `run_embedded_migrations` helpers; encryption vs unencrypted behavior remains constructor-only at the public adapter layer. - Shared lifecycle helpers: - `collect_db_path_health()` emits common parent/path health problems for DB-backed services. - `bootstrap_db_parent()` creates the resolved DB parent directory for repair/setup flows. @@ -48,14 +54,20 @@ the secret value. No plaintext fallback exists. The shared module is exported from `cli/src/services/mod.rs` and compile-checked. Current concrete wrappers: - `cli/src/services/local_db/mod.rs`: `LocalDb = TursoDb`, with `LocalDbSpec` resolving `local_db_path()` and declaring zero migrations. -- `cli/src/services/agent_trace_db/mod.rs`: `AgentTraceDb = TursoDb`, with `AgentTraceDbSpec` resolving `agent_trace_db_path()` and loading ordered Agent Trace migrations for `diff_traces` and `post_commit_patch_intersections`. +- `cli/src/services/agent_trace_db/mod.rs`: `AgentTraceDb = TursoDb`, with `AgentTraceDbSpec` resolving `agent_trace_db_path()` and loading ordered Agent Trace migrations (`001..014`) for `diff_traces`, `post_commit_patch_intersections`, `agent_traces`, `messages`, `parts`, indexes, and triggers. Setup/lifecycle initialization uses migration-running `AgentTraceDb::new()`; high-frequency hook runtime paths use no-migration `AgentTraceDb::open_for_hooks_without_migrations()` plus `ensure_schema_ready_for_hooks()` so hook invocations never run schema migrations on the hot path. - `cli/src/services/auth_db/mod.rs`: `AuthDb = EncryptedTursoDb`, with `AuthDbSpec` resolving `auth_db_path()` and loading ordered auth migrations where baseline SQL creates `auth_credentials` without `user_id`, with `updated_at`, and a trigger that auto-refreshes `updated_at` on row updates. All three database wrappers (local DB, auth DB, Agent Trace DB) have lifecycle providers. `lifecycle_providers(include_hooks)` registers database providers in order `LocalDbLifecycle` → `AuthDbLifecycle` → `AgentTraceDbLifecycle` before optional hooks, so setup initializes all three databases and doctor diagnoses/fixes all three canonical DB paths. ## Migration metadata -The shared `TursoConnectionCore` migration path creates a service-local `__sce_migrations` table before applying migrations. Each migration is skipped only when its ID is already recorded in that table; otherwise the SQL is executed and the ID is recorded after success. +`TursoDb::new()` opens the database through the same connection path as `open_without_migrations()`, then calls `run_migrations()`. The shared `TursoConnectionCore` migration path creates a service-local `__sce_migrations` table before applying migrations. Each migration is skipped only when its ID is already recorded in that table; otherwise the SQL is executed and the ID is recorded after success. + +`TursoDb::open_without_migrations()` is the explicit runtime-open seam for high-frequency callers that must not perform schema changes on their hot path. It still creates the parent directory and uses the configured connection-open retry policy; callers must verify schema readiness before query/write work. + +Migrations are deliberately outside the connection-open retry block. The constructors retry only local Turso open/connect; schema changes are not retried because migration SQL must not be replayed after partial execution. + +`TursoDb` and `EncryptedTursoDb` operation methods use the same config-driven query retry policy, resolved from `policies.database_retry..query` via `DATABASE_RETRY_CONFIG` `OnceLock` with fallback to hardcoded defaults (`5` attempts, `200ms` timeout, `25ms..100ms` backoff). `execute()` and `query()` convert caller parameters to owned Turso params before retry so each attempt can clone the same values. `query_map()` retries the initial query and full row-fetch loop, then runs caller-provided row mapping after retry completion so mapping failures are surfaced as logic errors and are not retried. Existing databases created before migration metadata are upgraded by re-applying the current idempotent migration list and recording each migration ID. This lets later `sce setup` / lifecycle initialization runs apply migrations added after the database file already existed, including Agent Trace DB schema/index additions. diff --git a/flake.lock b/flake.lock index 96ece7da..c2f12e54 100644 --- a/flake.lock +++ b/flake.lock @@ -149,17 +149,17 @@ ] }, "locked": { - "lastModified": 1778776475, - "narHash": "sha256-uOrvQZ16TlgRFt7kiKIPMT84S07PArVdw6OWzDt7rD8=", + "lastModified": 1780748666, + "narHash": "sha256-Cga+cPuoxu2o/oMwKNXJ1DdHW5EoyRTLr9xj2hoOtaE=", "owner": "tursodatabase", "repo": "turso", - "rev": "c349ca43a7eb1d68854f668c23b97a2f1791d55a", + "rev": "fed0b50b8aa697f56ab015aa9e905e0f56371092", "type": "github" }, "original": { "owner": "tursodatabase", "repo": "turso", - "rev": "c349ca43a7eb1d68854f668c23b97a2f1791d55a", + "rev": "fed0b50b8aa697f56ab015aa9e905e0f56371092", "type": "github" } } diff --git a/flake.nix b/flake.nix index 41ad4663..940faa3a 100644 --- a/flake.nix +++ b/flake.nix @@ -9,7 +9,7 @@ crane.url = "github:ipetkov/crane"; opencode.url = "github:anomalyco/opencode/dev"; opencode-nixpkgs.follows = "opencode/nixpkgs"; - turso.url = "github:tursodatabase/turso/c349ca43a7eb1d68854f668c23b97a2f1791d55a"; + turso.url = "github:tursodatabase/turso/fed0b50b8aa697f56ab015aa9e905e0f56371092"; turso.inputs.nixpkgs.follows = "nixpkgs"; turso.inputs.flake-utils.follows = "flake-utils"; turso.inputs.crane.follows = "crane"; @@ -274,7 +274,7 @@ tursoCargoArgs = { pname = "turso"; - version = "0.6.0"; + version = "0.7.0-pre.5"; src = turso; strictDeps = true; diff --git a/scripts/stress-test-conversation-trace-firehose-mixed.sh b/scripts/stress-test-conversation-trace-firehose-mixed.sh new file mode 100755 index 00000000..d1632c86 --- /dev/null +++ b/scripts/stress-test-conversation-trace-firehose-mixed.sh @@ -0,0 +1,350 @@ +#!/usr/bin/env bash +set -euo pipefail + +total_requests=100 +min_delay_ms=10 +max_delay_ms=300 +sce_bin="${SCE_BIN:-sce}" + +usage() { + cat <<'EOF' +Usage: + scripts/stress-test-conversation-trace-firehose-mixed.sh [flags] + +Firehose-style stress test for `sce hooks conversation-trace`. +Each request is launched in the background, with a random launch-to-launch +delay, and sends a random valid typed batch payload: +`message.updated` or `message.part.updated`. +The script prints a complete summary and exits non-zero if any request process +fails. + +Flags: + -n, --requests Total requests to launch. Default: 100 + -m, --min-delay-ms Minimum delay between launches. Default: 10 + -M, --max-delay-ms Maximum delay between launches. Default: 300 + --sce-bin Binary to invoke. Default: $SCE_BIN or sce + -h, --help Show this help text + +Examples: + scripts/stress-test-conversation-trace-firehose-mixed.sh -n 250 -m 0 -M 25 + SCE_BIN=./target/debug/sce scripts/stress-test-conversation-trace-firehose-mixed.sh --requests 1000 +EOF +} + +fail() { + printf 'error: %s\n' "$1" >&2 + exit 1 +} + +require_value() { + local flag="$1" + local value="${2:-}" + + if [[ -z "$value" || "$value" == -* ]]; then + fail "${flag} requires a value" + fi +} + +is_non_negative_integer() { + [[ "$1" =~ ^[0-9]+$ ]] +} + +while [[ $# -gt 0 ]]; do + case "$1" in + -n|--requests) + require_value "$1" "${2:-}" + total_requests="$2" + shift 2 + ;; + -m|--min-delay-ms) + require_value "$1" "${2:-}" + min_delay_ms="$2" + shift 2 + ;; + -M|--max-delay-ms) + require_value "$1" "${2:-}" + max_delay_ms="$2" + shift 2 + ;; + --sce-bin) + require_value "$1" "${2:-}" + sce_bin="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + fail "unknown flag '$1'" + ;; + esac +done + +is_non_negative_integer "$total_requests" || fail "requests must be a non-negative integer" +is_non_negative_integer "$min_delay_ms" || fail "min-delay-ms must be a non-negative integer" +is_non_negative_integer "$max_delay_ms" || fail "max-delay-ms must be a non-negative integer" + +if (( total_requests < 1 )); then + fail "requests must be at least 1" +fi + +if (( min_delay_ms > max_delay_ms )); then + fail "min-delay-ms must be less than or equal to max-delay-ms" +fi + +if ! command -v "$sce_bin" >/dev/null 2>&1; then + fail "sce binary '$sce_bin' was not found; set SCE_BIN or pass --sce-bin" +fi + +repo_root="$(git rev-parse --show-toplevel 2>/dev/null || pwd)" +cd "$repo_root" + +run_id="$(date -u +%Y%m%dT%H%M%SZ)-$$" +tmp_dir="$(mktemp -d)" +results_dir="$tmp_dir/results" +mkdir -p "$results_dir" + +cleanup() { + rm -rf "$tmp_dir" +} +trap cleanup EXIT + +now_ms() { + date +%s%3N +} + +random_between() { + local min="$1" + local max="$2" + local span=$((max - min + 1)) + local random_value=$((((RANDOM << 15) ^ RANDOM) & 0x3fffffff)) + + printf '%s\n' $((min + (random_value % span))) +} + +sleep_ms() { + local delay_ms="$1" + local seconds=$((delay_ms / 1000)) + local millis=$((delay_ms % 1000)) + + sleep "${seconds}.$(printf '%03d' "$millis")" +} + +random_role() { + if (( RANDOM % 2 == 0 )); then + printf 'user\n' + else + printf 'assistant\n' + fi +} + +random_part_type() { + case $((RANDOM % 3)) in + 0) printf 'text\n' ;; + 1) printf 'reasoning\n' ;; + *) printf 'patch\n' ;; + esac +} + +build_message_updated_payload() { + local request_index="$1" + local batch_size="$2" + local payload='{"type":"message.updated","payloads":[' + local item_index role session_id message_id generated_at_unix_ms separator + + for ((item_index = 1; item_index <= batch_size; item_index++)); do + role="$(random_role)" + session_id="stress-session-${run_id}-$((RANDOM % 25))" + message_id="stress-message-${run_id}-${request_index}-${item_index}" + generated_at_unix_ms="$(now_ms)" + separator=',' + if (( item_index == batch_size )); then + separator='' + fi + + payload+="{\"session_id\":\"${session_id}\",\"message_id\":\"${message_id}\",\"role\":\"${role}\",\"generated_at_unix_ms\":${generated_at_unix_ms}}${separator}" + done + + payload+=']}' + printf '%s\n' "$payload" +} + +build_message_part_updated_payload() { + local request_index="$1" + local batch_size="$2" + local payload='{"type":"message.part.updated","payloads":[' + local item_index part_type session_id message_id generated_at_unix_ms text separator + + for ((item_index = 1; item_index <= batch_size; item_index++)); do + part_type="$(random_part_type)" + session_id="stress-session-${run_id}-$((RANDOM % 25))" + message_id="stress-message-${run_id}-${request_index}-${item_index}" + generated_at_unix_ms="$(now_ms)" + text="stress ${part_type} payload request ${request_index} item ${item_index} random $RANDOM" + separator=',' + if (( item_index == batch_size )); then + separator='' + fi + + payload+="{\"session_id\":\"${session_id}\",\"message_id\":\"${message_id}\",\"part_type\":\"${part_type}\",\"text\":\"${text}\",\"generated_at_unix_ms\":${generated_at_unix_ms}}${separator}" + done + + payload+=']}' + printf '%s\n' "$payload" +} + +launch_request() { + local request_index="$1" + local event_type batch_size payload + + batch_size="$(random_between 1 4)" + if (( RANDOM % 2 == 0 )); then + event_type='message.updated' + payload="$(build_message_updated_payload "$request_index" "$batch_size")" + else + event_type='message.part.updated' + payload="$(build_message_part_updated_payload "$request_index" "$batch_size")" + fi + + { + local started_at_ms ended_at_ms output exit_code + started_at_ms="$(now_ms)" + if output="$(printf '%s\n' "$payload" | "$sce_bin" hooks conversation-trace 2>&1)"; then + exit_code=0 + else + exit_code=$? + fi + ended_at_ms="$(now_ms)" + + printf '%s\n' "$output" >"$results_dir/output-${request_index}.txt" + printf '%s\t%s\t%s\t%s\t%s\t%s\n' \ + "$request_index" \ + "$exit_code" \ + "$event_type" \ + "$batch_size" \ + "$started_at_ms" \ + "$ended_at_ms" \ + >"$results_dir/result-${request_index}.tsv" + } & +} + +printf 'Conversation-trace firehose stress test\n' +printf 'Repository: %s\n' "$repo_root" +printf 'Command: %s hooks conversation-trace\n' "$sce_bin" +printf 'Requests: %s\n' "$total_requests" +printf 'Launch delay: %sms to %sms\n' "$min_delay_ms" "$max_delay_ms" +printf 'Batch size: random 1 to 4 payload items per request\n' +printf 'Run ID: %s\n' "$run_id" +printf '\n' + +test_started_at_ms="$(now_ms)" +pids=() + +for ((request_index = 1; request_index <= total_requests; request_index++)); do + launch_request "$request_index" + pids+=("$!") + + if (( request_index < total_requests )); then + sleep_ms "$(random_between "$min_delay_ms" "$max_delay_ms")" + fi +done + +printf 'Launched %s requests. Waiting for background hook processes...\n' "$total_requests" + +for pid in "${pids[@]}"; do + wait "$pid" +done + +test_ended_at_ms="$(now_ms)" + +completed=0 +succeeded=0 +failed=0 +message_updated_requests=0 +message_part_updated_requests=0 +message_updated_items=0 +message_part_updated_items=0 +hook_attempted=0 +hook_persisted=0 +hook_skipped=0 +duration_total_ms=0 +failed_request_ids=() + +for ((request_index = 1; request_index <= total_requests; request_index++)); do + result_file="$results_dir/result-${request_index}.tsv" + output_file="$results_dir/output-${request_index}.txt" + + if [[ ! -f "$result_file" ]]; then + failed=$((failed + 1)) + failed_request_ids+=("${request_index}:missing-result") + continue + fi + + IFS=$'\t' read -r recorded_index exit_code event_type batch_size started_at_ms ended_at_ms <"$result_file" + completed=$((completed + 1)) + duration_total_ms=$((duration_total_ms + ended_at_ms - started_at_ms)) + + if [[ "$event_type" == 'message.updated' ]]; then + message_updated_requests=$((message_updated_requests + 1)) + message_updated_items=$((message_updated_items + batch_size)) + else + message_part_updated_requests=$((message_part_updated_requests + 1)) + message_part_updated_items=$((message_part_updated_items + batch_size)) + fi + + if [[ -f "$output_file" ]]; then + output="$(<"$output_file")" + if [[ "$output" =~ attempted=([0-9]+),[[:space:]]persisted=([0-9]+),[[:space:]]skipped=([0-9]+) ]]; then + hook_attempted=$((hook_attempted + BASH_REMATCH[1])) + hook_persisted=$((hook_persisted + BASH_REMATCH[2])) + hook_skipped=$((hook_skipped + BASH_REMATCH[3])) + fi + fi + + if (( exit_code == 0 )); then + succeeded=$((succeeded + 1)) + else + failed=$((failed + 1)) + failed_request_ids+=("${recorded_index}:exit-${exit_code}") + fi +done + +elapsed_ms=$((test_ended_at_ms - test_started_at_ms)) +average_duration_ms=0 +if (( completed > 0 )); then + average_duration_ms=$((duration_total_ms / completed)) +fi + +printf '\nResults\n' +printf ' Requests launched: %s\n' "$total_requests" +printf ' Requests completed: %s\n' "$completed" +printf ' Requests succeeded: %s\n' "$succeeded" +printf ' Requests failed: %s\n' "$failed" +printf ' message.updated requests: %s\n' "$message_updated_requests" +printf ' message.part.updated requests: %s\n' "$message_part_updated_requests" +printf ' message.updated payload items: %s\n' "$message_updated_items" +printf ' message.part payload items: %s\n' "$message_part_updated_items" +printf ' Hook attempted rows reported: %s\n' "$hook_attempted" +printf ' Hook persisted rows reported: %s\n' "$hook_persisted" +printf ' Hook skipped rows reported: %s\n' "$hook_skipped" +printf ' Total elapsed ms: %s\n' "$elapsed_ms" +printf ' Average hook duration ms: %s\n' "$average_duration_ms" + +if (( failed > 0 )); then + printf '\nFailed request samples\n' + sample_count=0 + for failure in "${failed_request_ids[@]}"; do + printf ' %s\n' "$failure" + sample_count=$((sample_count + 1)) + if (( sample_count >= 10 )); then + break + fi + done +fi + +if (( failed > 0 )); then + exit 1 +fi + +exit 0