diff --git a/scripts/triage/parse.ts b/scripts/triage/parse.ts index 6e13979..7d9706a 100644 --- a/scripts/triage/parse.ts +++ b/scripts/triage/parse.ts @@ -8,9 +8,69 @@ export function stripCodeFence(text: string): string { .trim(); } +// LLMs sometimes emit conversational preamble/postamble around the JSON array. +// Scan for the first balanced outermost [...] that parses as an array of objects, +// respecting quoted strings so brackets inside string values do not affect depth +// counting and so scalar arrays like [1,2] in prose are not mistaken for the +// decisions array. +function extractJsonArray(text: string): string { + let i = 0; + while (i < text.length) { + if (text[i] === "[") { + let depth = 0; + let j = i; + let inStr = false; + let esc = false; + while (j < text.length) { + const ch = text[j]; + if (esc) { + esc = false; + } else if (inStr) { + if (ch === "\\") esc = true; + else if (ch === '"') inStr = false; + } else { + if (ch === '"') inStr = true; + else if (ch === "[") depth++; + else if (ch === "]") { + depth--; + if (depth === 0) { + const candidate = text.slice(i, j + 1); + try { + const arr: unknown = JSON.parse(candidate); + if ( + Array.isArray(arr) && + arr.every( + (el: unknown) => typeof el === "object" && el !== null, + ) + ) { + return candidate; + } + } catch { + // not valid JSON, try next candidate + } + break; + } + } + } + j++; + } + i = j + 1; + } else { + i++; + } + } + return text; +} + export function parseDecisions(text: string): IssueDecision[] { const clean = stripCodeFence(text); - const parsed: unknown = JSON.parse(clean); + let parsed: unknown; + try { + parsed = JSON.parse(clean); + } catch { + // LLMs sometimes emit conversational preamble/postamble around the JSON array + parsed = JSON.parse(extractJsonArray(clean)); + } if (!Array.isArray(parsed)) { throw new Error("eval: expected JSON array of IssueDecision"); } diff --git a/test/triage-parse.test.ts b/test/triage-parse.test.ts new file mode 100644 index 0000000..b09d24c --- /dev/null +++ b/test/triage-parse.test.ts @@ -0,0 +1,77 @@ +import { describe, expect, it } from "vitest"; +import { parseDecisions, stripCodeFence } from "../scripts/triage/parse.js"; + +describe("stripCodeFence", () => { + it("removes ```json fence", () => { + expect(stripCodeFence("```json\n[]\n```")).toBe("[]"); + }); + + it("removes plain ``` fence", () => { + expect(stripCodeFence("```\n[]\n```")).toBe("[]"); + }); + + it("returns plain text unchanged", () => { + expect(stripCodeFence("[]")).toBe("[]"); + }); +}); + +describe("parseDecisions", () => { + const DECISIONS = [{ number: 1, decision: "ready", reason: "clear spec" }]; + + it("parses a bare JSON array", () => { + expect(parseDecisions(JSON.stringify(DECISIONS))).toEqual(DECISIONS); + }); + + it("extracts JSON array when model emits conversational preamble", () => { + const text = `I'll check each issue.\n${JSON.stringify(DECISIONS)}`; + expect(parseDecisions(text)).toEqual(DECISIONS); + }); + + it("extracts JSON array when model emits preamble and postamble", () => { + const text = `Sure! Here you go:\n${JSON.stringify(DECISIONS)}\nLet me know if you need more.`; + expect(parseDecisions(text)).toEqual(DECISIONS); + }); + + it("strips code fence then extracts JSON array with preamble", () => { + const text = `I'll evaluate now.\n\`\`\`json\n${JSON.stringify(DECISIONS)}\n\`\`\``; + expect(parseDecisions(text)).toEqual(DECISIONS); + }); + + it("handles preamble containing '['", () => { + const text = `Analyzing issues [#12, #34]:\n${JSON.stringify(DECISIONS)}`; + expect(parseDecisions(text)).toEqual(DECISIONS); + }); + + it("handles postamble containing ']'", () => { + const text = `${JSON.stringify(DECISIONS)}\nNote: sub-issues [#45] were skipped.`; + expect(parseDecisions(text)).toEqual(DECISIONS); + }); + + it("skips scalar array in preamble and extracts decisions array", () => { + const text = `See [1,2] for examples.\n${JSON.stringify(DECISIONS)}`; + expect(parseDecisions(text)).toEqual(DECISIONS); + }); + + it("handles closing bracket inside a reason string value", () => { + const d = [{ number: 1, decision: "ready", reason: "] edge case" }]; + expect(parseDecisions(`preamble\n${JSON.stringify(d)}`)).toEqual(d); + }); + + it("throws when input contains no JSON array", () => { + expect(() => parseDecisions("I'll check each issue.")).toThrow(SyntaxError); + }); + + it("throws a meaningful error when result is not an array", () => { + expect(() => parseDecisions(JSON.stringify({ not: "an array" }))).toThrow( + "eval: expected JSON array of IssueDecision", + ); + }); + + it("throws when decision.number is not a number", () => { + expect(() => + parseDecisions( + JSON.stringify([{ number: "1", decision: "ready", reason: "x" }]), + ), + ).toThrow("eval: decision.number must be a number"); + }); +});