feat: add build ai diagnosis and fix pipeline

.gitignore

@@ -10,6 +10,7 @@ infra/caddy/routes/
 bugs
 apps/api/index
 docker-compose.yml
+docker-compose.override.yml
 bump.sh
 scripts/workflow/bump.sh
 __pycache__

VERSION

@@ -1 +1 @@
-0.3.0
+0.4.0

apps/agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dequel-agent",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "private": true,
   "type": "module",
   "scripts": {

apps/api/Dockerfile.sandbox

@@ -0,0 +1,23 @@
+FROM oven/bun:1
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates \
+    git \
+    ripgrep \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /runner
+COPY package.sandbox.json ./package.json
+RUN bun install --production
+
+COPY src/fixdiag/sandbox-runner/ ./fixdiag/sandbox-runner/
+COPY src/fixdiag/types.ts ./fixdiag/types.ts
+COPY src/fixdiag/llm.ts ./fixdiag/llm.ts
+RUN rm -rf ./fixdiag/sandbox-runner/__tests__
+
+RUN mkdir -p /srv/jobs /srv/dequel-src /srv/project-src \
+    && chown -R bun:bun /srv/jobs /srv/dequel-src /srv/project-src
+
+USER bun
+
+CMD ["sleep", "infinity"]

apps/api/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dequel-api",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "private": true,
   "type": "module",
   "scripts": {
@@ -10,6 +10,7 @@
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.1130.0",
+    "@ax-llm/ax": "^25.0.0",
     "@elysiajs/cors": "^1.1.1",
     "@sinclair/typebox": "^0.34.13",
     "drizzle-orm": "^0.45.2",

apps/api/package.sandbox.json

@@ -0,0 +1,7 @@
+{
+	"name": "dequel-diag-sandbox",
+	"private": true,
+	"dependencies": {
+		"@ax-llm/ax": "^25.0.0"
+	}
+}

apps/api/src/api/index.ts

@@ -19,6 +19,8 @@ import { scalingRoutes } from "./scaling";
 import { serverInfoRoutes } from "./server-info";
 import { serversRoutes } from "./servers";
 import { settingsRoutes } from "./settings";
+import { llmSettingsRoutes } from "./settings/llm";
+import { fixdiagRoutes } from "../fixdiag/routes";
 import { sharedEnvLinksRoutes, sharedEnvVarsRoutes } from "./shared-env-vars";
 import { sshKeysRoutes } from "./ssh-keys";
 import { volumesRoutes } from "./volumes";
@@ -114,5 +116,7 @@ export const apiRoutes = new Elysia({
 	.use(alertsRoutes)
 	.use(githubRoutes)
 	.use(settingsRoutes)
+	.use(llmSettingsRoutes)
+	.use(fixdiagRoutes)
 	.use(routesRoutes)
 	.use(backupRoutes);

apps/api/src/api/settings/llm.ts

@@ -0,0 +1,118 @@
+import { Elysia } from "elysia";
+import {
+	LLM_PROVIDERS,
+	deleteLlmKey,
+	getDecryptedLlmKey,
+	getLlmKeyStatus,
+	updateLlmModels,
+	upsertLlmKey,
+} from "../../db/repo";
+import { fetchProviderModels, DEFAULT_PROVIDER_MODELS } from "../../fixdiag/provider-models";
+import { fail, ok } from "../response";
+
+export const llmSettingsRoutes = new Elysia({ prefix: "/settings" })
+	.get("/llm-keys", async () => ok(await getLlmKeyStatus()))
+	.get("/llm-default-models", async () => ok(DEFAULT_PROVIDER_MODELS))
+	.get("/llm-keys/:provider/models", async ({ params, query, set }: any) => {
+		const provider = String(params.provider);
+		if (!(LLM_PROVIDERS as readonly string[]).includes(provider)) {
+			set.status = 400;
+			return fail(`provider must be one of: ${LLM_PROVIDERS.join(", ")}`);
+		}
+		const existing = await getDecryptedLlmKey(provider);
+		if (!existing) {
+			set.status = 404;
+			return fail("Provider key not configured");
+		}
+		const refresh = query?.refresh === "true" || existing.models.length === 0;
+		if (refresh) {
+			const models = await fetchProviderModels({
+				provider,
+				apiKey: existing.apiKey,
+				baseUrl: existing.baseUrl,
+			});
+			if (models.length > 0) {
+				await updateLlmModels(provider, models);
+				return ok({ provider, models, cached: false });
+			}
+		}
+		return ok({ provider, models: existing.models, cached: true });
+	})
+	.post("/llm-keys/:provider/sync", async ({ params, set }: any) => {
+		const provider = String(params.provider);
+		if (!(LLM_PROVIDERS as readonly string[]).includes(provider)) {
+			set.status = 400;
+			return fail(`provider must be one of: ${LLM_PROVIDERS.join(", ")}`);
+		}
+		const existing = await getDecryptedLlmKey(provider);
+		if (!existing) {
+			set.status = 404;
+			return fail("Provider key not configured");
+		}
+		const models = await fetchProviderModels({
+			provider,
+			apiKey: existing.apiKey,
+			baseUrl: existing.baseUrl,
+		});
+		await updateLlmModels(provider, models);
+		return ok({ provider, models }, `Synced ${models.length} models for ${provider}`);
+	})
+	.put("/llm-keys", async ({ body, set }: any) => {
+		const provider = String(body?.provider ?? "");
+		if (!(LLM_PROVIDERS as readonly string[]).includes(provider)) {
+			set.status = 400;
+			return fail(`provider must be one of: ${LLM_PROVIDERS.join(", ")}`);
+		}
+		if (body?.apiKey !== undefined && typeof body.apiKey !== "string") {
+			set.status = 400;
+			return fail("apiKey must be a string");
+		}
+		const existing = await getDecryptedLlmKey(provider);
+		if (provider === "custom") {
+			if (!body?.baseURL && !existing?.baseUrl) {
+				set.status = 400;
+				return fail("baseURL is required for the custom provider");
+			}
+		}
+		const apiKey =
+			typeof body?.apiKey === "string" && body.apiKey.length > 0 ? body.apiKey : provider === "ollama" ? "ollama" : "";
+		const effectiveBaseUrl = body?.baseURL !== undefined ? body.baseURL : existing?.baseUrl;
+		const baseChanged = body?.baseURL !== undefined && body.baseURL !== existing?.baseUrl;
+		if (baseChanged && !apiKey && existing?.apiKey) {
+			set.status = 400;
+			return fail("apiKey is required when changing baseURL");
+		}
+		const effectiveApiKey = apiKey || existing?.apiKey || "";
+
+		let models: string[] | undefined = Array.isArray(body?.models) ? body.models.map(String) : undefined;
+		if (!models || models.length === 0) {
+			try {
+				const pulled = await fetchProviderModels({
+					provider,
+					apiKey: effectiveApiKey,
+					baseUrl: effectiveBaseUrl,
+				});
+				if (pulled.length > 0) {
+					models = pulled;
+				}
+			} catch {
+				models = existing?.models ?? undefined;
+			}
+		}
+
+		const status = await upsertLlmKey({
+			provider,
+			apiKey,
+			baseURL: body?.baseURL,
+			models,
+		});
+		return ok(status, "LLM provider key updated");
+	})
+	.delete("/llm-keys/:provider", async ({ params, set }: any) => {
+		const deleted = await deleteLlmKey(String(params.provider));
+		if (!deleted) {
+			set.status = 404;
+			return fail("LLM provider key not found");
+		}
+		return ok(null, "LLM provider key deleted");
+	});

apps/api/src/db/__tests__/diag-runs.test.ts

@@ -0,0 +1,84 @@
+import { afterAll, beforeAll, describe, expect, it, mock } from "bun:test";
+import { drizzle } from "drizzle-orm/node-postgres";
+import type { Pool } from "pg";
+import { setDbProvider } from "../db-provider";
+import * as schema from "../schema";
+import { createTestPool, truncateAllTables } from "../test-helper";
+import {
+	createDiagRun,
+	finishDiagRun,
+	getDiagRun,
+	getStagePayload,
+	listActiveDiagRuns,
+	listStageResults,
+	markInterruptedDiagRuns,
+	recordStageResult,
+} from "../repo/diag-runs";
+
+let pool: Pool;
+
+mock.restore();
+
+beforeAll(async () => {
+	pool = createTestPool();
+	const db = drizzle(pool, { schema });
+	setDbProvider(async () => db);
+	await truncateAllTables(pool);
+	await pool.query(
+		`INSERT INTO projects (id, name, source_type, created_at, updated_at)
+		 VALUES ('proj-diagruns', 'DiagRuns Project', 'git', NOW(), NOW()) ON CONFLICT DO NOTHING`,
+	);
+	await pool.query(
+		`INSERT INTO deployments (id, project_id, source_type, source_ref, status, created_at, updated_at)
+		 VALUES ('dep-x', 'proj-diagruns', 'git', 'https://github.com/test/repo.git', 'failed', NOW(), NOW()),
+		        ('dep-dup', 'proj-diagruns', 'git', 'https://github.com/test/repo.git', 'failed', NOW(), NOW())
+		 ON CONFLICT DO NOTHING`,
+	);
+});
+
+afterAll(async () => {
+	try {
+		await truncateAllTables(pool);
+	} finally {
+		await pool.end();
+	}
+});
+
+describe("diag-runs", () => {
+	it("creates, finds and finishes runs", async () => {
+		const run = await createDiagRun({ deploymentId: "dep-x", commitSha: "abc", provider: "groq", model: "m" });
+		expect(run.status).toBe("running");
+		expect(run.currentStage).toBeNull();
+
+		await recordStageResult(run.id, "triage", { failingStage: "build" });
+		expect(await getStagePayload(run.id, "triage")).toEqual({ failingStage: "build" });
+
+		await recordStageResult(run.id, "triage", { failingStage: "deploy" });
+		expect(await getStagePayload(run.id, "triage")).toEqual({ failingStage: "deploy" });
+		expect((await listStageResults(run.id)).filter((s) => s.stage === "triage")).toHaveLength(1);
+
+		await finishDiagRun(run.id, "done", "user-source", { cause: "user-source" }, null);
+		const done = await getDiagRun(run.id);
+		expect(done?.status).toBe("done");
+		expect(done?.cause).toBe("user-source");
+		expect(done?.report).toEqual({ cause: "user-source" });
+	});
+
+	it("rejects duplicate runs for the same deployment and commit", async () => {
+		await createDiagRun({ deploymentId: "dep-dup", commitSha: "", provider: "groq", model: "m" });
+		await expect(
+			createDiagRun({ deploymentId: "dep-dup", commitSha: "", provider: "groq", model: "m" }),
+		).rejects.toThrow();
+	});
+
+	it("lists active runs and marks interrupted ones as failed", async () => {
+		const run = await createDiagRun({ deploymentId: "dep-x", commitSha: "active-1", provider: "ollama", model: "m" });
+		const active = await listActiveDiagRuns();
+		const found = active.find((r) => r.id === run.id);
+		expect(found).toMatchObject({ deploymentId: "dep-x", projectName: "DiagRuns Project", provider: "ollama" });
+
+		expect(await markInterruptedDiagRuns()).toBeGreaterThanOrEqual(1);
+		expect((await getDiagRun(run.id))?.status).toBe("error");
+		expect(await listActiveDiagRuns()).toEqual([]);
+	});
+});

apps/api/src/db/__tests__/llm-keys.test.ts

@@ -0,0 +1,73 @@
+import { afterAll, beforeAll, describe, expect, it, mock } from "bun:test";
+import { drizzle } from "drizzle-orm/node-postgres";
+import type { Pool } from "pg";
+import { setDbProvider } from "../db-provider";
+import * as schema from "../schema";
+import { createTestPool, truncateAllTables } from "../test-helper";
+import { deleteLlmKey, getDecryptedLlmKey, getLlmKeyStatus, updateLlmModels, upsertLlmKey } from "../repo/llm-keys";
+
+let pool: Pool;
+
+mock.restore();
+
+beforeAll(async () => {
+	pool = createTestPool();
+	const db = drizzle(pool, { schema });
+	setDbProvider(async () => db);
+	await truncateAllTables(pool);
+});
+
+afterAll(async () => {
+	try {
+		await truncateAllTables(pool);
+	} finally {
+		await pool.end();
+	}
+});
+
+describe("llm-keys", () => {
+	it("stores encrypted and reports status without the secret", async () => {
+		const status = await upsertLlmKey({
+			provider: "groq",
+			apiKey: "gsk-secret-1",
+			models: ["llama-3.3-70b-versatile"],
+		});
+		expect(status.provider).toBe("groq");
+		expect(status.configured).toBe(true);
+		expect(status.models).toEqual(["llama-3.3-70b-versatile"]);
+		expect(JSON.stringify(status)).not.toContain("gsk-secret-1");
+
+		const all = await getLlmKeyStatus();
+		expect(all.find((s) => s.provider === "groq")?.configured).toBe(true);
+		expect(JSON.stringify(all)).not.toContain("gsk-secret-1");
+	});
+
+	it("round-trips the decrypted key", async () => {
+		await upsertLlmKey({ provider: "openai", apiKey: "sk-secret-2" });
+		const rec = await getDecryptedLlmKey("openai");
+		expect(rec?.apiKey).toBe("sk-secret-2");
+		expect(await getDecryptedLlmKey("missing")).toBeNull();
+	});
+
+	it("keeps the old key when apiKey is empty", async () => {
+		await upsertLlmKey({ provider: "groq", apiKey: "gsk-secret-1" });
+		const status = await upsertLlmKey({ provider: "groq", apiKey: "", models: ["other-model"] });
+		expect(status.configured).toBe(true);
+		expect(status.models).toEqual(["other-model"]);
+		expect((await getDecryptedLlmKey("groq"))?.apiKey).toBe("gsk-secret-1");
+	});
+
+	it("updates models for a provider", async () => {
+		await upsertLlmKey({ provider: "gemini", apiKey: "gem-sec-1" });
+		const updated = await updateLlmModels("gemini", ["gemini-2.5-flash", "gemini-2.5-pro"]);
+		expect(updated?.models).toEqual(["gemini-2.5-flash", "gemini-2.5-pro"]);
+		expect(await updateLlmModels("nonexistent", ["model"])).toBeNull();
+	});
+
+	it("deletes keys", async () => {
+		await upsertLlmKey({ provider: "anthropic", apiKey: "sk-ant-1" });
+		expect(await deleteLlmKey("anthropic")).toBe(true);
+		expect(await deleteLlmKey("anthropic")).toBe(false);
+		expect((await getLlmKeyStatus()).find((s) => s.provider === "anthropic")).toBeUndefined();
+	});
+});