diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f34a61d..5d3fb1f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -36,6 +36,7 @@ jobs: echo "API_IMAGE=ghcr.io/${REPO_LC}/api" >> $GITHUB_ENV echo "WEB_IMAGE=ghcr.io/${REPO_LC}/web" >> $GITHUB_ENV echo "AGENT_IMAGE=ghcr.io/${REPO_LC}/agent" >> $GITHUB_ENV + echo "SANDBOX_IMAGE=ghcr.io/${REPO_LC}/sandbox" >> $GITHUB_ENV env: REPO: ${{ github.repository }} @@ -56,6 +57,7 @@ jobs: push: true build-args: | BUILD_TIME=${{ github.sha }} + DEQUEL_POSTHOG_KEY=${{ secrets.DEQUEL_POSTHOG_KEY }} tags: | ${{ env.API_IMAGE }}:${{ steps.version.outputs.VERSION }} ${{ steps.version.outputs.IS_PRERELEASE == 'false' && format('{0}:latest', env.API_IMAGE) || format('{0}:next', env.API_IMAGE) }} @@ -89,6 +91,18 @@ jobs: cache-from: type=gha,scope=agent-${{ github.sha }} cache-to: type=gha,scope=agent-${{ github.sha }},mode=max + - name: Build and push diagnose sandbox image + uses: docker/build-push-action@v6 + with: + context: apps/api + file: apps/api/Dockerfile.sandbox + push: true + tags: | + ${{ env.SANDBOX_IMAGE }}:${{ steps.version.outputs.VERSION }} + ${{ steps.version.outputs.IS_PRERELEASE == 'false' && format('{0}:latest', env.SANDBOX_IMAGE) || format('{0}:next', env.SANDBOX_IMAGE) }} + cache-from: type=gha,scope=sandbox-${{ github.sha }} + cache-to: type=gha,scope=sandbox-${{ github.sha }},mode=max + - name: Build config tarball run: | VERSION="${{ steps.version.outputs.VERSION }}" diff --git a/.gitignore b/.gitignore index 5d04e06..2fbab5d 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ infra/caddy/routes/ bugs apps/api/index docker-compose.yml +docker-compose.override.yml bump.sh scripts/workflow/bump.sh __pycache__ diff --git a/AGENTS.md b/AGENTS.md index e4024ff..a732d09 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -178,7 +178,7 @@ Requires secrets: `VERCEL_TOKEN`, `VERCEL_ORG_ID`, `VERCEL_PROJECT_ID` | `DATABASE_URL` | `postgresql://dequel:dequel@localhost:5432/dequel` | PostgreSQL connection string | | `WORKSPACE_ROOT` | `./workspace` | Build staging | | `CADDY_ROUTES_DIR` | `./infra/caddy/routes` | Caddy route output | -| `CADDY_BASE_DOMAIN` | `localhost` | Base domain for deployment subdomains. Set to a real domain (e.g. `example.com`) for Let's Encrypt auto-SSL. | +| `CADDY_BASE_DOMAIN` | `localhost` | Base domain for deployment subdomains. Set to a real domain (e.g. `example.com`) for Let's Encrypt auto-SSL. Public links (e.g. failure email logs) derive their base URL from this. | | `CADDY_EMAIL` | _(empty)_ | Email for Let's Encrypt SSL certificate notifications | | `DOCKER_NETWORK` | `dequel_net` | Docker network for deployments | | `BUILDKIT_HOST` | `tcp://buildkit:1234` | Buildkit daemon | diff --git a/VERSION b/VERSION index 0d91a54..60a2d3e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.0 +0.4.0 \ No newline at end of file diff --git a/apps/agent/package.json b/apps/agent/package.json index da8553e..b0b2f89 100644 --- a/apps/agent/package.json +++ b/apps/agent/package.json @@ -1,6 +1,6 @@ { "name": "dequel-agent", - "version": "0.3.0", + "version": "0.4.0", "private": true, "type": "module", "scripts": { diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile index 69d785f..1f4b12b 100644 --- a/apps/api/Dockerfile +++ b/apps/api/Dockerfile @@ -1,6 +1,9 @@ FROM oven/bun:1 ARG BUILD_TIME=0 +ARG DEQUEL_POSTHOG_KEY="" + +ENV DEQUEL_POSTHOG_KEY=$DEQUEL_POSTHOG_KEY RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates \ diff --git a/apps/api/Dockerfile.sandbox b/apps/api/Dockerfile.sandbox new file mode 100644 index 0000000..f032e94 --- /dev/null +++ b/apps/api/Dockerfile.sandbox @@ -0,0 +1,23 @@ +FROM oven/bun:1 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + git \ + ripgrep \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /runner +COPY package.sandbox.json ./package.json +RUN bun install --production + +COPY src/fixdiag/sandbox-runner/ ./fixdiag/sandbox-runner/ +COPY src/fixdiag/types.ts ./fixdiag/types.ts +COPY src/fixdiag/llm.ts ./fixdiag/llm.ts +RUN rm -rf ./fixdiag/sandbox-runner/__tests__ + +RUN mkdir -p /srv/jobs /srv/dequel-src /srv/project-src \ + && chown -R bun:bun /srv/jobs /srv/dequel-src /srv/project-src + +USER bun + +CMD ["sleep", "infinity"] diff --git a/apps/api/package.json b/apps/api/package.json index 388f427..58d7fb5 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -1,6 +1,6 @@ { "name": "dequel-api", - "version": "0.3.0", + "version": "0.4.0", "private": true, "type": "module", "scripts": { @@ -10,6 +10,7 @@ }, "dependencies": { "@aws-sdk/client-s3": "^3.1130.0", + "@ax-llm/ax": "^25.0.0", "@elysiajs/cors": "^1.1.1", "@sinclair/typebox": "^0.34.13", "drizzle-orm": "^0.45.2", diff --git a/apps/api/package.sandbox.json b/apps/api/package.sandbox.json new file mode 100644 index 0000000..3b320e6 --- /dev/null +++ b/apps/api/package.sandbox.json @@ -0,0 +1,7 @@ +{ + "name": "dequel-diag-sandbox", + "private": true, + "dependencies": { + "@ax-llm/ax": "^25.0.0" + } +} diff --git a/apps/api/src/agents/job-channel.ts b/apps/api/src/agents/job-channel.ts index 8c0e4af..c64bb91 100644 --- a/apps/api/src/agents/job-channel.ts +++ b/apps/api/src/agents/job-channel.ts @@ -9,6 +9,7 @@ import { leaseNextAgentJob, listCancelledJobIds, listDeployments, + recordDeploymentFailure, updateAgentHeartbeat, updateDeploymentCommitSha, updateDeploymentStatus, @@ -119,8 +120,10 @@ export const processAgentJobUpdate = async ( } } } else { - await updateDeploymentStatus(deploymentId, "failed", { - failureReason: update.error || "Remote agent deployment failed", + await recordDeploymentFailure({ + deploymentId, + reason: update.error || "Remote agent deployment failed", + source: "job-channel", }); await appendLog(deploymentId, "system", `Remote deployment failed: ${update.error || "Unknown agent error"}`); } diff --git a/apps/api/src/api/__tests__/auth-routes.test.ts b/apps/api/src/api/__tests__/auth-routes.test.ts new file mode 100644 index 0000000..e3c213e --- /dev/null +++ b/apps/api/src/api/__tests__/auth-routes.test.ts @@ -0,0 +1,110 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, it } from "bun:test"; +import { drizzle } from "drizzle-orm/node-postgres"; +import type { Pool } from "pg"; +import { setDbProvider } from "../../db/db-provider"; +import * as schema from "../../db/schema"; +import { createTestPool, truncateAllTables } from "../../db/test-helper"; + +const TEST_SECRET = "test-jwt-secret-for-testing-purposes-only"; +let pool: Pool; + +beforeAll(async () => { + pool = createTestPool(); + const db = drizzle(pool, { schema }); + setDbProvider(async () => db); + const { initAuth } = await import("../../utils/auth"); + initAuth(TEST_SECRET); +}); + +beforeEach(async () => { + await truncateAllTables(pool); +}); + +afterAll(async () => { + await truncateAllTables(pool); + await pool.end(); +}); + +describe("auth routes", () => { + it("GET /auth/me returns expiresIn when access token is valid", async () => { + const { authRoutes } = await import("../auth/index"); + const { signAccessToken } = await import("../../utils/auth"); + const token = await signAccessToken("testuser"); + const res = await authRoutes.handle( + new Request("http://localhost/auth/me", { + headers: { + cookie: `dequel_session=${token}`, + }, + }), + ); + expect(res.status).toBe(200); + const json = (await res.json()) as { + status: string; + data: { authenticated: boolean; username: string; expiresIn: number }; + }; + expect(json.status).toBe("success"); + expect(json.data.authenticated).toBe(true); + expect(json.data.username).toBe("testuser"); + expect(typeof json.data.expiresIn).toBe("number"); + expect(json.data.expiresIn).toBeGreaterThan(0); + expect(json.data.expiresIn).toBeLessThanOrEqual(900); + }); + + it("GET /auth/me auto-refreshes when access token is missing but refresh token is valid", async () => { + const { authRoutes } = await import("../auth/index"); + const { generateRefreshToken, storeRefreshToken } = await import("../../utils/auth"); + const rt = generateRefreshToken(); + await storeRefreshToken("testuser", rt); + + const res = await authRoutes.handle( + new Request("http://localhost/auth/me", { + headers: { + cookie: `dequel_refresh=${rt}`, + }, + }), + ); + expect(res.status).toBe(200); + const json = (await res.json()) as { + status: string; + data: { authenticated: boolean; username: string; expiresIn: number }; + }; + expect(json.status).toBe("success"); + expect(json.data.authenticated).toBe(true); + expect(json.data.username).toBe("testuser"); + expect(json.data.expiresIn).toBe(900); + + const cookies = res.headers.get("set-cookie") || ""; + expect(cookies).toContain("dequel_session="); + expect(cookies).toContain("dequel_refresh="); + }); + + it("GET /auth/me returns unauthenticated when no valid tokens exist", async () => { + const { authRoutes } = await import("../auth/index"); + const res = await authRoutes.handle(new Request("http://localhost/auth/me")); + expect(res.status).toBe(200); + const json = (await res.json()) as { status: string; data: { authenticated: boolean } }; + expect(json.status).toBe("success"); + expect(json.data.authenticated).toBe(false); + }); + + it("POST /auth/refresh returns expiresIn: 900 and rotates tokens", async () => { + const { authRoutes } = await import("../auth/index"); + const { generateRefreshToken, storeRefreshToken } = await import("../../utils/auth"); + const rt = generateRefreshToken(); + await storeRefreshToken("testuser", rt); + + const res = await authRoutes.handle( + new Request("http://localhost/auth/refresh", { + method: "POST", + headers: { + cookie: `dequel_refresh=${rt}`, + }, + }), + ); + expect(res.status).toBe(200); + const json = (await res.json()) as { status: string; data: { username: string; expiresIn: number } }; + expect(json.status).toBe("success"); + expect(json.data.username).toBe("testuser"); + expect(json.data.expiresIn).toBe(900); + }); +}); diff --git a/apps/api/src/api/alerts/index.ts b/apps/api/src/api/alerts/index.ts index a1bc0c3..452cfa4 100644 --- a/apps/api/src/api/alerts/index.ts +++ b/apps/api/src/api/alerts/index.ts @@ -1,7 +1,12 @@ import { Elysia } from "elysia"; import { createAlert, deleteAlert, listAlerts, updateAlertEnabled } from "../../db/repo"; +import type { AlertChannel, AlertType } from "../../types"; +import { validateDestination } from "../../utils/destination"; import { created, fail, ok } from "../response"; +const ALERT_TYPES: ReadonlySet = new Set(["cpu", "memory", "downtime"]); +const ALERT_CHANNELS: ReadonlySet = new Set(["email", "slack", "webhook"]); + export const alertsRoutes = new Elysia() .get("/projects/:id/alerts", async ({ params }) => ok(await listAlerts(params.id))) .post("/projects/:id/alerts", async ({ params, body, set }: any) => { @@ -9,6 +14,21 @@ export const alertsRoutes = new Elysia() set.status = 400; return fail("type and channel are required"); } + if (!ALERT_TYPES.has(String(body.type))) { + set.status = 400; + return fail(`type must be one of: ${[...ALERT_TYPES].join(", ")}`); + } + if (!ALERT_CHANNELS.has(String(body.channel))) { + set.status = 400; + return fail(`channel must be one of: ${[...ALERT_CHANNELS].join(", ")}`); + } + if (body.channel !== "email") { + const destinationError = await validateDestination(String(body.destination ?? "")); + if (destinationError) { + set.status = 400; + return fail(destinationError); + } + } return created( await createAlert({ projectId: params.id, diff --git a/apps/api/src/api/auth/index.ts b/apps/api/src/api/auth/index.ts index 85ebb74..fa1c2c0 100644 --- a/apps/api/src/api/auth/index.ts +++ b/apps/api/src/api/auth/index.ts @@ -67,7 +67,7 @@ export const authRoutes = new Elysia() dequel_session.set({ ...SESSION_COOKIE_OPTS, secure: isSecure }); dequel_refresh.value = refreshToken; dequel_refresh.set({ ...REFRESH_COOKIE_OPTS, secure: isSecure }); - return ok({ username }, "Logged in"); + return ok({ username, expiresIn: 900 }, "Logged in"); }) .post("/auth/logout", async ({ cookie: { dequel_session, dequel_refresh } }) => { const rt = dequel_refresh.value; @@ -99,12 +99,39 @@ export const authRoutes = new Elysia() dequel_session.set({ ...SESSION_COOKIE_OPTS, secure: isSecure }); dequel_refresh.value = newRefreshToken; dequel_refresh.set({ ...REFRESH_COOKIE_OPTS, secure: isSecure }); - return ok({ username }, "Token refreshed"); + return ok({ username, expiresIn: 900 }, "Token refreshed"); }) - .get("/auth/me", async ({ cookie: { dequel_session } }) => { + .get("/auth/me", async ({ cookie: { dequel_session, dequel_refresh }, isSecure }) => { const token = dequel_session.value; - if (!token) return ok({ authenticated: false }); - const payload = await verifyAccessToken(token); - if (!payload) return ok({ authenticated: false }); - return ok({ authenticated: true, username: payload.sub }); + if (token) { + const payload = await verifyAccessToken(token); + if (payload) { + const current = Math.floor(Date.now() / 1000); + return ok({ + authenticated: true, + username: payload.sub, + expiresIn: Math.max(0, payload.exp - current), + }); + } + } + const rt = dequel_refresh.value; + if (rt) { + const username = await validateRefreshToken(rt); + if (username) { + await blacklistRefreshToken(rt); + const accessToken = await signAccessToken(username); + const newRefreshToken = generateRefreshToken(); + await storeRefreshToken(username, newRefreshToken); + dequel_session.value = accessToken; + dequel_session.set({ ...SESSION_COOKIE_OPTS, secure: isSecure }); + dequel_refresh.value = newRefreshToken; + dequel_refresh.set({ ...REFRESH_COOKIE_OPTS, secure: isSecure }); + return ok({ + authenticated: true, + username, + expiresIn: 900, + }); + } + } + return ok({ authenticated: false }); }); diff --git a/apps/api/src/api/backups/index.ts b/apps/api/src/api/backups/index.ts index 5bee6d6..b052cb0 100644 --- a/apps/api/src/api/backups/index.ts +++ b/apps/api/src/api/backups/index.ts @@ -1,7 +1,7 @@ import { Elysia } from "elysia"; import { BackupOrchestrator } from "../../backup/orchestrator"; -import { S3_BACKUP_PREFIX } from "../../backup/types"; import type { BackupTarget, StorageConfig } from "../../backup/types"; +import { S3_BACKUP_PREFIX } from "../../backup/types"; import { deleteBackupRecord, getBackupRecord, listBackupRecords } from "../../db/repo/backups"; import { getDatabaseById } from "../../db/repo/databases"; import { getBackupStorageSettings } from "../../db/repo/settings"; diff --git a/apps/api/src/api/databases/query.ts b/apps/api/src/api/databases/query.ts index d5c259b..9c05a5d 100644 --- a/apps/api/src/api/databases/query.ts +++ b/apps/api/src/api/databases/query.ts @@ -60,16 +60,56 @@ export const getDatabaseTables = async (dbRecord: Database): Promise 250 then break end; res[#res+1] = k .. '|||' .. (redis.call('TYPE', k)['ok'] or 'string') end; return res"; + const res = await dockerExec( + dbRecord.containerName, + ["redis-cli", ...authArgs, "--raw", "EVAL", script, "0"], + server, + ); + if (res.code === 0) { + const lines = res.stdout + .split("\n") + .map((l) => l.trim()) + .filter((l) => l.length > 0 && !l.startsWith("Warning:")); + if (lines.length > 0) { + return lines.map((line) => { + const [name, type] = line.split("|||"); + return { name: name || line, type: type || "string" }; + }); + } + } + + const fallbackRes = await dockerExec( + dbRecord.containerName, + ["redis-cli", ...authArgs, "--raw", "KEYS", "*"], + server, + ); + if (fallbackRes.code !== 0) return []; + const keys = fallbackRes.stdout .split("\n") .map((l) => l.trim()) - .filter(Boolean); - return keys.slice(0, 50).map((name) => ({ name, type: "key" })); + .filter((l) => l.length > 0 && !l.startsWith("Warning:")); + return keys.slice(0, 100).map((name) => ({ name, type: "string" })); } if (dbRecord.type === "mongodb") { + const evalScript = ` +(() => { + try { + const names = db.getCollectionNames(); + const res = names.map(name => { + let count = 0; + try { count = db[name].estimatedDocumentCount(); } catch {} + return { name, type: 'collection', rowCount: count }; + }); + return '__DEQUEL_JSON_START__' + EJSON.stringify(res) + '__DEQUEL_JSON_END__'; + } catch (e) { + return '__DEQUEL_JSON_START__' + EJSON.stringify(db.getCollectionNames().map(name => ({ name, type: 'collection' }))) + '__DEQUEL_JSON_END__'; + } +})() +`; const res = await dockerExec( dbRecord.containerName, [ @@ -83,11 +123,21 @@ export const getDatabaseTables = async (dbRecord: Database): Promise { + if ((arg.startsWith('"') && arg.endsWith('"')) || (arg.startsWith("'") && arg.endsWith("'"))) { + return arg.slice(1, -1); + } + return arg; + }); +} + +const normalizeBson = (obj: any): any => { + if (obj === null || obj === undefined) return obj; + if (typeof obj === "object") { + if (typeof obj.$oid === "string") return obj.$oid; + if (typeof obj.$date === "string") return obj.$date; + if (typeof obj.$numberLong === "string") return Number(obj.$numberLong); + if (Array.isArray(obj)) return obj.map(normalizeBson); + const clean: Record = {}; + for (const [k, v] of Object.entries(obj)) { + clean[k] = normalizeBson(v); + } + return clean; + } + return obj; +}; + export const executeDatabaseQuery = async (dbRecord: Database, query: string): Promise => { if (!dbRecord.containerName) { throw new Error("Database container is not active"); @@ -194,24 +271,67 @@ export const executeDatabaseQuery = async (dbRecord: Database, query: string): P } if (dbRecord.type === "redis") { - const parts = query.trim().split(/\s+/); - const res = await dockerExec(dbRecord.containerName, ["redis-cli", "-a", dbRecord.password, ...parts], server); - const executionTimeMs = Date.now() - startTime; + const rawLines = query + .split("\n") + .map((l) => l.trim()) + .filter((l) => l.length > 0 && !l.startsWith("#")); - if (res.code !== 0) { - throw new Error(res.stderr || res.stdout || "Command execution failed"); + if (rawLines.length === 0) { + return { rows: [], columns: ["command", "result"], executionTimeMs: 0 }; + } + + const results: Record[] = []; + for (const line of rawLines) { + const parts = parseRedisCommandArgs(line); + if (parts.length === 0) continue; + + const redisArgs = ["redis-cli"]; + if (dbRecord.password) { + redisArgs.push("-a", dbRecord.password); + } + redisArgs.push(...parts); + + const res = await dockerExec(dbRecord.containerName, redisArgs, server); + const cleanOut = (res.stdout || "").replace(/^Warning: Using a password[^\n]*\n?/gm, "").trim(); + const cleanErr = (res.stderr || "").replace(/^Warning: Using a password[^\n]*\n?/gm, "").trim(); + const output = res.code === 0 ? cleanOut : cleanErr || cleanOut || "ERROR"; + results.push({ + command: line, + result: output, + }); } - const output = res.stdout; + const executionTimeMs = Date.now() - startTime; return { - rows: [{ result: output }], - columns: ["result"], + rows: results, + columns: ["command", "result"], executionTimeMs, - rawOutput: output, + rawOutput: results.map((r) => `${r.command} => ${r.result}`).join("\n"), }; } if (dbRecord.type === "mongodb") { + const wrappedScript = ` +(() => { + try { + let __res = (${query}); + if (__res && typeof __res.toArray === 'function') { + __res = __res.toArray(); + } + return '__DEQUEL_JSON_START__' + EJSON.stringify(__res) + '__DEQUEL_JSON_END__'; + } catch (e) { + try { + let __res2 = eval(${JSON.stringify(query)}); + if (__res2 && typeof __res2.toArray === 'function') { + __res2 = __res2.toArray(); + } + return '__DEQUEL_JSON_START__' + EJSON.stringify(__res2) + '__DEQUEL_JSON_END__'; + } catch (e2) { + return '__DEQUEL_JSON_START__' + EJSON.stringify({ __mongo_error: e2.message || String(e2) }) + '__DEQUEL_JSON_END__'; + } + } +})() +`; const res = await dockerExec( dbRecord.containerName, [ @@ -225,26 +345,74 @@ export const executeDatabaseQuery = async (dbRecord: Database, query: string): P dbRecord.databaseName, "--quiet", "--eval", - query, + wrappedScript, ], server, ); const executionTimeMs = Date.now() - startTime; - if (res.code !== 0) { - throw new Error(res.stderr || res.stdout || "MongoDB query failed"); + if (res.code === 0) { + const startIdx = res.stdout.indexOf("__DEQUEL_JSON_START__"); + const endIdx = res.stdout.indexOf("__DEQUEL_JSON_END__"); + if (startIdx !== -1 && endIdx !== -1) { + const jsonStr = res.stdout.substring(startIdx + "__DEQUEL_JSON_START__".length, endIdx); + try { + const parsed = JSON.parse(jsonStr); + if (parsed && typeof parsed === "object" && "__mongo_error" in parsed) { + throw new Error(parsed.__mongo_error); + } + const normalized = normalizeBson(parsed); + const rows: Record[] = Array.isArray(normalized) + ? normalized.map((item) => (typeof item === "object" && item !== null ? item : { result: item })) + : typeof normalized === "object" && normalized !== null + ? [normalized] + : [{ result: normalized }]; + + const colSet = new Set(); + rows.forEach((r) => Object.keys(r).forEach((k) => colSet.add(k))); + const columns = Array.from(colSet); + return { + rows, + columns: columns.length > 0 ? columns : ["result"], + executionTimeMs, + rawOutput: JSON.stringify(normalized, null, 2), + }; + } catch (e: any) { + if (e.message && !e.message.includes("JSON")) throw e; + } + } } + const fallbackRes = await dockerExec( + dbRecord.containerName, + [ + "mongosh", + "-u", + dbRecord.username, + "-p", + dbRecord.password, + "--authenticationDatabase", + "admin", + dbRecord.databaseName, + "--quiet", + "--eval", + query, + ], + server, + ); + if (fallbackRes.code !== 0) { + throw new Error(fallbackRes.stderr || fallbackRes.stdout || "MongoDB query failed"); + } let rows: Record[] = []; try { - const parsed = JSON.parse(res.stdout); - rows = Array.isArray(parsed) ? parsed : [parsed]; + const parsed = JSON.parse(fallbackRes.stdout); + const norm = normalizeBson(parsed); + rows = Array.isArray(norm) ? norm : [norm]; } catch { - rows = [{ result: res.stdout }]; + rows = [{ result: fallbackRes.stdout.trim() }]; } - const columns = rows.length > 0 ? Object.keys(rows[0]) : ["result"]; - return { rows, columns, executionTimeMs, rawOutput: res.stdout }; + return { rows, columns, executionTimeMs: Date.now() - startTime, rawOutput: fallbackRes.stdout.trim() }; } throw new Error(`Unsupported database type: ${dbRecord.type}`); diff --git a/apps/api/src/api/deployments/index.ts b/apps/api/src/api/deployments/index.ts index 8146917..138e5c8 100644 --- a/apps/api/src/api/deployments/index.ts +++ b/apps/api/src/api/deployments/index.ts @@ -10,9 +10,11 @@ import { getProjectById, getServerById, listDeployments, + recordDeploymentFailure, } from "../../db/repo"; import { executorFor } from "../../executors/dispatch"; import { orchestrator } from "../../orchestrator"; +import { summarizeDeploymentError } from "../../orchestrator/deployment-errors"; import { logBus } from "../../orchestrator/log-bus"; import { config } from "../../utils/config"; import { isPrivateGitUrl } from "../../utils/validate"; @@ -31,8 +33,13 @@ const dispatchDeployment = async ( if (deployment.sourceType !== "git") throw new Error("Remote servers currently support Git deployments only"); const executor = executorFor(server.mode); if (server.mode === "ssh") { - void executor.deploy({ deployment, project, server }).catch((error) => { + void executor.deploy({ deployment, project, server }).catch(async (error) => { console.error(`[SSH Executor] Deployment ${deployment.id} failed:`, error); + await recordDeploymentFailure({ + deploymentId: deployment.id, + reason: summarizeDeploymentError(error), + source: "dispatch", + }).catch((e) => console.error(`[SSH Executor] Failed to record failure for ${deployment.id}:`, e)); }); return; } diff --git a/apps/api/src/api/index.ts b/apps/api/src/api/index.ts index b779741..571912f 100644 --- a/apps/api/src/api/index.ts +++ b/apps/api/src/api/index.ts @@ -1,4 +1,5 @@ import { Elysia } from "elysia"; +import { fail } from "./response"; import { agentRoutes } from "./agents"; import { alertsRoutes } from "./alerts"; import { apiKeysRoutes } from "./api-keys"; @@ -11,12 +12,15 @@ import { envVarsRoutes } from "./env-vars"; import { githubRoutes } from "./github"; import { healthRoutes } from "./health"; import { projectsRoutes } from "./projects"; +import { projectStatusRoutes } from "./projects/status"; import { prometheusRoutes } from "./prometheus"; import { routesRoutes } from "./routes"; import { scalingRoutes } from "./scaling"; import { serverInfoRoutes } from "./server-info"; import { serversRoutes } from "./servers"; import { settingsRoutes } from "./settings"; +import { llmSettingsRoutes } from "./settings/llm"; +import { fixdiagRoutes } from "../fixdiag/routes"; import { sharedEnvLinksRoutes, sharedEnvVarsRoutes } from "./shared-env-vars"; import { sshKeysRoutes } from "./ssh-keys"; import { volumesRoutes } from "./volumes"; @@ -44,7 +48,7 @@ const authMiddleware = (app: Elysia) => const payload = await verifyAccessToken(match[1]); if (payload) return; set.status = 401; - return { error: "Invalid session" }; + return fail("Invalid session"); } const authHeader = request.headers.get("authorization"); @@ -55,22 +59,47 @@ const authMiddleware = (app: Elysia) => const key = await validateApiKey(token); if (key) return; set.status = 401; - return { error: "Invalid API key" }; + return fail("Invalid API key"); } } set.status = 401; - return { error: "Authentication required" }; + return fail("Authentication required"); }); +const INTERNAL_ERROR = + /Failed query:|params:|getaddrinfo|ECONNREFUSED|ETIMEDOUT|EAI_AGAIN|EHOSTUNREACH|timeout exceeded|node:internal|Cannot read propert|is not a function|is not a constructor|Unexpected token/i; + +import { captureTelemetry } from "../utils/telemetry"; + export const apiRoutes = new Elysia({ prefix: "/api", }) + .onError(({ error, set, path }) => { + const err = error as { status?: number; message?: string; name?: string }; + set.status = typeof err?.status === "number" ? err.status : 500; + const message = err?.message ?? "Internal server error"; + + if (set.status >= 500) { + captureTelemetry("server_error", { + path, + status: set.status, + error_name: err?.name || "UnhandledServerError", + }).catch(() => {}); + } + + if (set.status >= 500 || INTERNAL_ERROR.test(message)) { + console.error("[API] Unhandled error:", error); + return fail("Internal server error"); + } + return fail(message); + }) .use(authRoutes) .use(authMiddleware) .use(agentRoutes) .use(healthRoutes) .use(projectsRoutes) + .use(projectStatusRoutes) .use(deploymentsRoutes) .use(envVarsRoutes) .use(sharedEnvVarsRoutes) @@ -87,5 +116,7 @@ export const apiRoutes = new Elysia({ .use(alertsRoutes) .use(githubRoutes) .use(settingsRoutes) + .use(llmSettingsRoutes) + .use(fixdiagRoutes) .use(routesRoutes) .use(backupRoutes); diff --git a/apps/api/src/api/projects/index.ts b/apps/api/src/api/projects/index.ts index acc10ee..fe698b2 100644 --- a/apps/api/src/api/projects/index.ts +++ b/apps/api/src/api/projects/index.ts @@ -6,7 +6,6 @@ import { deleteProjectCascade, getProjectById, getServerById, - listDomains, listProjects, updateProject, } from "../../db/repo"; @@ -15,7 +14,9 @@ import { reloadCaddy, tryRun } from "../../orchestrator/runtime"; import { config } from "../../utils/config"; import { dockerBin } from "../../utils/docker-bin"; import { removeFromCaddyRoute } from "../../utils/domain-verifier"; +import { buildProjectRequestHostRegex, caddyRequestLogSelector } from "../../utils/loki"; import { isPort, isPrivateGitUrl, SERVICE_NAME_RE, validateComposeServices } from "../../utils/validate"; +import { captureTelemetry } from "../../utils/telemetry"; import { created, fail, ok } from "../response"; const validateComposeFields = (body: any): string | null => { @@ -108,6 +109,12 @@ export const projectsRoutes = new Elysia() outputDir: body.outputDir || undefined, startCommand: body.startCommand || undefined, }); + + captureTelemetry("project_created", { + build_type: project.buildType, + project_type: project.projectType, + source_type: project.sourceType, + }).catch(() => {}); return created(project); }) .patch("/projects/:id", async ({ params: { id }, body, set }: any) => { @@ -194,22 +201,8 @@ export const projectsRoutes = new Elysia() set.status = 404; return fail("Project not found"); } - const slugify = (s: string) => - s - .toLowerCase() - .replace(/[^a-z0-9-]+/g, "-") - .replace(/^-+|-+$/g, "") - .slice(0, 63); - const slug = slugify(project.name); - const domains = [`${slug}.${config.caddyBaseDomain}`]; - const projectDomains = await listDomains(id); - const verified = projectDomains.filter((d) => d.validationStatus === "verified"); - for (const d of verified) { - domains.push(d.domain); - } - - const regexEscaped = domains.map((d) => d.replace(/[-/\\^$*+?.()|[\]{}]/g, "\\\\$&")).join("|"); - const queryStr = `{container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$"`; + const hostRegex = await buildProjectRequestHostRegex(id); + const queryStr = caddyRequestLogSelector(hostRegex); const startParam = (queryParams as any)?.start; const endParam = (queryParams as any)?.end; @@ -274,23 +267,9 @@ export const projectsRoutes = new Elysia() set.status = 404; return fail("Project not found"); } - const slugify = (s: string) => - s - .toLowerCase() - .replace(/[^a-z0-9-]+/g, "-") - .replace(/^-+|-+$/g, "") - .slice(0, 63); - const slug = slugify(project.name); - const domains = [`${slug}.${config.caddyBaseDomain}`]; - const projectDomains = await listDomains(id); - const verified = projectDomains.filter((d) => d.validationStatus === "verified"); - for (const d of verified) { - domains.push(d.domain); - } + const hostRegex = await buildProjectRequestHostRegex(id); - const regexEscaped = domains.map((d) => d.replace(/[-/\\^$*+?.()|[\]{}]/g, "\\\\$&")).join("|"); - - const query = `sum(count_over_time({container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$" [5m]))`; + const query = `sum(count_over_time(${caddyRequestLogSelector(hostRegex)} [5m]))`; const end = Math.floor(Date.now() / 1000); const start = end - 6 * 60 * 60; @@ -325,22 +304,8 @@ export const projectsRoutes = new Elysia() return fail("Project not found"); } const encoder = new TextEncoder(); - const slugify = (s: string) => - s - .toLowerCase() - .replace(/[^a-z0-9-]+/g, "-") - .replace(/^-+|-+$/g, "") - .slice(0, 63); - const slug = slugify(project.name); - const domains = [`${slug}.${config.caddyBaseDomain}`]; - const projectDomains = await listDomains(id); - const verified = projectDomains.filter((d) => d.validationStatus === "verified"); - for (const d of verified) { - domains.push(d.domain); - } - - const regexEscaped = domains.map((d) => d.replace(/[-/\\^$*+?.()|[\]{}]/g, "\\\\$&")).join("|"); - const query = `{container="dequel-caddy-1"} | json | request_host =~ "^(${regexEscaped})$"`; + const hostRegex = await buildProjectRequestHostRegex(id); + const query = caddyRequestLogSelector(hostRegex); let ws: WebSocket | null = null; let closed = false; diff --git a/apps/api/src/api/projects/status.ts b/apps/api/src/api/projects/status.ts new file mode 100644 index 0000000..eedf904 --- /dev/null +++ b/apps/api/src/api/projects/status.ts @@ -0,0 +1,29 @@ +import { Elysia } from "elysia"; +import { getProjectById } from "../../db/repo"; +import { getProjectStatus } from "../../monitoring/project-status"; +import { fail, ok } from "../response"; + +const DEFAULT_WINDOW_SECONDS = 3600; +const MIN_WINDOW_SECONDS = 60; +const MAX_WINDOW_SECONDS = 604800; + +const clampWindow = (raw: unknown): number => { + const value = Number(raw); + if (!Number.isFinite(value) || value <= 0) return DEFAULT_WINDOW_SECONDS; + return Math.min(MAX_WINDOW_SECONDS, Math.max(MIN_WINDOW_SECONDS, Math.floor(value))); +}; + +export const projectStatusRoutes = new Elysia().get("/projects/:id/status", async ({ params: { id }, query, set }) => { + const project = await getProjectById(id); + if (!project) { + set.status = 404; + return fail("Project not found"); + } + try { + return ok(await getProjectStatus(id, clampWindow((query as any)?.window))); + } catch (err) { + console.error(`[Status] Failed to build status for project ${id}:`, err); + set.status = 500; + return fail("Failed to build project status"); + } +}); diff --git a/apps/api/src/api/settings/index.ts b/apps/api/src/api/settings/index.ts index 61c5c81..79895c0 100644 --- a/apps/api/src/api/settings/index.ts +++ b/apps/api/src/api/settings/index.ts @@ -9,6 +9,7 @@ import { upsertBackupStorageSettings, upsertSmtpSettings, } from "../../db/repo"; +import { buildSmtpTestEmail } from "../../monitoring/templates"; import { failoverState } from "../../orchestrator/failover"; import { rerenderAllIngressRoutes } from "../../orchestrator/ingress-sync"; import { fail, ok } from "../response"; @@ -87,10 +88,12 @@ export const settingsRoutes = new Elysia({ prefix: "/settings" }) secure: settings.port === 465, auth: settings.user && settings.pass ? { user: settings.user, pass: settings.pass } : undefined, }); + const { subject, html } = buildSmtpTestEmail(); await transporter.sendMail({ from: settings.fromAddress, to: settings.fromAddress, - subject: "[Dequel] SMTP Test Email", + subject, + html, text: "This is a test email from Dequel. Your SMTP settings are working correctly.", }); return ok(null, "Test email sent"); diff --git a/apps/api/src/api/settings/llm.ts b/apps/api/src/api/settings/llm.ts new file mode 100644 index 0000000..d215073 --- /dev/null +++ b/apps/api/src/api/settings/llm.ts @@ -0,0 +1,118 @@ +import { Elysia } from "elysia"; +import { + LLM_PROVIDERS, + deleteLlmKey, + getDecryptedLlmKey, + getLlmKeyStatus, + updateLlmModels, + upsertLlmKey, +} from "../../db/repo"; +import { fetchProviderModels, DEFAULT_PROVIDER_MODELS } from "../../fixdiag/provider-models"; +import { fail, ok } from "../response"; + +export const llmSettingsRoutes = new Elysia({ prefix: "/settings" }) + .get("/llm-keys", async () => ok(await getLlmKeyStatus())) + .get("/llm-default-models", async () => ok(DEFAULT_PROVIDER_MODELS)) + .get("/llm-keys/:provider/models", async ({ params, query, set }: any) => { + const provider = String(params.provider); + if (!(LLM_PROVIDERS as readonly string[]).includes(provider)) { + set.status = 400; + return fail(`provider must be one of: ${LLM_PROVIDERS.join(", ")}`); + } + const existing = await getDecryptedLlmKey(provider); + if (!existing) { + set.status = 404; + return fail("Provider key not configured"); + } + const refresh = query?.refresh === "true" || existing.models.length === 0; + if (refresh) { + const models = await fetchProviderModels({ + provider, + apiKey: existing.apiKey, + baseUrl: existing.baseUrl, + }); + if (models.length > 0) { + await updateLlmModels(provider, models); + return ok({ provider, models, cached: false }); + } + } + return ok({ provider, models: existing.models, cached: true }); + }) + .post("/llm-keys/:provider/sync", async ({ params, set }: any) => { + const provider = String(params.provider); + if (!(LLM_PROVIDERS as readonly string[]).includes(provider)) { + set.status = 400; + return fail(`provider must be one of: ${LLM_PROVIDERS.join(", ")}`); + } + const existing = await getDecryptedLlmKey(provider); + if (!existing) { + set.status = 404; + return fail("Provider key not configured"); + } + const models = await fetchProviderModels({ + provider, + apiKey: existing.apiKey, + baseUrl: existing.baseUrl, + }); + await updateLlmModels(provider, models); + return ok({ provider, models }, `Synced ${models.length} models for ${provider}`); + }) + .put("/llm-keys", async ({ body, set }: any) => { + const provider = String(body?.provider ?? ""); + if (!(LLM_PROVIDERS as readonly string[]).includes(provider)) { + set.status = 400; + return fail(`provider must be one of: ${LLM_PROVIDERS.join(", ")}`); + } + if (body?.apiKey !== undefined && typeof body.apiKey !== "string") { + set.status = 400; + return fail("apiKey must be a string"); + } + const existing = await getDecryptedLlmKey(provider); + if (provider === "custom") { + if (!body?.baseURL && !existing?.baseUrl) { + set.status = 400; + return fail("baseURL is required for the custom provider"); + } + } + const apiKey = + typeof body?.apiKey === "string" && body.apiKey.length > 0 ? body.apiKey : provider === "ollama" ? "ollama" : ""; + const effectiveBaseUrl = body?.baseURL !== undefined ? body.baseURL : existing?.baseUrl; + const baseChanged = body?.baseURL !== undefined && body.baseURL !== existing?.baseUrl; + if (baseChanged && !apiKey && existing?.apiKey) { + set.status = 400; + return fail("apiKey is required when changing baseURL"); + } + const effectiveApiKey = apiKey || existing?.apiKey || ""; + + let models: string[] | undefined = Array.isArray(body?.models) ? body.models.map(String) : undefined; + if (!models || models.length === 0) { + try { + const pulled = await fetchProviderModels({ + provider, + apiKey: effectiveApiKey, + baseUrl: effectiveBaseUrl, + }); + if (pulled.length > 0) { + models = pulled; + } + } catch { + models = existing?.models ?? undefined; + } + } + + const status = await upsertLlmKey({ + provider, + apiKey, + baseURL: body?.baseURL, + models, + }); + return ok(status, "LLM provider key updated"); + }) + .delete("/llm-keys/:provider", async ({ params, set }: any) => { + const deleted = await deleteLlmKey(String(params.provider)); + if (!deleted) { + set.status = 404; + return fail("LLM provider key not found"); + } + return ok(null, "LLM provider key deleted"); + }); diff --git a/apps/api/src/api/shared-env-vars/index.ts b/apps/api/src/api/shared-env-vars/index.ts index e7b5118..d480956 100644 --- a/apps/api/src/api/shared-env-vars/index.ts +++ b/apps/api/src/api/shared-env-vars/index.ts @@ -1,4 +1,7 @@ +import { and, eq } from "drizzle-orm"; import { Elysia } from "elysia"; +import { getDb } from "../../db/db-provider"; +import { getRowsAffected } from "../../db/repo/helpers"; import { createSharedEnvVar, deleteSharedEnvVar, @@ -7,9 +10,9 @@ import { linkSharedEnvVarsToProject, listLinkedSharedEnvVars, listSharedEnvVars, - unlinkSharedEnvVarFromProject, updateSharedEnvVar, } from "../../db/repo/shared-env-vars"; +import { projectSharedEnvLinks } from "../../db/schema"; import { fail, ok } from "../response"; export const sharedEnvVarsRoutes = new Elysia() @@ -80,8 +83,15 @@ export const sharedEnvLinksRoutes = new Elysia() await linkSharedEnvVarsToProject(params.id, body.sharedEnvVarIds); return ok(null, "Linked"); }) - .delete("/projects/:id/shared-env-links/:linkId", async ({ params, set }: any) => { - const removed = await unlinkSharedEnvVarFromProject(params.id, params.linkId); + .delete("/projects/:id/shared-env-links/:linkId", async ({ params, set }) => { + const db = await getDb(); + const removed = + getRowsAffected( + await db + .delete(projectSharedEnvLinks) + .where(and(eq(projectSharedEnvLinks.id, params.linkId), eq(projectSharedEnvLinks.projectId, params.id))) + .execute(), + ) > 0; if (!removed) { set.status = 404; return fail("Link not found"); diff --git a/apps/api/src/backup/__tests__/cron.test.ts b/apps/api/src/backup/__tests__/cron.test.ts new file mode 100644 index 0000000..ae6f373 --- /dev/null +++ b/apps/api/src/backup/__tests__/cron.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, test } from "bun:test"; +import { matchesCron } from "../scheduler"; + +const at = (min: number, hour = 0, day = 1, month = 1) => new Date(2026, month - 1, day, hour, min); + +describe("matchesCron", () => { + test("star matches every value", () => { + for (const m of [0, 7, 59]) expect(matchesCron("* * * * *", at(m))).toBe(true); + }); + + test("step without base: */5 matches multiples of 5 only", () => { + expect(matchesCron("*/5 * * * *", at(0))).toBe(true); + expect(matchesCron("*/5 * * * *", at(5))).toBe(true); + expect(matchesCron("*/5 * * * *", at(55))).toBe(true); + expect(matchesCron("*/5 * * * *", at(3))).toBe(false); + expect(matchesCron("*/5 * * * *", at(7))).toBe(false); + }); + + test("range with step: 0-30/5 matches 0,5,...,30 but not 35", () => { + expect(matchesCron("0-30/5 * * * *", at(0))).toBe(true); + expect(matchesCron("0-30/5 * * * *", at(30))).toBe(true); + expect(matchesCron("0-30/5 * * * *", at(35))).toBe(false); + expect(matchesCron("0-30/5 * * * *", at(31))).toBe(false); + }); + + test("step with base: 5/15 matches 5,20,35,50 not 15,30", () => { + expect(matchesCron("5/15 * * * *", at(5))).toBe(true); + expect(matchesCron("5/15 * * * *", at(20))).toBe(true); + expect(matchesCron("5/15 * * * *", at(50))).toBe(true); + expect(matchesCron("5/15 * * * *", at(15))).toBe(false); + expect(matchesCron("5/15 * * * *", at(0))).toBe(false); + }); + + test("plain number matches only itself", () => { + expect(matchesCron("30 * * * *", at(30))).toBe(true); + expect(matchesCron("30 * * * *", at(29))).toBe(false); + }); + + test("plain range without step", () => { + expect(matchesCron("10-20 * * * *", at(15))).toBe(true); + expect(matchesCron("10-20 * * * *", at(21))).toBe(false); + expect(matchesCron("10-20 * * * *", at(9))).toBe(false); + }); + + test("range with step respects both bounds and offset", () => { + expect(matchesCron("0-30/7 * * * *", at(0))).toBe(true); + expect(matchesCron("0-30/7 * * * *", at(28))).toBe(true); + expect(matchesCron("0-30/7 * * * *", at(35))).toBe(false); + expect(matchesCron("5-25/10 * * * *", at(5))).toBe(true); + expect(matchesCron("5-25/10 * * * *", at(15))).toBe(true); + expect(matchesCron("5-25/10 * * * *", at(10))).toBe(false); + }); + + test("invalid step never matches", () => { + expect(matchesCron("*/0 * * * *", at(0))).toBe(false); + expect(matchesCron("*/x * * * *", at(0))).toBe(false); + }); + + test("hour expression: 0 */6 matches 0,6,12,18", () => { + expect(matchesCron("0 */6 * * *", at(0, 0))).toBe(true); + expect(matchesCron("0 */6 * * *", at(0, 6))).toBe(true); + expect(matchesCron("0 */6 * * *", at(0, 18))).toBe(true); + expect(matchesCron("0 */6 * * *", at(0, 3))).toBe(false); + }); + + test("comma list", () => { + expect(matchesCron("1,15 * * * *", at(1))).toBe(true); + expect(matchesCron("1,15 * * * *", at(15))).toBe(true); + expect(matchesCron("1,15 * * * *", at(2))).toBe(false); + }); + + test("wrong field count is invalid", () => { + expect(matchesCron("* * *", at(0))).toBe(false); + expect(matchesCron("* * * * * *", at(0))).toBe(false); + }); + + test("malformed tokens are rejected outright", () => { + expect(matchesCron("*/5oops * * * *", at(0))).toBe(false); + expect(matchesCron("*/5oops * * * *", at(5))).toBe(false); + expect(matchesCron("5/15/2 * * * *", at(5))).toBe(false); + expect(matchesCron("5/15/2 * * * *", at(20))).toBe(false); + expect(matchesCron("10-20-30 * * * *", at(15))).toBe(false); + expect(matchesCron("1x * * * *", at(1))).toBe(false); + expect(matchesCron("a * * * *", at(0))).toBe(false); + expect(matchesCron("1,*/3x * * * *", at(3))).toBe(false); + }); +}); diff --git a/apps/api/src/backup/scheduler.ts b/apps/api/src/backup/scheduler.ts index 4b7f340..eab7ab1 100644 --- a/apps/api/src/backup/scheduler.ts +++ b/apps/api/src/backup/scheduler.ts @@ -1,8 +1,8 @@ import { listAllDatabases } from "../db/repo/databases"; import { getBackupStorageSettings } from "../db/repo/settings"; import { BackupOrchestrator } from "./orchestrator"; -import { S3_BACKUP_PREFIX } from "./types"; import type { BackupTarget, StorageConfig } from "./types"; +import { S3_BACKUP_PREFIX } from "./types"; const lastFiredMinute = new Map(); @@ -71,9 +71,24 @@ function toStorageConfig(settings: { return { type: "local", path: settings.path || "/data/backups" }; } -function matchesCron(cron: string, date: Date): boolean { +const CRON_TOKEN = /^(\*|\d+)(?:-(\d+))?(?:\/(\d+))?$/; + +const isValidToken = (token: string): boolean => { + const match = CRON_TOKEN.exec(token); + if (!match) return false; + const [, start, end, step] = match; + if (start === "*" && end !== undefined) return false; + if (end !== undefined && Number(end) < Number(start)) return false; + if (step !== undefined && Number(step) < 1) return false; + return true; +}; + +const isValidField = (expr: string): boolean => expr.length > 0 && expr.split(",").every(isValidToken); + +export function matchesCron(cron: string, date: Date): boolean { const parts = cron.trim().split(/\s+/); if (parts.length !== 5) return false; + if (!parts.every(isValidField)) return false; const [minExpr, hourExpr, dayExpr, monthExpr, dowExpr] = parts; @@ -90,21 +105,28 @@ function matchField(expr: string, value: number): boolean { if (expr === "*") return true; for (const part of expr.split(",")) { - if (part.includes("-")) { - const [start, end] = part.split("-").map(Number); - if (value >= start && value <= end) return true; - } else if (part.includes("/")) { - const [range, step] = part.split("/"); - const stepNum = parseInt(step, 10); - if (range === "*") { - if (value % stepNum === 0) return true; - } else { - const start = parseInt(range, 10); - if (value >= start && value % stepNum === 0) return true; + if (!part) continue; + const [range, stepStr] = part.split("/"); + const step = stepStr === undefined ? 1 : Number.parseInt(stepStr, 10); + if (!Number.isInteger(step) || step <= 0) continue; + + let start = 0; + let end = Number.POSITIVE_INFINITY; + if (range.includes("-")) { + const [s, e] = range.split("-").map((n) => Number.parseInt(n, 10)); + if (!Number.isInteger(s) || !Number.isInteger(e)) continue; + start = s; + end = e; + } else if (range !== "*") { + const s = Number.parseInt(range, 10); + if (!Number.isInteger(s)) continue; + if (stepStr === undefined) { + if (value === s) return true; + continue; } - } else { - if (parseInt(part, 10) === value) return true; + start = s; } + if (value >= start && value <= end && (value - start) % step === 0) return true; } return false; diff --git a/apps/api/src/db/__tests__/deployment-events.test.ts b/apps/api/src/db/__tests__/deployment-events.test.ts new file mode 100644 index 0000000..49e88ff --- /dev/null +++ b/apps/api/src/db/__tests__/deployment-events.test.ts @@ -0,0 +1,201 @@ +import { afterAll, afterEach, beforeAll, describe, expect, it } from "bun:test"; +import { drizzle } from "drizzle-orm/node-postgres"; +import type { Pool } from "pg"; +import { setDbProvider } from "../db-provider"; +import * as schema from "../schema"; +import { createTestPool, truncateAllTables } from "../test-helper"; + +let pool: Pool; + +const seed = async () => { + await pool.query( + `INSERT INTO projects (id, name, source_type, created_at, updated_at) + VALUES ('proj-ev', 'Event Project', 'git', NOW(), NOW()) ON CONFLICT DO NOTHING`, + ); + await pool.query( + `INSERT INTO deployments (id, project_id, source_type, source_ref, status, branch, commit_sha, created_at, updated_at) + VALUES ('dep-ev-1', 'proj-ev', 'git', 'https://github.com/test/repo.git', 'pending', 'main', 'abc1234567890abcdef', NOW(), NOW()), + ('dep-ev-2', 'proj-ev', 'git', 'https://github.com/test/repo.git', 'pending', 'main', NULL, NOW(), NOW()) + ON CONFLICT DO NOTHING`, + ); +}; + +beforeAll(async () => { + pool = createTestPool(); + const db = drizzle(pool, { schema }); + setDbProvider(async () => db); + await truncateAllTables(pool); + await seed(); +}); + +afterEach(async () => { + await truncateAllTables(pool); + await seed(); +}); + +afterAll(async () => { + try { + await truncateAllTables(pool); + } finally { + await pool.end(); + } +}); + +const eventsFor = async (deploymentId: string) => + ( + await pool.query(`SELECT id, type, message, metadata, sent_at FROM deployment_events WHERE deployment_id = $1`, [ + deploymentId, + ]) + ).rows; + +const deploymentRow = async (deploymentId: string) => + (await pool.query(`SELECT status, failure_reason FROM deployments WHERE id = $1`, [deploymentId])).rows[0]; + +describe("deployment event backbone", () => { + it("records a failure exactly once under 5 concurrent calls", async () => { + const { recordDeploymentFailure } = await import("../repo/deployment-events"); + const results = await Promise.all( + Array.from({ length: 5 }, () => + recordDeploymentFailure({ deploymentId: "dep-ev-1", reason: "build exploded", source: "pipeline" }), + ), + ); + + expect(results.filter((r) => r.claimed)).toHaveLength(1); + expect(results.filter((r) => r.eventId !== null)).toHaveLength(1); + + const events = await eventsFor("dep-ev-1"); + expect(events).toHaveLength(1); + expect(events[0].type).toBe("failed"); + expect(events[0].message).toBe("build exploded"); + + const dep = await deploymentRow("dep-ev-1"); + expect(dep.status).toBe("failed"); + expect(dep.failure_reason).toBe("build exploded"); + }); + + it("keeps the first terminal event across failed -> pending -> failed", async () => { + const { recordDeploymentFailure } = await import("../repo/deployment-events"); + const { updateDeploymentStatus } = await import("../repo/deployments"); + + const first = await recordDeploymentFailure({ + deploymentId: "dep-ev-1", + reason: "first failure", + source: "pipeline", + }); + expect(first.claimed).toBe(true); + + await updateDeploymentStatus("dep-ev-1", "pending"); + + const second = await recordDeploymentFailure({ + deploymentId: "dep-ev-1", + reason: "second failure", + source: "ssh", + }); + expect(second.claimed).toBe(false); + + const events = await eventsFor("dep-ev-1"); + expect(events).toHaveLength(1); + expect(events[0].message).toBe("first failure"); + }); + + it("suppresses failure emails when the deployment was cancelled", async () => { + const { recordDeploymentCancellation, recordDeploymentFailure, listPendingFailureNotificationIds } = await import( + "../repo/deployment-events" + ); + + const cancel = await recordDeploymentCancellation({ + deploymentId: "dep-ev-1", + reason: "Cancelled", + source: "pipeline", + }); + expect(cancel.claimed).toBe(true); + + const fail = await recordDeploymentFailure({ + deploymentId: "dep-ev-1", + reason: "late agent error", + source: "job-channel", + }); + expect(fail.claimed).toBe(false); + + const events = await eventsFor("dep-ev-1"); + expect(events).toHaveLength(1); + expect(events[0].type).toBe("cancelled"); + + const dep = await deploymentRow("dep-ev-1"); + expect(dep.status).toBe("failed"); + expect(dep.failure_reason).toBe("Cancelled"); + + expect(await listPendingFailureNotificationIds()).toHaveLength(0); + }); + + it("does not let the trigger add a failed event after a cancellation", async () => { + const { recordDeploymentCancellation } = await import("../repo/deployment-events"); + await recordDeploymentCancellation({ deploymentId: "dep-ev-1", reason: "Cancelled", source: "pipeline" }); + + await pool.query(`UPDATE deployments SET status = 'pending', failure_reason = NULL WHERE id = 'dep-ev-1'`); + await pool.query(`UPDATE deployments SET status = 'failed', failure_reason = 'Cancelled' WHERE id = 'dep-ev-1'`); + + const events = await eventsFor("dep-ev-1"); + expect(events).toHaveLength(1); + expect(events[0].type).toBe("cancelled"); + }); + + it("trigger inserts a fallback failed event for raw status writes", async () => { + await pool.query( + `UPDATE deployments SET status = 'failed', failure_reason = 'raw write boom' WHERE id = 'dep-ev-2'`, + ); + + const events = await eventsFor("dep-ev-2"); + expect(events).toHaveLength(1); + expect(events[0].type).toBe("failed"); + expect(events[0].message).toBe("raw write boom"); + expect((events[0].metadata as any)?.source).toBe("status-trigger"); + expect(events[0].sent_at).toBeNull(); + }); + + it("claims, marks sent, and stops retrying", async () => { + const { + recordDeploymentFailure, + claimFailureNotification, + markFailureNotificationSent, + listPendingFailureNotificationIds, + } = await import("../repo/deployment-events"); + + const { claimed, eventId } = await recordDeploymentFailure({ + deploymentId: "dep-ev-1", + reason: "mail me", + source: "pipeline", + }); + expect(claimed).toBe(true); + + const pending = await listPendingFailureNotificationIds(); + expect(pending).toContain(eventId); + + const ctx = await claimFailureNotification(eventId!); + expect(ctx).not.toBeNull(); + expect(ctx!.deploymentId).toBe("dep-ev-1"); + expect(ctx!.projectName).toBe("Event Project"); + expect(ctx!.failureReason).toBe("mail me"); + expect(ctx!.commitSha).toBe("abc1234567890abcdef"); + expect(ctx!.attempt).toBe(1); + + await markFailureNotificationSent(eventId!); + expect(await claimFailureNotification(eventId!)).toBeNull(); + expect(await listPendingFailureNotificationIds()).not.toContain(eventId); + }); + + it("caps delivery attempts at 3", async () => { + const { recordDeploymentFailure, claimFailureNotification } = await import("../repo/deployment-events"); + + const { eventId } = await recordDeploymentFailure({ + deploymentId: "dep-ev-2", + reason: "flaky smtp", + source: "pipeline", + }); + + expect(await claimFailureNotification(eventId!)).not.toBeNull(); + expect(await claimFailureNotification(eventId!)).not.toBeNull(); + expect(await claimFailureNotification(eventId!)).not.toBeNull(); + expect(await claimFailureNotification(eventId!)).toBeNull(); + }); +}); diff --git a/apps/api/src/db/__tests__/diag-runs.test.ts b/apps/api/src/db/__tests__/diag-runs.test.ts new file mode 100644 index 0000000..88462a2 --- /dev/null +++ b/apps/api/src/db/__tests__/diag-runs.test.ts @@ -0,0 +1,84 @@ +import { afterAll, beforeAll, describe, expect, it, mock } from "bun:test"; +import { drizzle } from "drizzle-orm/node-postgres"; +import type { Pool } from "pg"; +import { setDbProvider } from "../db-provider"; +import * as schema from "../schema"; +import { createTestPool, truncateAllTables } from "../test-helper"; +import { + createDiagRun, + finishDiagRun, + getDiagRun, + getStagePayload, + listActiveDiagRuns, + listStageResults, + markInterruptedDiagRuns, + recordStageResult, +} from "../repo/diag-runs"; + +let pool: Pool; + +mock.restore(); + +beforeAll(async () => { + pool = createTestPool(); + const db = drizzle(pool, { schema }); + setDbProvider(async () => db); + await truncateAllTables(pool); + await pool.query( + `INSERT INTO projects (id, name, source_type, created_at, updated_at) + VALUES ('proj-diagruns', 'DiagRuns Project', 'git', NOW(), NOW()) ON CONFLICT DO NOTHING`, + ); + await pool.query( + `INSERT INTO deployments (id, project_id, source_type, source_ref, status, created_at, updated_at) + VALUES ('dep-x', 'proj-diagruns', 'git', 'https://github.com/test/repo.git', 'failed', NOW(), NOW()), + ('dep-dup', 'proj-diagruns', 'git', 'https://github.com/test/repo.git', 'failed', NOW(), NOW()) + ON CONFLICT DO NOTHING`, + ); +}); + +afterAll(async () => { + try { + await truncateAllTables(pool); + } finally { + await pool.end(); + } +}); + +describe("diag-runs", () => { + it("creates, finds and finishes runs", async () => { + const run = await createDiagRun({ deploymentId: "dep-x", commitSha: "abc", provider: "groq", model: "m" }); + expect(run.status).toBe("running"); + expect(run.currentStage).toBeNull(); + + await recordStageResult(run.id, "triage", { failingStage: "build" }); + expect(await getStagePayload(run.id, "triage")).toEqual({ failingStage: "build" }); + + await recordStageResult(run.id, "triage", { failingStage: "deploy" }); + expect(await getStagePayload(run.id, "triage")).toEqual({ failingStage: "deploy" }); + expect((await listStageResults(run.id)).filter((s) => s.stage === "triage")).toHaveLength(1); + + await finishDiagRun(run.id, "done", "user-source", { cause: "user-source" }, null); + const done = await getDiagRun(run.id); + expect(done?.status).toBe("done"); + expect(done?.cause).toBe("user-source"); + expect(done?.report).toEqual({ cause: "user-source" }); + }); + + it("rejects duplicate runs for the same deployment and commit", async () => { + await createDiagRun({ deploymentId: "dep-dup", commitSha: "", provider: "groq", model: "m" }); + await expect( + createDiagRun({ deploymentId: "dep-dup", commitSha: "", provider: "groq", model: "m" }), + ).rejects.toThrow(); + }); + + it("lists active runs and marks interrupted ones as failed", async () => { + const run = await createDiagRun({ deploymentId: "dep-x", commitSha: "active-1", provider: "ollama", model: "m" }); + const active = await listActiveDiagRuns(); + const found = active.find((r) => r.id === run.id); + expect(found).toMatchObject({ deploymentId: "dep-x", projectName: "DiagRuns Project", provider: "ollama" }); + + expect(await markInterruptedDiagRuns()).toBeGreaterThanOrEqual(1); + expect((await getDiagRun(run.id))?.status).toBe("error"); + expect(await listActiveDiagRuns()).toEqual([]); + }); +}); diff --git a/apps/api/src/db/__tests__/llm-keys.test.ts b/apps/api/src/db/__tests__/llm-keys.test.ts new file mode 100644 index 0000000..2e4bf8b --- /dev/null +++ b/apps/api/src/db/__tests__/llm-keys.test.ts @@ -0,0 +1,73 @@ +import { afterAll, beforeAll, describe, expect, it, mock } from "bun:test"; +import { drizzle } from "drizzle-orm/node-postgres"; +import type { Pool } from "pg"; +import { setDbProvider } from "../db-provider"; +import * as schema from "../schema"; +import { createTestPool, truncateAllTables } from "../test-helper"; +import { deleteLlmKey, getDecryptedLlmKey, getLlmKeyStatus, updateLlmModels, upsertLlmKey } from "../repo/llm-keys"; + +let pool: Pool; + +mock.restore(); + +beforeAll(async () => { + pool = createTestPool(); + const db = drizzle(pool, { schema }); + setDbProvider(async () => db); + await truncateAllTables(pool); +}); + +afterAll(async () => { + try { + await truncateAllTables(pool); + } finally { + await pool.end(); + } +}); + +describe("llm-keys", () => { + it("stores encrypted and reports status without the secret", async () => { + const status = await upsertLlmKey({ + provider: "groq", + apiKey: "gsk-secret-1", + models: ["llama-3.3-70b-versatile"], + }); + expect(status.provider).toBe("groq"); + expect(status.configured).toBe(true); + expect(status.models).toEqual(["llama-3.3-70b-versatile"]); + expect(JSON.stringify(status)).not.toContain("gsk-secret-1"); + + const all = await getLlmKeyStatus(); + expect(all.find((s) => s.provider === "groq")?.configured).toBe(true); + expect(JSON.stringify(all)).not.toContain("gsk-secret-1"); + }); + + it("round-trips the decrypted key", async () => { + await upsertLlmKey({ provider: "openai", apiKey: "sk-secret-2" }); + const rec = await getDecryptedLlmKey("openai"); + expect(rec?.apiKey).toBe("sk-secret-2"); + expect(await getDecryptedLlmKey("missing")).toBeNull(); + }); + + it("keeps the old key when apiKey is empty", async () => { + await upsertLlmKey({ provider: "groq", apiKey: "gsk-secret-1" }); + const status = await upsertLlmKey({ provider: "groq", apiKey: "", models: ["other-model"] }); + expect(status.configured).toBe(true); + expect(status.models).toEqual(["other-model"]); + expect((await getDecryptedLlmKey("groq"))?.apiKey).toBe("gsk-secret-1"); + }); + + it("updates models for a provider", async () => { + await upsertLlmKey({ provider: "gemini", apiKey: "gem-sec-1" }); + const updated = await updateLlmModels("gemini", ["gemini-2.5-flash", "gemini-2.5-pro"]); + expect(updated?.models).toEqual(["gemini-2.5-flash", "gemini-2.5-pro"]); + expect(await updateLlmModels("nonexistent", ["model"])).toBeNull(); + }); + + it("deletes keys", async () => { + await upsertLlmKey({ provider: "anthropic", apiKey: "sk-ant-1" }); + expect(await deleteLlmKey("anthropic")).toBe(true); + expect(await deleteLlmKey("anthropic")).toBe(false); + expect((await getLlmKeyStatus()).find((s) => s.provider === "anthropic")).toBeUndefined(); + }); +}); diff --git a/apps/api/src/db/__tests__/shared-env-links-runner.ts b/apps/api/src/db/__tests__/shared-env-links-runner.ts new file mode 100644 index 0000000..8b8fe62 --- /dev/null +++ b/apps/api/src/db/__tests__/shared-env-links-runner.ts @@ -0,0 +1,109 @@ +import { randomUUID } from "node:crypto"; +import { eq } from "drizzle-orm"; +import { drizzle } from "drizzle-orm/node-postgres"; +import { Pool } from "pg"; +import { setDbProvider } from "../db-provider"; +import { + linkSharedEnvVarsToProject, + listLinkedSharedEnvVars, + unlinkSharedEnvVarFromProject, +} from "../repo/shared-env-vars"; +import * as schema from "../schema"; + +const TEST_DATABASE_URL = process.env.TEST_DATABASE_URL ?? "postgresql://dequel:dequel@localhost:5433/dequel"; +const pool = new Pool({ connectionString: TEST_DATABASE_URL }); +const db = drizzle(pool, { schema }); +setDbProvider(async () => db); + +const cleanup = async () => { + await pool.query('DELETE FROM "project_shared_env_links" WHERE project_id LIKE $1', ["test-sel-%"]); + await pool.query('DELETE FROM "shared_env_vars" WHERE key LIKE $1', ["TEST_SEL_%"]); + await pool.query('DELETE FROM "projects" WHERE id LIKE $1', ["test-sel-%"]); +}; + +try { + await pool.query(` + CREATE TABLE IF NOT EXISTS "shared_env_vars" ( + "id" text PRIMARY KEY, + "key" text NOT NULL, + "value" text NOT NULL DEFAULT '', + "value_encrypted" text, + "value_iv" text, + "value_tag" text, + "environment" text DEFAULT 'production', + "description" text, + "tags" jsonb DEFAULT '[]', + "created_at" timestamp DEFAULT now(), + "updated_at" timestamp DEFAULT now() + ) + `); + await pool.query(` + CREATE TABLE IF NOT EXISTS "project_shared_env_links" ( + "id" text PRIMARY KEY, + "project_id" text NOT NULL REFERENCES "projects"("id") ON DELETE CASCADE, + "shared_env_var_id" text NOT NULL REFERENCES "shared_env_vars"("id") ON DELETE CASCADE, + "created_at" timestamp DEFAULT now() + ) + `); + await pool.query( + `CREATE UNIQUE INDEX IF NOT EXISTS "project_shared_env_links_project_shared_idx" ON "project_shared_env_links" ("project_id", "shared_env_var_id")`, + ); + await cleanup(); + + const projectId = `test-sel-${randomUUID().slice(0, 8)}`; + const varId1 = randomUUID(); + const varId2 = randomUUID(); + + await pool.query( + `INSERT INTO projects (id, name, source_type, created_at, updated_at) VALUES ($1, 'test', 'git', NOW(), NOW())`, + [projectId], + ); + await pool.query( + `INSERT INTO shared_env_vars (id, key, value, environment, created_at, updated_at) VALUES ($1, 'TEST_SEL_DB_URL', '', 'production', NOW(), NOW())`, + [varId1], + ); + await pool.query( + `INSERT INTO shared_env_vars (id, key, value, environment, created_at, updated_at) VALUES ($1, 'TEST_SEL_API_KEY', '', 'production', NOW(), NOW())`, + [varId2], + ); + + await linkSharedEnvVarsToProject(projectId, [varId1, varId2]); + let linked = await listLinkedSharedEnvVars(projectId); + const linkCount1 = linked.length; + const hasVar1 = linked.some((v) => v.key === "TEST_SEL_DB_URL"); + const hasVar2 = linked.some((v) => v.key === "TEST_SEL_API_KEY"); + const hasLinkId = linked.every((v) => typeof v.linkId === "string" && v.linkId.length > 0); + + await linkSharedEnvVarsToProject(projectId, [varId1]); + linked = await listLinkedSharedEnvVars(projectId); + const afterRelink = linked.length; + + const target = linked.find((v) => v.key === "TEST_SEL_API_KEY"); + const unlinkResult = await unlinkSharedEnvVarFromProject(projectId, target!.id); + linked = await listLinkedSharedEnvVars(projectId); + const afterUnlink = linked.length; + const stillHasVar1 = linked.some((v) => v.key === "TEST_SEL_DB_URL"); + + const badUnlink = await unlinkSharedEnvVarFromProject(projectId, "nonexistent"); + + await unlinkSharedEnvVarFromProject(projectId, varId1); + const empty = await listLinkedSharedEnvVars(projectId); + + console.log( + JSON.stringify({ + linkCount: linkCount1, + hasVar1, + hasVar2, + hasLinkId, + afterRelink, + unlinkResult, + afterUnlink, + stillHasVar1, + badUnlink, + emptyCount: empty.length, + }), + ); +} finally { + await cleanup(); + await pool.end(); +} diff --git a/apps/api/src/db/__tests__/shared-env-links.test.ts b/apps/api/src/db/__tests__/shared-env-links.test.ts new file mode 100644 index 0000000..7598ef1 --- /dev/null +++ b/apps/api/src/db/__tests__/shared-env-links.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, it } from "bun:test"; +import { spawnSync } from "node:child_process"; +import { join } from "node:path"; + +const runnerPath = join(import.meta.dir, "shared-env-links-runner.ts"); + +const runScenarios = (): any => { + const result = spawnSync("bun", [runnerPath], { + env: { + ...process.env, + DATABASE_URL: "postgresql://dequel:dequel@localhost:5433/dequel", + TEST_DATABASE_URL: "postgresql://dequel:dequel@localhost:5433/dequel", + }, + encoding: "utf8", + }); + if (result.status !== 0) { + throw new Error(`Shared env links test runner failed:\n${result.stdout}\n${result.stderr}`); + } + for (const line of result.stdout.split("\n").reverse()) { + try { + return JSON.parse(line); + } catch {} + } + throw new Error(`Shared env links test runner produced no JSON output:\n${result.stdout}`); +}; + +describe("Shared Env Var Linking", () => { + it("links shared vars and returns linkId", () => { + const result = runScenarios(); + expect(result.linkCount).toBe(2); + expect(result.hasVar1).toBe(true); + expect(result.hasVar2).toBe(true); + expect(result.hasLinkId).toBe(true); + }); + + it("deduplicates on re-link", () => { + const result = runScenarios(); + expect(result.afterRelink).toBe(2); + }); + + it("unlinks by key and redeploy data is correct", () => { + const result = runScenarios(); + expect(result.unlinkResult).toBe(true); + expect(result.afterUnlink).toBe(1); + expect(result.stillHasVar1).toBe(true); + }); + + it("returns false for non-existent unlink", () => { + const result = runScenarios(); + expect(result.badUnlink).toBe(false); + }); + + it("returns empty after all unlinked", () => { + const result = runScenarios(); + expect(result.emptyCount).toBe(0); + }); +}); diff --git a/apps/api/src/db/migrate.ts b/apps/api/src/db/migrate.ts index 5843151..8707c6d 100644 --- a/apps/api/src/db/migrate.ts +++ b/apps/api/src/db/migrate.ts @@ -2,7 +2,6 @@ import { migrate as drizzleMigrate } from "drizzle-orm/node-postgres/migrator"; import { config } from "../utils/config"; import { getDb } from "./client"; import { getGithubIntegration, setGithubIntegration } from "./repo/github"; -import { getSmtpSettings, upsertSmtpSettings } from "./repo/settings"; export const migrate = async () => { const db = await getDb(); @@ -51,17 +50,4 @@ const seedFromConfig = async () => { console.log("[Config] Seeded GitHub integration from config file"); } } - if (config.smtpHost) { - const existing = await getSmtpSettings(); - if (!existing) { - await upsertSmtpSettings({ - host: config.smtpHost, - port: config.smtpPort, - user: config.smtpUser, - pass: config.smtpPass, - fromAddress: config.smtpFrom, - }); - console.log("[Config] Seeded SMTP settings from config file"); - } - } }; diff --git a/apps/api/src/db/migrations/0034_deployment_event_backbone.sql b/apps/api/src/db/migrations/0034_deployment_event_backbone.sql new file mode 100644 index 0000000..3bd9932 --- /dev/null +++ b/apps/api/src/db/migrations/0034_deployment_event_backbone.sql @@ -0,0 +1,30 @@ +ALTER TABLE deployment_events ADD COLUMN sent_at timestamptz; +ALTER TABLE deployment_events ADD COLUMN attempts integer NOT NULL DEFAULT 0; + +UPDATE deployment_events SET sent_at = now() WHERE type = 'failed'; + +DELETE FROM deployment_events a USING deployment_events b + WHERE a.deployment_id = b.deployment_id AND a.type = b.type AND a.ctid < b.ctid + AND a.type IN ('failed','cancelled'); + +CREATE UNIQUE INDEX udep_events_failed ON deployment_events (deployment_id) WHERE type = 'failed'; +CREATE UNIQUE INDEX udep_events_cancelled ON deployment_events (deployment_id) WHERE type = 'cancelled'; + +CREATE FUNCTION deployment_terminal_event_fallback() RETURNS trigger AS $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM deployment_events + WHERE deployment_id = NEW.id AND type IN ('failed','cancelled')) THEN + INSERT INTO deployment_events (id, deployment_id, type, message, metadata) + VALUES (gen_random_uuid()::text, NEW.id, 'failed', NEW.failure_reason, + jsonb_build_object('source', 'status-trigger')) + ON CONFLICT DO NOTHING; + END IF; + RETURN NEW; +END $$ LANGUAGE plpgsql; + +CREATE TRIGGER trg_deployment_terminal_event +AFTER UPDATE OF status ON deployments +FOR EACH ROW WHEN (NEW.status = 'failed' AND OLD.status IS DISTINCT FROM 'failed') +EXECUTE FUNCTION deployment_terminal_event_fallback(); + +DELETE FROM alerts WHERE type = 'error_rate'; diff --git a/apps/api/src/db/migrations/0035_deployment_fk_cascade.sql b/apps/api/src/db/migrations/0035_deployment_fk_cascade.sql new file mode 100644 index 0000000..4ddc59e --- /dev/null +++ b/apps/api/src/db/migrations/0035_deployment_fk_cascade.sql @@ -0,0 +1,11 @@ +ALTER TABLE "agent_jobs" DROP CONSTRAINT IF EXISTS "agent_jobs_deployment_id_deployments_id_fk"; +ALTER TABLE "agent_jobs" ADD CONSTRAINT "agent_jobs_deployment_id_deployments_id_fk" + FOREIGN KEY ("deployment_id") REFERENCES "deployments"("id") ON DELETE CASCADE; + +ALTER TABLE "deployment_logs" DROP CONSTRAINT IF EXISTS "deployment_logs_deployment_id_deployments_id_fk"; +ALTER TABLE "deployment_logs" ADD CONSTRAINT "deployment_logs_deployment_id_deployments_id_fk" + FOREIGN KEY ("deployment_id") REFERENCES "deployments"("id") ON DELETE CASCADE; + +ALTER TABLE "deployment_events" DROP CONSTRAINT IF EXISTS "deployment_events_deployment_id_deployments_id_fk"; +ALTER TABLE "deployment_events" ADD CONSTRAINT "deployment_events_deployment_id_deployments_id_fk" + FOREIGN KEY ("deployment_id") REFERENCES "deployments"("id") ON DELETE CASCADE; diff --git a/apps/api/src/db/migrations/0036_fixdiag.sql b/apps/api/src/db/migrations/0036_fixdiag.sql new file mode 100644 index 0000000..51973af --- /dev/null +++ b/apps/api/src/db/migrations/0036_fixdiag.sql @@ -0,0 +1,47 @@ +CREATE TABLE llm_provider_keys ( + provider text PRIMARY KEY, + key_encrypted text, + key_iv text, + key_tag text, + base_url text, + models jsonb NOT NULL DEFAULT '[]', + updated_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE diag_runs ( + id text PRIMARY KEY, + deployment_id text NOT NULL REFERENCES deployments(id) ON DELETE CASCADE, + commit_sha text NOT NULL DEFAULT '', + provider text NOT NULL, + model text NOT NULL, + status text NOT NULL DEFAULT 'running', + current_stage text, + cause text, + report jsonb, + error text, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (deployment_id, commit_sha) +); + +CREATE TABLE diag_stages ( + id text PRIMARY KEY, + run_id text NOT NULL REFERENCES diag_runs(id) ON DELETE CASCADE, + stage text NOT NULL, + payload jsonb, + created_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (run_id, stage) +); + +CREATE TABLE diag_actions ( + key text PRIMARY KEY, + run_id text NOT NULL REFERENCES diag_runs(id) ON DELETE CASCADE, + kind text NOT NULL, + status text NOT NULL DEFAULT 'requested', + result jsonb, + error text, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now() +); + +CREATE UNIQUE INDEX diag_actions_one_done_per_kind ON diag_actions (run_id, kind) WHERE status = 'done'; diff --git a/apps/api/src/db/migrations/meta/_journal.json b/apps/api/src/db/migrations/meta/_journal.json index 6497c94..850b822 100644 --- a/apps/api/src/db/migrations/meta/_journal.json +++ b/apps/api/src/db/migrations/meta/_journal.json @@ -71,6 +71,27 @@ "when": 1788600000000, "tag": "0033_add_system_backup_settings", "breakpoints": true + }, + { + "idx": 10, + "version": "7", + "when": 1788600001000, + "tag": "0034_deployment_event_backbone", + "breakpoints": true + }, + { + "idx": 11, + "version": "7", + "when": 1790424535143, + "tag": "0035_deployment_fk_cascade", + "breakpoints": true + }, + { + "idx": 12, + "version": "7", + "when": 1790700000000, + "tag": "0036_fixdiag", + "breakpoints": true } ] -} +} \ No newline at end of file diff --git a/apps/api/src/db/repo/deployment-events.ts b/apps/api/src/db/repo/deployment-events.ts index 528dbad..7ffd3a6 100644 --- a/apps/api/src/db/repo/deployment-events.ts +++ b/apps/api/src/db/repo/deployment-events.ts @@ -1,7 +1,12 @@ import { randomUUID } from "node:crypto"; -import { eq } from "drizzle-orm"; +import { and, asc, desc, eq, inArray, isNull, lt, sql } from "drizzle-orm"; +import { emitDeploymentFailed } from "../../events"; +import type { FailureNotificationContext, RecordFailureInput, RecordFailureOutcome } from "../../types"; import { getDb } from "../db-provider"; -import { deploymentEvents } from "../schema"; +import { deploymentEvents, deployments, projects } from "../schema"; +import { applyStatusUpdate } from "./deployments"; + +const TERMINAL_TYPES = ["failed", "cancelled"]; export const createDeploymentEvent = async (input: { deploymentId: string; @@ -33,3 +38,180 @@ export const listDeploymentEvents = async (deploymentId: string) => { .orderBy(deploymentEvents.createdAt) .execute(); }; + +const recordTerminal = async ( + input: RecordFailureInput, + type: "failed" | "cancelled", +): Promise => { + const db = await getDb(); + const eventId = await db.transaction(async (tx) => { + const [dep] = await tx + .select({ id: deployments.id }) + .from(deployments) + .where(eq(deployments.id, input.deploymentId)) + .for("update") + .execute(); + if (!dep) return null; + + const [existing] = await tx + .select({ id: deploymentEvents.id }) + .from(deploymentEvents) + .where(and(eq(deploymentEvents.deploymentId, input.deploymentId), inArray(deploymentEvents.type, TERMINAL_TYPES))) + .limit(1) + .execute(); + if (existing) return null; + + const [inserted] = await tx + .insert(deploymentEvents) + .values({ + id: randomUUID(), + deploymentId: input.deploymentId, + type, + message: input.reason, + metadata: { source: input.source }, + }) + .onConflictDoNothing() + .returning({ id: deploymentEvents.id }) + .execute(); + if (!inserted) return null; + + await applyStatusUpdate(tx, input.deploymentId, "failed", { failureReason: input.reason }); + return inserted.id; + }); + + if (eventId && type === "failed") { + emitDeploymentFailed({ eventId, deploymentId: input.deploymentId }); + } + return { claimed: eventId !== null, eventId }; +}; + +export const recordDeploymentFailure = (input: RecordFailureInput): Promise => + recordTerminal({ ...input, cancel: false }, "failed"); + +export const recordDeploymentCancellation = (input: RecordFailureInput): Promise => + recordTerminal({ ...input, cancel: true }, "cancelled"); + +export const claimFailureNotification = async (eventId: string): Promise => { + const db = await getDb(); + const [claimed] = await db + .update(deploymentEvents) + .set({ attempts: sql`${deploymentEvents.attempts} + 1` }) + .where( + and( + eq(deploymentEvents.id, eventId), + eq(deploymentEvents.type, "failed"), + isNull(deploymentEvents.sentAt), + lt(deploymentEvents.attempts, 3), + ), + ) + .returning({ id: deploymentEvents.id }) + .execute(); + if (!claimed) return null; + + const [row] = await db + .select({ + eventId: deploymentEvents.id, + deploymentId: deploymentEvents.deploymentId, + attempts: deploymentEvents.attempts, + message: deploymentEvents.message, + projectId: deployments.projectId, + sourceRef: deployments.sourceRef, + commitSha: deployments.commitSha, + finishedAt: deployments.finishedAt, + projectName: projects.name, + }) + .from(deploymentEvents) + .innerJoin(deployments, eq(deployments.id, deploymentEvents.deploymentId)) + .leftJoin(projects, eq(projects.id, deployments.projectId)) + .where(eq(deploymentEvents.id, eventId)) + .execute(); + if (!row) return null; + + return { + eventId: row.eventId, + deploymentId: row.deploymentId, + projectId: row.projectId, + projectName: row.projectName ?? row.sourceRef, + failureReason: row.message, + commitSha: row.commitSha, + sourceRef: row.sourceRef, + finishedAt: row.finishedAt ? new Date(row.finishedAt).toISOString() : null, + attempt: row.attempts, + }; +}; + +export const markFailureNotificationSent = async (eventId: string): Promise => { + const db = await getDb(); + await db.update(deploymentEvents).set({ sentAt: new Date() }).where(eq(deploymentEvents.id, eventId)).execute(); +}; + +export const listPendingFailureNotificationIds = async (limit = 20): Promise => { + const db = await getDb(); + const rows = await db + .select({ id: deploymentEvents.id }) + .from(deploymentEvents) + .where(and(eq(deploymentEvents.type, "failed"), isNull(deploymentEvents.sentAt), lt(deploymentEvents.attempts, 3))) + .orderBy(asc(deploymentEvents.createdAt)) + .limit(limit) + .execute(); + return rows.map((r) => r.id); +}; + +export interface ProjectEventRow { + id: string; + deploymentId: string; + type: string; + message: string | null; + source: string | null; + createdAt: string; + deploymentStatus: string; + commitSha: string | null; + sourceRef: string; + finishedAt: string | null; + sentAt: string | null; +} + +export const listProjectEvents = async ( + projectId: string, + sinceIso: string, + limit = 200, +): Promise => { + const db = await getDb(); + const rows = await db + .select({ + id: deploymentEvents.id, + deploymentId: deploymentEvents.deploymentId, + type: deploymentEvents.type, + message: deploymentEvents.message, + metadata: deploymentEvents.metadata, + createdAt: deploymentEvents.createdAt, + deploymentStatus: deployments.status, + commitSha: deployments.commitSha, + sourceRef: deployments.sourceRef, + finishedAt: deployments.finishedAt, + sentAt: deploymentEvents.sentAt, + }) + .from(deploymentEvents) + .innerJoin(deployments, eq(deployments.id, deploymentEvents.deploymentId)) + .where(and(eq(deployments.projectId, projectId), sql`${deploymentEvents.createdAt} >= ${sinceIso}`)) + .orderBy(desc(deploymentEvents.createdAt)) + .limit(limit) + .execute(); + + return rows.map((r) => ({ + id: r.id, + deploymentId: r.deploymentId, + type: r.type, + message: r.message, + source: + r.metadata && typeof r.metadata === "object" && r.metadata !== null + ? ((r.metadata as Record).source as string | null) + : null, + createdAt: new Date(r.createdAt).toISOString(), + deploymentStatus: r.deploymentStatus, + commitSha: r.commitSha, + sourceRef: r.sourceRef, + finishedAt: r.finishedAt ? new Date(r.finishedAt).toISOString() : null, + sentAt: r.sentAt ? new Date(r.sentAt).toISOString() : null, + })); +}; diff --git a/apps/api/src/db/repo/deployments.ts b/apps/api/src/db/repo/deployments.ts index 4b5a135..c35c26e 100644 --- a/apps/api/src/db/repo/deployments.ts +++ b/apps/api/src/db/repo/deployments.ts @@ -91,13 +91,17 @@ const ACTIVE_STATUSES: DeploymentStatus[] = ["pending", "building", "deploying"] const STAMP_FINISHED_UNCONDITIONALLY: DeploymentStatus[] = ["running", "failed"]; -export const updateDeploymentStatus = async ( +type Tx = Parameters>["transaction"]>[0]>[0]; + +import { captureTelemetry } from "../../utils/telemetry"; + +export const applyStatusUpdate = async ( + tx: Tx | Awaited>, id: string, status: DeploymentStatus, patch: Partial> = {}, ) => { - const db = await getDb(); - const [existing] = await db + const [existing] = await tx .select({ finishedAt: deployments.finishedAt }) .from(deployments) .where(eq(deployments.id, id)) @@ -115,7 +119,23 @@ export const updateDeploymentStatus = async ( updates.finishedAt = now(); } } - await db.update(deployments).set(updates).where(eq(deployments.id, id)).execute(); + await tx.update(deployments).set(updates).where(eq(deployments.id, id)).execute(); + + if (status === "running" || status === "failed") { + captureTelemetry("deployment_executed", { + status, + has_failure_reason: Boolean(patch.failureReason), + }).catch(() => {}); + } +}; + +export const updateDeploymentStatus = async ( + id: string, + status: DeploymentStatus, + patch: Partial> = {}, +) => { + const db = await getDb(); + await applyStatusUpdate(db, id, status, patch); }; export const deleteDeploymentAndLogs = async (id: string): Promise => { diff --git a/apps/api/src/db/repo/diag-actions.ts b/apps/api/src/db/repo/diag-actions.ts new file mode 100644 index 0000000..3f4b928 --- /dev/null +++ b/apps/api/src/db/repo/diag-actions.ts @@ -0,0 +1,97 @@ +import { and, eq } from "drizzle-orm"; +import { getDb } from "../db-provider"; +import { diagActions } from "../schema"; +import { now } from "./helpers"; + +export type DiagActionKind = "pr" | "slack"; +export type DiagActionStatus = "requested" | "executing" | "done" | "failed"; + +export interface DiagAction { + key: string; + runId: string; + kind: DiagActionKind; + status: DiagActionStatus; + result: unknown; + error: string | null; + updatedAt: string; +} + +const STALE_MS = 5 * 60_000; + +const toAction = (row: typeof diagActions.$inferSelect): DiagAction => ({ + key: row.key, + runId: row.runId, + kind: row.kind as DiagActionKind, + status: row.status as DiagActionStatus, + result: row.result ?? null, + error: row.error, + updatedAt: row.updatedAt?.toISOString() ?? new Date().toISOString(), +}); + +export const getDiagAction = async (key: string): Promise => { + const db = await getDb(); + const [row] = await db.select().from(diagActions).where(eq(diagActions.key, key)).execute(); + return row ? toAction(row) : null; +}; + +export const claimDiagAction = async ( + key: string, + runId: string, + kind: DiagActionKind, +): Promise<{ action: DiagAction; fresh: boolean }> => { + const db = await getDb(); + const timestamp = now(); + const inserted = await db + .insert(diagActions) + .values({ key, runId, kind, status: "executing", updatedAt: timestamp }) + .onConflictDoNothing({ target: diagActions.key }) + .returning() + .execute(); + if (inserted.length === 1) return { action: toAction(inserted[0]), fresh: true }; + const [row] = await db.select().from(diagActions).where(eq(diagActions.key, key)).execute(); + if (!row) throw new Error("Failed to claim diagnosis action"); + if (row.runId !== runId || row.kind !== kind) throw new Error("Action key already used for a different intent"); + const action = toAction(row); + if (action.status === "done") return { action, fresh: false }; + if (action.status === "failed" || Date.now() - new Date(action.updatedAt).getTime() > STALE_MS) { + const [claimed] = await db + .update(diagActions) + .set({ status: "executing", error: null, updatedAt: timestamp }) + .where( + and(eq(diagActions.key, key), eq(diagActions.status, row.status), eq(diagActions.updatedAt, row.updatedAt)), + ) + .returning() + .execute(); + if (!claimed) return { action, fresh: false }; + return { action: toAction(claimed), fresh: true }; + } + return { action, fresh: false }; +}; + +export const completeDiagAction = async (key: string, result: unknown): Promise => { + const db = await getDb(); + await db + .update(diagActions) + .set({ status: "done", result: result as object, error: null, updatedAt: now() }) + .where(eq(diagActions.key, key)) + .execute(); +}; + +export const failDiagAction = async (key: string, error: string): Promise => { + const db = await getDb(); + await db + .update(diagActions) + .set({ status: "failed", error, updatedAt: now() }) + .where(eq(diagActions.key, key)) + .execute(); +}; + +export const findCompletedAction = async (runId: string, kind: DiagActionKind): Promise => { + const db = await getDb(); + const [row] = await db + .select() + .from(diagActions) + .where(and(eq(diagActions.runId, runId), eq(diagActions.kind, kind), eq(diagActions.status, "done"))) + .execute(); + return row ? toAction(row) : null; +}; diff --git a/apps/api/src/db/repo/diag-runs.ts b/apps/api/src/db/repo/diag-runs.ts new file mode 100644 index 0000000..9d57807 --- /dev/null +++ b/apps/api/src/db/repo/diag-runs.ts @@ -0,0 +1,158 @@ +import { randomUUID } from "node:crypto"; +import { and, eq } from "drizzle-orm"; +import type { CauseKind, DiagRun, DiagStageName, DiagStatus, Proposal } from "../../fixdiag/types"; +import { getDb } from "../db-provider"; +import { deployments, diagRuns, diagStages, projects } from "../schema"; +import { now } from "./helpers"; + +const toRun = (row: typeof diagRuns.$inferSelect): DiagRun => ({ + id: row.id, + deploymentId: row.deploymentId, + commitSha: row.commitSha, + provider: row.provider as DiagRun["provider"], + model: row.model, + status: row.status as DiagStatus, + currentStage: row.currentStage as DiagStageName | null, + cause: row.cause as CauseKind | null, + report: (row.report as Proposal | null) ?? null, + error: row.error, + createdAt: row.createdAt?.toISOString() ?? new Date().toISOString(), +}); + +export const findDiagRun = async (deploymentId: string, commitSha: string): Promise => { + const db = await getDb(); + const [row] = await db + .select() + .from(diagRuns) + .where(and(eq(diagRuns.deploymentId, deploymentId), eq(diagRuns.commitSha, commitSha))) + .execute(); + return row ? toRun(row) : null; +}; + +export const getDiagRun = async (runId: string): Promise => { + const db = await getDb(); + const [row] = await db.select().from(diagRuns).where(eq(diagRuns.id, runId)).execute(); + return row ? toRun(row) : null; +}; + +export const createDiagRun = async (input: { + deploymentId: string; + commitSha: string; + provider: DiagRun["provider"]; + model: string; +}): Promise => { + const db = await getDb(); + const [inserted] = await db + .insert(diagRuns) + .values({ + id: randomUUID(), + deploymentId: input.deploymentId, + commitSha: input.commitSha, + provider: input.provider, + model: input.model, + }) + .returning() + .execute(); + return toRun(inserted); +}; + +export const deleteDiagRun = async (runId: string): Promise => { + const db = await getDb(); + await db.delete(diagRuns).where(eq(diagRuns.id, runId)).execute(); +}; + +export const recordStageResult = async (runId: string, stage: DiagStageName, payload: unknown): Promise => { + const db = await getDb(); + await db + .insert(diagStages) + .values({ id: randomUUID(), runId, stage, payload: payload as object }) + .onConflictDoUpdate({ target: [diagStages.runId, diagStages.stage], set: { payload: payload as object } }) + .execute(); + await db.update(diagRuns).set({ currentStage: stage, updatedAt: now() }).where(eq(diagRuns.id, runId)).execute(); +}; + +export const getStagePayload = async (runId: string, stage: DiagStageName): Promise => { + const db = await getDb(); + const [row] = await db + .select() + .from(diagStages) + .where(and(eq(diagStages.runId, runId), eq(diagStages.stage, stage))) + .execute(); + return row?.payload ?? null; +}; + +export const listStageResults = async (runId: string): Promise<{ stage: DiagStageName; payload: unknown }[]> => { + const db = await getDb(); + const rows = await db.select().from(diagStages).where(eq(diagStages.runId, runId)).execute(); + return rows.map((r) => ({ stage: r.stage as DiagStageName, payload: r.payload ?? null })); +}; + +export const finishDiagRun = async ( + runId: string, + status: DiagStatus, + cause: CauseKind | null, + report: Proposal | null, + error: string | null, +): Promise => { + const db = await getDb(); + await db + .update(diagRuns) + .set({ status, cause, report: report as object | null, error, updatedAt: now() }) + .where(eq(diagRuns.id, runId)) + .execute(); +}; + +export interface ActiveDiagRun { + id: string; + deploymentId: string; + projectId: string | null; + projectName: string | null; + provider: string; + model: string; + currentStage: DiagStageName | null; + createdAt: string; +} + +export const listActiveDiagRuns = async (): Promise => { + const db = await getDb(); + const rows = await db + .select({ + id: diagRuns.id, + deploymentId: diagRuns.deploymentId, + projectId: deployments.projectId, + projectName: projects.name, + provider: diagRuns.provider, + model: diagRuns.model, + currentStage: diagRuns.currentStage, + createdAt: diagRuns.createdAt, + }) + .from(diagRuns) + .leftJoin(deployments, eq(diagRuns.deploymentId, deployments.id)) + .leftJoin(projects, eq(deployments.projectId, projects.id)) + .where(eq(diagRuns.status, "running")) + .execute(); + return rows.map((row) => ({ + id: row.id, + deploymentId: row.deploymentId, + projectId: row.projectId, + projectName: row.projectName, + provider: row.provider, + model: row.model, + currentStage: row.currentStage as DiagStageName | null, + createdAt: row.createdAt?.toISOString() ?? new Date().toISOString(), + })); +}; + +export const markInterruptedDiagRuns = async (): Promise => { + const active = await listActiveDiagRuns(); + for (const run of active) { + await finishDiagRun( + run.id, + "error", + null, + null, + "Diagnosis interrupted by API restart; start it again from the deployment.", + ); + } + return active.length; +}; diff --git a/apps/api/src/db/repo/github-sessions.ts b/apps/api/src/db/repo/github-sessions.ts index c74e286..0f3154d 100644 --- a/apps/api/src/db/repo/github-sessions.ts +++ b/apps/api/src/db/repo/github-sessions.ts @@ -11,6 +11,13 @@ export const getGithubSession = async (id: string): Promise => { return decryptValue(row.accessTokenEncrypted, row.accessTokenIv, row.accessTokenTag, config.envEncryptionKey); }; +export const getGithubTokenFromCookie = async (cookie: string | null): Promise => { + if (!cookie) return null; + const match = cookie.match(/(?:^|;\s*)github_session=([^;]+)/); + if (!match) return null; + return getGithubSession(match[1]); +}; + export const createGithubSession = async (id: string, accessToken: string): Promise => { const db = await getDb(); const enc = encryptValue(accessToken, config.envEncryptionKey); diff --git a/apps/api/src/db/repo/index.ts b/apps/api/src/db/repo/index.ts index fa4369b..d99287a 100644 --- a/apps/api/src/db/repo/index.ts +++ b/apps/api/src/db/repo/index.ts @@ -29,7 +29,16 @@ export { updateDatabaseSettings, updateDatabaseStatus, } from "./databases"; -export { createDeploymentEvent, listDeploymentEvents } from "./deployment-events"; +export { + claimFailureNotification, + createDeploymentEvent, + listDeploymentEvents, + listPendingFailureNotificationIds, + listProjectEvents, + markFailureNotificationSent, + recordDeploymentCancellation, + recordDeploymentFailure, +} from "./deployment-events"; export { appendLog, countDeployments, @@ -41,6 +50,26 @@ export { updateDeploymentCommitSha, updateDeploymentStatus, } from "./deployments"; +export { + createDiagRun, + deleteDiagRun, + findDiagRun, + finishDiagRun, + getDiagRun, + getStagePayload, + listActiveDiagRuns, + listStageResults, + markInterruptedDiagRuns, + recordStageResult, +} from "./diag-runs"; +export type { DiagAction, DiagActionKind, DiagActionStatus } from "./diag-actions"; +export { + claimDiagAction, + completeDiagAction, + failDiagAction, + findCompletedAction, + getDiagAction, +} from "./diag-actions"; export { createDomain, deleteDomain, @@ -59,7 +88,12 @@ export { updateEnvironmentVariable, } from "./env-vars"; export { getGithubIntegration, setGithubIntegration } from "./github"; -export { createGithubSession, deleteGithubSession, getGithubSession } from "./github-sessions"; +export { + createGithubSession, + deleteGithubSession, + getGithubSession, + getGithubTokenFromCookie, +} from "./github-sessions"; export { getPlatformSettings, setIngressServer } from "./platform-settings"; export type { ProjectCleanupInfo } from "./projects"; export { @@ -95,6 +129,15 @@ export { } from "./servers"; export type { SmtpSettingsData } from "./settings"; export { getBackupStorageSettings, getSmtpSettings, upsertBackupStorageSettings, upsertSmtpSettings } from "./settings"; +export type { LlmKeyInput, LlmKeyStatus, LlmProvider } from "./llm-keys"; +export { + LLM_PROVIDERS, + deleteLlmKey, + getDecryptedLlmKey, + getLlmKeyStatus, + updateLlmModels, + upsertLlmKey, +} from "./llm-keys"; export { createSharedEnvVar, deleteSharedEnvVar, diff --git a/apps/api/src/db/repo/llm-keys.ts b/apps/api/src/db/repo/llm-keys.ts new file mode 100644 index 0000000..c60f287 --- /dev/null +++ b/apps/api/src/db/repo/llm-keys.ts @@ -0,0 +1,110 @@ +import { eq } from "drizzle-orm"; +import { config } from "../../utils/config"; +import { decryptValue, encryptValue } from "../../utils/crypto"; +import { getDb } from "../db-provider"; +import { llmProviderKeys } from "../schema"; +import { now } from "./helpers"; + +export const LLM_PROVIDERS = ["openai", "anthropic", "gemini", "groq", "ollama", "custom"] as const; +export type LlmProvider = (typeof LLM_PROVIDERS)[number]; + +export interface LlmKeyStatus { + provider: string; + configured: boolean; + baseUrl: string | null; + models: string[]; +} + +export interface LlmKeyInput { + provider: string; + apiKey: string; + baseURL?: string; + models?: string[]; +} + +const toStatus = (row: typeof llmProviderKeys.$inferSelect): LlmKeyStatus => ({ + provider: row.provider, + configured: !!(row.keyEncrypted && row.keyIv && row.keyTag), + baseUrl: row.baseUrl, + models: (row.models as string[]) ?? [], +}); + +export const getLlmKeyStatus = async (): Promise => { + const db = await getDb(); + const rows = await db.select().from(llmProviderKeys).execute(); + return rows.map(toStatus); +}; + +export const getDecryptedLlmKey = async ( + provider: string, +): Promise<{ apiKey: string; baseUrl: string | null; models: string[] } | null> => { + const db = await getDb(); + const [row] = await db.select().from(llmProviderKeys).where(eq(llmProviderKeys.provider, provider)).execute(); + if (!row?.keyEncrypted || !row.keyIv || !row.keyTag) return null; + return { + apiKey: decryptValue(row.keyEncrypted, row.keyIv, row.keyTag, config.envEncryptionKey), + baseUrl: row.baseUrl, + models: (row.models as string[]) ?? [], + }; +}; + +export const upsertLlmKey = async (input: LlmKeyInput): Promise => { + const db = await getDb(); + const encrypted = input.apiKey ? encryptValue(input.apiKey, config.envEncryptionKey) : null; + const timestamp = now(); + const [existing] = await db + .select() + .from(llmProviderKeys) + .where(eq(llmProviderKeys.provider, input.provider)) + .execute(); + if (existing) { + const [updated] = await db + .update(llmProviderKeys) + .set({ + keyEncrypted: encrypted?.encrypted ?? existing.keyEncrypted, + keyIv: encrypted?.iv ?? existing.keyIv, + keyTag: encrypted?.tag ?? existing.keyTag, + baseUrl: input.baseURL ?? existing.baseUrl, + models: input.models ?? existing.models, + updatedAt: timestamp, + }) + .where(eq(llmProviderKeys.provider, input.provider)) + .returning() + .execute(); + return toStatus(updated); + } + const [inserted] = await db + .insert(llmProviderKeys) + .values({ + provider: input.provider, + keyEncrypted: encrypted?.encrypted ?? null, + keyIv: encrypted?.iv ?? null, + keyTag: encrypted?.tag ?? null, + baseUrl: input.baseURL ?? null, + models: input.models ?? [], + updatedAt: timestamp, + }) + .returning() + .execute(); + return toStatus(inserted); +}; + +export const deleteLlmKey = async (provider: string): Promise => { + const db = await getDb(); + const deleted = await db.delete(llmProviderKeys).where(eq(llmProviderKeys.provider, provider)).returning().execute(); + return deleted.length > 0; +}; + +export const updateLlmModels = async (provider: string, models: string[]): Promise => { + const db = await getDb(); + const [updated] = await db + .update(llmProviderKeys) + .set({ + models, + updatedAt: now(), + }) + .where(eq(llmProviderKeys.provider, provider)) + .returning() + .execute(); + return updated ? toStatus(updated) : null; +}; diff --git a/apps/api/src/db/repo/projects.ts b/apps/api/src/db/repo/projects.ts index 994debf..34d53c1 100644 --- a/apps/api/src/db/repo/projects.ts +++ b/apps/api/src/db/repo/projects.ts @@ -48,9 +48,7 @@ const mapProject = (row: typeof projects.$inferSelect): Project => ({ installCommand: row.installCommand ?? null, outputDir: row.outputDir ?? null, startCommand: row.startCommand ?? null, - githubTokenEncrypted: row.githubTokenEncrypted ?? null, - githubTokenIv: row.githubTokenIv ?? null, - githubTokenTag: row.githubTokenTag ?? null, + hasGithubToken: !!(row.githubTokenEncrypted && row.githubTokenIv && row.githubTokenTag), createdAt: formatTimestamp(row.createdAt), updatedAt: formatTimestamp(row.updatedAt), }); diff --git a/apps/api/src/db/repo/shared-env-vars.ts b/apps/api/src/db/repo/shared-env-vars.ts index dc7d048..ee34d54 100644 --- a/apps/api/src/db/repo/shared-env-vars.ts +++ b/apps/api/src/db/repo/shared-env-vars.ts @@ -127,19 +127,21 @@ export const linkSharedEnvVarsToProject = async (projectId: string, sharedEnvVar export const unlinkSharedEnvVarFromProject = async (projectId: string, sharedEnvVarId: string): Promise => { const db = await getDb(); + const links = await db + .select() + .from(projectSharedEnvLinks) + .where( + and(eq(projectSharedEnvLinks.projectId, projectId), eq(projectSharedEnvLinks.sharedEnvVarId, sharedEnvVarId)), + ) + .execute(); + if (links.length === 0) return false; return ( - getRowsAffected( - await db - .delete(projectSharedEnvLinks) - .where( - and(eq(projectSharedEnvLinks.projectId, projectId), eq(projectSharedEnvLinks.sharedEnvVarId, sharedEnvVarId)), - ) - .execute(), - ) > 0 + getRowsAffected(await db.delete(projectSharedEnvLinks).where(eq(projectSharedEnvLinks.id, links[0].id)).execute()) > + 0 ); }; -export const listLinkedSharedEnvVars = async (projectId: string): Promise => { +export const listLinkedSharedEnvVars = async (projectId: string): Promise<(SharedEnvVar & { linkId: string })[]> => { const db = await getDb(); const links = await db .select() @@ -156,9 +158,12 @@ export const listLinkedSharedEnvVars = async (projectId: string): Promise [r.id, r])); return links - .map((l) => rowMap.get(l.sharedEnvVarId)) - .filter(Boolean) - .map((r) => mapSharedEnvVar(r!)); + .map((l) => { + const row = rowMap.get(l.sharedEnvVarId); + if (!row) return null; + return { ...mapSharedEnvVar(row), linkId: l.id }; + }) + .filter(Boolean) as (SharedEnvVar & { linkId: string })[]; }; export const listSharedEnvVarsForDeploy = async ( diff --git a/apps/api/src/db/schema.ts b/apps/api/src/db/schema.ts index 93a085a..cea82ef 100644 --- a/apps/api/src/db/schema.ts +++ b/apps/api/src/db/schema.ts @@ -104,6 +104,8 @@ export const deploymentEvents = pgTable( type: text().notNull(), message: text(), metadata: jsonb("metadata"), + sentAt: timestamp("sent_at", { withTimezone: true }), + attempts: integer().notNull().default(0), createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), }, (table) => [ @@ -363,6 +365,68 @@ export const backupStorageSettings = pgTable("backup_storage_settings", { updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), }); +export const llmProviderKeys = pgTable("llm_provider_keys", { + provider: text().primaryKey(), + keyEncrypted: text("key_encrypted"), + keyIv: text("key_iv"), + keyTag: text("key_tag"), + baseUrl: text("base_url"), + models: jsonb().notNull().default([]), + updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), +}); + +export const diagRuns = pgTable( + "diag_runs", + { + id: text().primaryKey(), + deploymentId: text("deployment_id").notNull(), + commitSha: text("commit_sha").notNull().default(""), + provider: text().notNull(), + model: text().notNull(), + status: text().notNull().default("running"), + currentStage: text("current_stage"), + cause: text(), + report: jsonb(), + error: text(), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), + updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), + }, + (table) => [ + foreignKey({ columns: [table.deploymentId], foreignColumns: [deployments.id], onDelete: "cascade" }), + uniqueIndex("diag_runs_deployment_commit").on(table.deploymentId, table.commitSha), + ], +); + +export const diagStages = pgTable( + "diag_stages", + { + id: text().primaryKey(), + runId: text("run_id").notNull(), + stage: text().notNull(), + payload: jsonb(), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), + }, + (table) => [ + foreignKey({ columns: [table.runId], foreignColumns: [diagRuns.id], onDelete: "cascade" }), + uniqueIndex("diag_stages_run_stage").on(table.runId, table.stage), + ], +); + +export const diagActions = pgTable( + "diag_actions", + { + key: text().primaryKey(), + runId: text("run_id").notNull(), + kind: text().notNull(), + status: text().notNull().default("requested"), + result: jsonb(), + error: text(), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), + updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), + }, + (table) => [foreignKey({ columns: [table.runId], foreignColumns: [diagRuns.id], onDelete: "cascade" })], +); + export const sharedEnvVars = pgTable("shared_env_vars", { id: text().primaryKey(), key: text().notNull(), diff --git a/apps/api/src/db/test-helper.ts b/apps/api/src/db/test-helper.ts index aee307f..c56d0e1 100644 --- a/apps/api/src/db/test-helper.ts +++ b/apps/api/src/db/test-helper.ts @@ -15,9 +15,13 @@ const TABLE_NAMES = [ "deployment_logs", "deployments", "databases", + "diag_actions", + "diag_runs", + "diag_stages", "domains", "environment_variables", "github_integrations", + "llm_provider_keys", "platform_settings", "projects", "refresh_tokens", diff --git a/apps/api/src/events.ts b/apps/api/src/events.ts new file mode 100644 index 0000000..4530ac7 --- /dev/null +++ b/apps/api/src/events.ts @@ -0,0 +1,23 @@ +export interface DeploymentFailedSignal { + eventId: string; + deploymentId: string; +} + +type Handler = (signal: DeploymentFailedSignal) => void; + +const handlers = new Set(); + +export const onDeploymentFailed = (handler: Handler): (() => void) => { + handlers.add(handler); + return () => handlers.delete(handler); +}; + +export const emitDeploymentFailed = (signal: DeploymentFailedSignal): void => { + for (const handler of handlers) { + try { + handler(signal); + } catch (err) { + console.error("[Events] deployment-failed handler error:", err); + } + } +}; diff --git a/apps/api/src/executors/agent.ts b/apps/api/src/executors/agent.ts index 8a9bb09..9c69cbf 100644 --- a/apps/api/src/executors/agent.ts +++ b/apps/api/src/executors/agent.ts @@ -1,4 +1,5 @@ import type { Deployment, Project, Server } from "../types"; +import { CANCELLED_FAILURE_REASON } from "../utils/failure-outcome"; import { routeNamesFor } from "../utils/routes"; import type { DeploymentExecutor, @@ -106,10 +107,14 @@ export const agentExecutor: DeploymentExecutor = { }, async cancel({ deployment }: ExecutorCancelInput) { - const { cancelAgentJobsByDeploymentId, updateDeploymentStatus, appendLog } = await getRepo(); + const { cancelAgentJobsByDeploymentId, recordDeploymentCancellation, appendLog } = await getRepo(); if (deployment.status !== "pending" && deployment.status !== "building") return; await cancelAgentJobsByDeploymentId(deployment.id); - await updateDeploymentStatus(deployment.id, "failed", { failureReason: "Cancelled" }); + await recordDeploymentCancellation({ + deploymentId: deployment.id, + reason: CANCELLED_FAILURE_REASON, + source: "agent", + }); await appendLog(deployment.id, "system", "Deployment cancelled by user"); }, }; diff --git a/apps/api/src/executors/ssh.ts b/apps/api/src/executors/ssh.ts index dcac72b..3192468 100644 --- a/apps/api/src/executors/ssh.ts +++ b/apps/api/src/executors/ssh.ts @@ -1,6 +1,7 @@ import { summarizeDeploymentError } from "../orchestrator/deployment-errors"; import type { Deployment, Project, Server } from "../types"; import { config } from "../utils/config"; +import { CANCELLED_FAILURE_REASON } from "../utils/failure-outcome"; import { removeRemoteCaddyRoute, runRemoteScript, syncRemoteCaddyRoute } from "../utils/ssh"; import { emitLog } from "./logging"; import { buildRemoteDeployScript, parseRemoteBuildResult } from "./ssh-build-script"; @@ -254,10 +255,10 @@ const deployComposeRemote = async (deployment: Deployment, project: Project, ser }; const markFailed = async (deploymentId: string, error: unknown) => { - const { updateDeploymentStatus } = await getRepo(); + const { recordDeploymentFailure } = await getRepo(); const message = summarizeDeploymentError(error); await emitLog(deploymentId, "system", `Deployment failed: ${message}`); - await updateDeploymentStatus(deploymentId, "failed", { failureReason: message }); + await recordDeploymentFailure({ deploymentId, reason: message, source: "ssh" }); }; export const sshExecutor: DeploymentExecutor = { @@ -268,7 +269,11 @@ export const sshExecutor: DeploymentExecutor = { if (!project) throw new Error("Deployment requires a project"); if (project.buildType === "compose") { - await deployComposeRemote(deployment, project, server); + try { + await deployComposeRemote(deployment, project, server); + } catch (error) { + await markFailed(deployment.id, error); + } return; } @@ -353,7 +358,9 @@ export const sshExecutor: DeploymentExecutor = { } catch (error) { const message = summarizeDeploymentError(error); await emitLog(deployment.id, "system", `Rollback failed: ${message}`); - await updateDeploymentStatus(deployment.id, "failed", { failureReason: message }); + const { recordDeploymentFailure, updateDeploymentStatus } = await getRepo(); + const r = await recordDeploymentFailure({ deploymentId: deployment.id, reason: message, source: "rollback" }); + if (!r.claimed) await updateDeploymentStatus(deployment.id, "failed", { failureReason: message }); throw error; } }, @@ -400,9 +407,13 @@ export const sshExecutor: DeploymentExecutor = { }, async cancel({ deployment }: ExecutorCancelInput) { - const { updateDeploymentStatus } = await getRepo(); + const { recordDeploymentCancellation } = await getRepo(); if (deployment.status !== "pending" && deployment.status !== "building") return; - await updateDeploymentStatus(deployment.id, "failed", { failureReason: "Cancelled" }); + await recordDeploymentCancellation({ + deploymentId: deployment.id, + reason: CANCELLED_FAILURE_REASON, + source: "ssh", + }); await emitLog(deployment.id, "system", "Deployment cancelled by user (remote build may continue on the server)"); }, }; diff --git a/apps/api/src/fixdiag/__tests__/actions.test.ts b/apps/api/src/fixdiag/__tests__/actions.test.ts new file mode 100644 index 0000000..ff2369a --- /dev/null +++ b/apps/api/src/fixdiag/__tests__/actions.test.ts @@ -0,0 +1,134 @@ +import { afterAll, afterEach, beforeAll, describe, expect, it, mock } from "bun:test"; +import { drizzle } from "drizzle-orm/node-postgres"; +import type { Pool } from "pg"; +import { setDbProvider } from "../../db/db-provider"; +import * as schema from "../../db/schema"; +import { createTestPool, truncateAllTables } from "../../db/test-helper"; +import { createDiagRun, finishDiagRun } from "../../db/repo/diag-runs"; +import { getDiagAction } from "../../db/repo/diag-actions"; +import { createGithubSession, deleteGithubSession } from "../../db/repo/github-sessions"; +import { approveFixPr, approveSlackPost } from "../actions"; +import * as sandboxHost from "../sandbox-host"; +import type { Proposal } from "../types"; + +mock.restore(); + +mock.module("../sandbox-host", () => ({ + ...sandboxHost, + ensureSandbox: async () => { + throw new Error("sandbox unavailable"); + }, +})); + +let pool: Pool; + +const seed = async () => { + await pool.query( + `INSERT INTO projects (id, name, source_type, created_at, updated_at) + VALUES ('proj-act', 'Act Project', 'git', NOW(), NOW()) ON CONFLICT DO NOTHING`, + ); + await pool.query( + `INSERT INTO deployments (id, project_id, source_type, source_ref, status, branch, commit_sha, created_at, updated_at) + VALUES ('dep-git-1', 'proj-act', 'git', 'https://github.com/acme/shop.git', 'failed', 'main', 'aaaabbbbcccc', NOW(), NOW()), + ('dep-zip-1', 'proj-act', 'upload', '/tmp/nowhere', 'failed', 'main', '', NOW(), NOW()), + ('dep-run-1', 'proj-act', 'git', 'https://github.com/acme/shop.git', 'running', 'main', NULL, NOW(), NOW()) + ON CONFLICT DO NOTHING`, + ); +}; + +const userReport: Proposal = { + cause: "user-source", + userFix: { title: "Fix it", body: "Do the thing", suggestedDiff: "diff --git a/x b/x\n" }, +}; + +const dequelReport: Proposal = { + cause: "dequel-source", + dequelReport: { problem: "p", cause: "c", proposedFix: "f" }, +}; + +const doneRun = async (deploymentId: string, commitSha: string, report: Proposal) => { + const run = await createDiagRun({ deploymentId, commitSha, provider: "groq", model: "m" }); + await finishDiagRun(run.id, "done", report.cause, report, null); + return run; +}; + +beforeAll(async () => { + pool = createTestPool(); + const db = drizzle(pool, { schema }); + setDbProvider(async () => db); + await truncateAllTables(pool); + await seed(); + await deleteGithubSession("sess-test").catch(() => {}); + await createGithubSession("sess-test", "tok-test"); +}); + +afterEach(async () => { + await truncateAllTables(pool); + await seed(); +}); + +afterAll(async () => { + try { + await truncateAllTables(pool); + } finally { + await pool.end(); + } +}); + +describe("approveFixPr guards", () => { + const cookie = "github_session=sess-test"; + const authed = { validateToken: async () => true }; + + it("requires an idempotency key and configured git credentials first", async () => { + expect(await approveFixPr("nope", "", null)).toMatchObject({ ok: false, status: 400 }); + expect(await approveFixPr("nope", "k-1", null)).toMatchObject({ ok: false, status: 401 }); + expect(await approveFixPr("nope", "k-1", "github_session=missing")).toMatchObject({ ok: false, status: 401 }); + expect(await approveFixPr("nope", "k-unknown-run", cookie, authed)).toMatchObject({ ok: false, status: 404 }); + }); + + it("rejects unfinished runs and non-git projects before touching the sandbox", async () => { + const running = await createDiagRun({ + deploymentId: "dep-git-1", + commitSha: "aaaabbbbcccc", + provider: "groq", + model: "m", + }); + expect(await approveFixPr(running.id, "k-2", cookie, authed)).toMatchObject({ ok: false, status: 409 }); + const zip = await doneRun("dep-zip-1", "", userReport); + expect(await approveFixPr(zip.id, "k-zip", cookie, authed)).toMatchObject({ ok: false, status: 409 }); + }); + + it("fails cleanly when the sandbox is unavailable", async () => { + const noPatch = await doneRun("dep-git-1", "aaaabbbbcccc", { + cause: "user-source", + userFix: { title: "t", body: "b", suggestedDiff: null }, + }); + expect(await approveFixPr(noPatch.id, "k-nopatch", cookie, authed)).toMatchObject({ ok: false, status: 502 }); + }); + + it("rejects stale runs and blocks unauthenticated clicks without side effects", async () => { + const stale = await doneRun("dep-git-1", "oldsameaning", userReport); + expect(await approveFixPr(stale.id, "k-stale", cookie, authed)).toMatchObject({ ok: false, status: 409 }); + const fresh = await doneRun("dep-git-1", "aaaabbbbcccc", userReport); + expect(await approveFixPr(fresh.id, "k-auth", null)).toMatchObject({ ok: false, status: 401 }); + expect(await getDiagAction("k-auth")).toBeNull(); + }); + + it("replays the same key and rejects cross-intent reuse", async () => { + const fresh = await doneRun("dep-git-1", "aaaabbbbcccc", userReport); + const first = await approveFixPr(fresh.id, "k-replay", null); + const second = await approveFixPr(fresh.id, "k-replay", null); + expect(first).toEqual(second); + expect(await approveSlackPost(fresh.id, "k-replay")).toMatchObject({ ok: false, status: 409 }); + }); +}); + +describe("approveSlackPost guards", () => { + it("requires a finished dequel-source run and a configured webhook", async () => { + expect(await approveSlackPost("nope", "s-1")).toMatchObject({ ok: false, status: 404 }); + const user = await doneRun("dep-git-1", "bbbbccccdddd", userReport); + expect(await approveSlackPost(user.id, "s-2")).toMatchObject({ ok: false, status: 409 }); + const deq = await doneRun("dep-git-1", "aaaabbbbcccc", dequelReport); + expect(await approveSlackPost(deq.id, "s-3")).toMatchObject({ ok: false, status: 409 }); + }); +}); diff --git a/apps/api/src/fixdiag/__tests__/context.test.ts b/apps/api/src/fixdiag/__tests__/context.test.ts new file mode 100644 index 0000000..a998aad --- /dev/null +++ b/apps/api/src/fixdiag/__tests__/context.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it, mock } from "bun:test"; +import { failureReasonOf, logTextOf, tailLogs } from "../context"; +import type { LogLine } from "../types"; + +mock.restore(); + +const line = (sequence: number, message: string): LogLine => ({ sequence, stage: "build", message }); + +describe("tailLogs", () => { + it("keeps the last lines within limits", () => { + const logs = Array.from({ length: 500 }, (_, i) => line(i + 1, `line ${i + 1}`)); + const tail = tailLogs(logs); + expect(tail).toHaveLength(400); + expect(tail[0].sequence).toBe(101); + }); + + it("caps total characters from the front", () => { + const logs = [line(1, "a".repeat(20000)), line(2, "b".repeat(20000))]; + const tail = tailLogs(logs, 400, 24000); + expect(tail.map((l) => l.sequence)).toEqual([2]); + }); +}); + +describe("logTextOf", () => { + it("prefixes stage names", () => { + expect(logTextOf([line(1, "boom")])).toBe("[build] boom"); + }); + + it("never returns an empty string", () => { + expect(logTextOf([]).trim().length).toBeGreaterThan(0); + }); +}); + +describe("failureReasonOf", () => { + it("falls back to a placeholder", () => { + expect(failureReasonOf("boom")).toBe("boom"); + expect(failureReasonOf(null).trim().length).toBeGreaterThan(0); + expect(failureReasonOf(" ").trim().length).toBeGreaterThan(0); + }); +}); diff --git a/apps/api/src/fixdiag/__tests__/machine.test.ts b/apps/api/src/fixdiag/__tests__/machine.test.ts new file mode 100644 index 0000000..024f231 --- /dev/null +++ b/apps/api/src/fixdiag/__tests__/machine.test.ts @@ -0,0 +1,211 @@ +import { afterAll, afterEach, beforeAll, describe, expect, it, mock } from "bun:test"; +import { drizzle } from "drizzle-orm/node-postgres"; +import type { Pool } from "pg"; +import { setDbProvider } from "../../db/db-provider"; +import * as schema from "../../db/schema"; +import { createTestPool, truncateAllTables } from "../../db/test-helper"; +import { upsertLlmKey } from "../../db/repo/llm-keys"; +import { getDiagRun, listStageResults, recordStageResult } from "../../db/repo/diag-runs"; +import { DiagRunMachine, parseGithubRepo } from "../machine"; +import { diagBus } from "../stream"; +import type { FixdiagPrograms, InvestigateFn, Investigation, StageEvent } from "../types"; + +mock.restore(); + +let pool: Pool; + +const seed = async () => { + await pool.query( + `INSERT INTO projects (id, name, source_type, created_at, updated_at) + VALUES ('proj-diag', 'Diag Project', 'git', NOW(), NOW()) ON CONFLICT DO NOTHING`, + ); + await pool.query( + `INSERT INTO deployments (id, project_id, source_type, source_ref, status, branch, commit_sha, failure_reason, created_at, updated_at) + VALUES ('dep-failed-1', 'proj-diag', 'upload', '/tmp/does-not-exist', 'failed', 'main', 'deadbeef', 'boom', NOW(), NOW()), + ('dep-running-1', 'proj-diag', 'upload', '/tmp/does-not-exist', 'running', 'main', NULL, NULL, NOW(), NOW()) + ON CONFLICT DO NOTHING`, + ); + await pool.query( + `INSERT INTO deployment_logs (deployment_id, sequence, stage, message) + VALUES ('dep-failed-1', 1, 'build', 'Step 1/2'), ('dep-failed-1', 2, 'build', 'boom') ON CONFLICT DO NOTHING`, + ); +}; + +const fakePrograms: FixdiagPrograms = { + triageLogs: async () => ({ failingStage: "build", signalLines: ["boom"], confidence: "high" }), + explainFix: async () => ({ summary: "s", fixSteps: ["do x"], patchHint: null }), + draftProposal: async () => ({ cause: "user-source", userFix: { title: "t", body: "b", suggestedDiff: null } }), +}; + +const fakeInvestigation: Investigation = { + cause: "user-source", + culpritPaths: ["Dockerfile"], + rationale: "r", + keyEvidence: ["boom"], + dequelRev: "v0.3.0 @ abc1234", + dequelStale: false, +}; + +const fakeInvestigator: InvestigateFn = async () => fakeInvestigation; + +beforeAll(async () => { + pool = createTestPool(); + const db = drizzle(pool, { schema }); + setDbProvider(async () => db); + await truncateAllTables(pool); + await seed(); + await upsertLlmKey({ provider: "groq", apiKey: "gsk-test" }); +}); + +afterEach(async () => { + await truncateAllTables(pool); + await seed(); + await upsertLlmKey({ provider: "groq", apiKey: "gsk-test" }); +}); + +afterAll(async () => { + try { + await truncateAllTables(pool); + } finally { + await pool.end(); + } +}); + +describe("DiagRunMachine.start", () => { + it("rejects unknown deployments, non-failed deployments and bad input", async () => { + expect(await DiagRunMachine.start({ deploymentId: "nope", provider: "groq", model: "m" })).toMatchObject({ + ok: false, + status: 404, + }); + expect(await DiagRunMachine.start({ deploymentId: "dep-running-1", provider: "groq", model: "m" })).toMatchObject({ + ok: false, + status: 409, + }); + expect(await DiagRunMachine.start({ deploymentId: "dep-failed-1", provider: "nope", model: "m" })).toMatchObject({ + ok: false, + status: 400, + }); + expect(await DiagRunMachine.start({ deploymentId: "dep-failed-1", provider: "groq", model: " " })).toMatchObject({ + ok: false, + status: 400, + }); + expect(await DiagRunMachine.start({ deploymentId: "dep-failed-1", provider: "openai", model: "m" })).toMatchObject({ + ok: false, + status: 400, + }); + }); + + it("creates once and returns the existing run on repeat", async () => { + const first = await DiagRunMachine.start({ deploymentId: "dep-failed-1", provider: "groq", model: "m" }); + expect(first.ok && first.created).toBe(true); + const second = await DiagRunMachine.start({ deploymentId: "dep-failed-1", provider: "groq", model: "m" }); + expect(second.ok && !second.created && second.run.id === (first.ok && first.run.id)).toBe(true); + }); +}); + +describe("DiagRunMachine.drive", () => { + it("runs all stages, persists the report and emits events", async () => { + const started = await DiagRunMachine.start({ deploymentId: "dep-failed-1", provider: "groq", model: "m" }); + if (!started.ok) throw new Error("start failed"); + const events: StageEvent[] = []; + const unsub = diagBus.subscribe(started.run.id, (e) => events.push(e)); + try { + await DiagRunMachine.drive(started.run.id, fakePrograms, fakeInvestigator); + } finally { + unsub(); + } + const run = await getDiagRun(started.run.id); + expect(run?.status).toBe("done"); + expect(run?.cause).toBe("user-source"); + expect(run?.report).toEqual({ cause: "user-source", userFix: { title: "t", body: "b", suggestedDiff: null } }); + expect((await listStageResults(started.run.id)).map((s) => s.stage)).toEqual([ + "triage", + "investigate", + "explain", + "propose", + ]); + expect(events.filter((e) => e.type === "stage")).toHaveLength(4); + expect(events.some((e) => e.type === "done")).toBe(true); + }); + + it("resumes after the last completed stage", async () => { + const started = await DiagRunMachine.start({ deploymentId: "dep-failed-1", provider: "groq", model: "m" }); + if (!started.ok) throw new Error("start failed"); + await recordStageResult(started.run.id, "triage", { failingStage: "build", signalLines: [], confidence: "high" }); + const calls: string[] = []; + const counting: FixdiagPrograms = { + triageLogs: async (...args) => { + calls.push("triage"); + return fakePrograms.triageLogs(...args); + }, + explainFix: async (...args) => { + calls.push("explain"); + return fakePrograms.explainFix(...args); + }, + draftProposal: async (...args) => { + calls.push("propose"); + return fakePrograms.draftProposal(...args); + }, + }; + const investigator: InvestigateFn = async (...args) => { + calls.push("investigate"); + return fakeInvestigator(...args); + }; + await DiagRunMachine.drive(started.run.id, counting, investigator); + expect(calls).toEqual(["investigate", "explain", "propose"]); + expect((await getDiagRun(started.run.id))?.status).toBe("done"); + }); + + it("marks the run errored when the investigator throws", async () => { + const started = await DiagRunMachine.start({ deploymentId: "dep-failed-1", provider: "groq", model: "m" }); + if (!started.ok) throw new Error("start failed"); + const failing: InvestigateFn = async () => Promise.reject(new Error("sandbox down")); + const events: StageEvent[] = []; + const unsub = diagBus.subscribe(started.run.id, (e) => events.push(e)); + try { + await DiagRunMachine.drive(started.run.id, fakePrograms, failing); + } finally { + unsub(); + } + const run = await getDiagRun(started.run.id); + expect(run?.status).toBe("error"); + expect(run?.error).toBe("investigate failed: sandbox down"); + expect(events.some((e) => e.type === "error")).toBe(true); + }); + + it("auto-posts dequel-source reports to Slack without failing the run", async () => { + const started = await DiagRunMachine.start({ deploymentId: "dep-failed-1", provider: "groq", model: "m" }); + if (!started.ok) throw new Error("start failed"); + const dequelPrograms: FixdiagPrograms = { + ...fakePrograms, + draftProposal: async () => ({ + cause: "dequel-source", + dequelReport: { problem: "p", cause: "c", proposedFix: "f" }, + }), + }; + await DiagRunMachine.drive(started.run.id, dequelPrograms, fakeInvestigator); + const run = await getDiagRun(started.run.id); + expect(run?.status).toBe("done"); + expect(run?.cause).toBe("dequel-source"); + const { rows } = await pool.query("SELECT status FROM diag_actions WHERE run_id = $1 AND kind = 'slack'", [ + started.run.id, + ]); + expect(rows.length).toBe(1); + }); +}); + +describe("parseGithubRepo", () => { + it("parses https and ssh urls", () => { + expect(parseGithubRepo("https://github.com/acme/shop.git", "main")).toEqual({ + owner: "acme", + repo: "shop", + base: "main", + }); + expect(parseGithubRepo("git@github.com:acme/shop.git", null)).toEqual({ + owner: "acme", + repo: "shop", + base: "main", + }); + expect(parseGithubRepo("https://example.com/acme/shop", "dev")).toBeNull(); + }); +}); diff --git a/apps/api/src/fixdiag/__tests__/programs.test.ts b/apps/api/src/fixdiag/__tests__/programs.test.ts new file mode 100644 index 0000000..afe71ba --- /dev/null +++ b/apps/api/src/fixdiag/__tests__/programs.test.ts @@ -0,0 +1,66 @@ +import { describe, expect, it, mock } from "bun:test"; +import { createPrograms } from "../programs"; +import { heuristicSignalLines } from "../programs"; +import type { DiagLlm } from "../llm"; + +mock.restore(); + +const throwingLlm = (): DiagLlm => + ({ + chat: async () => { + throw new Error("Generate failed: boom"); + }, + }) as unknown as DiagLlm; + +describe("program fallbacks", () => { + it("triage falls back to heuristic signal lines", async () => { + const progs = createPrograms(throwingLlm()); + const out = await progs.triageLogs({ + failureReason: "boom", + logText: "[build] ok\n[build] ERROR: dial tcp: lookup ghcr.io: server misbehaving\n[system] done", + }); + expect(out.confidence).toBe("low"); + expect(out.failingStage).toBe("unknown"); + expect(out.signalLines).toEqual(["[build] ERROR: dial tcp: lookup ghcr.io: server misbehaving"]); + }); + + it("explain falls back to the localization rationale", async () => { + const progs = createPrograms(throwingLlm()); + const out = await progs.explainFix({ + verdict: { failingStage: "x", signalLines: [], confidence: "low" }, + localization: { cause: "dequel-source", culpritPaths: [], rationale: "dns broke" }, + }); + expect(out.summary).toBe("dns broke"); + expect(out.fixSteps).toEqual([]); + expect(out.patchHint).toBeNull(); + }); + + it("propose falls back to a minimal dequel report", async () => { + const progs = createPrograms(throwingLlm()); + const out = await progs.draftProposal({ + localization: { cause: "dequel-source", culpritPaths: [], rationale: "dns broke" }, + explanation: { summary: "s", fixSteps: [], patchHint: null }, + repo: null, + }); + expect(out.cause).toBe("dequel-source"); + expect(out.dequelReport?.problem).toBe("dns broke"); + }); + + it("propose skips the model for unknown cause", async () => { + const progs = createPrograms(throwingLlm()); + const out = await progs.draftProposal({ + localization: { cause: "unknown", culpritPaths: [], rationale: "" }, + explanation: { summary: "s", fixSteps: [], patchHint: null }, + repo: null, + }); + expect(out).toEqual({ cause: "unknown" }); + }); +}); + +describe("heuristicSignalLines", () => { + it("prefers error lines, else the tail", () => { + expect(heuristicSignalLines("[build] ok\n[build] ERROR: x\n[system] done")).toEqual(["[build] ERROR: x"]); + expect(heuristicSignalLines("a\nb")).toEqual(["a", "b"]); + expect(heuristicSignalLines("")).toEqual([]); + }); +}); diff --git a/apps/api/src/fixdiag/actions.ts b/apps/api/src/fixdiag/actions.ts new file mode 100644 index 0000000..b3f5213 --- /dev/null +++ b/apps/api/src/fixdiag/actions.ts @@ -0,0 +1,329 @@ +import { execFile } from "node:child_process"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { config } from "../utils/config"; +import { claimDiagAction, completeDiagAction, failDiagAction, findCompletedAction } from "../db/repo/diag-actions"; +import { getDiagRun, getStagePayload } from "../db/repo/diag-runs"; +import { getDeploymentById } from "../db/repo/deployments"; +import { getGithubTokenFromCookie } from "../db/repo/github-sessions"; +import { getProjectById } from "../db/repo/projects"; +import { parseGithubRepo } from "./repo-url"; +import { + clearProjectSource, + ensureSandbox, + readLocalVersion, + runFixAgent, + syncDequelSource, + syncProjectSource, +} from "./sandbox-host"; + +const execFileAsync = promisify(execFile); + +export type ActionResult = { ok: true; data: unknown } | { ok: false; status: number; message: string }; + +const short = (s: string, n = 500): string => (s.length > n ? `${s.slice(0, n)}…` : s); + +const git = async (args: string[], cwd: string, token: string) => { + const auth = Buffer.from(`x-access-token:${token}`).toString("base64"); + try { + return await execFileAsync("git", args, { + timeout: 120_000, + cwd, + maxBuffer: 4 * 1024 * 1024, + env: { + ...process.env, + GIT_TERMINAL_PROMPT: "0", + GIT_CONFIG_COUNT: "1", + GIT_CONFIG_KEY_0: "http.https://github.com/.extraHeader", + GIT_CONFIG_VALUE_0: `Authorization: Basic ${auth}`, + }, + }); + } catch (err) { + const stderr = String((err as { stderr?: unknown }).stderr ?? "") + .split(token) + .join("***") + .trim(); + throw new Error(`git ${args.join(" ")} failed: ${short(stderr || "command failed")}`); + } +}; + +const githubFetch = async (token: string, path: string, init?: RequestInit) => { + const res = await fetch(`https://api.github.com${path}`, { + ...init, + headers: { + Authorization: `Bearer ${token}`, + Accept: "application/vnd.github.v3+json", + "User-Agent": "dequel", + "Content-Type": "application/json", + ...init?.headers, + }, + signal: AbortSignal.timeout(30_000), + }); + return res; +}; + +const validGithubToken = async (token: string): Promise => { + try { + const res = await githubFetch(token, "/user"); + return res.ok; + } catch { + return false; + } +}; + +export const approveFixPr = async ( + runId: string, + key: string, + cookie: string | null, + opts?: { validateToken?: (token: string) => Promise }, +): Promise => { + if (!key.trim()) return { ok: false, status: 400, message: "idempotencyKey is required" }; + const trimmedKey = key.trim(); + const validate = opts?.validateToken ?? validGithubToken; + const token = await getGithubTokenFromCookie(cookie); + if (!token || !(await validate(token))) { + return { ok: false, status: 401, message: "Connect GitHub first (Settings → GitHub Integration)" }; + } + const run = await getDiagRun(runId); + if (!run) return { ok: false, status: 404, message: "Diagnosis not found" }; + let claim; + try { + claim = await claimDiagAction(trimmedKey, runId, "pr"); + } catch { + return { ok: false, status: 409, message: "Action key already used for a different intent" }; + } + if (!claim.fresh) { + if (claim.action.status === "done") return { ok: true, data: claim.action.result }; + return { ok: false, status: 409, message: "Action already in progress" }; + } + const completed = await findCompletedAction(runId, "pr"); + if (completed) { + await completeDiagAction(trimmedKey, completed.result); + return { ok: true, data: completed.result }; + } + const fail = async (status: number, message: string): Promise => { + await failDiagAction(trimmedKey, message); + return { ok: false, status, message }; + }; + + if (run.status !== "done") return fail(409, "Diagnosis is not complete yet"); + if (run.cause !== "user-source" || !run.report?.userFix) { + return fail(409, "This diagnosis has no user-code fix to apply"); + } + const fix = run.report.userFix; + const dep = await getDeploymentById(run.deploymentId); + if (!dep || dep.sourceType !== "git") return fail(409, "Fix PRs are only available for Git projects"); + if ((dep.commitSha ?? "") !== run.commitSha) { + return fail(409, "The deployment has moved on since this diagnosis (stale)"); + } + const repo = parseGithubRepo(dep.sourceRef, dep.branch); + if (!repo) return fail(422, "Project source is not a GitHub repository"); + + let patch: string; + try { + await ensureSandbox(); + const version = await readLocalVersion(); + const { rev, stale } = await syncDequelSource(version); + const project = await syncProjectSource( + { + deploymentId: dep.id, + projectId: dep.projectId, + sourceType: dep.sourceType, + sourceRef: dep.sourceRef, + branch: dep.branch, + commitSha: dep.commitSha ?? "", + failureReason: dep.failureReason, + }, + runId, + ); + if (!project.available) { + await clearProjectSource(runId); + return fail(422, "Project source could not be pulled into the sandbox"); + } + try { + const investigation = (await getStagePayload(runId, "investigate")) as { + culpritPaths?: unknown; + rationale?: unknown; + } | null; + const explanation = (await getStagePayload(runId, "explain")) as { + summary?: unknown; + fixSteps?: unknown; + } | null; + const lines = [ + `Deployment failed: ${dep.failureReason ?? run.deploymentId}`, + `Diagnosis: ${typeof explanation?.summary === "string" ? explanation.summary : ""}`, + `Culprit files: ${Array.isArray(investigation?.culpritPaths) ? investigation.culpritPaths.filter((p): p is string => typeof p === "string").join(", ") : ""}`, + `Fix steps: ${Array.isArray(explanation?.fixSteps) ? explanation.fixSteps.filter((s): s is string => typeof s === "string").join(" / ") : ""}`, + ]; + if (fix.suggestedDiff?.trim()) { + lines.push( + `Starting suggestion (verify against the files, do not apply blindly):\n${fix.suggestedDiff.slice(0, 8000)}`, + ); + } + const result = await runFixAgent({ + runId, + provider: run.provider, + model: run.model, + fixBrief: lines.join("\n"), + dequelRev: rev, + dequelStale: stale, + onProgress: () => {}, + }); + patch = result.patch; + } finally { + await clearProjectSource(runId); + } + } catch (err) { + return fail(502, `Agent fix failed: ${short(err instanceof Error ? err.message : String(err))}`); + } + + const branch = `dequel-fix/${run.id.slice(0, 8)}`; + const tmp = await mkdtemp(join(tmpdir(), "dequel-fix-")); + const repoDir = join(tmp, "repo"); + try { + const remote = `https://github.com/${repo.owner}/${repo.repo}.git`; + await git(["clone", "--depth", "1", "--branch", repo.base, remote, repoDir], tmpdir(), token); + if (run.commitSha) { + try { + await git(["fetch", "--depth", "1", "origin", run.commitSha], repoDir, token); + await git(["checkout", run.commitSha], repoDir, token); + } catch { + return fail(409, "The diagnosed commit is no longer available (stale)"); + } + } + const branchExists = + (await git(["rev-parse", "--verify", `refs/heads/${branch}`], repoDir, token).catch(() => null)) !== null; + await git(["checkout", ...(branchExists ? [branch] : ["-b", branch])], repoDir, token); + const patchPath = join(tmp, "fix.diff"); + await writeFile(patchPath, patch); + try { + await execFileAsync("git", ["apply", "--check", patchPath], { timeout: 30_000, cwd: repoDir }); + await execFileAsync("git", ["apply", patchPath], { timeout: 30_000, cwd: repoDir }); + } catch (err) { + const detail = short(err instanceof Error ? err.message : String(err)); + return fail(422, `Patch does not apply cleanly: ${detail}`); + } + await git(["add", "-A"], repoDir, token); + await execFileAsync( + "git", + ["-c", "user.name=Dequel", "-c", "user.email=dequel@localhost", "commit", "-m", fix.title, "-m", fix.body], + { + timeout: 30_000, + cwd: repoDir, + }, + ); + try { + await git(["push", "origin", branch], repoDir, token); + } catch (err) { + return fail(422, `Push failed: ${short(err instanceof Error ? err.message : String(err))}`); + } + const prRes = await githubFetch(token, `/repos/${repo.owner}/${repo.repo}/pulls`, { + method: "POST", + body: JSON.stringify({ title: fix.title, body: fix.body, head: branch, base: repo.base }), + }); + if (prRes.status === 201) { + const pr = (await prRes.json()) as { html_url?: string }; + const result = { prUrl: pr.html_url ?? "" }; + await completeDiagAction(trimmedKey, result); + return { ok: true, data: result }; + } + const prBody = short(await prRes.text().catch(() => "")); + if (prRes.status === 422 && prBody.includes("already exists")) { + const existing = await githubFetch( + token, + `/repos/${repo.owner}/${repo.repo}/pulls?head=${repo.owner}:${branch}&state=open`, + ); + const list = (await existing.json().catch(() => [])) as { html_url?: string }[]; + if (list[0]?.html_url) { + const result = { prUrl: list[0].html_url }; + await completeDiagAction(trimmedKey, result); + return { ok: true, data: result }; + } + } + return fail(502, `GitHub rejected the pull request: ${prBody || prRes.status}`); + } catch (err) { + return fail(500, short(err instanceof Error ? err.message : String(err))); + } finally { + await rm(tmp, { recursive: true, force: true }); + } +}; + +export const approveSlackPost = async (runId: string, key: string): Promise => { + if (!key.trim()) return { ok: false, status: 400, message: "idempotencyKey is required" }; + const trimmedKey = key.trim(); + const run = await getDiagRun(runId); + if (!run) return { ok: false, status: 404, message: "Diagnosis not found" }; + let claim; + try { + claim = await claimDiagAction(trimmedKey, runId, "slack"); + } catch { + return { ok: false, status: 409, message: "Action key already used for a different intent" }; + } + if (!claim.fresh) { + if (claim.action.status === "done") return { ok: true, data: claim.action.result }; + return { ok: false, status: 409, message: "Action already in progress" }; + } + const completed = await findCompletedAction(runId, "slack"); + if (completed) { + await completeDiagAction(trimmedKey, completed.result); + return { ok: true, data: completed.result }; + } + const fail = async (status: number, message: string): Promise => { + await failDiagAction(trimmedKey, message); + return { ok: false, status, message }; + }; + + if (run.status !== "done") return fail(409, "Diagnosis is not complete yet"); + if (run.cause !== "dequel-source" || !run.report?.dequelReport) { + return fail(409, "This diagnosis has no Dequel report to post"); + } + if (!config.dequelSlackWebhookUrl) return fail(409, "Dequel Slack is not configured"); + + const report = run.report.dequelReport; + const dep = await getDeploymentById(run.deploymentId); + const project = dep?.projectId ? await getProjectById(dep.projectId).catch(() => null) : null; + const investigation = (await getStagePayload(runId, "investigate")) as { + dequelRev?: unknown; + dequelStale?: unknown; + } | null; + const version = + typeof investigation?.dequelRev === "string" && investigation.dequelRev + ? investigation.dequelRev + (investigation.dequelStale === true ? " (possibly stale)" : "") + : "unknown"; + const payload = { + text: `Dequel diagnosis: ${project?.name ?? run.deploymentId}`, + blocks: [ + { type: "header", text: { type: "plain_text", text: `Dequel diagnosis: ${project?.name ?? "unknown project"}` } }, + { type: "section", text: { type: "mrkdwn", text: `*Problem*\n${report.problem}` } }, + { type: "section", text: { type: "mrkdwn", text: `*Dequel version*\n${version}` } }, + { type: "section", text: { type: "mrkdwn", text: `*Root cause*\n${report.cause}` } }, + { type: "section", text: { type: "mrkdwn", text: `*Proposed fix*\n${report.proposedFix}` } }, + { + type: "context", + elements: [ + { + type: "mrkdwn", + text: `Deployment ${run.deploymentId.slice(0, 8)} · diagnosed with ${run.provider}/${run.model}`, + }, + ], + }, + ], + }; + try { + const res = await fetch(config.dequelSlackWebhookUrl, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(payload), + redirect: "manual", + signal: AbortSignal.timeout(10_000), + }); + if (!res.ok) return fail(502, `Slack webhook returned ${res.status}`); + } catch (err) { + return fail(502, short(err instanceof Error ? err.message : String(err))); + } + const result = { posted: true, channel: config.dequelSlackChannel || undefined }; + await completeDiagAction(trimmedKey, result); + return { ok: true, data: result }; +}; diff --git a/apps/api/src/fixdiag/context.ts b/apps/api/src/fixdiag/context.ts new file mode 100644 index 0000000..a49ffee --- /dev/null +++ b/apps/api/src/fixdiag/context.ts @@ -0,0 +1,49 @@ +import { getDeploymentById, getLogs } from "../db/repo/deployments"; +import { getDecryptedLlmKey } from "../db/repo/llm-keys"; +import { buildLlm, type DiagLlm } from "./llm"; +import type { EvidenceBundle, LlmProvider, LogLine } from "./types"; + +export const tailLogs = (logs: LogLine[], maxLines = 400, maxChars = 24000): LogLine[] => { + const tail = logs.slice(-maxLines); + const out: LogLine[] = []; + let chars = 0; + for (let i = tail.length - 1; i >= 0; i--) { + const len = tail[i].message.length; + if (out.length > 0 && chars + len > maxChars) break; + out.unshift(tail[i]); + chars += len; + } + return out; +}; + +export const logTextOf = (logs: LogLine[]): string => { + if (!logs.length) return "(no build logs were recorded for this deployment)"; + return logs.map((l) => `[${l.stage}] ${l.message}`).join("\n"); +}; + +export const failureReasonOf = (reason: string | null): string => + reason?.trim() ? reason : "(no failure reason was recorded)"; + +export const collectFailureContext = async (deploymentId: string): Promise => { + const dep = await getDeploymentById(deploymentId); + if (!dep) throw new Error(`Deployment ${deploymentId} not found`); + const logs = await getLogs(deploymentId); + return { + deployment: { + deploymentId: dep.id, + projectId: dep.projectId, + sourceType: dep.sourceType, + sourceRef: dep.sourceRef, + branch: dep.branch, + commitSha: dep.commitSha ?? "", + failureReason: dep.failureReason, + }, + logs: tailLogs(logs.map((l) => ({ sequence: l.sequence, stage: l.stage, message: l.message }))), + }; +}; + +export const resolveModel = async (provider: LlmProvider, model: string): Promise => { + const rec = await getDecryptedLlmKey(provider); + if (!rec) throw new Error(`LLM provider "${provider}" is not configured`); + return buildLlm(provider, rec.apiKey, rec.baseUrl, model); +}; diff --git a/apps/api/src/fixdiag/llm.ts b/apps/api/src/fixdiag/llm.ts new file mode 100644 index 0000000..951f44a --- /dev/null +++ b/apps/api/src/fixdiag/llm.ts @@ -0,0 +1,37 @@ +import { ai } from "@ax-llm/ax"; +import type { CauseKind, LlmProvider, TriageConfidence } from "./types"; + +export const buildLlm = (provider: LlmProvider, apiKey: string, baseUrl: string | null, model: string) => { + switch (provider) { + case "openai": + return ai({ name: "openai", apiKey, config: { model } }); + case "anthropic": + return ai({ name: "anthropic", apiKey, config: { model } }); + case "gemini": + return ai({ name: "google-gemini", apiKey, config: { model } }); + case "groq": + return ai({ name: "groq", apiKey, config: { model } }); + case "ollama": + return ai({ + name: "openai", + apiKey: apiKey || "ollama", + apiURL: baseUrl ?? "http://host.docker.internal:11435/v1", + config: { model, maxTokens: 8192 }, + }); + case "custom": + return ai({ name: "openai", apiKey, apiURL: baseUrl ?? undefined, config: { model } }); + } +}; + +export type DiagLlm = ReturnType; + +export const asString = (value: unknown, fallback = ""): string => (typeof value === "string" ? value : fallback); + +export const asStringArray = (value: unknown, max = 32): string[] => + Array.isArray(value) ? value.filter((v): v is string => typeof v === "string").slice(0, max) : []; + +export const asConfidence = (value: unknown): TriageConfidence => + value === "low" || value === "medium" || value === "high" ? value : "low"; + +export const asCause = (value: unknown): CauseKind => + value === "user-source" || value === "dequel-source" || value === "unknown" ? value : "unknown"; diff --git a/apps/api/src/fixdiag/machine.ts b/apps/api/src/fixdiag/machine.ts new file mode 100644 index 0000000..f2ec4ce --- /dev/null +++ b/apps/api/src/fixdiag/machine.ts @@ -0,0 +1,192 @@ +import { + createDiagRun, + deleteDiagRun, + findDiagRun, + finishDiagRun, + getDiagRun, + getStagePayload, + recordStageResult, +} from "../db/repo/diag-runs"; +import { getDeploymentById } from "../db/repo/deployments"; +import { getDecryptedLlmKey, LLM_PROVIDERS } from "../db/repo/llm-keys"; +import { collectFailureContext, failureReasonOf, logTextOf, resolveModel } from "./context"; +import { approveSlackPost } from "./actions"; +import { createPrograms } from "./programs"; +import { diagBus } from "./stream"; +import type { + DiagRun, + DiagStageName, + EvidenceBundle, + Explanation, + FixdiagPrograms, + InvestigateFn, + Investigation, + LlmProvider, + Localization, + Proposal, + TriageVerdict, +} from "./types"; +import { + clearProjectSource, + ensureSandbox, + investigateInSandbox, + readLocalVersion, + syncDequelSource, + syncProjectSource, +} from "./sandbox-host"; +import { parseGithubRepo } from "./repo-url"; + +export { parseGithubRepo }; + +const STAGES: DiagStageName[] = ["triage", "investigate", "explain", "propose"]; +const activeDrives = new Set(); + +const defaultInvestigator: InvestigateFn = async ({ run, deployment, logText, verdict, onProgress }) => { + await ensureSandbox(); + const version = await readLocalVersion(); + const { rev, stale } = await syncDequelSource(version); + const project = await syncProjectSource(deployment, run.id); + try { + return await investigateInSandbox({ + runId: run.id, + provider: run.provider, + model: run.model, + deployment, + failureReason: failureReasonOf(deployment.failureReason), + logText, + verdict, + dequelRev: rev, + dequelStale: stale, + projectAvailable: project.available, + onProgress, + }); + } finally { + await clearProjectSource(run.id); + } +}; + +const runStage = async ( + programs: FixdiagPrograms, + investigate: InvestigateFn, + stage: DiagStageName, + evidence: EvidenceBundle, + run: DiagRun, + runId: string, + emitProgress: (line: string) => void, +): Promise => { + switch (stage) { + case "triage": + return programs.triageLogs({ + failureReason: failureReasonOf(evidence.deployment.failureReason), + logText: logTextOf(evidence.logs), + }); + case "investigate": + return investigate({ + run, + deployment: evidence.deployment, + logText: logTextOf(evidence.logs), + verdict: (await getStagePayload(runId, "triage")) as TriageVerdict, + onProgress: emitProgress, + }); + case "explain": { + const verdict = (await getStagePayload(runId, "triage")) as TriageVerdict; + const investigation = (await getStagePayload(runId, "investigate")) as Investigation; + const localization: Localization = { + cause: investigation.cause, + culpritPaths: investigation.culpritPaths, + rationale: investigation.rationale, + }; + return programs.explainFix({ verdict, localization }); + } + case "propose": { + const investigation = (await getStagePayload(runId, "investigate")) as Investigation; + const localization: Localization = { + cause: investigation.cause, + culpritPaths: investigation.culpritPaths, + rationale: investigation.rationale, + }; + const explanation = (await getStagePayload(runId, "explain")) as Explanation; + return programs.draftProposal({ + localization, + explanation, + repo: parseGithubRepo(evidence.deployment.sourceRef, evidence.deployment.branch), + }); + } + } +}; + +export const DiagRunMachine = { + start: async (input: { + deploymentId: string; + provider: string; + model: string; + }): Promise<{ ok: true; run: DiagRun; created: boolean } | { ok: false; status: number; message: string }> => { + if (!(LLM_PROVIDERS as readonly string[]).includes(input.provider)) { + return { ok: false, status: 400, message: `provider must be one of: ${LLM_PROVIDERS.join(", ")}` }; + } + if (!input.model?.trim()) return { ok: false, status: 400, message: "model is required" }; + const dep = await getDeploymentById(input.deploymentId); + if (!dep) return { ok: false, status: 404, message: "Deployment not found" }; + if (dep.status !== "failed") { + return { ok: false, status: 409, message: "Diagnosis is only available for failed deployments" }; + } + const key = await getDecryptedLlmKey(input.provider); + if (!key) return { ok: false, status: 400, message: `LLM provider "${input.provider}" is not configured` }; + const commitSha = dep.commitSha ?? ""; + const existing = await findDiagRun(input.deploymentId, commitSha); + if (existing) { + if (existing.status === "error") await deleteDiagRun(existing.id); + else return { ok: true, run: existing, created: false }; + } + const run = await createDiagRun({ + deploymentId: input.deploymentId, + commitSha, + provider: input.provider as LlmProvider, + model: input.model.trim(), + }); + return { ok: true, run, created: true }; + }, + + drive: async (runId: string, programs?: FixdiagPrograms, investigate?: InvestigateFn): Promise => { + if (activeDrives.has(runId)) return; + activeDrives.add(runId); + try { + const run = await getDiagRun(runId); + if (!run || run.status !== "running") return; + try { + const llm = await resolveModel(run.provider, run.model); + const progs = programs ?? createPrograms(llm); + const investigateFn = investigate ?? defaultInvestigator; + const evidence = await collectFailureContext(run.deploymentId); + const startIdx = run.currentStage ? STAGES.indexOf(run.currentStage) + 1 : 0; + for (let i = Math.max(0, startIdx); i < STAGES.length; i++) { + const stage = STAGES[i]; + let payload: unknown; + try { + payload = await runStage(progs, investigateFn, stage, evidence, run, runId, (line) => + diagBus.emit(runId, { type: "token", runId, stage, delta: line }), + ); + } catch (err) { + throw new Error(`${stage} failed: ${err instanceof Error ? err.message : String(err)}`); + } + await recordStageResult(runId, stage, payload); + diagBus.emit(runId, { type: "stage", runId, stage, payload }); + } + const proposal = (await getStagePayload(runId, "propose")) as Proposal | null; + const finalProposal: Proposal = proposal && proposal.cause ? proposal : { cause: "unknown" }; + await finishDiagRun(runId, "done", finalProposal.cause, finalProposal, null); + if (finalProposal.cause === "dequel-source") { + const posted = await approveSlackPost(runId, `auto-${runId}`).catch(() => null); + if (!posted?.ok) console.log(`[Fixdiag] Slack auto-post skipped for ${runId}: ${posted?.message ?? "error"}`); + } + diagBus.emit(runId, { type: "done", runId }); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + await finishDiagRun(runId, "error", null, null, message).catch(() => {}); + diagBus.emit(runId, { type: "error", runId, message }); + } + } finally { + activeDrives.delete(runId); + } + }, +}; diff --git a/apps/api/src/fixdiag/programs.ts b/apps/api/src/fixdiag/programs.ts new file mode 100644 index 0000000..95a96d5 --- /dev/null +++ b/apps/api/src/fixdiag/programs.ts @@ -0,0 +1,145 @@ +import { ax } from "@ax-llm/ax"; +import { asCause, asConfidence, asString, asStringArray, type DiagLlm } from "./llm"; +import type { + DequelReport, + ExplainInput, + Explanation, + FixdiagPrograms, + Localization, + Proposal, + ProposeInput, + TriageInput, + TriageVerdict, + UserFix, +} from "./types"; + +const nonEmpty = (value: string, label: string): string => + value.trim() ? value : `(${label} unavailable — evidence partial or empty)`; + +const clip = (value: string, max = 12_000): string => + value.length > max ? `${value.slice(0, max)}\n…[truncated ${value.length - max} chars]` : value; + +const ERROR_HINT = + /error|fail|exception|denied|refused|timeout|timed out|not found|missing|unable|cannot|invalid|misbehaving|unrecognized/i; + +export const heuristicSignalLines = (logText: string, max = 5): string[] => { + const lines = logText + .split("\n") + .map((line) => line.trim()) + .filter(Boolean); + const hits = lines.filter((line) => ERROR_HINT.test(line)); + return (hits.length > 0 ? hits : lines).slice(-max); +}; + +const PROGRAM_TIMEOUT_MS = 90_000; + +export const createPrograms = (llm: DiagLlm): FixdiagPrograms => { + const opts = { timeout: PROGRAM_TIMEOUT_MS, maxRetries: 1 } as const; + + const triage = ax( + 'failureReason:string, logText:string -> failingStage:string, signalLines:string[], confidence:class "low,medium,high"', + { + description: + 'You triage a failed Dequel deployment. Identify which build stage failed and quote the 1-5 most telling log lines. Reply with one line per field and no bullet dashes, for example:\nFailing Stage: Docker Build\nSignal Lines: ["[build] ERROR: ..."]\nConfidence: high', + }, + ); + + const explain = ax("verdict:string, localization:string -> summary:string, fixSteps:string[], patchHint:string", { + description: + "You explain a diagnosed build failure to the developer who owns the deployment. summary is 2-4 sentences. fixSteps are concrete actions. patchHint is a unified diff when the fix is a small code change, else the exact text (no diff). Reply with one line per field and no bullet dashes.", + }); + + const proposeUserFix = ax( + 'localization:string, explanation:string, repo:string -> cause:class "user-source,unknown", title:string, body:string, suggestedDiff:string', + { + description: + "You draft a pull request fixing a diagnosed failure in the user's own source. Always fill title, body, and suggestedDiff; write the exact text (no diff) for suggestedDiff when no code change applies. Answer unknown for cause when the evidence does not support a user-source fix. Reply with one line per field and no bullet dashes.", + }, + ); + + const proposeDequelReport = ax( + 'localization:string, explanation:string, repo:string -> cause:class "dequel-source,unknown", problem:string, fixCause:string, proposedFix:string', + { + description: + "You write a bug report for the Dequel platform team about a failure in Dequel's own tooling. Always fill problem, fixCause, and proposedFix with concrete content. Answer unknown for cause when the evidence does not support a Dequel-source report. Reply with one line per field and no bullet dashes.", + }, + ); + + return { + triageLogs: async (input: TriageInput): Promise => { + const failureReason = nonEmpty(input.failureReason ?? "", "failure reason"); + const logText = clip(nonEmpty(input.logText, "build logs")); + try { + const out = await triage.forward(llm, { failureReason, logText }, opts); + return { + failingStage: asString(out.failingStage, "unknown"), + signalLines: asStringArray(out.signalLines, 8), + confidence: asConfidence(out.confidence), + }; + } catch { + return { failingStage: "unknown", signalLines: heuristicSignalLines(logText), confidence: "low" }; + } + }, + + explainFix: async (input: ExplainInput): Promise => { + const verdict = JSON.stringify(input.verdict); + const localization = JSON.stringify(input.localization); + try { + const out = await explain.forward(llm, { verdict, localization }, opts); + const patchHint = asString(out.patchHint); + return { + summary: asString(out.summary), + fixSteps: asStringArray(out.fixSteps, 12), + patchHint: patchHint.trim() ? patchHint : null, + }; + } catch { + return { + summary: asString((input.localization as Localization).rationale, "Explanation unavailable."), + fixSteps: [], + patchHint: null, + }; + } + }, + + draftProposal: async (input: ProposeInput): Promise => { + const base = { + localization: JSON.stringify(input.localization), + explanation: JSON.stringify(input.explanation), + repo: JSON.stringify(input.repo), + }; + if (input.localization.cause === "user-source") { + try { + const out = await proposeUserFix.forward(llm, base, opts); + const userFix: UserFix = { + title: asString(out.title, "Fix build failure"), + body: asString(out.body), + suggestedDiff: asString(out.suggestedDiff).trim() || null, + }; + return { cause: asCause(out.cause) === "unknown" ? "unknown" : "user-source", userFix }; + } catch { + return { + cause: "user-source", + userFix: { title: "Fix build failure", body: input.localization.rationale, suggestedDiff: null }, + }; + } + } + if (input.localization.cause === "dequel-source") { + try { + const out = await proposeDequelReport.forward(llm, base, opts); + const dequelReport: DequelReport = { + problem: asString(out.problem), + cause: asString(out.fixCause), + proposedFix: asString(out.proposedFix), + }; + return { cause: asCause(out.cause) === "unknown" ? "unknown" : "dequel-source", dequelReport }; + } catch { + return { + cause: "dequel-source", + dequelReport: { problem: input.localization.rationale, cause: "", proposedFix: "" }, + }; + } + } + return { cause: "unknown" }; + }, + }; +}; diff --git a/apps/api/src/fixdiag/provider-models.ts b/apps/api/src/fixdiag/provider-models.ts new file mode 100644 index 0000000..308efa0 --- /dev/null +++ b/apps/api/src/fixdiag/provider-models.ts @@ -0,0 +1,251 @@ +export interface FetchModelsOptions { + provider: string; + apiKey?: string; + baseUrl?: string | null; + timeoutMs?: number; +} + +export const DEFAULT_PROVIDER_MODELS: Record = { + openai: ["gpt-4o", "gpt-4o-mini", "o1", "o3-mini", "gpt-4-turbo", "gpt-6.1-sol", "gpt-6-astra", "gpt-5.6-sol"], + anthropic: [ + "claude-3-5-sonnet-latest", + "claude-3-5-haiku-latest", + "claude-3-opus-latest", + "claude-sonnet-5", + "claude-opus-5", + ], + gemini: [ + "gemini-2.5-flash", + "gemini-2.5-pro", + "gemini-1.5-pro", + "gemini-1.5-flash", + "gemini-3.8-flash", + "gemini-3.7-flash", + ], + groq: [ + "llama-3.3-70b-versatile", + "llama-3.1-8b-instant", + "meta-llama/llama-4-maverick-17b-128e-instruct", + "moonshotai/kimi-k2-instruct", + "openai/gpt-oss-120b", + ], + ollama: ["llama3:latest", "qwen2.5-coder:latest", "mistral:latest", "deepseek-r1:latest", "qwen3.5:0.8b"], + custom: [], +}; + +const EXCLUDED_OPENAI_PATTERNS = + /embedding|whisper|tts|dall-e|moderation|babbage|davinci|realtime|transcription|audio|canary|search/i; + +const fetchOpenAiModels = async (apiKey: string, baseUrl?: string | null, timeoutMs = 8000): Promise => { + const root = baseUrl ? baseUrl.replace(/\/+$/, "") : "https://api.openai.com/v1"; + const url = root.endsWith("/models") ? root : `${root}/models`; + const res = await fetch(url, { + method: "GET", + headers: { + Authorization: `Bearer ${apiKey}`, + "Content-Type": "application/json", + }, + signal: AbortSignal.timeout(timeoutMs), + }); + if (!res.ok) { + throw new Error(`OpenAI models request failed: ${res.status} ${res.statusText}`); + } + const body = (await res.json()) as { data?: Array<{ id: string }> }; + const rawList = Array.isArray(body?.data) ? body.data : []; + const filtered = rawList + .map((m) => m.id) + .filter((id): id is string => typeof id === "string" && !EXCLUDED_OPENAI_PATTERNS.test(id)); + + filtered.sort((a, b) => { + const aRank = a.includes("gpt-4o") || a.startsWith("o1") || a.startsWith("o3") ? 0 : 1; + const bRank = b.includes("gpt-4o") || b.startsWith("o1") || b.startsWith("o3") ? 0 : 1; + if (aRank !== bRank) return aRank - bRank; + return a.localeCompare(b); + }); + return filtered; +}; + +const fetchAnthropicModels = async (apiKey: string, timeoutMs = 8000): Promise => { + const res = await fetch("https://api.anthropic.com/v1/models", { + method: "GET", + headers: { + "x-api-key": apiKey, + "anthropic-version": "2023-06-01", + "Content-Type": "application/json", + }, + signal: AbortSignal.timeout(timeoutMs), + }); + if (!res.ok) { + throw new Error(`Anthropic models request failed: ${res.status} ${res.statusText}`); + } + const body = (await res.json()) as { data?: Array<{ id: string }> }; + const rawList = Array.isArray(body?.data) ? body.data : []; + const filtered = rawList.map((m) => m.id).filter((id): id is string => typeof id === "string"); + + filtered.sort((a, b) => { + const aRank = a.includes("sonnet") ? 0 : a.includes("opus") ? 1 : a.includes("haiku") ? 2 : 3; + const bRank = b.includes("sonnet") ? 0 : b.includes("opus") ? 1 : b.includes("haiku") ? 2 : 3; + if (aRank !== bRank) return aRank - bRank; + return b.localeCompare(a); + }); + return filtered; +}; + +const fetchGeminiModels = async (apiKey: string, timeoutMs = 8000): Promise => { + const url = `https://generativelanguage.googleapis.com/v1beta/models?key=${encodeURIComponent(apiKey)}`; + const res = await fetch(url, { + method: "GET", + headers: { + "Content-Type": "application/json", + }, + signal: AbortSignal.timeout(timeoutMs), + }); + if (!res.ok) { + throw new Error(`Gemini models request failed: ${res.status} ${res.statusText}`); + } + const body = (await res.json()) as { + models?: Array<{ name: string; supportedGenerationMethods?: string[] }>; + }; + const rawList = Array.isArray(body?.models) ? body.models : []; + const filtered = rawList + .filter((m) => { + const methods = Array.isArray(m.supportedGenerationMethods) ? m.supportedGenerationMethods : []; + return methods.includes("generateContent"); + }) + .map((m) => (typeof m.name === "string" ? m.name.replace(/^models\//, "") : "")) + .filter((id): id is string => !!id && !/embedding|aqa|bison|gecko/i.test(id)); + + filtered.sort((a, b) => { + const aRank = a.includes("flash") ? 0 : a.includes("pro") ? 1 : 2; + const bRank = b.includes("flash") ? 0 : b.includes("pro") ? 1 : 2; + if (aRank !== bRank) return aRank - bRank; + return a.localeCompare(b); + }); + return filtered; +}; + +const fetchGroqModels = async (apiKey: string, baseUrl?: string | null, timeoutMs = 8000): Promise => { + const root = baseUrl ? baseUrl.replace(/\/+$/, "") : "https://api.groq.com/openai/v1"; + const url = root.endsWith("/models") ? root : `${root}/models`; + const res = await fetch(url, { + method: "GET", + headers: { + Authorization: `Bearer ${apiKey}`, + "Content-Type": "application/json", + }, + signal: AbortSignal.timeout(timeoutMs), + }); + if (!res.ok) { + throw new Error(`Groq models request failed: ${res.status} ${res.statusText}`); + } + const body = (await res.json()) as { data?: Array<{ id: string; active?: boolean }> }; + const rawList = Array.isArray(body?.data) ? body.data : []; + const filtered = rawList + .filter((m) => m.active !== false) + .map((m) => m.id) + .filter((id): id is string => typeof id === "string" && !/whisper|tts/i.test(id)); + + filtered.sort((a, b) => a.localeCompare(b)); + return filtered; +}; + +const fetchOllamaModels = async (baseUrl?: string | null, timeoutMs = 8000): Promise => { + const raw = baseUrl || "http://host.docker.internal:11435"; + const root = raw.replace(/\/v1\/?$/, "").replace(/\/+$/, ""); + let models: string[] = []; + + try { + const res = await fetch(`${root}/api/tags`, { + method: "GET", + signal: AbortSignal.timeout(timeoutMs), + }); + if (res.ok) { + const body = (await res.json()) as { models?: Array<{ name: string }> }; + if (Array.isArray(body?.models)) { + models = body.models.map((m) => m.name).filter((n): n is string => typeof n === "string"); + } + } + } catch {} + + if (models.length === 0) { + try { + const res = await fetch(`${root}/v1/models`, { + method: "GET", + signal: AbortSignal.timeout(timeoutMs), + }); + if (res.ok) { + const body = (await res.json()) as { data?: Array<{ id: string }> }; + if (Array.isArray(body?.data)) { + models = body.data.map((m) => m.id).filter((n): n is string => typeof n === "string"); + } + } + } catch {} + } + + return models; +}; + +const fetchCustomModels = async (apiKey?: string, baseUrl?: string | null, timeoutMs = 8000): Promise => { + if (!baseUrl) return []; + const cleanBase = baseUrl.replace(/\/+$/, ""); + const headers: Record = { "Content-Type": "application/json" }; + if (apiKey) headers.Authorization = `Bearer ${apiKey}`; + + const endpoints = cleanBase.endsWith("/models") + ? [cleanBase] + : cleanBase.endsWith("/v1") + ? [`${cleanBase}/models`] + : [`${cleanBase}/models`, `${cleanBase}/v1/models`]; + + for (const endpoint of endpoints) { + try { + const res = await fetch(endpoint, { + method: "GET", + headers, + signal: AbortSignal.timeout(timeoutMs), + }); + if (res.ok) { + const body = (await res.json()) as any; + const list = Array.isArray(body?.data) ? body.data : Array.isArray(body) ? body : []; + const extracted = list + .map((m: any) => (typeof m === "string" ? m : m?.id || m?.name)) + .filter((id: unknown): id is string => typeof id === "string"); + if (extracted.length > 0) return extracted; + } + } catch {} + } + return []; +}; + +export const fetchProviderModels = async (options: FetchModelsOptions): Promise => { + const { provider, apiKey = "", baseUrl, timeoutMs = 8000 } = options; + try { + let models: string[] = []; + switch (provider) { + case "openai": + if (apiKey) models = await fetchOpenAiModels(apiKey, baseUrl, timeoutMs); + break; + case "anthropic": + if (apiKey) models = await fetchAnthropicModels(apiKey, timeoutMs); + break; + case "gemini": + if (apiKey) models = await fetchGeminiModels(apiKey, timeoutMs); + break; + case "groq": + if (apiKey) models = await fetchGroqModels(apiKey, baseUrl, timeoutMs); + break; + case "ollama": + models = await fetchOllamaModels(baseUrl, timeoutMs); + break; + case "custom": + models = await fetchCustomModels(apiKey, baseUrl, timeoutMs); + break; + } + + if (models.length > 0) { + return Array.from(new Set(models)); + } + } catch {} + + return DEFAULT_PROVIDER_MODELS[provider] ?? []; +}; diff --git a/apps/api/src/fixdiag/repo-url.ts b/apps/api/src/fixdiag/repo-url.ts new file mode 100644 index 0000000..ec718eb --- /dev/null +++ b/apps/api/src/fixdiag/repo-url.ts @@ -0,0 +1,8 @@ +export const parseGithubRepo = ( + sourceRef: string, + branch: string | null, +): { owner: string; repo: string; base: string } | null => { + const match = /github\.com[/:]([^/]+)\/([^/]+?)(?:\.git)?\/?$/i.exec(sourceRef.trim()); + if (!match) return null; + return { owner: match[1], repo: match[2], base: branch ?? "main" }; +}; diff --git a/apps/api/src/fixdiag/routes.ts b/apps/api/src/fixdiag/routes.ts new file mode 100644 index 0000000..0b630ab --- /dev/null +++ b/apps/api/src/fixdiag/routes.ts @@ -0,0 +1,136 @@ +import { Elysia } from "elysia"; +import { findCompletedAction } from "../db/repo/diag-actions"; +import { getDiagRun, listActiveDiagRuns, listStageResults } from "../db/repo/diag-runs"; +import { created, fail, ok } from "../api/response"; +import { approveFixPr, approveSlackPost } from "./actions"; +import { DiagRunMachine } from "./machine"; +import { diagBus } from "./stream"; +import type { StageEvent } from "./types"; + +export const fixdiagRoutes = new Elysia() + .post("/deployments/:id/diagnose", async ({ params: { id }, body, set }: any) => { + const started = await DiagRunMachine.start({ + deploymentId: String(id), + provider: String(body?.provider ?? ""), + model: String(body?.model ?? ""), + }); + if (!started.ok) { + set.status = started.status; + return fail(started.message); + } + if (started.created) { + void DiagRunMachine.drive(started.run.id).catch((err) => { + console.error("[Fixdiag] Drive failed:", err); + }); + return created(started.run, "Diagnosis started"); + } + if (started.run.status === "running") { + void DiagRunMachine.drive(started.run.id).catch((err) => { + console.error("[Fixdiag] Drive failed:", err); + }); + } + return ok(started.run, "Diagnosis already exists"); + }) + .get("/diagnoses/active", async () => ok(await listActiveDiagRuns())) + .get("/diagnoses/:id", async ({ params: { id }, set }: any) => { + const run = await getDiagRun(String(id)); + if (!run) { + set.status = 404; + return fail("Diagnosis not found"); + } + const slack = await findCompletedAction(run.id, "slack").catch(() => null); + return ok({ run, stages: await listStageResults(run.id), slackPosted: !!slack }); + }) + .post("/diagnoses/:id/approve-pr", async ({ params: { id }, body, request, set }: any) => { + const res = await approveFixPr(String(id), String(body?.idempotencyKey ?? ""), request.headers.get("cookie")); + if (!res.ok) { + set.status = res.status; + return fail(res.message); + } + return ok(res.data, "Fix PR created"); + }) + .post("/diagnoses/:id/approve-slack", async ({ params: { id }, body, set }: any) => { + const res = await approveSlackPost(String(id), String(body?.idempotencyKey ?? "")); + if (!res.ok) { + set.status = res.status; + return fail(res.message); + } + return ok(res.data, "Report posted to Dequel Slack"); + }) + .get("/diagnoses/:id/stream", async ({ params: { id }, request, set }: any) => { + const run = await getDiagRun(String(id)); + if (!run) { + set.status = 404; + return fail("Diagnosis not found"); + } + const encoder = new TextEncoder(); + let unsubscribe: () => void = () => {}; + let heartbeat: ReturnType | null = null; + let closed = false; + const stop = () => { + if (closed) return; + closed = true; + unsubscribe(); + if (heartbeat) clearInterval(heartbeat); + }; + const runId = run.id; + const stream = new ReadableStream({ + async start(controller) { + const send = (eventName: string, payload: unknown) => { + if (closed) return; + controller.enqueue(encoder.encode(`event: ${eventName}\ndata: ${JSON.stringify(payload)}\n\n`)); + }; + const handle = (event: StageEvent) => { + if (closed) return; + if (event.type === "stage") send("stage", event); + else if (event.type === "done") { + send("done", event); + stop(); + controller.close(); + } else if (event.type === "error") { + send("error", event); + stop(); + controller.close(); + } + }; + send("ready", { runId }); + const buffered: StageEvent[] = []; + let replaying = true; + unsubscribe = diagBus.subscribe(runId, (event) => { + if (event.type === "token") return; + if (replaying) buffered.push(event); + else handle(event); + }); + for (const stage of await listStageResults(runId)) { + send("stage", { runId, stage: stage.stage, payload: stage.payload }); + } + replaying = false; + for (const event of buffered) handle(event); + if (closed) return; + const latest = (await getDiagRun(runId)) ?? run; + if (closed) return; + if (latest.status === "done") { + send("done", { runId }); + stop(); + controller.close(); + return; + } + if (latest.status === "error") { + send("error", { runId, message: latest.error }); + stop(); + controller.close(); + return; + } + heartbeat = setInterval(() => send("heartbeat", { at: new Date().toISOString() }), 15000); + }, + cancel: stop, + }); + request.signal.addEventListener("abort", stop, { once: true }); + return new Response(stream, { + headers: { + "Content-Type": "text/event-stream", + "Cache-Control": "no-cache", + Connection: "keep-alive", + }, + }); + }); diff --git a/apps/api/src/fixdiag/sandbox-host.ts b/apps/api/src/fixdiag/sandbox-host.ts new file mode 100644 index 0000000..df9412f --- /dev/null +++ b/apps/api/src/fixdiag/sandbox-host.ts @@ -0,0 +1,352 @@ +import { execFile } from "node:child_process"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { promisify } from "node:util"; +import { getDecryptedLlmKey } from "../db/repo/llm-keys"; +import { config } from "../utils/config"; +import type { DeploymentFacts, Investigation, TriageVerdict } from "./types"; + +const execFileAsync = promisify(execFile); + +export const SANDBOX_CONTAINER = "dequel-diag-sandbox"; +const RUNNER = "/runner/fixdiag/sandbox-runner/runner.js"; + +const short = (s: string, n = 500): string => (s.length > n ? `${s.slice(0, n)}…` : s); + +export const readLocalVersion = async (): Promise => { + const roots = [resolve(import.meta.dir, "../../../.."), "/app"]; + for (const root of roots) { + try { + const rev = await readFile(join(root, "VERSION"), "utf8").catch(() => null); + const text = (typeof rev === "string" ? rev : (rev as unknown as { toString(): string })?.toString())?.trim(); + if (text) return text; + } catch {} + } + const manifests = [resolve(import.meta.dir, "../../package.json"), "/app/package.json"]; + for (const manifest of manifests) { + try { + const raw = await readFile(manifest, "utf8").catch(() => null); + const text = typeof raw === "string" ? raw : (raw as unknown as { toString(): string })?.toString(); + if (!text) continue; + const version = (JSON.parse(text) as { version?: unknown }).version; + if (typeof version === "string" && version.trim()) return version.trim(); + } catch {} + } + return "unknown"; +}; + +const docker = async (args: string[], timeoutMs = 60_000): Promise => { + try { + const out = await execFileAsync("docker", args, { timeout: timeoutMs, maxBuffer: 8 * 1024 * 1024 }); + return String(out.stdout ?? ""); + } catch (err) { + throw new Error(short(err instanceof Error ? err.message : String(err))); + } +}; + +const OWN_DIRS = "/srv/jobs /srv/dequel-src /srv/project-src"; + +const normalizeOwnership = async (): Promise => { + await docker( + [ + "exec", + "-u", + "0", + SANDBOX_CONTAINER, + "sh", + "-c", + `u=$(id -u bun); for d in ${OWN_DIRS}; do [ "$(stat -c %u "$d" 2>/dev/null)" = "$u" ] || chown -R bun:bun "$d"; done`, + ], + 120_000, + ).catch(() => {}); +}; + +export const ensureSandbox = async (): Promise => { + const running = await docker(["inspect", "-f", "{{.State.Running}}", SANDBOX_CONTAINER], 15_000) + .then((out) => out.trim() === "true") + .catch(() => false); + if (!running) { + const exists = await docker(["inspect", "-f", "{{.Id}}", SANDBOX_CONTAINER], 15_000) + .then((out) => out.trim().length > 0) + .catch(() => false); + if (!exists) throw new Error("Diagnosis sandbox is not running (start the diagnose-sandbox service)"); + await docker(["start", SANDBOX_CONTAINER], 60_000); + } + await normalizeOwnership(); +}; + +export const syncDequelSource = async (version: string): Promise<{ rev: string; stale: boolean }> => { + const tag = version.startsWith("v") ? version : `v${version}`; + const hasRepo = await docker([ + "exec", + SANDBOX_CONTAINER, + "sh", + "-c", + "test -d /srv/dequel-src/.git && echo yes || echo no", + ]).then((out) => out.trim() === "yes"); + if (!hasRepo) { + await docker( + [ + "exec", + SANDBOX_CONTAINER, + "git", + "clone", + "--depth", + "1", + "--branch", + tag, + config.dequelSourceRepoUrl, + "/srv/dequel-src", + ], + 180_000, + ).catch(() => + docker( + ["exec", SANDBOX_CONTAINER, "git", "clone", "--depth", "1", config.dequelSourceRepoUrl, "/srv/dequel-src"], + 180_000, + ), + ); + } + const fetched = await docker([ + "exec", + SANDBOX_CONTAINER, + "sh", + "-c", + `git -C /srv/dequel-src fetch --depth 1 origin ${tag} && git -C /srv/dequel-src checkout -q ${tag}`, + ]) + .then(() => true) + .catch(() => false); + const sha = await docker( + ["exec", SANDBOX_CONTAINER, "git", "-C", "/srv/dequel-src", "rev-parse", "--short", "HEAD"], + 15_000, + ) + .then((out) => out.trim()) + .catch(() => ""); + const rev = sha ? `${tag} @ ${sha}` : tag; + return { rev, stale: !fetched }; +}; + +const PROJECT_SRC_PARENT = "/srv/project-src"; + +const sanitizeRunId = (runId: string): string => runId.replace(/[^A-Za-z0-9_-]/g, "") || "run"; + +export const projectSrcDir = (runId: string): string => `${PROJECT_SRC_PARENT}/${sanitizeRunId(runId)}`; + +export const syncProjectSource = async ( + facts: DeploymentFacts, + runId: string, +): Promise<{ available: boolean; srcDir: string }> => { + const srcDir = projectSrcDir(runId); + await docker( + [ + "exec", + "-u", + "0", + SANDBOX_CONTAINER, + "sh", + "-c", + `rm -rf '${srcDir}' && mkdir -p '${srcDir}' && chown bun:bun '${srcDir}'`, + ], + 30_000, + ); + if (facts.sourceType === "git") { + const base = facts.branch ? ["--branch", facts.branch] : []; + const cloned = await docker( + ["exec", SANDBOX_CONTAINER, "git", "clone", "--depth", "1", ...base, facts.sourceRef, srcDir], + 180_000, + ) + .then(() => true) + .catch(() => false); + if (!cloned) return { available: false, srcDir }; + if (facts.commitSha) { + await docker( + ["exec", SANDBOX_CONTAINER, "git", "-C", srcDir, "fetch", "--depth", "1", "origin", facts.commitSha], + 120_000, + ).catch(() => ""); + await docker(["exec", SANDBOX_CONTAINER, "git", "-C", srcDir, "checkout", "-q", facts.commitSha], 30_000).catch( + () => "", + ); + } + return { available: true, srcDir }; + } + if (facts.sourceType === "upload") { + const dir = join(config.workspaceRoot, facts.deploymentId); + try { + await docker(["cp", `${dir}/.`, `${SANDBOX_CONTAINER}:${srcDir}/`], 120_000); + await docker(["exec", "-u", "0", SANDBOX_CONTAINER, "chown", "-R", "bun:bun", srcDir], 60_000).catch(() => {}); + return { available: true, srcDir }; + } catch { + return { available: false, srcDir }; + } + } + return { available: false, srcDir }; +}; + +export const clearProjectSource = async (runId: string): Promise => { + await docker(["exec", "-u", "0", SANDBOX_CONTAINER, "rm", "-rf", projectSrcDir(runId)], 30_000).catch(() => {}); +}; + +export const investigateInSandbox = async (input: { + runId: string; + provider: string; + model: string; + deployment: DeploymentFacts; + failureReason: string; + logText: string; + verdict: TriageVerdict; + dequelRev: string; + dequelStale: boolean; + projectAvailable: boolean; + onProgress: (line: string) => void; +}): Promise => { + const key = await getDecryptedLlmKey(input.provider); + if (!key) throw new Error(`LLM provider "${input.provider}" is not configured`); + await ensureSandbox(); + const taskBrief = [ + `Dequel platform source is mounted at scope "dequel" (${input.dequelRev}${input.dequelStale ? ", possibly stale" : ""}).`, + input.projectAvailable + ? 'The deployed project\'s source is mounted at scope "project".' + : "No project source is available; diagnose from the logs and the Dequel source only.", + `Triage already read the logs (confidence ${input.verdict.confidence}, failing stage "${input.verdict.failingStage}") and flagged these signal lines: ${input.verdict.signalLines.slice(0, 6).join(" | ")}. Start from them instead of re-discovering the failure.`, + "Use listFiles to discover layout, readFile for targeted reads, searchText to trace error strings.", + ].join(" "); + const raw = await runAgentJob({ + jobDir: `/srv/jobs/${sanitizeRunId(input.runId)}`, + input: { + mode: "investigate", + failureReason: input.failureReason, + logText: input.logText, + taskBrief, + provider: input.provider, + model: input.model, + apiKey: key.apiKey, + baseUrl: key.baseUrl, + hasProjectSource: input.projectAvailable, + projectSrcDir: input.projectAvailable ? projectSrcDir(input.runId) : null, + dequelRev: input.dequelRev, + dequelStale: input.dequelStale, + deadlineMs: 9 * 60_000, + }, + execTimeoutMs: 10 * 60_000, + onProgress: input.onProgress, + }); + return validateInvestigation(raw); +}; + +export interface FixAgentResult { + patch: string; + summary: string; + changedFiles: string[]; +} + +export const runFixAgent = async (input: { + runId: string; + provider: string; + model: string; + fixBrief: string; + dequelRev: string; + dequelStale: boolean; + onProgress: (line: string) => void; +}): Promise => { + const key = await getDecryptedLlmKey(input.provider); + if (!key) throw new Error(`LLM provider "${input.provider}" is not configured`); + await ensureSandbox(); + const srcDir = projectSrcDir(input.runId); + const raw = (await runAgentJob({ + jobDir: `/srv/jobs/fix-${sanitizeRunId(input.runId)}`, + input: { + mode: "fix", + fixBrief: input.fixBrief, + provider: input.provider, + model: input.model, + apiKey: key.apiKey, + baseUrl: key.baseUrl, + hasProjectSource: true, + projectSrcDir: srcDir, + dequelRev: input.dequelRev, + dequelStale: input.dequelStale, + deadlineMs: 5.5 * 60_000, + }, + execTimeoutMs: 6 * 60_000, + onProgress: input.onProgress, + })) as Record; + const summary = typeof raw.summary === "string" ? raw.summary : ""; + const changedFiles = Array.isArray(raw.changedFiles) + ? raw.changedFiles.filter((v): v is string => typeof v === "string").slice(0, 32) + : []; + await docker(["exec", SANDBOX_CONTAINER, "git", "-C", srcDir, "add", "-A"], 30_000).catch(() => ""); + const patch = await docker( + ["exec", SANDBOX_CONTAINER, "git", "-C", srcDir, "diff", "--cached", "--no-color", "--binary"], + 30_000, + ) + .then((out) => out) + .catch(() => ""); + if (!patch.trim()) throw new Error("agent made no changes to the project source"); + if (patch.length > 128_000) throw new Error("agent fix is too large to apply as a pull request"); + return { patch, summary, changedFiles }; +}; + +const runAgentJob = async (opts: { + jobDir: string; + input: unknown; + execTimeoutMs: number; + onProgress: (line: string) => void; +}): Promise => { + const hostTmp = await mkdtemp(join(tmpdir(), "dequel-diag-job-")); + try { + await writeFile(join(hostTmp, "input.json"), JSON.stringify(opts.input)); + await docker(["exec", "-u", "0", SANDBOX_CONTAINER, "mkdir", "-p", opts.jobDir], 15_000); + await docker(["cp", join(hostTmp, "input.json"), `${SANDBOX_CONTAINER}:${opts.jobDir}/input.json`], 30_000); + await docker(["exec", "-u", "0", SANDBOX_CONTAINER, "chown", "-R", "bun:bun", opts.jobDir], 15_000); + const deadline = Date.now() + opts.execTimeoutMs; + let exited = false; + const run = docker(["exec", SANDBOX_CONTAINER, "bun", RUNNER, `${opts.jobDir}/input.json`], opts.execTimeoutMs) + .then(() => { + exited = true; + }) + .catch((err) => { + exited = true; + throw new Error(`sandbox agent failed: ${short(String(err))}`); + }); + run.catch(() => {}); + let seen = 0; + while (Date.now() < deadline && !exited) { + await new Promise((resolve) => setTimeout(resolve, 2000)); + const lines = await docker( + ["exec", SANDBOX_CONTAINER, "sh", "-c", `cat ${opts.jobDir}/progress.jsonl 2>/dev/null || true`], + 15_000, + ) + .then((out) => out.split("\n").filter(Boolean)) + .catch(() => [] as string[]); + for (const line of lines.slice(seen)) opts.onProgress(line); + seen = lines.length; + } + await run; + const report = await docker( + ["exec", SANDBOX_CONTAINER, "sh", "-c", `cat ${opts.jobDir}/report.json 2>/dev/null || true`], + 15_000, + ) + .then((out) => out.trim()) + .catch(() => ""); + if (!report) throw new Error("sandbox agent produced no report"); + return JSON.parse(report); + } finally { + await docker(["exec", "-u", "0", SANDBOX_CONTAINER, "rm", "-rf", opts.jobDir], 30_000).catch(() => {}); + await rm(hostTmp, { recursive: true, force: true }).catch(() => {}); + } +}; + +export const validateInvestigation = (raw: unknown): Investigation => { + const record = (raw ?? {}) as Record; + const cause = record.cause === "user-source" || record.cause === "dequel-source" ? record.cause : "unknown"; + const strings = (value: unknown, max = 8): string[] => + Array.isArray(value) ? value.filter((v): v is string => typeof v === "string").slice(0, max) : []; + return { + cause, + culpritPaths: strings(record.culpritPaths), + rationale: typeof record.rationale === "string" ? record.rationale : "", + keyEvidence: strings(record.keyEvidence), + dequelRev: typeof record.dequelRev === "string" ? record.dequelRev : "unknown", + dequelStale: record.dequelStale === true, + }; +}; diff --git a/apps/api/src/fixdiag/sandbox-runner/__tests__/agent-def.test.ts b/apps/api/src/fixdiag/sandbox-runner/__tests__/agent-def.test.ts new file mode 100644 index 0000000..bd35bb8 --- /dev/null +++ b/apps/api/src/fixdiag/sandbox-runner/__tests__/agent-def.test.ts @@ -0,0 +1,11 @@ +import { describe, expect, it } from "bun:test"; +import { createFixer, createInvestigator } from "../agent-def"; + +const ctx = { progressPath: "/tmp/progress.jsonl", projectRoot: "/srv/project-src", dequelRef: "v0.0.0" }; + +describe("agent construction", () => { + it("builds the investigator and fixer without throwing", () => { + expect(() => createInvestigator(ctx)).not.toThrow(); + expect(() => createFixer(ctx)).not.toThrow(); + }); +}); diff --git a/apps/api/src/fixdiag/sandbox-runner/__tests__/forward.test.ts b/apps/api/src/fixdiag/sandbox-runner/__tests__/forward.test.ts new file mode 100644 index 0000000..97fe5bd --- /dev/null +++ b/apps/api/src/fixdiag/sandbox-runner/__tests__/forward.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it, mock } from "bun:test"; +import { forwardWithFallback, type Forwardable } from "../forward"; + +mock.restore(); + +const answerOnly = (): Forwardable => ({ + forward: async () => ({ cause: "unknown" }), +}); + +describe("forwardWithFallback", () => { + it("returns the first result when the program finishes in budget", async () => { + const prog: Forwardable = { + forward: async () => ({ cause: "dequel-source" }), + }; + const out = await forwardWithFallback(prog, answerOnly(), {}, { q: "x" }, "/tmp/nonexistent-progress.log"); + expect(out).toEqual({ cause: "dequel-source" }); + }); + + it("retries answer-only on shared memory after max steps", async () => { + const calls: Record[] = []; + let n = 0; + const prog: Forwardable = { + forward: async (_llm, _values, opts) => { + calls.push(opts ?? {}); + n += 1; + if (n === 1) throw new Error("Generate failed: Max steps reached: 20"); + return { cause: "unknown" }; + }, + }; + const answerCalls: Record[] = []; + const fallback: Forwardable = { + forward: async (_llm, _values, opts) => { + answerCalls.push(opts ?? {}); + return { cause: "unknown" }; + }, + }; + const out = await forwardWithFallback(prog, fallback, {}, { q: "x" }, "/tmp/nonexistent-progress.log"); + expect(out).toEqual({ cause: "unknown" }); + expect(calls).toHaveLength(1); + expect(answerCalls).toHaveLength(1); + expect(answerCalls[0]).toMatchObject({ functionCall: "none" }); + expect((answerCalls[0] as Record).mem).toBe((calls[0] as Record).mem); + }); + + it("rethrows errors other than max steps", async () => { + const prog: Forwardable = { + forward: async () => { + throw new Error("Generate failed: HTTP 429 - Too Many Requests"); + }, + }; + await expect( + forwardWithFallback(prog, answerOnly(), {}, { q: "x" }, "/tmp/nonexistent-progress.log"), + ).rejects.toThrow(/429/); + }); + + it("detects max steps buried in a wrapper error chain", async () => { + const prog: Forwardable = { + forward: async () => { + throw new Error("Generate failed", { cause: new Error("Max steps reached: 20") }); + }, + }; + const out = await forwardWithFallback(prog, answerOnly(), {}, { q: "x" }, "/tmp/nonexistent-progress.log"); + expect(out).toEqual({ cause: "unknown" }); + }); +}); diff --git a/apps/api/src/fixdiag/sandbox-runner/__tests__/tools.test.ts b/apps/api/src/fixdiag/sandbox-runner/__tests__/tools.test.ts new file mode 100644 index 0000000..117a065 --- /dev/null +++ b/apps/api/src/fixdiag/sandbox-runner/__tests__/tools.test.ts @@ -0,0 +1,103 @@ +import { afterAll, beforeAll, describe, expect, it, mock } from "bun:test"; +import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + buildFileTools, + buildFixTools, + editFileAt, + listFilesIn, + readFileAt, + searchInScope, + writeFileAt, + type ToolContext, +} from "../tools"; + +mock.restore(); + +let root = ""; +let ctx: ToolContext; + +beforeAll(async () => { + root = await mkdtemp(join(tmpdir(), "diag-tools-")); + ctx = { + roots: { dequel: join(root, "dequel"), project: join(root, "project") }, + progressPath: join(root, "progress.jsonl"), + }; + await mkdir(join(root, "dequel", "apps/api/src"), { recursive: true }); + await writeFile(join(root, "dequel", "apps/api/src/pipeline.ts"), "export const run = () => { buildImage(); };\n"); + await writeFile(join(root, "dequel", "README.md"), "Dequel\n"); + await mkdir(join(root, "dequel", "node_modules"), { recursive: true }); + await writeFile(join(root, "dequel", "node_modules/blob.js"), "x".repeat(100)); + await mkdir(join(root, "project"), { recursive: true }); + await writeFile(join(root, "project", "Dockerfile"), "FROM node:20\nRUN npm ci\n"); +}); + +afterAll(async () => { + if (root) await rm(root, { recursive: true, force: true }); +}); + +describe("sandbox file tools", () => { + it("lists files while skipping dependency dirs", async () => { + const paths = await listFilesIn(ctx, "dequel"); + expect(paths).toContain("apps/api/src/pipeline.ts"); + expect(paths).toContain("README.md"); + expect(paths.some((p) => p.includes("node_modules"))).toBe(false); + expect(await listFilesIn(ctx, "dequel", "apps/api/src")).toEqual(["apps/api/src/pipeline.ts"]); + }); + + it("reads capped file content", async () => { + const out = await readFileAt(ctx, "project", "Dockerfile"); + expect(out).toContain("FROM node:20"); + await expect(readFileAt(ctx, "project", "missing.txt")).rejects.toThrow(); + }); + + it("rejects paths escaping the roots", async () => { + await expect(readFileAt(ctx, "dequel", "../../etc/passwd")).rejects.toThrow(/sandbox roots/); + await expect(readFileAt(ctx, "dequel", "/srv/jobs/x/input.json")).rejects.toThrow(/sandbox roots/); + await expect(readFileAt(ctx, "nope", "x")).rejects.toThrow(); + await expect(searchInScope(ctx, "dequel", " ")).rejects.toThrow(); + }); + + it("searches contents across the tree", async () => { + const hits = await searchInScope(ctx, "dequel", "buildImage"); + expect(hits).toHaveLength(1); + expect(hits[0]).toContain("apps/api/src/pipeline.ts:1:"); + expect(await searchInScope(ctx, "project", "zzz-no-match")).toEqual([]); + }); + + it("edits files with exact matches only", async () => { + await writeFile(join(root, "project", "app.ts"), "const a = 1;\nconst b = 1;\n"); + await expect(editFileAt(ctx, "app.ts", "missing", "x")).rejects.toThrow(/not found/); + await expect(editFileAt(ctx, "const", "x")).rejects.toThrow(); + await expect(editFileAt(ctx, "app.ts", "= 1;", "= 2;")).rejects.toThrow(/2 times/); + await expect(editFileAt(ctx, "app.ts", "const a = 1;", "const a = 2;")).resolves.toContain("edited"); + await expect(editFileAt(ctx, "app.ts", "const a = 1;", "const a = 1;")).rejects.toThrow(/identical/); + await expect(editFileAt(ctx, "app.ts", "", "x")).rejects.toThrow(/must not be empty/); + }); + + it("creates new files but never overwrites or escapes", async () => { + await expect(writeFileAt(ctx, "new/nested/file.ts", "hello\n")).resolves.toContain("created"); + await expect(writeFileAt(ctx, "new/nested/file.ts", "again")).rejects.toThrow(/already exists/); + await expect(writeFileAt(ctx, "../../evil.ts", "x")).rejects.toThrow(/sandbox roots/); + await expect(writeFileAt(ctx, "big.ts", "x".repeat(100_001))).rejects.toThrow(/exceeds/); + }); + + it("returns handler errors as results instead of throwing", async () => { + const tools = [...buildFileTools(ctx.roots, ctx.progressPath), ...buildFixTools(ctx.roots, ctx.progressPath)]; + const call = async (name: string, args: Record) => { + const tool = tools.find((t) => t.name === name) as unknown as { func: (a: unknown) => Promise }; + return tool.func(args); + }; + await expect(call("readFile", { scope: "dequel", path: "/srv/jobs/x/input.json" })).resolves.toMatch( + /sandbox roots/, + ); + await expect(call("listFiles", { scope: "dequel", prefix: "../../etc" })).resolves.toEqual([ + expect.stringMatching(/sandbox roots/), + ]); + await expect(call("searchText", { scope: "dequel", query: " " })).resolves.toEqual([ + expect.stringMatching(/Error:/), + ]); + await expect(call("readFile", { scope: "nope", path: "x" })).resolves.toMatch(/Error:/); + }); +}); diff --git a/apps/api/src/fixdiag/sandbox-runner/agent-def.ts b/apps/api/src/fixdiag/sandbox-runner/agent-def.ts new file mode 100644 index 0000000..29ccbb5 --- /dev/null +++ b/apps/api/src/fixdiag/sandbox-runner/agent-def.ts @@ -0,0 +1,40 @@ +import { ax } from "@ax-llm/ax"; +import { buildFileTools, buildFixTools } from "./tools"; + +export interface InvestigatorContext { + progressPath: string; + projectRoot: string | null; + dequelRef: string; +} + +const INVESTIGATOR_MAX_STEPS = 20; +const FIXER_MAX_STEPS = 10; + +export const createInvestigator = (ctx: InvestigatorContext, withTools = true) => { + const functions = withTools + ? buildFileTools({ dequel: "/srv/dequel-src", project: ctx.projectRoot }, ctx.progressPath) + : []; + return ax( + 'failureReason:string, logText:string, taskBrief:string -> cause:class "user-source,dequel-source,unknown", culpritPaths:string[], rationale:string, keyEvidence:string[]', + { + functions, + maxSteps: INVESTIGATOR_MAX_STEPS, + description: withTools + ? "Investigate a failed Dequel deployment using the provided read-only file tools. Scope dequel is Dequel's own platform source; scope project is the deployed project's source and may be absent. Decide first whether the cause is in the user's source or Dequel's source, then gather evidence with a few targeted calls. Failures in Dequel's build environment itself (BuildKit, Docker daemon/network/DNS, railpack image pulls) are dequel-source even when no Dequel code file references the failing artifact; reserve unknown for cases where neither side is implicated. Be economical: prefer searchText over broad lists, read only the files that matter, never re-read a file, never repeat a search that returned 0 hits, and after at most 8 tool calls stop calling tools and write your final answer from what you have. You cannot write files, run commands, or reach the network. Prefer unknown over guessing." + : "Write your final investigation answer NOW from the conversation so far. Tool use is disabled: do not emit any tool calls, answer directly with cause, culpritPaths, rationale, and keyEvidence. Prefer unknown over guessing.", + }, + ); +}; + +export const createFixer = (ctx: InvestigatorContext, withTools = true) => { + const functions = withTools + ? buildFixTools({ dequel: "/srv/dequel-src", project: ctx.projectRoot }, ctx.progressPath) + : []; + return ax("fixBrief:string -> summary:string, changedFiles:string[]", { + functions, + maxSteps: FIXER_MAX_STEPS, + description: withTools + ? "Fix a diagnosed build failure in the project source. You have read tools over the dequel scope (reference only, never modify) and read plus edit tools over the project scope. Make the smallest change that fixes the diagnosed failure: prefer editFile with exact matches, create files only when necessary, never touch lockfiles, vendored code, or anything outside the project scope. End by summarizing the change and listing every file you modified." + : "Write your final fix summary NOW from the conversation so far. Tool use is disabled: do not emit any tool calls, answer directly with summary and changedFiles.", + }); +}; diff --git a/apps/api/src/fixdiag/sandbox-runner/forward.ts b/apps/api/src/fixdiag/sandbox-runner/forward.ts new file mode 100644 index 0000000..66bba4d --- /dev/null +++ b/apps/api/src/fixdiag/sandbox-runner/forward.ts @@ -0,0 +1,74 @@ +import { appendFile } from "node:fs/promises"; +import { AxMemory } from "@ax-llm/ax"; + +export const FORWARD_OPTS = { timeout: 120_000, maxRetries: 1 } as const; + +export type Forwardable = { + forward: ( + llm: unknown, + values: Record, + opts?: Record, + ) => Promise>; +}; + +export const chainText = (err: unknown): string => { + const parts: string[] = []; + const seen = new Set(); + const visit = (cur: unknown): void => { + if (!cur || seen.has(cur)) return; + seen.add(cur); + if (typeof cur === "string") { + if (cur.trim()) parts.push(cur.trim()); + return; + } + if (typeof cur !== "object") return; + const e = cur as { message?: unknown; responseBody?: unknown; cause?: unknown; errors?: unknown }; + if (typeof e.message === "string" && e.message.trim()) parts.push(e.message.trim()); + if (e.responseBody != null) { + const text = typeof e.responseBody === "string" ? e.responseBody : JSON.stringify(e.responseBody); + if (text && text.trim() && text.trim() !== "{}") parts.push(text.trim()); + } + if (Array.isArray(e.errors)) for (const sub of e.errors) visit(sub); + visit(e.cause); + }; + visit(err); + return parts.join(" | ").slice(0, 960); +}; + +export const describeError = (err: unknown): string => { + let message = ""; + let body = ""; + const seen = new Set(); + let cur: unknown = err; + while (cur && typeof cur === "object" && !seen.has(cur)) { + seen.add(cur); + const e = cur as { message?: unknown; responseBody?: unknown; cause?: unknown }; + if (!message && typeof e.message === "string" && e.message.trim()) message = e.message.trim(); + if (!body && e.responseBody != null) { + const text = typeof e.responseBody === "string" ? e.responseBody : JSON.stringify(e.responseBody); + if (text && text.trim() && text.trim() !== "{}") body = text.trim(); + } + cur = e.cause; + } + if (body && !message.includes(body)) return `${body} | ${message}`.slice(0, 480); + return (message || String(err)).slice(0, 480); +}; + +export const forwardWithFallback = async ( + prog: Forwardable, + answerOnlyProg: Forwardable, + llm: unknown, + values: Record, + progressPath: string, +): Promise> => { + const mem = new AxMemory(); + try { + return await prog.forward(llm, values, { ...FORWARD_OPTS, mem }); + } catch (err) { + if (!/max steps/i.test(chainText(err))) throw err; + await appendFile(progressPath, "step budget exhausted; writing final answer without further tool calls\n").catch( + () => {}, + ); + return await answerOnlyProg.forward(llm, values, { ...FORWARD_OPTS, mem, functionCall: "none", maxSteps: 6 }); + } +}; diff --git a/apps/api/src/fixdiag/sandbox-runner/runner.ts b/apps/api/src/fixdiag/sandbox-runner/runner.ts new file mode 100644 index 0000000..11d0192 --- /dev/null +++ b/apps/api/src/fixdiag/sandbox-runner/runner.ts @@ -0,0 +1,115 @@ +import { appendFile, readFile, writeFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { asCause, asString, asStringArray, buildLlm } from "../llm"; +import { describeError, forwardWithFallback } from "./forward"; +import type { LlmProvider } from "../types"; +import { createFixer, createInvestigator } from "./agent-def"; + +interface BaseInput { + mode: "investigate" | "fix"; + provider: LlmProvider; + model: string; + apiKey: string; + baseUrl: string | null; + hasProjectSource: boolean; + projectSrcDir?: string | null; + dequelRev: string; + dequelStale: boolean; + deadlineMs?: number; +} + +interface InvestigateInput extends BaseInput { + mode: "investigate"; + failureReason: string; + logText: string; + taskBrief: string; +} + +interface FixInput extends BaseInput { + mode: "fix"; + fixBrief: string; +} + +type RunnerInput = InvestigateInput | FixInput; + +const MAX_LOG_CHARS = 3_000; +const FORWARD_OPTS = { timeout: 120_000, maxRetries: 1 } as const; + +const clipTail = (value: string, max: number): string => + value.length > max ? `…[last ${max} of ${value.length} chars]\n${value.slice(-max)}` : value; + +const main = async (): Promise => { + const [inputPath] = Bun.argv.slice(2); + if (!inputPath) { + console.error("usage: runner.js "); + process.exit(2); + } + const dir = dirname(inputPath); + const progressPath = join(dir, "progress.jsonl"); + const raw = JSON.parse(await readFile(inputPath, "utf8")) as RunnerInput; + const deadline = setTimeout( + () => { + console.error("agent deadline exceeded"); + process.exit(2); + }, + raw.deadlineMs ?? 9 * 60_000, + ); + try { + const llm = buildLlm(raw.provider, raw.apiKey, raw.baseUrl ?? null, raw.model); + const ctx = { + progressPath, + projectRoot: raw.hasProjectSource ? (raw.projectSrcDir ?? "/srv/project-src") : null, + dequelRef: raw.dequelRev, + }; + if (raw.mode === "fix") { + await appendFile(progressPath, "fix started\n").catch(() => {}); + const out = await forwardWithFallback( + createFixer(ctx), + createFixer(ctx, false), + llm, + { fixBrief: raw.fixBrief }, + progressPath, + ); + await writeFile( + join(dir, "report.json"), + JSON.stringify({ + summary: asString(out.summary), + changedFiles: asStringArray(out.changedFiles, 32), + }), + ); + await appendFile(progressPath, "fix complete\n").catch(() => {}); + return; + } + await appendFile(progressPath, "investigation started\n").catch(() => {}); + const out = await forwardWithFallback( + createInvestigator(ctx), + createInvestigator(ctx, false), + llm, + { + failureReason: clipTail(raw.failureReason, 2_000), + logText: clipTail(raw.logText, MAX_LOG_CHARS), + taskBrief: raw.taskBrief, + }, + progressPath, + ); + await writeFile( + join(dir, "report.json"), + JSON.stringify({ + cause: asCause(out.cause), + culpritPaths: asStringArray(out.culpritPaths, 8), + rationale: asString(out.rationale), + keyEvidence: asStringArray(out.keyEvidence, 8), + dequelRev: raw.dequelRev, + dequelStale: raw.dequelStale, + }), + ); + await appendFile(progressPath, "investigation complete\n").catch(() => {}); + } finally { + clearTimeout(deadline); + } +}; + +await main().catch((err) => { + console.error(describeError(err)); + process.exit(1); +}); diff --git a/apps/api/src/fixdiag/sandbox-runner/tools.ts b/apps/api/src/fixdiag/sandbox-runner/tools.ts new file mode 100644 index 0000000..bb043ab --- /dev/null +++ b/apps/api/src/fixdiag/sandbox-runner/tools.ts @@ -0,0 +1,302 @@ +import { appendFile, mkdir, readdir, readFile, realpath, stat, writeFile } from "node:fs/promises"; +import { dirname, join, resolve } from "node:path"; +import { f, fn } from "@ax-llm/ax"; +const MAX_LIST = 80; +const MAX_READ_BYTES = 2_400; +const MAX_WRITE_BYTES = 100_000; +const MAX_HITS = 10; +const MAX_SCAN_FILES = 500; +const MAX_SCAN_BYTES = 100_000; +const MAX_LIST_CHARS = 1_800; +const MAX_SEARCH_CHARS = 1_200; + +const SKIP_DIRS = new Set([ + "node_modules", + ".git", + "dist", + "build", + ".next", + "__MACOSX", + ".venv", + "target", + ".cache", + "coverage", +]); + +export interface SandboxRoots { + dequel: string; + project: string | null; +} + +export interface ToolContext { + roots: SandboxRoots; + progressPath: string; + readBytes?: number; +} + +const capByChars = (items: string[], maxChars: number): string[] => { + const out: string[] = []; + let used = 0; + for (const item of items) { + if (used + item.length > maxChars) { + out.push("…[truncated — narrow with prefix or a new query]"); + break; + } + out.push(item); + used += item.length; + } + return out; +}; + +const toBytes = (data: unknown): Buffer | null => { + if (Buffer.isBuffer(data)) return data; + if (typeof data === "string") return Buffer.from(data, "utf8"); + if (data instanceof Uint8Array) return Buffer.from(data.buffer, data.byteOffset, data.byteLength); + return null; +}; + +const noteProgress = async (progressPath: string, message: string): Promise => { + await appendFile(progressPath, `${message}\n`).catch(() => {}); +}; + +const errText = (err: unknown): string => + `Error: ${err instanceof Error ? err.message : String(err)} (adjust the call and try again)`; + +const asTextResult = + (handler: (...args: A) => Promise) => + async (...args: A): Promise => { + try { + return await handler(...args); + } catch (err) { + return errText(err); + } + }; + +const asListResult = + (handler: (...args: A) => Promise) => + async (...args: A): Promise => { + try { + return await handler(...args); + } catch (err) { + return [errText(err)]; + } + }; + +const baseFor = (roots: SandboxRoots, scope: string): string => { + if (scope === "dequel") return roots.dequel; + if (scope === "project") { + if (!roots.project) throw new Error("project source is not available for this diagnosis (logs only)"); + return roots.project; + } + throw new Error(`unknown scope "${scope}" (use "dequel" or "project")`); +}; + +const confinePath = async (roots: SandboxRoots, scope: string, rel: string): Promise => { + const base = baseFor(roots, scope); + const rootResolved = resolve(base); + const abs = resolve(rootResolved, rel); + if (abs !== rootResolved && !abs.startsWith(`${rootResolved}/`)) + throw new Error("path escapes the sandbox roots (use a repo-relative path, not an absolute path or ..)"); + const real = await realpath(abs).catch(() => null); + if (!real || (real !== rootResolved && !real.startsWith(`${rootResolved}/`))) { + throw new Error(`cannot access path: ${rel} (missing, or outside the ${scope} source root)`); + } + return abs; +}; + +const collectPaths = async (dir: string, rootResolved: string, out: string[]): Promise => { + if (out.length >= MAX_LIST) return; + const entries = await readdir(dir, { withFileTypes: true }).catch(() => []); + for (const entry of entries) { + if (out.length >= MAX_LIST) return; + const abs = join(dir, entry.name); + if (entry.isDirectory()) { + if (SKIP_DIRS.has(entry.name)) continue; + out.push(`${abs.slice(rootResolved.length + 1)}/`); + await collectPaths(abs, rootResolved, out); + } else if (entry.isFile()) { + out.push(abs.slice(rootResolved.length + 1)); + } + } +}; + +export const listFilesIn = async (ctx: ToolContext, scope: string, prefix?: string): Promise => { + const abs = await confinePath(ctx.roots, scope, prefix ?? ""); + const st = await stat(abs).catch(() => null); + if (!st?.isDirectory()) throw new Error(`not a directory: ${prefix ?? ""}`); + const out: string[] = []; + await collectPaths(abs, resolve(baseFor(ctx.roots, scope)), out); + const capped = capByChars(out, MAX_LIST_CHARS); + await noteProgress(ctx.progressPath, `list ${scope}/${prefix ?? ""} (${out.length} paths, ${capped.length} shown)`); + return capped; +}; + +export const readFileAt = async (ctx: ToolContext, scope: string, path: string): Promise => { + const abs = await confinePath(ctx.roots, scope, path); + const st = await stat(abs).catch(() => null); + if (!st?.isFile() || st.size > 1024 * 1024) throw new Error(`not a readable file: ${path}`); + const buf = toBytes(await readFile(abs).catch(() => null)); + if (!buf || buf.length === 0) throw new Error(`cannot read file: ${path}`); + if (buf.subarray(0, 8000).includes(0)) return `(binary file, skipped: ${path})`; + const take = Math.min(buf.length, ctx.readBytes ?? MAX_READ_BYTES); + await noteProgress(ctx.progressPath, `read ${scope}/${path} (${take} bytes)`); + return `${buf.length > take ? "(truncated) " : ""}${path}\n---\n${buf.subarray(0, take).toString("utf8")}`; +}; + +export const searchInScope = async ( + ctx: ToolContext, + scope: string, + query: string, + prefix?: string, +): Promise => { + if (!query.trim()) throw new Error("query must not be empty"); + const start = await confinePath(ctx.roots, scope, prefix ?? ""); + const rootResolved = resolve(baseFor(ctx.roots, scope)); + const hits: string[] = []; + let scanned = 0; + const visit = async (dir: string): Promise => { + if (hits.length >= MAX_HITS || scanned >= MAX_SCAN_FILES) return; + const entries = await readdir(dir, { withFileTypes: true }).catch(() => []); + for (const entry of entries) { + if (hits.length >= MAX_HITS || scanned >= MAX_SCAN_FILES) return; + const abs = join(dir, entry.name); + if (entry.isDirectory()) { + if (!SKIP_DIRS.has(entry.name)) await visit(abs); + } else if (entry.isFile()) { + scanned++; + const st = await stat(abs).catch(() => null); + if (!st?.isFile() || st.size > 256 * 1024) continue; + const buf = toBytes(await readFile(abs).catch(() => null)); + if (!buf || buf.length === 0 || buf.subarray(0, 8000).includes(0)) continue; + const text = buf.subarray(0, MAX_SCAN_BYTES).toString("utf8"); + const needle = query.toLowerCase(); + const lines = text.split("\n"); + for (let i = 0; i < lines.length && hits.length < MAX_HITS; i++) { + if (lines[i].toLowerCase().includes(needle)) { + hits.push(`${abs.slice(rootResolved.length + 1)}:${i + 1}: ${lines[i].trim().slice(0, 120)}`); + } + } + } + } + }; + await visit(start); + await noteProgress(ctx.progressPath, `search ${scope} for "${query.slice(0, 60)}" (${hits.length} hits)`); + return capByChars(hits, MAX_SEARCH_CHARS); +}; + +export const buildFileTools = (roots: SandboxRoots, progressPath: string, readBytes?: number) => { + const ctx: ToolContext = { roots, progressPath, readBytes }; + const listFiles = fn("listFiles") + .description( + "List files under a source scope. Scope dequel is Dequel's own platform source; scope project is the deployed project's source. Call with a narrow prefix to discover layout before reading.", + ) + .arg("scope", f.string('Source scope: "dequel" or "project"')) + .arg("prefix", f.string("Optional subdirectory to list, e.g. apps/api/src/orchestrator").optional()) + .returns(f.string("Repo-relative paths, directories end with /").array()) + .handler( + asListResult(async ({ scope, prefix }: { scope: string; prefix?: string }) => listFilesIn(ctx, scope, prefix)), + ) + .build(); + + const readFileTool = fn("readFile") + .description( + `Read one source file from the start, capped at ${((readBytes ?? MAX_READ_BYTES) / 1000).toFixed(1)}KB with a truncation flag. Prefer searchText to locate the exact region before reading.`, + ) + .arg("scope", f.string('Source scope: "dequel" or "project"')) + .arg("path", f.string("Repo-relative file path, e.g. apps/api/src/orchestrator/pipeline.ts")) + .returns(f.string("File content prefixed with a header line")) + .handler(asTextResult(async ({ scope, path }: { scope: string; path: string }) => readFileAt(ctx, scope, path))) + .build(); + + const searchText = fn("searchText") + .description( + "Search file contents for a fixed string (case-insensitive) under a scope. Use this to find error strings, function definitions, or config keys across the tree.", + ) + .arg("scope", f.string('Source scope: "dequel" or "project"')) + .arg("query", f.string("Fixed string to search for")) + .arg("prefix", f.string("Optional subdirectory to search under").optional()) + .returns(f.string('Matching lines as "path:line: text"').array()) + .handler( + asListResult(async ({ scope, query, prefix }: { scope: string; query: string; prefix?: string }) => + searchInScope(ctx, scope, query, prefix), + ), + ) + .build(); + + return [listFiles, readFileTool, searchText]; +}; + +export const editFileAt = async ( + ctx: ToolContext, + path: string, + oldString: string, + newString: string, +): Promise => { + if (!oldString) throw new Error("oldString must not be empty"); + if (newString === oldString) throw new Error("newString is identical to oldString"); + const abs = await confinePath(ctx.roots, "project", path); + const st = await stat(abs).catch(() => null); + if (!st?.isFile()) throw new Error(`not a file: ${path}`); + const buf = toBytes(await readFile(abs).catch(() => null)); + if (!buf) throw new Error(`cannot read file: ${path}`); + const content = buf.toString("utf8"); + const occurrences = content.split(oldString).length - 1; + if (occurrences === 0) throw new Error(`oldString not found in ${path}`); + if (occurrences > 1) throw new Error(`oldString matches ${occurrences} times in ${path}; include more context`); + await writeFile(abs, content.replace(oldString, newString)); + await noteProgress(ctx.progressPath, `edit ${path}`); + return `edited ${path}`; +}; + +export const writeFileAt = async (ctx: ToolContext, path: string, content: string): Promise => { + if (content.length > MAX_WRITE_BYTES) throw new Error(`content exceeds ${MAX_WRITE_BYTES} bytes`); + if (!ctx.roots.project) throw new Error("project source is not available for this diagnosis (logs only)"); + const rootResolved = resolve(ctx.roots.project); + const abs = resolve(rootResolved, path); + if (abs !== rootResolved && !abs.startsWith(`${rootResolved}/`)) + throw new Error("path escapes the sandbox roots (use a repo-relative path, not an absolute path or ..)"); + const parent = dirname(abs); + await mkdir(parent, { recursive: true }); + const realParent = await realpath(parent).catch(() => null); + if (!realParent || (realParent !== rootResolved && !realParent.startsWith(`${rootResolved}/`))) { + throw new Error("path escapes the sandbox roots (use a repo-relative path, not an absolute path or ..)"); + } + const exists = await stat(abs).catch(() => null); + if (exists) throw new Error(`${path} already exists; use editFile to modify it`); + await mkdir(parent, { recursive: true }); + await writeFile(abs, content); + await noteProgress(ctx.progressPath, `create ${path} (${content.length} bytes)`); + return `created ${path}`; +}; + +export const buildFixTools = (roots: SandboxRoots, progressPath: string) => { + const ctx: ToolContext = { roots, progressPath, readBytes: 8_000 }; + const reads = buildFileTools(roots, progressPath, 8_000); + + const editFile = fn("editFile") + .description( + "Replace one exact block of text in a project file. oldString must match exactly once. Use for surgical fixes; prefer this over rewriting whole files.", + ) + .arg("path", f.string("Project-relative file path")) + .arg("oldString", f.string("Exact text to replace (must occur exactly once)")) + .arg("newString", f.string("Replacement text")) + .returns(f.string("Confirmation")) + .handler( + asTextResult(async ({ path, oldString, newString }: { path: string; oldString: string; newString: string }) => + editFileAt(ctx, path, oldString, newString), + ), + ) + .build(); + + const createFile = fn("createFile") + .description("Create a NEW project file. Fails if the file already exists. Keep new files small and necessary.") + .arg("path", f.string("Project-relative file path")) + .arg("content", f.string("Full file content")) + .returns(f.string("Confirmation")) + .handler( + asTextResult(async ({ path, content }: { path: string; content: string }) => writeFileAt(ctx, path, content)), + ) + .build(); + + return [...reads, editFile, createFile]; +}; diff --git a/apps/api/src/fixdiag/stream.ts b/apps/api/src/fixdiag/stream.ts new file mode 100644 index 0000000..c8c9c45 --- /dev/null +++ b/apps/api/src/fixdiag/stream.ts @@ -0,0 +1,30 @@ +import type { StageEvent } from "./types"; + +type Listener = (event: StageEvent) => void; + +class DiagBus { + private listeners = new Map>(); + + subscribe(runId: string, listener: Listener): () => void { + let set = this.listeners.get(runId); + if (!set) { + set = new Set(); + this.listeners.set(runId, set); + } + set.add(listener); + return () => { + set.delete(listener); + if (set.size === 0) this.listeners.delete(runId); + }; + } + + emit(runId: string, event: StageEvent) { + for (const listener of this.listeners.get(runId) ?? []) { + try { + listener(event); + } catch {} + } + } +} + +export const diagBus = new DiagBus(); diff --git a/apps/api/src/fixdiag/types.ts b/apps/api/src/fixdiag/types.ts new file mode 100644 index 0000000..086c84f --- /dev/null +++ b/apps/api/src/fixdiag/types.ts @@ -0,0 +1,127 @@ +export type LlmProvider = "openai" | "anthropic" | "gemini" | "groq" | "ollama" | "custom"; + +export type DiagStageName = "triage" | "investigate" | "explain" | "propose"; + +export type CauseKind = "user-source" | "dequel-source" | "unknown"; + +export type TriageConfidence = "low" | "medium" | "high"; + +export type DiagStatus = "running" | "done" | "error"; + +export interface LogLine { + sequence: number; + stage: string; + message: string; +} + +export interface DeploymentFacts { + deploymentId: string; + projectId: string | null; + sourceType: string; + sourceRef: string; + branch: string | null; + commitSha: string; + failureReason: string | null; +} + +export interface EvidenceBundle { + deployment: DeploymentFacts; + logs: LogLine[]; +} + +export interface TriageVerdict { + failingStage: string; + signalLines: string[]; + confidence: TriageConfidence; +} + +export interface Localization { + cause: CauseKind; + culpritPaths: string[]; + rationale: string; +} + +export interface Explanation { + summary: string; + fixSteps: string[]; + patchHint: string | null; +} + +export interface UserFix { + title: string; + body: string; + suggestedDiff: string | null; +} + +export interface DequelReport { + problem: string; + cause: string; + proposedFix: string; +} + +export interface Proposal { + cause: CauseKind; + userFix?: UserFix; + dequelReport?: DequelReport; +} + +export interface DiagRun { + id: string; + deploymentId: string; + commitSha: string; + provider: LlmProvider; + model: string; + status: DiagStatus; + currentStage: DiagStageName | null; + cause: CauseKind | null; + report: Proposal | null; + error: string | null; + createdAt: string; +} + +export type StageEvent = + | { type: "stage"; runId: string; stage: DiagStageName; payload: unknown } + | { type: "token"; runId: string; stage: DiagStageName; delta: string } + | { type: "done"; runId: string } + | { type: "error"; runId: string; message: string }; + +export interface TriageInput { + failureReason: string | null; + logText: string; +} + +export interface ExplainInput { + verdict: TriageVerdict; + localization: Localization; +} + +export interface ProposeInput { + localization: Localization; + explanation: Explanation; + repo: { owner: string; repo: string; base: string } | null; +} + +export interface Investigation { + cause: CauseKind; + culpritPaths: string[]; + rationale: string; + keyEvidence: string[]; + dequelRev: string; + dequelStale: boolean; +} + +export interface InvestigateInput { + run: DiagRun; + deployment: DeploymentFacts; + logText: string; + verdict: TriageVerdict; + onProgress: (line: string) => void; +} + +export type InvestigateFn = (input: InvestigateInput) => Promise; + +export interface FixdiagPrograms { + triageLogs(input: TriageInput): Promise; + explainFix(input: ExplainInput): Promise; + draftProposal(input: ProposeInput): Promise; +} diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index cf5dc6a..24f8a7d 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -8,8 +8,10 @@ import { startDatabaseMonitoring } from "./databases/manager"; import { getDb } from "./db/db-provider"; import { migrate } from "./db/migrate"; import { ensureLocalServer } from "./db/repo"; +import { markInterruptedDiagRuns } from "./db/repo/diag-runs"; import { deployments } from "./db/schema"; import { alertEvaluator } from "./monitoring/evaluator"; +import { startFailureNotifier } from "./monitoring/failure-notifier"; import { orchestrator } from "./orchestrator"; import { startBuildCleanup } from "./orchestrator/cleanup"; import { startFailoverMonitor } from "./orchestrator/failover"; @@ -21,6 +23,9 @@ import { config } from "./utils/config"; import { startDomainPolling } from "./utils/domain-verifier"; import { loadOrCreateJwtSecret } from "./utils/secrets"; +import { projects } from "./db/schema"; +import { captureTelemetry } from "./utils/telemetry"; + const bootstrap = async () => { await mkdir(config.workspaceRoot, { recursive: true }); await mkdir(config.caddyRoutesDir, { recursive: true }); @@ -30,12 +35,15 @@ const bootstrap = async () => { await migrate(); await ensureLocalServer(); + const interrupted = await markInterruptedDiagRuns().catch(() => 0); + if (interrupted > 0) console.log(`[Fixdiag] Marked ${interrupted} interrupted diagnosis run(s) as failed`); await orchestrator.reconcileState(); orchestrator.startWorker(); scalingEngine.start(); serverManager.start(); startDomainPolling(); alertEvaluator.start(); + startFailureNotifier(); startBuildCleanup(); startFailoverMonitor(); startDatabaseMonitoring(); @@ -47,6 +55,24 @@ const bootstrap = async () => { cleanupExpiredTokens().catch(() => {}); }, 60_000); + captureTelemetry("instance_boot").catch(() => {}); + + // 24-hour anonymous heartbeat + setInterval( + async () => { + try { + const db = await getDb(); + const [projectsRes] = await db.select({ count: count() }).from(projects); + const [deploymentsRes] = await db.select({ count: count() }).from(deployments); + captureTelemetry("instance_heartbeat", { + projects_count: Number(projectsRes?.count ?? 0), + deployments_count: Number(deploymentsRes?.count ?? 0), + }).catch(() => {}); + } catch {} + }, + 24 * 60 * 60 * 1000, + ); + const metrics = { requestsTotal: 0, activeDeployments: 0, diff --git a/apps/api/src/monitoring/__tests__/alert-guard.test.ts b/apps/api/src/monitoring/__tests__/alert-guard.test.ts new file mode 100644 index 0000000..6fb9ca8 --- /dev/null +++ b/apps/api/src/monitoring/__tests__/alert-guard.test.ts @@ -0,0 +1,77 @@ +import { describe, expect, it } from "bun:test"; +import { scalingGuard } from "../alert-guard"; + +const enabled = (maxReplicas = 5) => ({ enabled: true, maxReplicas }); + +describe("scalingGuard", () => { + it("does not touch downtime or unknown alert types", () => { + expect(scalingGuard("downtime", { policy: null, cpuLimit: null, currentReplicas: null })).toEqual({ + suppress: false, + suggestion: null, + }); + }); + + it("fires with an enable prompt when no policy exists", () => { + expect(scalingGuard("cpu", { policy: null, cpuLimit: 1, currentReplicas: 1 })).toEqual({ + suppress: false, + suggestion: { kind: "enable_autoscaling" }, + }); + }); + + it("fires with an enable prompt when the policy is disabled", () => { + expect( + scalingGuard("memory", { + policy: { enabled: false, maxReplicas: 5 }, + cpuLimit: 1, + currentReplicas: 1, + }), + ).toEqual({ suppress: false, suggestion: { kind: "enable_autoscaling" } }); + }); + + it("fires with an enable prompt when no cpu limit is set", () => { + expect(scalingGuard("cpu", { policy: enabled(), cpuLimit: null, currentReplicas: 1 })).toEqual({ + suppress: false, + suggestion: { kind: "enable_autoscaling" }, + }); + expect(scalingGuard("cpu", { policy: enabled(), cpuLimit: 0, currentReplicas: 1 })).toEqual({ + suppress: false, + suggestion: { kind: "enable_autoscaling" }, + }); + }); + + it("fires with an increase-replicas prompt when at the ceiling", () => { + expect(scalingGuard("cpu", { policy: enabled(3), cpuLimit: 1, currentReplicas: 3 })).toEqual({ + suppress: false, + suggestion: { kind: "increase_max_replicas", current: 3, maxReplicas: 3 }, + }); + }); + + it("treats an unknown replica count as a single replica", () => { + expect(scalingGuard("cpu", { policy: enabled(1), cpuLimit: 1, currentReplicas: null })).toEqual({ + suppress: false, + suggestion: { kind: "increase_max_replicas", current: 1, maxReplicas: 1 }, + }); + expect(scalingGuard("cpu", { policy: enabled(5), cpuLimit: 1, currentReplicas: null })).toEqual({ + suppress: true, + suggestion: null, + }); + }); + + it("suppresses cpu when autoscaling is on and replicas are below the ceiling", () => { + expect(scalingGuard("cpu", { policy: enabled(5), cpuLimit: 1, currentReplicas: 1 })).toEqual({ + suppress: true, + suggestion: null, + }); + }); + + it("never suppresses memory alerts", () => { + expect(scalingGuard("memory", { policy: enabled(5), cpuLimit: 1, currentReplicas: 4 })).toEqual({ + suppress: false, + suggestion: null, + }); + expect(scalingGuard("memory", { policy: enabled(3), cpuLimit: 1, currentReplicas: 3 })).toEqual({ + suppress: false, + suggestion: { kind: "increase_max_replicas", current: 3, maxReplicas: 3 }, + }); + }); +}); diff --git a/apps/api/src/monitoring/__tests__/health.test.ts b/apps/api/src/monitoring/__tests__/health.test.ts new file mode 100644 index 0000000..3abf9e1 --- /dev/null +++ b/apps/api/src/monitoring/__tests__/health.test.ts @@ -0,0 +1,104 @@ +import { describe, expect, it } from "bun:test"; +import { evaluateHealth } from "../health"; + +const base = { + server: null as { status: string; lastHeartbeatAgeMs: number | null } | null, + runningDeployments: 1, + hasDeployments: true, + routeErrors: [] as string[], + http: { available: true, errorRate: 0 }, +}; + +const check = (result: ReturnType, name: string) => result.checks.find((c) => c.name === name)!; + +describe("evaluateHealth", () => { + it("is healthy when everything is fine locally", () => { + const result = evaluateHealth(base); + expect(result.overall).toBe("healthy"); + expect(result.checks).toHaveLength(4); + expect(check(result, "server").status).toBe("ok"); + expect(check(result, "containers").status).toBe("ok"); + expect(check(result, "ingress").status).toBe("ok"); + expect(check(result, "http").status).toBe("ok"); + }); + + it("treats a missing server row as local ok", () => { + const result = evaluateHealth({ ...base, server: null }); + expect(check(result, "server").status).toBe("ok"); + }); + + it("fails on a stale remote heartbeat", () => { + const result = evaluateHealth({ ...base, server: { status: "connected", lastHeartbeatAgeMs: 120_000 } }); + expect(check(result, "server").status).toBe("fail"); + expect(check(result, "server").detail).toContain("Heartbeat stale"); + expect(result.overall).toBe("down"); + }); + + it("fails when a server never heartbeated", () => { + const result = evaluateHealth({ ...base, server: { status: "pending", lastHeartbeatAgeMs: null } }); + expect(check(result, "server").status).toBe("fail"); + expect(result.overall).toBe("down"); + }); + + it("passes on a fresh heartbeat", () => { + const result = evaluateHealth({ ...base, server: { status: "connected", lastHeartbeatAgeMs: 5_000 } }); + expect(check(result, "server").status).toBe("ok"); + }); + + it("warns when there are no deployments yet", () => { + const result = evaluateHealth({ ...base, hasDeployments: false, runningDeployments: 0 }); + expect(check(result, "containers").status).toBe("warn"); + expect(check(result, "containers").detail).toBe("No deployments yet"); + expect(result.overall).toBe("degraded"); + }); + + it("fails when deployments exist but none are running", () => { + const result = evaluateHealth({ ...base, runningDeployments: 0 }); + expect(check(result, "containers").status).toBe("fail"); + expect(result.overall).toBe("down"); + }); + + it("warns on route errors with the first error as detail", () => { + const result = evaluateHealth({ ...base, routeErrors: ["upstream unreachable", "second"] }); + expect(check(result, "ingress").status).toBe("warn"); + expect(check(result, "ingress").detail).toBe("upstream unreachable"); + expect(result.overall).toBe("degraded"); + }); + + it("is unknown when Loki is unavailable", () => { + const result = evaluateHealth({ ...base, http: { available: false, errorRate: null } }); + expect(check(result, "http").status).toBe("unknown"); + expect(check(result, "http").detail).toBe("Loki unavailable"); + expect(result.overall).toBe("degraded"); + }); + + it("is unknown when there is no traffic", () => { + const result = evaluateHealth({ ...base, http: { available: true, errorRate: null } }); + expect(check(result, "http").status).toBe("unknown"); + expect(check(result, "http").detail).toBe("No traffic"); + expect(result.overall).toBe("degraded"); + }); + + it("warns above a 5% error rate", () => { + const result = evaluateHealth({ ...base, http: { available: true, errorRate: 0.06 } }); + expect(check(result, "http").status).toBe("warn"); + expect(check(result, "http").detail).toBe("6.0% error rate"); + expect(result.overall).toBe("degraded"); + }); + + it("stays ok at or below a 5% error rate", () => { + const result = evaluateHealth({ ...base, http: { available: true, errorRate: 0.05 } }); + expect(check(result, "http").status).toBe("ok"); + expect(result.overall).toBe("healthy"); + }); + + it("lets fail win over warn for overall status", () => { + const result = evaluateHealth({ + ...base, + runningDeployments: 0, + routeErrors: ["broken"], + http: { available: true, errorRate: 0.5 }, + }); + expect(result.overall).toBe("down"); + }); +}); diff --git a/apps/api/src/monitoring/__tests__/http-error-rate.test.ts b/apps/api/src/monitoring/__tests__/http-error-rate.test.ts new file mode 100644 index 0000000..15e5038 --- /dev/null +++ b/apps/api/src/monitoring/__tests__/http-error-rate.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it } from "bun:test"; +import { parseHttpErrorRate } from "../http-error-rate"; + +const sample = [ + { metric: { status: "200" }, value: ["1758000000", "480"] }, + { metric: { status: "204" }, value: ["1758000000", "20"] }, + { metric: { status: "404" }, value: ["1758000000", "12"] }, + { metric: { status: "500" }, value: ["1758000000", "8"] }, +]; + +describe("parseHttpErrorRate", () => { + it("derives totals and rate from status buckets", () => { + const result = parseHttpErrorRate(sample, 3600); + expect(result.available).toBe(true); + expect(result.windowSeconds).toBe(3600); + expect(result.totalRequests).toBe(520); + expect(result.errorRequests).toBe(20); + expect(result.errorRate).toBeCloseTo(20 / 520, 10); + expect(result.byStatus).toHaveLength(4); + }); + + it("returns errorRate null on no traffic", () => { + const result = parseHttpErrorRate([], 300); + expect(result.available).toBe(true); + expect(result.totalRequests).toBe(0); + expect(result.errorRate).toBeNull(); + }); + + it("marks unavailable payloads instead of reporting 0", () => { + for (const payload of [null, undefined, "nope", { not: "an array" }]) { + const result = parseHttpErrorRate(payload, 300); + expect(result.available).toBe(false); + expect(result.errorRate).toBeNull(); + expect(result.totalRequests).toBe(0); + expect(result.byStatus).toHaveLength(0); + } + }); + + it("filters malformed bucket entries", () => { + const result = parseHttpErrorRate( + [ + { metric: { status: "200" }, value: ["1", "10"] }, + { metric: {}, value: ["1", "5"] }, + { metric: { status: "201" }, value: ["1", "not-a-number"] }, + null, + ], + 60, + ); + expect(result.available).toBe(true); + expect(result.totalRequests).toBe(10); + expect(result.byStatus).toHaveLength(1); + }); + + it("counts 4xx and 5xx as errors but not 3xx", () => { + const result = parseHttpErrorRate( + [ + { metric: { status: "301" }, value: ["1", "100"] }, + { metric: { status: "404" }, value: ["1", "1"] }, + ], + 60, + ); + expect(result.errorRequests).toBe(1); + expect(result.errorRate).toBeCloseTo(1 / 101, 10); + }); +}); diff --git a/apps/api/src/monitoring/__tests__/incident-policy.test.ts b/apps/api/src/monitoring/__tests__/incident-policy.test.ts new file mode 100644 index 0000000..a89fc9b --- /dev/null +++ b/apps/api/src/monitoring/__tests__/incident-policy.test.ts @@ -0,0 +1,143 @@ +import { describe, expect, it } from "bun:test"; +import { type Incident, DEFAULT_POLICY, backoffMs, commitAfterSend, decide } from "../incident-policy"; + +const P = DEFAULT_POLICY; +const MIN = 60_000; + +const breach = (state: Incident | null, now: number) => decide(state, { kind: "breach" }, now, P); +const clear = (state: Incident | null, now: number) => decide(state, { kind: "clear" }, now, P); +const noData = (state: Incident | null, now: number) => decide(state, { kind: "no_data" }, now, P); + +const firstSend = (now: number): Incident => { + const d = breach(null, now); + if (d.kind !== "send") throw new Error("expected send"); + return d.next; +}; + +describe("backoffMs", () => { + it("grows tenfold per send and clamps at the cap", () => { + expect(backoffMs(1, P)).toBe(5 * MIN); + expect(backoffMs(2, P)).toBe(50 * MIN); + expect(backoffMs(3, P)).toBe(500 * MIN); + expect(backoffMs(4, P)).toBe(P.capMs); + expect(backoffMs(5, P)).toBe(P.capMs); + expect(backoffMs(99, P)).toBe(P.capMs); + }); +}); + +describe("decide — breach", () => { + it("first breach sends immediately with the incident opened now", () => { + const d = breach(null, 1_000); + expect(d.kind).toBe("send"); + if (d.kind !== "send") return; + expect(d.next).toEqual({ + status: "active", + since: 1_000, + sends: 1, + nextDueAt: 1_000 + 5 * MIN, + clearSince: null, + }); + }); + + it("does not send again before the backoff is due", () => { + const state = firstSend(0); + expect(breach(state, 4 * MIN + 59_000).kind).toBe("noop"); + }); + + it("sends the next rung once due, stretching the gap tenfold", () => { + const state = firstSend(0); + const d = breach(state, 5 * MIN); + expect(d.kind).toBe("send"); + if (d.kind !== "send") return; + expect(d.next.sends).toBe(2); + expect(d.next.nextDueAt).toBe(5 * MIN + 50 * MIN); + }); + + it("sends exactly 3 emails over an 8h sustained breach", () => { + let state: Incident | null = null; + let sends = 0; + for (let minute = 0; minute <= 8 * 60; minute++) { + const d = breach(state, minute * MIN); + if (d.kind === "send") { + sends++; + state = d.next; + } else if (d.kind === "commit") { + state = d.next; + } + } + expect(sends).toBe(3); + }); +}); + +describe("decide — no_data and clear", () => { + it("no_data never fires and never mutates state", () => { + const state = firstSend(0); + expect(noData(state, 10 * MIN).kind).toBe("noop"); + expect(noData(null, 10 * MIN).kind).toBe("noop"); + }); + + it("clear with no incident is a noop", () => { + expect(clear(null, 0).kind).toBe("noop"); + }); + + it("clear starts a recovery grace window without sending", () => { + const state = firstSend(0); + const d = clear(state, 2 * MIN); + expect(d.kind).toBe("commit"); + if (d.kind !== "commit" || !d.next) return; + expect(d.next.clearSince).toBe(2 * MIN); + expect(d.next.sends).toBe(1); + }); + + it("clear inside the grace keeps the incident, past the grace deletes it", () => { + const state = { ...firstSend(0), clearSince: 1 * MIN } as Incident; + const inside = clear(state, 1 * MIN + P.recoveryGraceMs - 1_000); + expect(inside.kind).toBe("noop"); + const past = clear(state, 1 * MIN + P.recoveryGraceMs); + expect(past.kind).toBe("commit"); + if (past.kind !== "commit") return; + expect(past.next).toBeNull(); + }); + + it("a re-breach inside the grace cancels recovery and keeps the schedule", () => { + const state = { ...firstSend(0), clearSince: 2 * MIN } as Incident; + const d = breach(state, 3 * MIN); + expect(d.kind).toBe("commit"); + if (d.kind !== "commit" || !d.next) return; + expect(d.next.clearSince).toBeNull(); + expect(d.next.sends).toBe(1); + expect(d.next.since).toBe(0); + }); + + it("a sustained clear then fresh breach restarts at send #1", () => { + const state = firstSend(0); + const started = clear(state, 6 * MIN); + if (started.kind !== "commit" || !started.next) throw new Error("expected grace start"); + const recovered = clear(started.next, 6 * MIN + P.recoveryGraceMs); + if (recovered.kind !== "commit" || recovered.next !== null) throw new Error("expected recovery"); + const fresh = breach(null, 10 * MIN); + expect(fresh.kind).toBe("send"); + if (fresh.kind !== "send") return; + expect(fresh.next.sends).toBe(1); + expect(fresh.next.since).toBe(10 * MIN); + }); +}); + +describe("commitAfterSend", () => { + const decision = () => { + const d = breach(null, 0); + if (d.kind !== "send") throw new Error("expected send"); + return d; + }; + + it("persists the new incident only on a confirmed send", () => { + expect(commitAfterSend(null, decision(), { status: "sent" })).toEqual(decision().next); + }); + + it("keeps the prior state when the send failed or was skipped", () => { + const prior = firstSend(0); + expect(commitAfterSend(prior, decision(), { status: "failed", error: "x" })).toBe(prior); + expect(commitAfterSend(prior, decision(), { status: "skipped", reason: "no_recipient" })).toBe(prior); + expect(commitAfterSend(null, decision(), { status: "failed", error: "x" })).toBeNull(); + }); +}); diff --git a/apps/api/src/monitoring/__tests__/incident-tracker.test.ts b/apps/api/src/monitoring/__tests__/incident-tracker.test.ts new file mode 100644 index 0000000..59b0e9f --- /dev/null +++ b/apps/api/src/monitoring/__tests__/incident-tracker.test.ts @@ -0,0 +1,114 @@ +import { describe, expect, it } from "bun:test"; +import type { Incident } from "../incident-policy"; +import { createIncidentTracker, type IncidentStore } from "../incident-tracker"; + +const MIN = 60_000; + +const makeStore = () => { + const data = new Map(); + let loadError = false; + const store: IncidentStore = { + async load(id) { + if (loadError) throw new Error("redis down"); + return data.get(id) ?? null; + }, + async save(id, state) { + if (state) data.set(id, state); + else data.delete(id); + }, + }; + return { store, data, failLoad: () => (loadError = true) }; +}; + +const makeSend = () => { + const calls = { count: 0 }; + let failuresLeft = 0; + const send = async () => { + calls.count++; + if (failuresLeft > 0) { + failuresLeft--; + return { status: "failed", error: "smtp down" } as const; + } + return { status: "sent" } as const; + }; + return { calls, send, failNext: () => (failuresLeft = 1) }; +}; + +describe("incident tracker", () => { + it("sends the first breach and stores the incident", async () => { + const { store, data } = makeStore(); + const { calls, send } = makeSend(); + const tracker = createIncidentTracker(store); + + await tracker.step({ alertId: "a1", observation: { kind: "breach" }, now: 0, send }); + + expect(calls.count).toBe(1); + expect(data.get("a1")?.sends).toBe(1); + }); + + it("suppresses repeat sends until the backoff is due", async () => { + const { store, data } = makeStore(); + const { calls, send } = makeSend(); + const tracker = createIncidentTracker(store); + + await tracker.step({ alertId: "a1", observation: { kind: "breach" }, now: 0, send }); + await tracker.step({ alertId: "a1", observation: { kind: "breach" }, now: 4 * MIN, send }); + expect(calls.count).toBe(1); + + await tracker.step({ alertId: "a1", observation: { kind: "breach" }, now: 5 * MIN, send }); + expect(calls.count).toBe(2); + expect(data.get("a1")?.sends).toBe(2); + }); + + it("keeps the prior state when the send fails, then retries next tick", async () => { + const { store, data } = makeStore(); + const { calls, send, failNext } = makeSend(); + const tracker = createIncidentTracker(store); + + failNext(); + await tracker.step({ alertId: "a1", observation: { kind: "breach" }, now: 0, send }); + expect(calls.count).toBe(1); + expect(data.has("a1")).toBe(false); + + await tracker.step({ alertId: "a1", observation: { kind: "breach" }, now: MIN, send }); + expect(calls.count).toBe(2); + expect(data.get("a1")?.sends).toBe(1); + }); + + it("deletes the incident after a sustained clear", async () => { + const { store, data } = makeStore(); + const { send } = makeSend(); + const tracker = createIncidentTracker(store); + + await tracker.step({ alertId: "a1", observation: { kind: "breach" }, now: 0, send }); + await tracker.step({ alertId: "a1", observation: { kind: "clear" }, now: 2 * MIN, send }); + expect(data.get("a1")?.clearSince).toBe(2 * MIN); + + await tracker.step({ alertId: "a1", observation: { kind: "clear" }, now: 8 * MIN, send }); + expect(data.has("a1")).toBe(false); + }); + + it("refreshes the stored incident on a suppressed tick", async () => { + const { store, data } = makeStore(); + const { calls, send } = makeSend(); + const tracker = createIncidentTracker(store); + + await tracker.step({ alertId: "a1", observation: { kind: "breach" }, now: 0, send }); + const stored = data.get("a1"); + await tracker.step({ alertId: "a1", observation: { kind: "breach" }, now: MIN, send }); + expect(calls.count).toBe(1); + expect(data.get("a1")).toEqual(stored); + }); + + it("aborts before sending when the store is unavailable", async () => { + const { store, failLoad } = makeStore(); + const { calls, send } = makeSend(); + const tracker = createIncidentTracker(store); + + failLoad(); + await expect(tracker.step({ alertId: "a1", observation: { kind: "breach" }, now: 0, send })).rejects.toThrow( + "redis down", + ); + expect(calls.count).toBe(0); + }); +}); diff --git a/apps/api/src/monitoring/__tests__/slack-message.test.ts b/apps/api/src/monitoring/__tests__/slack-message.test.ts new file mode 100644 index 0000000..74c4b1c --- /dev/null +++ b/apps/api/src/monitoring/__tests__/slack-message.test.ts @@ -0,0 +1,96 @@ +import { describe, expect, test } from "bun:test"; +import { buildSlackMessage } from "../slack-message"; + +describe("buildSlackMessage", () => { + const findBlock = (blocks: any[], type: string) => blocks.find((b) => b.type === type); + + test("cpu alert formats threshold and value as percentages", () => { + const { text, blocks } = buildSlackMessage("cpu", "tabi", 80, 51.6); + expect(text).toBe("tabi: cpu alert"); + const fields = findBlock(blocks, "section").fields.map((f: any) => f.text); + expect(fields).toContain("*Type:* cpu"); + expect(fields).toContain("*Threshold:* 80%"); + expect(fields).toContain("*Current:* 51.6%"); + }); + + test("memory alert shows percentages for current value and threshold", () => { + const { blocks } = buildSlackMessage("memory", "tabi", 85, 91.5); + const fields = findBlock(blocks, "section").fields.map((f: any) => f.text); + expect(fields).toContain("*Current:* 91.5%"); + expect(fields).toContain("*Threshold:* 85%"); + }); + + test("memory containers render as percentages", () => { + const { blocks } = buildSlackMessage("memory", "tabi", 85, 91.5, { + containers: [{ name: "tabi-abc", value: 91.5 }], + }); + const section = blocks.find((b: any) => b.type === "section" && b.text?.text?.includes("tabi-abc")); + expect(section.text.text).toContain("`tabi-abc` 91.5%"); + }); + + test("downtime alert shows service down", () => { + const { blocks } = buildSlackMessage("downtime", "tabi", null, 1); + const fields = findBlock(blocks, "section").fields.map((f: any) => f.text); + expect(fields).toContain("*Type:* downtime"); + expect(fields).toContain("*Threshold:* N/A"); + expect(fields).toContain("*Current:* Service down"); + }); + + test("includes per-container values when details present", () => { + const { blocks } = buildSlackMessage("cpu", "tabi", 80, 51.6, { + containers: [ + { name: "tabi-abc", value: 51.6 }, + { name: "tabi-def", value: 49.2 }, + ], + }); + const section = blocks.find((b: any) => b.type === "section" && b.text?.text?.includes("tabi-abc")); + expect(section).toBeDefined(); + expect(section.text.text).toContain("`tabi-abc` 51.6%"); + expect(section.text.text).toContain("`tabi-def` 49.2%"); + }); + + test("scaling suggestion enable_autoscaling renders title and button", () => { + const { blocks } = buildSlackMessage("cpu", "tabi", 80, 51.6, { + scaling: { kind: "enable_autoscaling", url: "https://dequel.example/project/1?tab=scaling" }, + }); + const section = blocks.find((b: any) => b.type === "section" && b.text?.text?.includes("Autoscaling is off")); + expect(section).toBeDefined(); + expect(section.text.text).toContain("Enable autoscaling"); + const actions = blocks.filter((b: any) => b.type === "actions"); + const cta = actions.flatMap((a: any) => a.elements).find((e: any) => e.text.text === "Set up autoscaling"); + expect(cta).toBeDefined(); + expect(cta.url).toBe("https://dequel.example/project/1?tab=scaling"); + }); + + test("scaling suggestion increase_max_replicas shows replica limit and counts", () => { + const { blocks } = buildSlackMessage("cpu", "tabi", 80, 51.6, { + scaling: { kind: "increase_max_replicas", current: 3, maxReplicas: 3, url: "https://x/scaling" }, + }); + const section = blocks.find((b: any) => b.text?.text?.includes("Replica limit reached")); + expect(section.text.text).toContain("(3/3)"); + const cta = blocks + .filter((b: any) => b.type === "actions") + .flatMap((a: any) => a.elements) + .find((e: any) => e.text.text === "Adjust scaling"); + expect(cta).toBeDefined(); + }); + + test("projectUrl renders Open project button", () => { + const { blocks } = buildSlackMessage("cpu", "tabi", 80, 51.6, { + projectUrl: "https://dequel.example/project/42", + }); + const btn = blocks + .filter((b: any) => b.type === "actions") + .flatMap((a: any) => a.elements) + .find((e: any) => e.text.text === "Open project"); + expect(btn).toBeDefined(); + expect(btn.url).toBe("https://dequel.example/project/42"); + }); + + test("no scaling block and no project button without details", () => { + const { blocks } = buildSlackMessage("cpu", "tabi", 80, 51.6); + expect(blocks.filter((b: any) => b.type === "actions")).toHaveLength(0); + expect(blocks.some((b: any) => b.text?.text?.includes("Autoscaling"))).toBe(false); + expect(blocks).toHaveLength(2); + }); +}); diff --git a/apps/api/src/monitoring/__tests__/templates.test.ts b/apps/api/src/monitoring/__tests__/templates.test.ts new file mode 100644 index 0000000..fe441ad --- /dev/null +++ b/apps/api/src/monitoring/__tests__/templates.test.ts @@ -0,0 +1,160 @@ +import { describe, expect, it } from "bun:test"; +import { buildDeploymentFailureEmail, buildEmail, buildSmtpTestEmail } from "../templates"; + +describe("Email Templates", () => { + describe("buildDeploymentFailureEmail", () => { + it("generates a deployment failure email matching Dequel design system", () => { + const { subject, html } = buildDeploymentFailureEmail({ + projectName: "clinsight-be", + failureReason: "fix(docker): add build dependencies for pycairo compilation (#29) (#30)", + commitSha: "a1b2c3d4e5f67890", + sourceRef: "main", + finishedAt: "2026-09-24T01:43:00Z", + logsUrl: "https://dequel.app/project/proj-123?tab=deployments", + }); + + expect(subject).toBe("deploy failed for clinsight-be"); + expect(html).toContain("Dequel"); + expect(html).toContain("DEPLOY FAILED"); + expect(html).toContain("clinsight-be"); + expect(html).toContain("a1b2c3d4e5f6"); + expect(html).toContain("main"); + expect(html).toContain("fix(docker): add build dependencies"); + expect(html).toContain("View Logs"); + expect(html).toContain("https://dequel.app/project/proj-123?tab=deployments"); + expect(html).toContain("Learn more"); + expect(html).toContain("troubleshooting deploys on"); + expect(html).toContain('The Dequel team'); + // Check WebP Logo & HTML table Grid graphic presence + expect(html).toContain("logo_xzvwej.webp"); + expect(html).toContain("border-spacing:3px"); + }); + + it("handles missing optional context fields gracefully", () => { + const { subject, html } = buildDeploymentFailureEmail({ + projectName: "api-service", + failureReason: null, + commitSha: null, + sourceRef: "dev", + finishedAt: null, + }); + + expect(subject).toBe("deploy failed for api-service"); + expect(html).toContain("DEPLOY FAILED"); + expect(html).not.toContain("View Logs"); + }); + + it("escapes HTML in project name, failure reason and source ref", () => { + const { html } = buildDeploymentFailureEmail({ + projectName: "", + failureReason: "a < b & c > d", + commitSha: null, + sourceRef: "main", + finishedAt: null, + }); + + expect(html).not.toContain(""); + expect(html).toContain("<img src=x>"); + expect(html).toContain("<b>main</b>"); + expect(html).toContain("a < b & c > d"); + }); + }); + + describe("buildEmail (monitoring alerts)", () => { + it("generates CPU alert email", () => { + const { subject, html } = buildEmail("cpu", "web-app", 80, 94.2, { + containers: [{ name: "web-app-1", value: 94.2 }], + appUrl: "https://web-app.example.com", + }); + + expect(subject).toBe("High CPU on web-app (94%)"); + expect(html).toContain("CPU ALERT"); + expect(html).toContain("94.2%"); + expect(html).toContain("View Application"); + }); + + it("generates Memory alert email", () => { + const { subject, html } = buildEmail("memory", "worker-app", 85, 91.0); + + expect(subject).toBe("High memory on worker-app (91%)"); + expect(html).toContain("MEMORY ALERT"); + expect(html).toContain("91.0%"); + }); + + it("renders the enable-autoscaling suggestion", () => { + const { html } = buildEmail("cpu", "web-app", 80, 94.2, { + scaling: { kind: "enable_autoscaling", url: "https://dequel.local/project/p1?tab=scaling" }, + }); + + expect(html).toContain("Autoscaling is off"); + expect(html).toContain("Set up autoscaling"); + expect(html).not.toContain("SCALING_HTML"); + }); + + it("renders the increase-replicas suggestion", () => { + const { html } = buildEmail("memory", "web-app", 85, 91.0, { + scaling: { + kind: "increase_max_replicas", + current: 3, + maxReplicas: 3, + url: "https://dequel.local/project/p1?tab=scaling", + }, + }); + + expect(html).toContain("Replica limit reached"); + expect(html).toContain("(3/3)"); + expect(html).toContain("Adjust scaling"); + }); + + it("renders no scaling box without a suggestion", () => { + const { html } = buildEmail("cpu", "web-app", 80, 94.2); + expect(html).not.toContain("SCALING_HTML"); + expect(html).not.toContain("Autoscaling is off"); + }); + + it("generates downtime alert email", () => { + const { subject, html } = buildEmail("downtime", "db-proxy", null, 0, { + lastRunningAt: "2026-09-26T08:00:00Z", + logsUrl: "https://dequel.app/logs", + }); + + expect(subject).toBe("db-proxy is down"); + expect(html).toContain("SERVICE DOWN"); + expect(html).toContain("Offline"); + expect(html).toContain("View Logs"); + }); + + it("generates cert expiry alert email", () => { + const { subject, html } = buildEmail("cert_expiry", "my-domain.com", null, 14); + + expect(subject).toBe("SSL certificate for my-domain.com expires in 14 days"); + expect(html).toContain("CERTIFICATE EXPIRY"); + expect(html).toContain("14"); + }); + + it("generates default fallback alert email", () => { + const { subject, html } = buildEmail("disk_space", "storage-node", 90, 95); + + expect(subject).toBe("[Dequel] disk_space alert — storage-node"); + expect(html).toContain("ALERT"); + }); + + it("escapes the project name in alert email bodies", () => { + const { html } = buildEmail("cpu", "a & c", 80, 94.2); + + expect(html).not.toContain("a & c"); + expect(html).toContain("a<b> & c"); + }); + }); + + describe("buildSmtpTestEmail", () => { + it("generates a formatted SMTP test email", () => { + const { subject, html } = buildSmtpTestEmail(); + + expect(subject).toBe("[Dequel] SMTP Test Email"); + expect(html).toContain("SMTP TEST"); + expect(html).toContain("SMTP Transport Verified"); + expect(html).toContain("Dequel"); + }); + }); +}); diff --git a/apps/api/src/monitoring/alert-guard.ts b/apps/api/src/monitoring/alert-guard.ts new file mode 100644 index 0000000..b56f99b --- /dev/null +++ b/apps/api/src/monitoring/alert-guard.ts @@ -0,0 +1,31 @@ +export type ScalingSuggestion = + | { kind: "enable_autoscaling" } + | { kind: "increase_max_replicas"; current: number; maxReplicas: number }; + +export type ScalingContext = { + policy: { enabled: boolean; maxReplicas: number } | null; + cpuLimit: number | null; + currentReplicas: number | null; +}; + +export type ScalingGuard = + | { suppress: true; suggestion: null } + | { suppress: false; suggestion: ScalingSuggestion | null }; + +export const scalingGuard = (alertType: string, ctx: ScalingContext): ScalingGuard => { + if (alertType !== "cpu" && alertType !== "memory") { + return { suppress: false, suggestion: null }; + } + if (!ctx.policy || !ctx.policy.enabled || !ctx.cpuLimit || ctx.cpuLimit <= 0) { + return { suppress: false, suggestion: { kind: "enable_autoscaling" } }; + } + const current = ctx.currentReplicas ?? 1; + if (current >= ctx.policy.maxReplicas) { + return { + suppress: false, + suggestion: { kind: "increase_max_replicas", current, maxReplicas: ctx.policy.maxReplicas }, + }; + } + if (alertType === "memory") return { suppress: false, suggestion: null }; + return { suppress: true, suggestion: null }; +}; diff --git a/apps/api/src/monitoring/container-stats.ts b/apps/api/src/monitoring/container-stats.ts new file mode 100644 index 0000000..8db7779 --- /dev/null +++ b/apps/api/src/monitoring/container-stats.ts @@ -0,0 +1,87 @@ +import { getServerById } from "../db/repo"; +import { run } from "../orchestrator/runtime"; +import type { Deployment, Server } from "../types"; +import { dockerBin } from "../utils/docker-bin"; + +const AGENT_OFFLINE_MS = 90_000; + +export interface ContainerStats { + cpuPercent: number; + memoryMb: number; + memoryPercent: number | null; +} + +const parseMemToMb = (mem: string): number => { + const match = mem.match(/^([\d.]+)(\w+)$/); + if (!match) return 0; + const val = parseFloat(match[1]); + switch (match[2]) { + case "GiB": + case "GB": + return val * 1024; + case "MiB": + case "MB": + return val; + case "KiB": + case "KB": + return val / 1024; + default: + return val; + } +}; + +const parseStatsJson = (statsJson: string): ContainerStats | null => { + try { + const stats = JSON.parse(statsJson); + const memoryPercent = parseFloat(String(stats.MemPerc ?? "").replace("%", "")); + return { + cpuPercent: parseFloat(stats.CPUPerc?.replace("%", "") ?? "0"), + memoryMb: parseMemToMb(stats.MemUsage?.split("/")[0]?.trim() ?? "0B"), + memoryPercent: Number.isFinite(memoryPercent) ? memoryPercent : null, + }; + } catch { + return null; + } +}; + +const isAgentOffline = (server: Server | null): boolean => { + if (!server?.lastHeartbeat) return true; + return Date.now() - new Date(server.lastHeartbeat).getTime() > AGENT_OFFLINE_MS; +}; + +export const getDeploymentContainerStats = async (deployment: Deployment): Promise => { + const server = + deployment.serverId && deployment.serverId !== "local" + ? await getServerById(deployment.serverId).catch(() => null) + : null; + const mode = server?.mode ?? "local"; + if (mode === "agent") { + if (isAgentOffline(server)) return null; + const { agentStatsCache } = await import("../agents/stats-cache"); + const containers = await agentStatsCache.get(server!.id); + const stat = containers.get(deployment.containerName ?? ""); + return stat ? { cpuPercent: stat.cpuPercent, memoryMb: stat.memoryMb, memoryPercent: null } : null; + } + try { + const statsJson = await run( + dockerBin, + ["stats", "--no-stream", "--format", "{{json .}}", deployment.containerName ?? ""], + server, + ); + return parseStatsJson(statsJson); + } catch { + return null; + } +}; + +export const collectContainerStats = async ( + deployments: Deployment[], +): Promise<{ name: string; cpuPercent: number; memoryMb: number }[]> => { + const out: { name: string; cpuPercent: number; memoryMb: number }[] = []; + for (const dep of deployments) { + if (dep.status !== "running" || !dep.containerName) continue; + const stats = await getDeploymentContainerStats(dep); + if (stats) out.push({ name: dep.containerName, cpuPercent: stats.cpuPercent, memoryMb: stats.memoryMb }); + } + return out; +}; diff --git a/apps/api/src/monitoring/evaluator.ts b/apps/api/src/monitoring/evaluator.ts index b756452..17056b7 100644 --- a/apps/api/src/monitoring/evaluator.ts +++ b/apps/api/src/monitoring/evaluator.ts @@ -1,115 +1,34 @@ import { eq } from "drizzle-orm"; import Redis from "ioredis"; import { getDb } from "../db/db-provider"; -import { getProjectById, getServerById, listDeployments } from "../db/repo"; +import { getProjectById, getScalingPolicy, listDeployments } from "../db/repo"; import { alerts } from "../db/schema"; -import { run } from "../orchestrator/runtime"; -import type { Deployment, Server } from "../types"; +import { scalingEngine } from "../scaling/engine"; +import type { Deployment } from "../types"; import { config } from "../utils/config"; -import { dockerBin } from "../utils/docker-bin"; +import { appBaseUrl } from "../utils/routes"; +import { scalingGuard } from "./alert-guard"; +import { type ContainerStats, getDeploymentContainerStats } from "./container-stats"; +import { type Observation } from "./incident-policy"; +import { createIncidentTracker, createRedisIncidentStore } from "./incident-tracker"; import { sendNotification } from "./notifier"; +import type { AlertDetails } from "./templates"; -const NOTIFICATION_KEY = "dequel:alert:notified"; -const NOTIFICATION_COOLDOWN_MS = 300_000; // 5 min between same alert -const AGENT_OFFLINE_MS = 90_000; - -interface ContainerStats { - cpuPercent: number; - memoryMb: number; -} - -const parseMemToMb = (mem: string): number => { - const match = mem.match(/^([\d.]+)(\w+)$/); - if (!match) return 0; - const val = parseFloat(match[1]); - switch (match[2]) { - case "GiB": - case "GB": - return val * 1024; - case "MiB": - case "MB": - return val; - case "KiB": - case "KB": - return val / 1024; - default: - return val; - } -}; - -const parseStatsJson = (statsJson: string): ContainerStats | null => { - try { - const stats = JSON.parse(statsJson); - return { - cpuPercent: parseFloat(stats.CPUPerc?.replace("%", "") ?? "0"), - memoryMb: parseMemToMb(stats.MemUsage?.split("/")[0]?.trim() ?? "0B"), - }; - } catch { - return null; - } -}; - -const isAgentOffline = (server: Server | null): boolean => { - if (!server?.lastHeartbeat) return true; - return Date.now() - new Date(server.lastHeartbeat).getTime() > AGENT_OFFLINE_MS; -}; - -const getContainerStats = async (deployment: Deployment): Promise => { - const server = - deployment.serverId && deployment.serverId !== "local" - ? await getServerById(deployment.serverId).catch(() => null) - : null; - const mode = server?.mode ?? "local"; - if (mode === "agent") { - if (isAgentOffline(server)) return null; - const { agentStatsCache } = await import("../agents/stats-cache"); - const containers = await agentStatsCache.get(server!.id); - const stat = containers.get(deployment.containerName ?? ""); - return stat ? { cpuPercent: stat.cpuPercent, memoryMb: stat.memoryMb } : null; - } - try { - const statsJson = await run( - dockerBin, - ["stats", "--no-stream", "--format", "{{json .}}", deployment.containerName ?? ""], - server, - ); - return parseStatsJson(statsJson); - } catch { - return null; - } -}; - -const getMetricValue = async (alertType: string, _projectId: string, deployments: Deployment[]): Promise => { - if (alertType === "cpu" || alertType === "memory") { - let total = 0; - let count = 0; - for (const dep of deployments) { - if (dep.status !== "running" || !dep.containerName) continue; - const stats = await getContainerStats(dep); - if (stats) { - total += alertType === "cpu" ? stats.cpuPercent : stats.memoryMb; - count++; - } - } - return count > 0 ? total / count : 0; - } - if (alertType === "downtime") { - const running = deployments.filter((d) => d.status === "running"); - return running.length === 0 ? 1 : 0; - } - if (alertType === "error_rate") { - const failed = deployments.filter((d) => d.status === "failed"); - return failed.length > 0 ? failed.length : 0; - } - return 0; +const memoryPercentOf = (stats: ContainerStats, limitMb: number | null): number | null => { + if (stats.memoryPercent !== null) return stats.memoryPercent; + if (limitMb && limitMb > 0) return (stats.memoryMb / limitMb) * 100; + return null; }; class AlertEvaluator { private redis: Redis; + private incidents: ReturnType; private interval: ReturnType | null = null; + private ticking = false; constructor() { this.redis = new Redis(config.redisUrl, { maxRetriesPerRequest: null, enableOfflineQueue: false }); + this.incidents = createIncidentTracker(createRedisIncidentStore(this.redis)); } start() { @@ -128,6 +47,8 @@ class AlertEvaluator { } private async tick() { + if (this.ticking) return; + this.ticking = true; try { const db = await getDb(); const alertRows = await db.select().from(alerts).where(eq(alerts.enabled, true)).execute(); @@ -156,52 +77,144 @@ class AlertEvaluator { } } catch (err) { console.error("[Alerts] Tick error:", err); + } finally { + this.ticking = false; } } - private async evaluate(alert: any, project: { id: string; name: string }, deployments: Deployment[]) { - const currentValue = await getMetricValue(alert.type, project.id, deployments); - if (currentValue === 0) return; - + private async evaluate( + alert: any, + project: { + id: string; + name: string; + liveUrl?: string | null; + cpuLimit?: number | null; + memoryLimitMb?: number | null; + }, + deployments: Deployment[], + ) { const threshold = alert.threshold ?? (alert.type === "memory" ? 85 : 70); - let breached = false; - - switch (alert.type) { - case "cpu": - breached = currentValue > threshold; - break; - case "memory": - breached = currentValue > threshold; - break; - case "downtime": - breached = currentValue > 0; - break; - case "error_rate": - breached = currentValue > 0; - break; - case "cert_expiry": - break; + const memoryLimitMb = project.memoryLimitMb ?? null; + let { observation, currentValue } = await this.probe(alert.type, deployments, threshold, memoryLimitMb); + let scaling: AlertDetails["scaling"]; + + if (observation.kind === "breach") { + const guard = await this.evaluateScalingGuard(alert.type, project); + if (guard.suppress) { + observation = { kind: "no_data" }; + } else if (guard.suggestion) { + scaling = { + ...guard.suggestion, + url: `${appBaseUrl()}/project/${project.id}?tab=scaling`, + }; + } } - if (!breached) return; - - const notifiedKey = `${NOTIFICATION_KEY}:${alert.id}`; - const lastNotified = await this.redis - .get(notifiedKey) - .then((v) => (v ? Number(v) : 0)) - .catch(() => 0); - if (Date.now() - lastNotified < NOTIFICATION_COOLDOWN_MS) return; - - await sendNotification({ - channel: alert.channel, - destination: alert.destination, - projectName: project.name, - alertType: alert.type, - threshold, - currentValue, + await this.incidents.step({ + alertId: alert.id, + observation, + send: async () => { + const details = await this.buildDetails(alert.type, deployments, memoryLimitMb); + return sendNotification({ + channel: alert.channel, + destination: alert.destination, + projectName: project.name, + alertType: alert.type, + threshold, + currentValue, + details: { + ...details, + scaling, + projectUrl: `${appBaseUrl()}/project/${project.id}`, + }, + }); + }, }); + } + + private async evaluateScalingGuard(alertType: string, project: { id: string; cpuLimit?: number | null }) { + const policy = await getScalingPolicy(project.id).catch(() => null); + const canScale = !!policy?.enabled && !!project.cpuLimit && project.cpuLimit > 0; + const replicas = canScale ? await scalingEngine.getProjectReplicas(project.id) : null; + return scalingGuard(alertType, { + policy: policy ? { enabled: policy.enabled, maxReplicas: policy.maxReplicas } : null, + cpuLimit: project.cpuLimit ?? null, + currentReplicas: replicas?.current ?? null, + }); + } + + private async probe( + alertType: string, + deployments: Deployment[], + threshold: number, + memoryLimitMb: number | null, + ): Promise<{ observation: Observation; currentValue: number }> { + if (alertType === "downtime") { + if (!deployments.length) return { observation: { kind: "no_data" }, currentValue: 0 }; + const running = deployments.some((d) => d.status === "running"); + return { + observation: { kind: running ? "clear" : "breach" }, + currentValue: running ? 0 : 1, + }; + } + + if (alertType !== "cpu" && alertType !== "memory") { + return { observation: { kind: "no_data" }, currentValue: 0 }; + } + + let total = 0; + let count = 0; + for (const dep of deployments) { + if (dep.status !== "running" || !dep.containerName) continue; + const stats = await getDeploymentContainerStats(dep); + if (!stats) continue; + if (alertType === "cpu") { + total += stats.cpuPercent; + count++; + continue; + } + const percent = memoryPercentOf(stats, memoryLimitMb); + if (percent === null) continue; + total += percent; + count++; + } + if (count === 0) return { observation: { kind: "no_data" }, currentValue: 0 }; + + const value = total / count; + return { + observation: { kind: value > threshold ? "breach" : "clear" }, + currentValue: value, + }; + } + + private async buildDetails( + alertType: string, + deployments: Deployment[], + memoryLimitMb: number | null, + ): Promise { + const details: AlertDetails = {}; + + if (alertType === "cpu" || alertType === "memory") { + const containers: { name: string; value: number }[] = []; + for (const dep of deployments) { + if (dep.status !== "running" || !dep.containerName) continue; + const stats = await getDeploymentContainerStats(dep); + if (!stats) continue; + const value = alertType === "cpu" ? stats.cpuPercent : memoryPercentOf(stats, memoryLimitMb); + if (value === null) continue; + containers.push({ name: dep.containerName, value }); + } + details.containers = containers; + } + + if (alertType === "downtime") { + const lastFinished = deployments + .filter((d) => d.finishedAt) + .sort((a, b) => new Date(b.finishedAt!).getTime() - new Date(a.finishedAt!).getTime())[0]; + details.lastRunningAt = lastFinished?.finishedAt ?? null; + } - await this.redis.set(notifiedKey, String(Date.now())).catch(() => {}); + return details; } } diff --git a/apps/api/src/monitoring/failure-notifier.ts b/apps/api/src/monitoring/failure-notifier.ts new file mode 100644 index 0000000..19388c9 --- /dev/null +++ b/apps/api/src/monitoring/failure-notifier.ts @@ -0,0 +1,85 @@ +import { + claimFailureNotification, + listPendingFailureNotificationIds, + markFailureNotificationSent, +} from "../db/repo/deployment-events"; +import { onDeploymentFailed } from "../events"; +import { config } from "../utils/config"; +import { isSmtpConfigured, sendDeploymentFailureEmail } from "./notifier"; + +let queue: string[] = []; +const pending = new Set(); +let draining = false; +let sweepTimer: ReturnType | null = null; +let unsubscribe: (() => void) | null = null; +let configuredWarned = false; + +const enqueue = (eventId: string) => { + if (pending.has(eventId)) return; + pending.add(eventId); + queue.push(eventId); + void drain(); +}; + +const drain = async () => { + if (draining) return; + draining = true; + try { + while (queue.length > 0) { + const eventId = queue.shift()!; + pending.delete(eventId); + try { + if (!(await isSmtpConfigured())) { + if (!configuredWarned) { + console.warn("[FailureNotifier] SMTP not configured — leaving failure events pending"); + configuredWarned = true; + } + continue; + } + configuredWarned = false; + const ctx = await claimFailureNotification(eventId); + if (!ctx) continue; + const delivery = await sendDeploymentFailureEmail(ctx); + if (delivery.status === "sent") { + await markFailureNotificationSent(eventId); + console.log(`[FailureNotifier] failure email sent for ${ctx.deploymentId} (attempt ${ctx.attempt})`); + } else { + console.warn( + `[FailureNotifier] ${ctx.deploymentId}: ${delivery.status}`, + "reason" in delivery ? delivery.reason : delivery.error, + ); + } + } catch (err) { + console.error(`[FailureNotifier] ${eventId}:`, err); + } + } + } finally { + draining = false; + } +}; + +const sweep = async () => { + try { + const ids = await listPendingFailureNotificationIds(); + for (const id of ids) enqueue(id); + } catch (err) { + console.error("[FailureNotifier] sweep failed:", err); + } +}; + +export const startFailureNotifier = (): void => { + if (unsubscribe) return; + unsubscribe = onDeploymentFailed((signal) => enqueue(signal.eventId)); + sweepTimer = setInterval(() => void sweep(), config.failureSweepIntervalMs); + void sweep(); + console.log("[FailureNotifier] started"); +}; + +export const stopFailureNotifier = (): void => { + unsubscribe?.(); + unsubscribe = null; + if (sweepTimer) clearInterval(sweepTimer); + sweepTimer = null; + queue = []; + pending.clear(); +}; diff --git a/apps/api/src/monitoring/health.ts b/apps/api/src/monitoring/health.ts new file mode 100644 index 0000000..59a19e1 --- /dev/null +++ b/apps/api/src/monitoring/health.ts @@ -0,0 +1,66 @@ +import type { HealthCheck, OverallHealth } from "../types"; + +const SERVER_HEARTBEAT_FAIL_MS = 90_000; +const HTTP_ERROR_RATE_WARN = 0.05; + +export const evaluateHealth = (input: { + server: { status: string; lastHeartbeatAgeMs: number | null } | null; + runningDeployments: number; + hasDeployments: boolean; + routeErrors: string[]; + http: { available: boolean; errorRate: number | null }; +}): { overall: OverallHealth; checks: HealthCheck[] } => { + const checks: HealthCheck[] = []; + + if (!input.server) { + checks.push({ name: "server", status: "ok", detail: null }); + } else if (input.server.lastHeartbeatAgeMs === null) { + checks.push({ name: "server", status: "fail", detail: "Server never heartbeated" }); + } else if (input.server.lastHeartbeatAgeMs > SERVER_HEARTBEAT_FAIL_MS) { + checks.push({ + name: "server", + status: "fail", + detail: `Heartbeat stale for ${Math.round(input.server.lastHeartbeatAgeMs / 1000)}s`, + }); + } else { + checks.push({ name: "server", status: "ok", detail: null }); + } + + if (!input.hasDeployments) { + checks.push({ name: "containers", status: "warn", detail: "No deployments yet" }); + } else if (input.runningDeployments === 0) { + checks.push({ name: "containers", status: "fail", detail: "No running containers" }); + } else { + checks.push({ + name: "containers", + status: "ok", + detail: `${input.runningDeployments} running`, + }); + } + + if (input.routeErrors.length > 0) { + checks.push({ name: "ingress", status: "warn", detail: input.routeErrors[0] }); + } else { + checks.push({ name: "ingress", status: "ok", detail: null }); + } + + if (!input.http.available) { + checks.push({ name: "http", status: "unknown", detail: "Loki unavailable" }); + } else if (input.http.errorRate === null) { + checks.push({ name: "http", status: "unknown", detail: "No traffic" }); + } else if (input.http.errorRate > HTTP_ERROR_RATE_WARN) { + checks.push({ + name: "http", + status: "warn", + detail: `${(input.http.errorRate * 100).toFixed(1)}% error rate`, + }); + } else { + checks.push({ name: "http", status: "ok", detail: null }); + } + + let overall: OverallHealth = "healthy"; + if (checks.some((c) => c.status === "fail")) overall = "down"; + else if (checks.some((c) => c.status === "warn" || c.status === "unknown")) overall = "degraded"; + + return { overall, checks }; +}; diff --git a/apps/api/src/monitoring/http-error-rate.ts b/apps/api/src/monitoring/http-error-rate.ts new file mode 100644 index 0000000..a2c6364 --- /dev/null +++ b/apps/api/src/monitoring/http-error-rate.ts @@ -0,0 +1,46 @@ +import type { HttpErrorRate } from "../types"; +import { buildProjectRequestHostRegex, caddyRequestLogSelector, lokiInstantQuery } from "../utils/loki"; + +const unavailable = (windowSeconds: number): HttpErrorRate => ({ + source: "loki", + available: false, + windowSeconds, + totalRequests: 0, + errorRequests: 0, + errorRate: null, + byStatus: [], +}); + +export const parseHttpErrorRate = (result: unknown, windowSeconds: number): HttpErrorRate => { + if (!Array.isArray(result)) return unavailable(windowSeconds); + const byStatus: { status: string; count: number }[] = []; + for (const entry of result as any[]) { + const status = String(entry?.metric?.status ?? ""); + const count = Number(entry?.value?.[1]); + if (!status || !Number.isFinite(count)) continue; + byStatus.push({ status, count }); + } + const totalRequests = byStatus.reduce((sum, b) => sum + b.count, 0); + const errorRequests = byStatus.reduce((sum, b) => sum + (Number(b.status) >= 400 ? b.count : 0), 0); + return { + source: "loki", + available: true, + windowSeconds, + totalRequests, + errorRequests, + errorRate: totalRequests > 0 ? errorRequests / totalRequests : null, + byStatus, + }; +}; + +export const getHttpErrorRate = async (projectId: string, windowSeconds: number): Promise => { + try { + const hostRegex = await buildProjectRequestHostRegex(projectId); + const query = `sum by (status) (count_over_time(${caddyRequestLogSelector(hostRegex)} [${windowSeconds}s]))`; + const result = await lokiInstantQuery(query); + if (result === null) return unavailable(windowSeconds); + return parseHttpErrorRate(result, windowSeconds); + } catch { + return unavailable(windowSeconds); + } +}; diff --git a/apps/api/src/monitoring/incident-policy.ts b/apps/api/src/monitoring/incident-policy.ts new file mode 100644 index 0000000..7fee917 --- /dev/null +++ b/apps/api/src/monitoring/incident-policy.ts @@ -0,0 +1,74 @@ +import type { MailDelivery } from "../types"; + +export type Observation = { kind: "breach" } | { kind: "clear" } | { kind: "no_data" }; + +export type Incident = { + status: "active"; + since: number; + sends: number; + nextDueAt: number; + clearSince: number | null; +}; + +export type IncidentPolicy = { + baseMs: number; + factor: number; + capMs: number; + recoveryGraceMs: number; +}; + +export type Decision = { kind: "noop" } | { kind: "commit"; next: Incident | null } | { kind: "send"; next: Incident }; + +export const DEFAULT_POLICY: IncidentPolicy = { + baseMs: 300_000, + factor: 10, + capMs: 43_200_000, + recoveryGraceMs: 300_000, +}; + +export const backoffMs = (sends: number, p: IncidentPolicy): number => { + const exponent = Math.max(0, sends - 1); + if (exponent > 30) return p.capMs; + return Math.min(p.baseMs * p.factor ** exponent, p.capMs); +}; + +export const decide = (state: Incident | null, obs: Observation, now: number, p: IncidentPolicy): Decision => { + if (obs.kind === "no_data") return { kind: "noop" }; + + if (obs.kind === "clear") { + if (!state) return { kind: "noop" }; + if (state.clearSince === null) return { kind: "commit", next: { ...state, clearSince: now } }; + if (now - state.clearSince >= p.recoveryGraceMs) return { kind: "commit", next: null }; + return { kind: "noop" }; + } + + if (!state) { + return { + kind: "send", + next: { + status: "active", + since: now, + sends: 1, + nextDueAt: now + backoffMs(1, p), + clearSince: null, + }, + }; + } + + const reentered = state.clearSince !== null ? { ...state, clearSince: null } : state; + if (now >= reentered.nextDueAt) { + const sends = reentered.sends + 1; + return { + kind: "send", + next: { ...reentered, sends, nextDueAt: now + backoffMs(sends, p) }, + }; + } + if (reentered !== state) return { kind: "commit", next: reentered }; + return { kind: "noop" }; +}; + +export const commitAfterSend = ( + prior: Incident | null, + decision: Extract, + delivery: MailDelivery, +): Incident | null => (delivery.status === "sent" ? decision.next : prior); diff --git a/apps/api/src/monitoring/incident-tracker.ts b/apps/api/src/monitoring/incident-tracker.ts new file mode 100644 index 0000000..b7bc617 --- /dev/null +++ b/apps/api/src/monitoring/incident-tracker.ts @@ -0,0 +1,82 @@ +import type { Redis } from "ioredis"; +import type { MailDelivery } from "../types"; +import { + type Decision, + type Incident, + type IncidentPolicy, + type Observation, + DEFAULT_POLICY, + commitAfterSend, + decide, +} from "./incident-policy"; + +const KEY_PREFIX = "dequel:alert:incident:"; +const TTL_SECONDS = 604_800; + +export type StepInput = { + alertId: string; + observation: Observation; + now?: number; + send: () => Promise; +}; + +export type IncidentStore = { + load(alertId: string): Promise; + save(alertId: string, state: Incident | null): Promise; +}; + +export type IncidentTracker = { step(input: StepInput): Promise }; + +const isIncident = (value: unknown): value is Incident => { + if (!value || typeof value !== "object") return false; + const v = value as Record; + return ( + v.status === "active" && + typeof v.since === "number" && + typeof v.sends === "number" && + typeof v.nextDueAt === "number" && + (v.clearSince === null || typeof v.clearSince === "number") + ); +}; + +export const createRedisIncidentStore = (redis: Redis): IncidentStore => ({ + async load(alertId) { + const raw = await redis.get(KEY_PREFIX + alertId); + if (!raw) return null; + try { + const parsed = JSON.parse(raw); + return isIncident(parsed) ? parsed : null; + } catch { + return null; + } + }, + async save(alertId, state) { + const key = KEY_PREFIX + alertId; + if (!state) { + await redis.del(key); + return; + } + await redis.set(key, JSON.stringify(state), "EX", TTL_SECONDS); + }, +}); + +export const createIncidentTracker = ( + store: IncidentStore, + policy: IncidentPolicy = DEFAULT_POLICY, +): IncidentTracker => ({ + async step(input) { + const state = await store.load(input.alertId); + const decision: Decision = decide(state, input.observation, input.now ?? Date.now(), policy); + + if (decision.kind === "send") { + const delivery = await input.send(); + await store.save(input.alertId, commitAfterSend(state, decision, delivery)); + return; + } + if (decision.kind === "commit") { + await store.save(input.alertId, decision.next); + return; + } + if (state) await store.save(input.alertId, state); + }, +}); diff --git a/apps/api/src/monitoring/notifier.ts b/apps/api/src/monitoring/notifier.ts index 34df23e..81a3ed6 100644 --- a/apps/api/src/monitoring/notifier.ts +++ b/apps/api/src/monitoring/notifier.ts @@ -1,5 +1,10 @@ import nodemailer from "nodemailer"; -import { config } from "../utils/config"; +import { getSmtpSettings } from "../db/repo/settings"; +import type { FailureNotificationContext, MailDelivery } from "../types"; +import { appBaseUrl } from "../utils/routes"; +import { validateDestination } from "../utils/destination"; +import { buildSlackMessage } from "./slack-message"; +import { type AlertDetails, buildDeploymentFailureEmail, buildEmail } from "./templates"; type NotifyOpts = { channel: string; @@ -8,45 +13,66 @@ type NotifyOpts = { alertType: string; threshold: number | null; currentValue: number; + details?: AlertDetails; +}; + +type SmtpConfig = { + host: string; + port: number; + user: string; + pass: string; + from: string; +}; + +const loadSmtpConfig = async (): Promise => { + try { + const db = await getSmtpSettings(); + if (db?.host) { + return { host: db.host, port: db.port, user: db.user, pass: db.pass, from: db.fromAddress }; + } + } catch {} + return null; }; let transporter: nodemailer.Transporter | null = null; +let transporterKey = ""; -const getTransporter = () => { - if (transporter) return transporter; - if (!config.smtpHost) return null; +const getTransporter = async (smtp: SmtpConfig) => { + const key = `${smtp.host}:${smtp.port}:${smtp.user}:${smtp.pass}`; + if (transporter && transporterKey === key) return transporter; transporter = nodemailer.createTransport({ - host: config.smtpHost, - port: config.smtpPort, - secure: config.smtpPort === 465, - auth: config.smtpUser && config.smtpPass ? { user: config.smtpUser, pass: config.smtpPass } : undefined, + host: smtp.host, + port: smtp.port, + secure: smtp.port === 465, + auth: smtp.user && smtp.pass ? { user: smtp.user, pass: smtp.pass } : undefined, + connectionTimeout: 10_000, + greetingTimeout: 10_000, + socketTimeout: 30_000, }); + transporterKey = key; return transporter; }; -const subject = (projectName: string, alertType: string) => - `[Dequel] ${alertType.toUpperCase()} alert — ${projectName}`; - -const textBody = (projectName: string, alertType: string, threshold: number | null, currentValue: number) => - `Alert: ${projectName}\n\nType: ${alertType}\nThreshold: ${threshold ?? "N/A"}\nCurrent value: ${currentValue}\n\nThis is an automated notification from Dequel.`; - const sendEmail = async ( to: string, projectName: string, alertType: string, threshold: number | null, currentValue: number, + details?: AlertDetails, ) => { - const t = getTransporter(); - if (!t) { + const smtp = await loadSmtpConfig(); + if (!smtp) { console.warn(`[Notifier] SMTP not configured — skipping email to ${to}`); return; } + const { subject, html } = buildEmail(alertType, projectName, threshold, currentValue, details); + const t = await getTransporter(smtp); await t.sendMail({ - from: config.smtpFrom, + from: smtp.from, to, - subject: subject(projectName, alertType), - text: textBody(projectName, alertType, threshold, currentValue), + subject, + html, }); }; @@ -56,27 +82,17 @@ const sendSlack = async ( alertType: string, threshold: number | null, currentValue: number, + details?: AlertDetails, ) => { - const payload = { - text: subject(projectName, alertType), - blocks: [ - { type: "header", text: { type: "plain_text", text: `⚠️ Dequel Alert: ${projectName}` } }, - { - type: "section", - fields: [ - { type: "mrkdwn", text: `*Type:* ${alertType}` }, - { type: "mrkdwn", text: `*Threshold:* ${threshold ?? "N/A"}` }, - { type: "mrkdwn", text: `*Current:* ${currentValue}` }, - ], - }, - ], - }; + const payload = buildSlackMessage(alertType, projectName, threshold, currentValue, details); const res = await fetch(webhookUrl, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(payload), + redirect: "manual", + signal: AbortSignal.timeout(10_000), }); - if (!res.ok) console.warn(`[Notifier] Slack webhook returned ${res.status}`); + if (!res.ok) throw new Error(`Slack webhook returned ${res.status}`); }; const sendWebhook = async ( @@ -98,26 +114,60 @@ const sendWebhook = async ( method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(payload), + redirect: "manual", + signal: AbortSignal.timeout(10_000), }); - if (!res.ok) console.warn(`[Notifier] Webhook returned ${res.status}`); + if (!res.ok) throw new Error(`Webhook returned ${res.status}`); }; -export const sendNotification = async (opts: NotifyOpts): Promise => { - const { channel, destination, projectName, alertType, threshold, currentValue } = opts; +export const sendNotification = async (opts: NotifyOpts): Promise => { + const { channel, projectName, alertType, threshold, currentValue, details } = opts; + let destination = opts.destination; + if (!destination && channel === "email") { + const smtp = await loadSmtpConfig(); + destination = smtp?.from ?? null; + } if (!destination) { console.warn(`[Notifier] No destination for ${channel} alert — skipping`); - return; - } - const fn = - channel === "email" ? sendEmail : channel === "slack" ? sendSlack : channel === "webhook" ? sendWebhook : null; - if (!fn) { - console.warn(`[Notifier] Unknown channel: ${channel}`); - return; + return { status: "skipped", reason: "no_recipient" }; } try { - await fn(destination, projectName, alertType, threshold, currentValue); + if (channel === "slack" || channel === "webhook") { + const destinationError = await validateDestination(destination); + if (destinationError) throw new Error(destinationError); + } + if (channel === "email") { + await sendEmail(destination, projectName, alertType, threshold, currentValue, details); + } else if (channel === "slack") { + await sendSlack(destination, projectName, alertType, threshold, currentValue, details); + } else if (channel === "webhook") { + await sendWebhook(destination, projectName, alertType, threshold, currentValue); + } else { + console.warn(`[Notifier] Unknown channel: ${channel}`); + return { status: "failed", error: `unknown channel: ${channel}` }; + } console.log(`[Notifier] ${channel} alert sent to ${destination}`); + return { status: "sent" }; } catch (err) { console.error(`[Notifier] Failed to send ${channel} alert:`, err); + return { status: "failed", error: err instanceof Error ? err.message : String(err) }; + } +}; + +export const isSmtpConfigured = async (): Promise => (await loadSmtpConfig()) !== null; + +export const sendDeploymentFailureEmail = async (ctx: FailureNotificationContext): Promise => { + const smtp = await loadSmtpConfig(); + if (!smtp) return { status: "skipped", reason: "no_smtp" }; + const to = smtp.from; + if (!to) return { status: "skipped", reason: "no_recipient" }; + try { + const logsUrl = ctx.projectId ? `${appBaseUrl()}/project/${ctx.projectId}?tab=deployments` : undefined; + const { subject, html } = buildDeploymentFailureEmail({ ...ctx, logsUrl }); + const t = await getTransporter(smtp); + await t.sendMail({ from: smtp.from, to, subject, html }); + return { status: "sent" }; + } catch (err) { + return { status: "failed", error: err instanceof Error ? err.message : String(err) }; } }; diff --git a/apps/api/src/monitoring/project-status.ts b/apps/api/src/monitoring/project-status.ts new file mode 100644 index 0000000..9536729 --- /dev/null +++ b/apps/api/src/monitoring/project-status.ts @@ -0,0 +1,95 @@ +import { getProjectById, getServerById, listDeployments, listProjectEvents, listRoutesByDeployment } from "../db/repo"; +import { scalingEngine } from "../scaling/engine"; +import type { ProjectStatus, StatusFailure, StatusHistoryEntry } from "../types"; +import { collectContainerStats } from "./container-stats"; +import { getHttpErrorRate } from "./http-error-rate"; +import { evaluateHealth } from "./health"; + +export const getProjectStatus = async (projectId: string, windowSeconds: number): Promise => { + const project = await getProjectById(projectId); + if (!project) throw new Error(`Project ${projectId} not found`); + + const sinceIso = new Date(Date.now() - windowSeconds * 1000).toISOString(); + const deployments = await listDeployments(projectId); + const running = deployments.filter((d) => d.status === "running"); + + const [events, replicas, containers, http] = await Promise.all([ + listProjectEvents(projectId, sinceIso), + scalingEngine.getProjectReplicas(projectId), + collectContainerStats(deployments), + getHttpErrorRate(projectId, windowSeconds), + ]); + + const serverId = running[0]?.serverId ?? project.serverId ?? null; + const server = serverId && serverId !== "local" ? await getServerById(serverId).catch(() => null) : null; + + const routeErrors: string[] = []; + for (const dep of running) { + const routes = await listRoutesByDeployment(dep.id).catch(() => []); + for (const route of routes) { + if (route.lastError) routeErrors.push(route.lastError); + else if (route.status === "failed") routeErrors.push(`Route ${route.hostname} is ${route.status}`); + } + } + + const latestRunningAt = running.reduce((max, d) => { + const t = new Date(d.createdAt).getTime(); + return max === null || t > max ? t : max; + }, null); + const createdAtById = new Map(deployments.map((d) => [d.id, new Date(d.createdAt).getTime()])); + + const failures: StatusFailure[] = events + .filter((e) => e.type === "failed") + .map((e) => ({ + deploymentId: e.deploymentId, + message: e.message, + commitSha: e.commitSha, + sourceRef: e.sourceRef, + finishedAt: e.finishedAt, + recovered: + latestRunningAt !== null && + latestRunningAt > (createdAtById.get(e.deploymentId) ?? new Date(e.createdAt).getTime()), + notifiedAt: e.sentAt, + })); + + const history: StatusHistoryEntry[] = events.map((e) => ({ + deploymentId: e.deploymentId, + type: e.type, + message: e.message, + at: e.createdAt, + })); + + const health = evaluateHealth({ + server: server + ? { + status: server.status, + lastHeartbeatAgeMs: server.lastHeartbeat ? Date.now() - new Date(server.lastHeartbeat).getTime() : null, + } + : null, + runningDeployments: running.length, + hasDeployments: deployments.length > 0, + routeErrors, + http: { available: http.available, errorRate: http.errorRate }, + }); + + return { + projectId, + windowSeconds, + health, + failures, + history, + replicas, + resources: { + server: server + ? { + status: server.status, + cpuUsedPercent: server.cpuUsedPercent, + memoryTotalMb: server.memoryTotalMb, + lastHeartbeatAt: server.lastHeartbeat, + } + : null, + containers, + }, + http, + }; +}; diff --git a/apps/api/src/monitoring/slack-message.ts b/apps/api/src/monitoring/slack-message.ts new file mode 100644 index 0000000..d0a4f0d --- /dev/null +++ b/apps/api/src/monitoring/slack-message.ts @@ -0,0 +1,73 @@ +import type { AlertDetails } from "./templates"; + +const formatValue = (alertType: string, value: number): string => { + if (alertType === "downtime") return "Service down"; + return `${value.toFixed(1)}%`; +}; + +const formatThreshold = (alertType: string, threshold: number | null): string => { + if (alertType === "downtime" || threshold === null) return "N/A"; + return `${threshold}%`; +}; + +const formatContainer = (c: { name: string; value: number }): string => `\`${c.name}\` ${c.value.toFixed(1)}%`; + +type SlackBlock = + | { type: "header"; text: { type: string; text: string } } + | { + type: "section"; + text?: { type: string; text: string }; + fields?: { type: string; text: string }[]; + } + | { type: "actions"; elements: { type: string; text: { type: string; text: string }; url: string }[] }; + +export const buildSlackMessage = ( + alertType: string, + projectName: string, + threshold: number | null, + currentValue: number, + details?: AlertDetails, +): { text: string; blocks: SlackBlock[] } => { + const blocks: SlackBlock[] = [ + { type: "header", text: { type: "plain_text", text: `Dequel alert: ${projectName}` } }, + { + type: "section", + fields: [ + { type: "mrkdwn", text: `*Type:* ${alertType.replace("_", " ")}` }, + { type: "mrkdwn", text: `*Threshold:* ${formatThreshold(alertType, threshold)}` }, + { type: "mrkdwn", text: `*Current:* ${formatValue(alertType, currentValue)}` }, + ], + }, + ]; + const containers = details?.containers ?? []; + if (containers.length > 0) { + blocks.push({ + type: "section", + text: { + type: "mrkdwn", + text: containers.map((c) => `• ${formatContainer(c)}`).join("\n"), + }, + }); + } + if (details?.scaling) { + const s = details.scaling; + const title = s.kind === "enable_autoscaling" ? "Autoscaling is off" : "Replica limit reached"; + const body = + s.kind === "enable_autoscaling" + ? "Enable autoscaling so Dequel adds capacity automatically while load stays high." + : `At the replica limit (${s.current}/${s.maxReplicas}) — raise max replicas so autoscaling can add capacity.`; + const cta = s.kind === "enable_autoscaling" ? "Set up autoscaling" : "Adjust scaling"; + blocks.push({ type: "section", text: { type: "mrkdwn", text: `*${title}*\n${body}` } }); + blocks.push({ + type: "actions", + elements: [{ type: "button", text: { type: "plain_text", text: cta }, url: s.url }], + }); + } + if (details?.projectUrl) { + blocks.push({ + type: "actions", + elements: [{ type: "button", text: { type: "plain_text", text: "Open project" }, url: details.projectUrl }], + }); + } + return { text: `${projectName}: ${alertType} alert`, blocks }; +}; diff --git a/apps/api/src/monitoring/templates.ts b/apps/api/src/monitoring/templates.ts new file mode 100644 index 0000000..a4f0539 --- /dev/null +++ b/apps/api/src/monitoring/templates.ts @@ -0,0 +1,222 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import type { ScalingSuggestion } from "./alert-guard"; + +export interface AlertDetails { + containers?: { name: string; value: number }[]; + lastRunningAt?: string | null; + logsUrl?: string; + appUrl?: string; + projectUrl?: string; + scaling?: (ScalingSuggestion & { url: string }) | null; +} + +export const EMAIL_ATTACHMENTS = [ + { + filename: "logo.webp", + path: join(import.meta.dir, "templates", "logo.webp"), + cid: "dequel-logo", + }, +]; + +const THEME = { + cardBorder: "#e4e4e7", + text: "#18181b", + textMuted: "#71717a", + accent: "#ea580c", + link: "#7c3aed", + red: "#dc2626", + redBg: "#fef2f2", + amber: "#d97706", + amberBg: "#fff7ed", + green: "#059669", + greenBg: "#ecfdf5", +}; + +const loadHtml = (filename: string): string => { + const filepath = join(import.meta.dir, "templates", filename); + return readFileSync(filepath, "utf-8"); +}; + +const layoutHtml = loadHtml("layout.html"); +const deployFailureTpl = loadHtml("deploy-failure.html"); +const alertMetricTpl = loadHtml("alert-metric.html"); +const alertDowntimeTpl = loadHtml("alert-downtime.html"); +const alertCertExpiryTpl = loadHtml("alert-cert-expiry.html"); +const alertDefaultTpl = loadHtml("alert-default.html"); +const smtpTestTpl = loadHtml("smtp-test.html"); + +const truncated = (s: string, n = 160) => (s.length > n ? `${s.slice(0, n)}…` : s); + +const escapeHtml = (s: string): string => + s.replace(/[&<>"']/g, (c) => + c === "&" ? "&" : c === "<" ? "<" : c === ">" ? ">" : c === '"' ? """ : "'", + ); + +const renderBadge = (text: string, bg: string, textColor: string) => { + if (!text) return ""; + return `
+ ${text} +
`; +}; + +const renderButton = (href?: string, text?: string) => { + if (!href || !text) return ""; + return ``; +}; + +const renderRow = (label: string, value: string) => ` +
+ ${label} + ${value} +
`; + +const renderScalingSuggestion = (s: AlertDetails["scaling"]): string => { + if (!s) return ""; + const enable = s.kind === "enable_autoscaling"; + const title = enable ? "Autoscaling is off" : "Replica limit reached"; + const text = enable + ? "Enable autoscaling for this project and Dequel will add capacity automatically while load stays high." + : `This project is at its replica limit (${s.current}/${s.maxReplicas}). Raise max replicas so autoscaling can add capacity.`; + const cta = enable ? "Set up autoscaling" : "Adjust scaling"; + return `
+
${title}
+
${text}
+ ${cta} +
`; +}; + +const renderShell = (badgeText: string, badgeBg: string, badgeTextColor: string, bodyContent: string): string => { + const badgeHtml = renderBadge(badgeText, badgeBg, badgeTextColor); + return layoutHtml.replace("{{BADGE_HTML}}", badgeHtml).replace("{{BODY_CONTENT}}", bodyContent); +}; + +export const buildEmail = ( + alertType: string, + projectName: string, + threshold: number | null, + currentValue: number, + details?: AlertDetails, +): { subject: string; html: string } => { + const project = escapeHtml(projectName); + switch (alertType) { + case "cpu": + case "memory": { + const isCpu = alertType === "cpu"; + const unit = "%"; + const containers = details?.containers ?? []; + const containerRows = containers.map((c) => renderRow(c.name, `${c.value.toFixed(1)}${unit}`)).join(""); + const actionButton = details?.appUrl ? renderButton(details.appUrl, "View Application") : ""; + + const body = alertMetricTpl + .replace(/{{METRIC_TYPE}}/g, isCpu ? "CPU" : "Memory") + .replace(/{{PROJECT_NAME}}/g, project) + .replace(/{{CURRENT_VALUE}}/g, currentValue.toFixed(1)) + .replace(/{{THRESHOLD}}/g, String(threshold ?? "N/A")) + .replace(/{{UNIT}}/g, unit) + .replace("{{CONTAINER_ROWS}}", containerRows) + .replace("{{SCALING_HTML}}", renderScalingSuggestion(details?.scaling)) + .replace("{{ACTION_BUTTON}}", actionButton); + + return { + subject: `${isCpu ? "High CPU" : "High memory"} on ${projectName} (${currentValue.toFixed(0)}${unit})`, + html: renderShell(isCpu ? "CPU ALERT" : "MEMORY ALERT", THEME.amberBg, THEME.amber, body), + }; + } + + case "downtime": { + const lastRunningRow = details?.lastRunningAt + ? renderRow("Last running", new Date(details.lastRunningAt).toUTCString()) + : ""; + const actionButton = details?.logsUrl ? renderButton(details.logsUrl, "View Logs") : ""; + + const body = alertDowntimeTpl + .replace(/{{PROJECT_NAME}}/g, project) + .replace("{{LAST_RUNNING_ROW}}", lastRunningRow) + .replace("{{ACTION_BUTTON}}", actionButton); + + return { + subject: `${projectName} is down`, + html: renderShell("SERVICE DOWN", THEME.redBg, THEME.red, body), + }; + } + + case "cert_expiry": { + const daysRow = renderRow("Days remaining", String(currentValue)); + const actionButton = details?.appUrl ? renderButton(details.appUrl, "View Site") : ""; + + const body = alertCertExpiryTpl + .replace(/{{PROJECT_NAME}}/g, project) + .replace(/{{CURRENT_VALUE}}/g, String(currentValue)) + .replace("{{DAYS_ROW}}", daysRow) + .replace("{{ACTION_BUTTON}}", actionButton); + + return { + subject: `SSL certificate for ${projectName} expires in ${currentValue} days`, + html: renderShell("CERTIFICATE EXPIRY", THEME.amberBg, THEME.amber, body), + }; + } + + default: { + const detailsRows = [ + renderRow("Type", alertType), + renderRow("Threshold", threshold !== null ? String(threshold) : "N/A"), + renderRow("Current value", String(currentValue)), + ].join(""); + + const body = alertDefaultTpl.replace(/{{PROJECT_NAME}}/g, project).replace("{{DETAILS_ROWS}}", detailsRows); + + return { + subject: `[Dequel] ${alertType} alert — ${projectName}`, + html: renderShell("ALERT", THEME.amberBg, THEME.amber, body), + }; + } + } +}; + +export const buildDeploymentFailureEmail = (ctx: { + projectName: string; + failureReason: string | null; + commitSha: string | null; + sourceRef: string; + finishedAt: string | null; + logsUrl?: string; +}): { subject: string; html: string } => { + const commitItem = ctx.commitSha + ? `
  • Commit: ${ + ctx.failureReason + ? `${escapeHtml(truncated(ctx.failureReason, 140))} (${ctx.commitSha.slice(0, 12)})` + : `${ctx.commitSha.slice(0, 12)}` + }
    • ` + : ""; + const sourceItem = ctx.sourceRef + ? `
  • Source: ${escapeHtml(ctx.sourceRef)}
    • ` + : ""; + const reasonItem = + ctx.failureReason && !ctx.commitSha + ? `
  • Reason: ${escapeHtml(truncated(ctx.failureReason, 160))}
    • ` + : ""; + + const detailsItems = `${commitItem}${sourceItem}${reasonItem}`; + const actionButton = ctx.logsUrl ? renderButton(ctx.logsUrl, "View Logs") : ""; + + const body = deployFailureTpl + .replace(/{{PROJECT_NAME}}/g, escapeHtml(ctx.projectName)) + .replace(/{{LOGS_URL}}/g, ctx.logsUrl || "#") + .replace("{{DETAILS_ITEMS}}", detailsItems) + .replace("{{ACTION_BUTTON}}", actionButton); + + return { + subject: `deploy failed for ${ctx.projectName}`, + html: renderShell("DEPLOY FAILED", THEME.redBg, THEME.red, body), + }; +}; + +export const buildSmtpTestEmail = (): { subject: string; html: string } => { + return { + subject: "[Dequel] SMTP Test Email", + html: renderShell("SMTP TEST", THEME.greenBg, THEME.green, smtpTestTpl), + }; +}; diff --git a/apps/api/src/monitoring/templates/alert-cert-expiry.html b/apps/api/src/monitoring/templates/alert-cert-expiry.html new file mode 100644 index 0000000..2230b9b --- /dev/null +++ b/apps/api/src/monitoring/templates/alert-cert-expiry.html @@ -0,0 +1,8 @@ +

    + The SSL certificate for {{PROJECT_NAME}} expires in {{CURRENT_VALUE}} days. +

    +{{DAYS_ROW}} +{{ACTION_BUTTON}} +

    + Learn more about SSL certificates on Dequel. +

    diff --git a/apps/api/src/monitoring/templates/alert-default.html b/apps/api/src/monitoring/templates/alert-default.html new file mode 100644 index 0000000..ff916ca --- /dev/null +++ b/apps/api/src/monitoring/templates/alert-default.html @@ -0,0 +1,7 @@ +

    + An alert was triggered for {{PROJECT_NAME}}. +

    +{{DETAILS_ROWS}} +

    + Learn more about alerts on Dequel. +

    diff --git a/apps/api/src/monitoring/templates/alert-downtime.html b/apps/api/src/monitoring/templates/alert-downtime.html new file mode 100644 index 0000000..83a82ce --- /dev/null +++ b/apps/api/src/monitoring/templates/alert-downtime.html @@ -0,0 +1,12 @@ +

    + {{PROJECT_NAME}} has no running deployments. The service appears to be down. +

    +
    +
    Offline
    +
    0 running deployments
    +
    +{{LAST_RUNNING_ROW}} +{{ACTION_BUTTON}} +

    + Learn more about service availability on Dequel. +

    diff --git a/apps/api/src/monitoring/templates/alert-metric.html b/apps/api/src/monitoring/templates/alert-metric.html new file mode 100644 index 0000000..118a2a5 --- /dev/null +++ b/apps/api/src/monitoring/templates/alert-metric.html @@ -0,0 +1,13 @@ +

    + {{METRIC_TYPE}} usage on {{PROJECT_NAME}} has exceeded your configured threshold. +

    +
    +
    {{CURRENT_VALUE}}{{UNIT}}
    +
    Current {{METRIC_TYPE}} — threshold {{THRESHOLD}}{{UNIT}}
    +
    +{{CONTAINER_ROWS}} +{{SCALING_HTML}} +{{ACTION_BUTTON}} +

    + Learn more about monitoring on Dequel. +

    diff --git a/apps/api/src/monitoring/templates/deploy-failure.html b/apps/api/src/monitoring/templates/deploy-failure.html new file mode 100644 index 0000000..cfcb7dc --- /dev/null +++ b/apps/api/src/monitoring/templates/deploy-failure.html @@ -0,0 +1,11 @@ +

    + We encountered an error during the deploy process for {{PROJECT_NAME}}. + This means your deploy didn't complete successfully and your latest changes may not be live. +

    +
      + {{DETAILS_ITEMS}} +
    +{{ACTION_BUTTON}} +

    + Learn more about troubleshooting deploys on Dequel. +

    diff --git a/apps/api/src/monitoring/templates/layout.html b/apps/api/src/monitoring/templates/layout.html new file mode 100644 index 0000000..2fb0743 --- /dev/null +++ b/apps/api/src/monitoring/templates/layout.html @@ -0,0 +1,116 @@ + + + + + + + Dequel Notification + + + + + + + +
    + + + + + + + + + + + + + + + +
    + + + + + +
    + + + + + +
    + Dequel Logo + + Dequel +
    +     		+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
               
               
               
    +
    +
    + {{BADGE_HTML}} + {{BODY_CONTENT}} +
    +

    + The Dequel team +

    +

    + Don't want to receive these emails? You can change your notification settings for your workspace or just this service. +

    +

    + Need more help? Contact our Support team. +

    +
    + © 2026 Dequel +
    +
    +
    + + + \ No newline at end of file diff --git a/apps/api/src/monitoring/templates/logo.webp b/apps/api/src/monitoring/templates/logo.webp new file mode 100644 index 0000000..1df87a6 Binary files /dev/null and b/apps/api/src/monitoring/templates/logo.webp differ diff --git a/apps/api/src/monitoring/templates/smtp-test.html b/apps/api/src/monitoring/templates/smtp-test.html new file mode 100644 index 0000000..197ada5 --- /dev/null +++ b/apps/api/src/monitoring/templates/smtp-test.html @@ -0,0 +1,11 @@ +

    + This is a test email sent from your Dequel instance. + If you are receiving this, your SMTP settings have been configured successfully. +

    +
    +
    SMTP Transport Verified
    +
    Notifications and deployment alerts are ready to be delivered via email.
    +
    +

    + Learn more about managing notifications on Dequel. +

    diff --git a/apps/api/src/orchestrator/__tests__/pipeline-cleanup.test.ts b/apps/api/src/orchestrator/__tests__/pipeline-cleanup.test.ts index e9330d7..311c3ee 100644 --- a/apps/api/src/orchestrator/__tests__/pipeline-cleanup.test.ts +++ b/apps/api/src/orchestrator/__tests__/pipeline-cleanup.test.ts @@ -48,6 +48,8 @@ const mockDb = { listEnvironmentVariablesForDeploy: mock(() => Promise.resolve([])), listVolumes: mock(() => Promise.resolve([])), listDeployments: mock(() => Promise.resolve([])), + listProjectEvents: mock(() => Promise.resolve([])), + listRoutesByDeployment: mock(() => Promise.resolve([])), listAllDatabases: mock(() => Promise.resolve([])), deleteDeploymentAndLogs: mock(() => Promise.resolve()), getScalingPolicy: mock(() => Promise.resolve(null)), @@ -65,11 +67,14 @@ const mockDb = { updateDomainValidation: mock(() => Promise.resolve()), listDomains: mock(() => Promise.resolve([])), createDeploymentEvent: mock(() => Promise.resolve()), + recordDeploymentFailure: mock(() => Promise.resolve({ outcome: "recorded" })), + recordDeploymentCancellation: mock(() => Promise.resolve({ outcome: "recorded" })), }; mock.module(fileUrl("../../db/repo"), () => mockDb); mock.module(fileUrl("../runtime"), () => ({ + run: mock(() => Promise.resolve("")), deployContainer: mock(() => Promise.resolve({ containerName: "test-project-abc12345", diff --git a/apps/api/src/orchestrator/pipeline.ts b/apps/api/src/orchestrator/pipeline.ts index 7ebd033..d3d9222 100644 --- a/apps/api/src/orchestrator/pipeline.ts +++ b/apps/api/src/orchestrator/pipeline.ts @@ -13,10 +13,14 @@ import { listDeployments, listEnvironmentVariablesForDeploy, listVolumes, + recordDeploymentCancellation, + recordDeploymentFailure, updateDeploymentCommitSha, updateDeploymentStatus, } from "../db/repo"; import { deployments } from "../db/schema"; +import { config } from "../utils/config"; +import { CANCELLED_FAILURE_REASON } from "../utils/failure-outcome"; import { ensureProjectDashboard } from "../utils/grafana"; import { buildWithCompose, destroyComposeStack } from "./compose"; import { deployComposeStack } from "./compose-deploy"; @@ -88,13 +92,12 @@ export class PipelineOrchestrator { await Promise.all([ this.queue.remove(deploymentId), - updateDeploymentStatus(deploymentId, "failed", { failureReason: "Cancelled" }), - appendLog(deploymentId, "system", "Deployment cancelled by user"), - createDeploymentEvent({ + recordDeploymentCancellation({ deploymentId, - type: "cancelled", - message: "Deployment cancelled by user", + reason: CANCELLED_FAILURE_REASON, + source: "pipeline", }), + appendLog(deploymentId, "system", "Deployment cancelled by user"), ]); logBus.publish({ deploymentId, @@ -466,13 +469,7 @@ export class PipelineOrchestrator { const message = summarizeDeploymentError(error); console.error(`[Orchestrator] Deployment ${deploymentId} failed:`, error); await emitLog(deploymentId, "system", `Deployment failed: ${message}`); - await updateDeploymentStatus(deploymentId, "failed", { failureReason: message }); - await createDeploymentEvent({ - deploymentId, - type: "failed", - message, - metadata: { stage: "unknown" }, - }); + await recordDeploymentFailure({ deploymentId, reason: message, source: "pipeline" }); if (!deployed) { await emitLog(deploymentId, "system", "Cleaning up Docker resources from failed deployment"); @@ -604,7 +601,12 @@ export class PipelineOrchestrator { const message = summarizeDeploymentError(error); console.error(`[Orchestrator] Rollback of ${targetDeploymentId} failed:`, error); await emitLog(targetDeploymentId, "system", `Rollback failed: ${message}`); - await updateDeploymentStatus(targetDeploymentId, "failed", { failureReason: message }); + const r = await recordDeploymentFailure({ + deploymentId: targetDeploymentId, + reason: message, + source: "rollback", + }); + if (!r.claimed) await updateDeploymentStatus(targetDeploymentId, "failed", { failureReason: message }); throw error; } } diff --git a/apps/api/src/orchestrator/reconciliation.ts b/apps/api/src/orchestrator/reconciliation.ts index 278c59e..1261f71 100644 --- a/apps/api/src/orchestrator/reconciliation.ts +++ b/apps/api/src/orchestrator/reconciliation.ts @@ -1,6 +1,7 @@ import { and, eq, lt } from "drizzle-orm"; import { getDb } from "../db/db-provider"; -import { agentJobs, deployments, servers } from "../db/schema"; +import { recordDeploymentFailure } from "../db/repo"; +import { agentJobs, servers } from "../db/schema"; const LEASE_RECOVERY_INTERVAL_MS = 30_000; const STALE_AGENT_THRESHOLD_MS = 5 * 60 * 1000; @@ -113,15 +114,11 @@ const cleanAbandonedJobs = async () => { .execute(); if (job.deploymentId) { - await db - .update(deployments) - .set({ - status: "failed", - failureReason: "Agent job abandoned", - finishedAt: new Date(), - }) - .where(eq(deployments.id, job.deploymentId)) - .execute(); + await recordDeploymentFailure({ + deploymentId: job.deploymentId, + reason: "Agent job abandoned", + source: "reconciler", + }).catch((err) => console.error(`[Reconciliation] Failed to record failure for ${job.deploymentId}:`, err)); } console.log(`[Reconciliation] Cleaned abandoned job ${job.id} (started ${job.startedAt})`); diff --git a/apps/api/src/scaling/engine.ts b/apps/api/src/scaling/engine.ts index c4a7b48..a4faafc 100644 --- a/apps/api/src/scaling/engine.ts +++ b/apps/api/src/scaling/engine.ts @@ -6,6 +6,7 @@ import type { Server } from "../types"; import { config } from "../utils/config"; import { DEQUEL_MANAGED_LABEL } from "../utils/dequel-labels"; import { dockerBin } from "../utils/docker-bin"; +import { slugify } from "../utils/routes"; import { execDockerSshCommand, syncRemoteCaddyRoute } from "../utils/ssh"; import { run, tryRun } from "./docker-utils"; @@ -232,7 +233,10 @@ class ScalingEngine { const containers = await agentStatsCache.get(target.server!.id); let count = 0; for (const stat of containers.values()) { - if (stat.replica && dep.id && stat.deploymentId === dep.id) count++; + if (!stat.replica || !dep.id) continue; + if (stat.containerName === dep.containerName || stat.containerName.startsWith(`deploy-${dep.id}`)) { + count++; + } } if (containers.has(dep.containerName ?? "")) count++; return Math.max(1, count); @@ -242,7 +246,7 @@ class ScalingEngine { "ps", "-q", "--filter", - "label=com.dequel.managed=1", + `label=${DEQUEL_MANAGED_LABEL}`, "--filter", `name=deploy-${dep.id}-replica-`, ]); @@ -269,7 +273,10 @@ class ScalingEngine { const containers = new Set(); for (const m of matches) { const parts = m.replace("reverse_proxy", "").trim().split(/\s+/); - for (const p of parts) containers.add(p.split(":")[0]); + for (const p of parts) { + if (!p.includes(":") || p.startsWith("{")) continue; + containers.add(p.split(":")[0]); + } } return Math.max(1, containers.size); } catch { @@ -277,6 +284,21 @@ class ScalingEngine { } } + async getProjectReplicas(projectId: string): Promise<{ current: number } | null> { + try { + const deployments = await listDeployments(projectId); + const runningDep = deployments.find((d) => d.status === "running") ?? deployments.find((d) => d.containerName); + if (!runningDep) return null; + const project = await getProjectById(projectId); + const target = await this.resolveTarget(runningDep); + const slug = project ? slugify(project.name) : projectId; + return { current: await this.getCurrentReplicas(slug, target, runningDep) }; + } catch (err) { + console.warn(`[Scaling] Failed to get replicas for project ${projectId}:`, err); + return null; + } + } + private async scaleUp( dep: { id: string; projectId: string | null; containerName: string | null; serverId?: string | null }, maxReplicas: number, @@ -499,6 +521,7 @@ class ScalingEngine { await import("../utils/ingress"); const ingressServer = await getIngressServer(); const viaIngress = shouldRouteViaIngress(target.server ?? null, ingressServer); + const { caddyReverseProxy, caddySite } = await import("../utils/caddy-site"); const caddySnippet = viaIngress ? projectServerSite( `${slug}.${baseDomain}`, @@ -506,7 +529,7 @@ class ScalingEngine { targets.map((t) => t.split(":")[0]), true, ) - : `${slug}.${baseDomain} {\n reverse_proxy ${targets.join(" ")} {\n header_up Host {upstream_hostport}\n }\n}\n`; + : caddySite(`${slug}.${baseDomain}`, caddyReverseProxy(targets.join(" "))); if (target.mode === "ssh") { await syncRemoteCaddyRoute(target.server!, `${slug}.caddy`, caddySnippet); diff --git a/apps/api/src/types.ts b/apps/api/src/types.ts index 4f1ef2b..89f9b99 100644 --- a/apps/api/src/types.ts +++ b/apps/api/src/types.ts @@ -16,7 +16,7 @@ export type SslStatus = "pending" | "provisioned" | "failed"; export type ServerStatus = "pending" | "connected" | "disconnected" | "failed"; export type ServerMode = "local" | "ssh" | "agent" | "docker_tcp"; export type AlertChannel = "email" | "slack" | "webhook"; -export type AlertType = "cpu" | "memory" | "error_rate" | "downtime" | "cert_expiry"; +export type AlertType = "cpu" | "memory" | "downtime" | "cert_expiry"; export interface Project { id: string; @@ -40,9 +40,7 @@ export interface Project { installCommand: string | null; outputDir: string | null; startCommand: string | null; - githubTokenEncrypted: string | null; - githubTokenIv: string | null; - githubTokenTag: string | null; + hasGithubToken: boolean; createdAt: string; updatedAt: string; } @@ -290,6 +288,92 @@ export interface CreateAlertInput { destination?: string; } +export type FailureSource = "pipeline" | "rollback" | "ssh" | "agent" | "job-channel" | "reconciler" | "dispatch"; + +export interface RecordFailureInput { + deploymentId: string; + reason: string; + source: FailureSource; + cancel?: boolean; +} + +export interface RecordFailureOutcome { + claimed: boolean; + eventId: string | null; +} + +export interface FailureNotificationContext { + eventId: string; + deploymentId: string; + projectId: string | null; + projectName: string; + failureReason: string | null; + commitSha: string | null; + sourceRef: string; + finishedAt: string | null; + attempt: number; +} + +export type MailDelivery = + | { status: "sent" } + | { status: "skipped"; reason: "no_smtp" | "no_recipient" } + | { status: "failed"; error: string }; + +export type HealthStatus = "ok" | "warn" | "fail" | "unknown"; +export type OverallHealth = "healthy" | "degraded" | "down"; + +export interface HealthCheck { + name: "server" | "containers" | "ingress" | "http"; + status: HealthStatus; + detail: string | null; +} + +export interface StatusFailure { + deploymentId: string; + message: string | null; + commitSha: string | null; + sourceRef: string; + finishedAt: string | null; + recovered: boolean; + notifiedAt: string | null; +} + +export interface StatusHistoryEntry { + deploymentId: string; + type: string; + message: string | null; + at: string; +} + +export interface HttpErrorRate { + source: "loki"; + available: boolean; + windowSeconds: number; + totalRequests: number; + errorRequests: number; + errorRate: number | null; + byStatus: { status: string; count: number }[]; +} + +export interface ProjectStatus { + projectId: string; + windowSeconds: number; + health: { overall: OverallHealth; checks: HealthCheck[] }; + failures: StatusFailure[]; + history: StatusHistoryEntry[]; + replicas: { current: number } | null; + resources: { + server: { + status: string; + cpuUsedPercent: number | null; + memoryTotalMb: number | null; + lastHeartbeatAt: string | null; + } | null; + containers: { name: string; cpuPercent: number; memoryMb: number }[]; + }; + http: HttpErrorRate; +} + export interface Deployment { id: string; projectId: string | null; diff --git a/apps/api/src/utils/__tests__/destination.test.ts b/apps/api/src/utils/__tests__/destination.test.ts new file mode 100644 index 0000000..f9ca856 --- /dev/null +++ b/apps/api/src/utils/__tests__/destination.test.ts @@ -0,0 +1,24 @@ +import { describe, expect, it, mock } from "bun:test"; +import { validateDestination } from "../destination"; + +mock.restore(); + +describe("validateDestination", () => { + it("rejects non-http protocols and credentials", async () => { + expect(await validateDestination("ftp://example.com/x")).toMatch(/http or https/); + expect(await validateDestination("https://user:pass@example.com/")).toMatch(/credentials/); + expect(await validateDestination("not-a-url")).toMatch(/valid URL/); + }); + + it("rejects literal private IPs without DNS", async () => { + expect(await validateDestination("http://10.0.0.5/hook")).toMatch(/public address/); + expect(await validateDestination("http://192.168.1.10/hook")).toMatch(/public address/); + expect(await validateDestination("http://127.0.0.1:9/hook")).toMatch(/public address/); + expect(await validateDestination("http://[::1]/hook")).toMatch(/public address/); + }); + + it("does not reject domain names before DNS resolution", async () => { + const res = await validateDestination("https://webhook.site/db-test-123"); + expect(res === null || res === "destination hostname could not be resolved").toBe(true); + }); +}); diff --git a/apps/api/src/utils/__tests__/telemetry.test.ts b/apps/api/src/utils/__tests__/telemetry.test.ts new file mode 100644 index 0000000..99c9ea2 --- /dev/null +++ b/apps/api/src/utils/__tests__/telemetry.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, it, spyOn } from "bun:test"; +import { captureTelemetry, getInstanceId } from "../telemetry"; + +describe("Telemetry Utility", () => { + it("generates and persists an instance ID", () => { + const instanceId = getInstanceId(); + expect(typeof instanceId).toBe("string"); + expect(instanceId.length).toBeGreaterThan(0); + expect(getInstanceId()).toBe(instanceId); + }); + + it("respects DEQUEL_TELEMETRY_DISABLED=1", async () => { + process.env.DEQUEL_TELEMETRY_DISABLED = "1"; + const fetchSpy = spyOn(globalThis, "fetch"); + + await captureTelemetry("test_event", { foo: "bar" }); + expect(fetchSpy).not.toHaveBeenCalled(); + + delete process.env.DEQUEL_TELEMETRY_DISABLED; + fetchSpy.mockRestore(); + }); +}); diff --git a/apps/api/src/utils/caddy-site.ts b/apps/api/src/utils/caddy-site.ts new file mode 100644 index 0000000..6898bc7 --- /dev/null +++ b/apps/api/src/utils/caddy-site.ts @@ -0,0 +1,10 @@ +export const CADDY_ACCESS_LOG_BLOCK = ` log { + output stdout + format json + }`; + +export const caddySite = (hosts: string, reverseProxy: string): string => + `${hosts} {\n${CADDY_ACCESS_LOG_BLOCK}\n${reverseProxy}\n}\n`; + +export const caddyReverseProxy = (targets: string): string => + ` reverse_proxy ${targets} {\n header_up Host {upstream_hostport}\n }`; diff --git a/apps/api/src/utils/config-loader.ts b/apps/api/src/utils/config-loader.ts index 58ac104..02a81a6 100644 --- a/apps/api/src/utils/config-loader.ts +++ b/apps/api/src/utils/config-loader.ts @@ -18,11 +18,6 @@ export interface FileConfig { queueConcurrency?: number; queueRetryMax?: number; queueRetryBaseMs?: number; - smtpHost?: string; - smtpPort?: number; - smtpUser?: string; - smtpPass?: string; - smtpFrom?: string; alertEvalIntervalMs?: number; githubClientId?: string; githubClientSecret?: string; diff --git a/apps/api/src/utils/config.ts b/apps/api/src/utils/config.ts index 2acb8a2..1067759 100644 --- a/apps/api/src/utils/config.ts +++ b/apps/api/src/utils/config.ts @@ -16,6 +16,7 @@ const SYSTEM = { dockerNetwork: "dequel_net", buildkitHost: "tcp://buildkit:1234", redisUrl: "redis://redis:6379", + dequelSourceRepoUrl: "https://github.com/Lftobs/dequel.git", } as const; export const config = { @@ -29,15 +30,13 @@ export const config = { agentTunnelUrl: withFile("AGENT_TUNNEL_URL", ""), appInternalPort: withFile("APP_INTERNAL_PORT", "17476", Number), envEncryptionKey: withFile("ENV_ENCRYPTION_KEY", "dev-env-key-change-me"), + dequelSlackWebhookUrl: withFile("DEQUEL_SLACK_WEBHOOK_URL", ""), + dequelSlackChannel: withFile("DEQUEL_SLACK_CHANNEL", ""), queueConcurrency: withFile("QUEUE_CONCURRENCY", "3", Number), queueRetryMax: withFile("QUEUE_RETRY_MAX", "5", Number), queueRetryBaseMs: withFile("QUEUE_RETRY_BASE_MS", "5000", Number), - smtpHost: withFile("SMTP_HOST", ""), - smtpPort: withFile("SMTP_PORT", "587", Number), - smtpUser: withFile("SMTP_USER", ""), - smtpPass: withFile("SMTP_PASS", ""), - smtpFrom: withFile("SMTP_FROM", "dequel@localhost"), alertEvalIntervalMs: withFile("ALERT_EVAL_INTERVAL_MS", "60000", Number), + failureSweepIntervalMs: withFile("FAILURE_SWEEP_INTERVAL_MS", "60000", Number), githubClientId: withFile("GITHUB_CLIENT_ID", ""), githubClientSecret: withFile("GITHUB_CLIENT_SECRET", ""), githubAppName: withFile("GITHUB_APP_NAME", "Dequel"), diff --git a/apps/api/src/utils/destination.ts b/apps/api/src/utils/destination.ts new file mode 100644 index 0000000..1327b93 --- /dev/null +++ b/apps/api/src/utils/destination.ts @@ -0,0 +1,55 @@ +import { lookup } from "node:dns/promises"; + +const isBlockedIpv4 = (ip: string): boolean => { + const parts = ip.split(".").map(Number); + if (parts.length !== 4 || parts.some((n) => !Number.isInteger(n) || n < 0 || n > 255)) return true; + const [a, b] = parts; + if (a === 0 || a === 10 || a === 127) return true; + if (a === 100 && b >= 64 && b <= 127) return true; + if (a === 169 && b === 254) return true; + if (a === 172 && b >= 16 && b <= 31) return true; + if (a === 192 && b === 168) return true; + if (a >= 224) return true; + return false; +}; + +const isBlockedIpv6 = (ip: string): boolean => { + const addr = ip.toLowerCase().split("%")[0]; + if (addr === "::" || addr === "::1") return true; + if (addr.startsWith("::ffff:")) return isBlockedIpv4(addr.slice(7)); + if (/^fe[89ab]/.test(addr)) return true; + if (/^f[cd]/.test(addr)) return true; + if (addr.startsWith("ff")) return true; + return false; +}; + +export const isBlockedAddress = (ip: string): boolean => { + const clean = ip.replace(/^\[|\]$/g, ""); + return clean.includes(":") ? isBlockedIpv6(clean) : isBlockedIpv4(clean); +}; + +const PUBLIC_REQUIRED = "destination must resolve to a public address"; + +export const validateDestination = async (raw: string): Promise => { + let url: URL; + try { + url = new URL(raw); + } catch { + return "destination must be a valid URL"; + } + if (url.protocol !== "http:" && url.protocol !== "https:") return "destination must use http or https"; + if (url.username || url.password) return "destination must not contain credentials"; + const hostname = url.hostname; + if (!hostname) return "destination must be a valid URL"; + const looksLikeIp = /^\d{1,3}(\.\d{1,3}){3}$/.test(hostname) || hostname.includes(":"); + if (looksLikeIp && isBlockedAddress(hostname)) return PUBLIC_REQUIRED; + let addresses: { address: string }[]; + try { + addresses = await lookup(hostname, { all: true }); + } catch { + return "destination hostname could not be resolved"; + } + if (!addresses.length) return "destination hostname could not be resolved"; + if (addresses.some((entry) => isBlockedAddress(entry.address))) return PUBLIC_REQUIRED; + return null; +}; diff --git a/apps/api/src/utils/domain-verifier.ts b/apps/api/src/utils/domain-verifier.ts index dc5a08a..05c7d2f 100644 --- a/apps/api/src/utils/domain-verifier.ts +++ b/apps/api/src/utils/domain-verifier.ts @@ -5,6 +5,7 @@ import { getDb } from "../db/db-provider"; import { getProjectById, listDomains, listEnvironmentVariablesForDeploy, updateDomainValidation } from "../db/repo"; import { domains } from "../db/schema"; import { reloadCaddy } from "../orchestrator/runtime"; +import { caddyReverseProxy, caddySite } from "./caddy-site"; import { config } from "./config"; import { resolveServerIp, validateDomain } from "./dns"; @@ -205,9 +206,7 @@ export const buildCaddySnippet = async ( } } const tPort = d.targetPort || port; - customBlocks.push( - `${entryDomain} {\n log {\n output stdout\n format json\n }\n reverse_proxy ${targetContainer}:${tPort} {\n header_up Host {upstream_hostport}\n }\n}\n`, - ); + customBlocks.push(caddySite(entryDomain, caddyReverseProxy(`${targetContainer}:${tPort}`))); } else { if (!defaultDomains.includes(entryDomain)) defaultDomains.push(entryDomain); } @@ -246,7 +245,7 @@ export const buildCaddySnippet = async ( } } - const primaryBlock = `${defaultDomains.join(", ")} {\n log {\n output stdout\n format json\n }\n reverse_proxy ${containerName}:${port} {\n header_up Host {upstream_hostport}\n }\n}\n`; + const primaryBlock = caddySite(defaultDomains.join(", "), caddyReverseProxy(`${containerName}:${port}`)); return [primaryBlock, ...customBlocks].join("\n"); }; diff --git a/apps/api/src/utils/failure-outcome.ts b/apps/api/src/utils/failure-outcome.ts new file mode 100644 index 0000000..9ea15f2 --- /dev/null +++ b/apps/api/src/utils/failure-outcome.ts @@ -0,0 +1,6 @@ +export const CANCELLED_FAILURE_REASON = "Cancelled"; + +export type FailureOutcome = "failure" | "cancelled"; + +export const classifyFailureOutcome = (reason: string | null | undefined): FailureOutcome => + reason === CANCELLED_FAILURE_REASON ? "cancelled" : "failure"; diff --git a/apps/api/src/utils/grafana.ts b/apps/api/src/utils/grafana.ts index b954b0b..2e9fd86 100644 --- a/apps/api/src/utils/grafana.ts +++ b/apps/api/src/utils/grafana.ts @@ -1,5 +1,6 @@ -import { listDomains } from "../db/repo"; import { config } from "./config"; +import { buildProjectRequestHostRegex } from "./loki"; +import { slugify } from "./routes"; interface GrafanaDashboard { dashboard: { @@ -64,24 +65,8 @@ export async function ensureProjectDashboard( projectName: string, containerRegex: string, ): Promise { - const slug = projectName - .toLowerCase() - .replace(/[^a-z0-9-]+/g, "-") - .replace(/^-+|-+$/g, "") - .slice(0, 63); - - const domains = [`${slug}.${config.caddyBaseDomain}`]; - try { - const projectDomains = await listDomains(projectId); - const verified = projectDomains.filter((d) => d.validationStatus === "verified"); - for (const d of verified) { - domains.push(d.domain); - } - } catch (e) { - console.warn("[Grafana] Failed to list domains for dashboard query:", e); - } - - const regexEscaped = domains.map((d) => d.replace(/[-/\\^$*+?.()|[\]{}]/g, "\\\\$&")).join("|"); + const slug = slugify(projectName); + const regexEscaped = await buildProjectRequestHostRegex(projectId); const dashboard: GrafanaDashboard = { dashboard: { diff --git a/apps/api/src/utils/ingress.ts b/apps/api/src/utils/ingress.ts index d969b92..929b04a 100644 --- a/apps/api/src/utils/ingress.ts +++ b/apps/api/src/utils/ingress.ts @@ -1,6 +1,7 @@ import { rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { createAgentJob, getPlatformSettings, getServerById, upsertRoute } from "../db/repo"; +import { caddyReverseProxy, caddySite } from "./caddy-site"; import { config } from "./config"; import { removeRemoteCaddyRoute, syncRemoteCaddyRoute } from "./ssh"; @@ -38,10 +39,11 @@ export const projectServerSite = ( viaIngress: boolean, ): string => { const targets = containers.map((c) => `${c}:${port}`).join(" "); + const proxy = caddyReverseProxy(targets); if (viaIngress) { - return `:80 {\n reverse_proxy ${targets} {\n header_up Host {upstream_hostport}\n }\n}\n`; + return caddySite(":80", proxy); } - return `${hostname} {\n reverse_proxy ${targets} {\n header_up Host {upstream_hostport}\n }\n}\n`; + return caddySite(hostname, proxy); }; export const ingressSite = (hostname: string, upstreamHost: string): string => diff --git a/apps/api/src/utils/loki.ts b/apps/api/src/utils/loki.ts new file mode 100644 index 0000000..253835f --- /dev/null +++ b/apps/api/src/utils/loki.ts @@ -0,0 +1,42 @@ +import { getProjectById, listDomains } from "../db/repo"; +import { config } from "./config"; +import { slugify } from "./routes"; + +const LOKI_URL = "http://loki:3100"; +const CADDY_LOG_STREAM = '{container="dequel-caddy-1"}'; + +export const buildProjectRequestHostRegex = async (projectId: string): Promise => { + const project = await getProjectById(projectId); + if (!project) throw new Error(`Project ${projectId} not found`); + const slug = slugify(project.name); + const domains = [`${slug}.${config.caddyBaseDomain}`]; + try { + const projectDomains = await listDomains(projectId); + for (const d of projectDomains) { + if (d.validationStatus === "verified") domains.push(d.domain); + } + } catch (err) { + console.warn("[Loki] Failed to list domains for host regex:", err); + } + return domains.map((d) => d.replace(/[-/\\^$*+?.()|[\]{}]/g, "\\\\$&")).join("|"); +}; + +export const caddyRequestLogSelector = (hostRegex: string): string => + `${CADDY_LOG_STREAM} | json | request_host =~ "^(${hostRegex})$"`; + +export const lokiInstantQuery = async (query: string, timeoutMs = 5000): Promise => { + const url = `${LOKI_URL}/loki/api/v1/query?query=${encodeURIComponent(query)}`; + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + try { + const response = await fetch(url, { signal: controller.signal }); + if (!response.ok) return null; + const data = (await response.json()) as any; + if (data.status !== "success") return null; + return data.data?.result ?? null; + } catch { + return null; + } finally { + clearTimeout(timeout); + } +}; diff --git a/apps/api/src/utils/routes.ts b/apps/api/src/utils/routes.ts index 8d9573e..b515cdb 100644 --- a/apps/api/src/utils/routes.ts +++ b/apps/api/src/utils/routes.ts @@ -9,6 +9,9 @@ export const slugify = (s: string) => export const baseDomainFor = () => (config.caddyBaseDomain === "localhost" ? "localhost:80" : config.caddyBaseDomain); +export const appBaseUrl = () => + config.caddyBaseDomain === "localhost" ? "http://localhost" : `https://${config.caddyBaseDomain}`; + export const routeNamesFor = (projectName: string | null, projectId: string | null, deploymentId: string) => { const slug = slugify(projectName || projectId || deploymentId); return { diff --git a/apps/api/src/utils/telemetry.ts b/apps/api/src/utils/telemetry.ts new file mode 100644 index 0000000..1c8895f --- /dev/null +++ b/apps/api/src/utils/telemetry.ts @@ -0,0 +1,62 @@ +import { randomUUID } from "node:crypto"; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import packageJson from "../../package.json"; +import { config } from "./config"; + +const getPosthogKey = () => config.posthogKey || process.env.DEQUEL_POSTHOG_KEY || "phc_dequel_telemetry_public_key"; +const getPosthogHost = () => config.telemetryHost || process.env.DEQUEL_TELEMETRY_HOST || "https://us.i.posthog.com"; + +const getTelemetryFilePath = () => { + const dataDir = join(config.workspaceRoot, "..", "data"); + return join(dataDir, ".instance-id"); +}; + +export const getInstanceId = (): string => { + try { + const filePath = getTelemetryFilePath(); + if (existsSync(filePath)) { + const id = readFileSync(filePath, "utf-8").trim(); + if (id) return id; + } + const newId = randomUUID(); + mkdirSync(dirname(filePath), { recursive: true }); + writeFileSync(filePath, newId, "utf-8"); + return newId; + } catch { + return "anonymous-instance"; + } +}; + +export const captureTelemetry = async (event: string, properties: Record = {}): Promise => { + if (process.env.DEQUEL_TELEMETRY_DISABLED === "1" || process.env.DO_NOT_TRACK === "1") { + return; + } + + try { + const instanceId = getInstanceId(); + const key = getPosthogKey(); + const host = getPosthogHost(); + const payload = { + api_key: key, + event, + distinct_id: instanceId, + properties: { + ...properties, + instance_id: instanceId, + version: packageJson.version, + platform: process.platform, + arch: process.arch, + bun_version: Bun.version, + }, + }; + + await fetch(`${host.replace(/\/$/, "")}/capture/`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(payload), + }).catch(() => {}); + } catch { + // Silent catch — telemetry should never affect platform execution + } +}; diff --git a/apps/docs/.astro/astro/content.d.ts b/apps/docs/.astro/astro/content.d.ts index a7e7267..dfde2ca 100644 --- a/apps/docs/.astro/astro/content.d.ts +++ b/apps/docs/.astro/astro/content.d.ts @@ -147,6 +147,13 @@ declare module "astro:content" { collection: "changelogs"; data: any; } & { render(): Render[".md"] }; + "v0.4.0.md": { + id: "v0.4.0.md"; + slug: "v040"; + body: string; + collection: "changelogs"; + data: any; + } & { render(): Render[".md"] }; }; docs: { "agent-caddy-routes.md": { @@ -156,6 +163,13 @@ declare module "astro:content" { collection: "docs"; data: any; } & { render(): Render[".md"] }; + "ai-diagnosis.md": { + id: "ai-diagnosis.md"; + slug: "ai-diagnosis"; + body: string; + collection: "docs"; + data: any; + } & { render(): Render[".md"] }; "auth.md": { id: "auth.md"; slug: "auth"; diff --git a/apps/docs/package.json b/apps/docs/package.json index d5066f1..8fe9783 100644 --- a/apps/docs/package.json +++ b/apps/docs/package.json @@ -1,7 +1,7 @@ { "name": "dequel-docs", "type": "module", - "version": "0.3.0", + "version": "0.4.0", "scripts": { "dev": "astro dev", "start": "astro dev", diff --git a/apps/docs/src/content/changelogs/v0.4.0.md b/apps/docs/src/content/changelogs/v0.4.0.md new file mode 100644 index 0000000..0eb4236 --- /dev/null +++ b/apps/docs/src/content/changelogs/v0.4.0.md @@ -0,0 +1,35 @@ +--- +version: 0.4.0 +date: "2026-10-03" +--- + +## What's Changed +### Features +- Add AI-powered deployment diagnosis ([903d9c9](https://github.com/Lftobs/dequel/commit/903d9c9594152355f6e48668939b79f7d4742cf7)) +- Add automated database backup and restore system ([3b477a9](https://github.com/Lftobs/dequel/commit/3b477a95fbd9c47cb2019929bedb00c84c6cc36b)) +- Database studio data viewer, database settings UI, backup configuration, and capsule tabs ([0fd77c1](https://github.com/Lftobs/dequel/commit/0fd77c186dd9e48ae2cab9687e6ab17afffacc6a)) +- Add shared environment variables and SSH key pool management ([8c7f76b](https://github.com/Lftobs/dequel/commit/8c7f76b82de60db7f7a8a086fe39e1ce50d7819f)) +- Add shared environment variable link support and UI components ([86205cb](https://github.com/Lftobs/dequel/commit/86205cb4f8ab7e5a82f9aa752df30b6da72e6be0)) +- Add project health and failure alerts ([a42c2b3](https://github.com/Lftobs/dequel/commit/a42c2b3e3526dbf98ea83d902971b4e09075739d)) +- Track anonymous usage events ([8abdf74](https://github.com/Lftobs/dequel/commit/8abdf7420877621e10ed277ccb63a70df97a3523)) + + +### Improvements +- Redesign keys and settings pages with tabbed navigation ([765a199](https://github.com/Lftobs/dequel/commit/765a199f6b578149358e1529429260325edf4ad8)) +- Remove delete projects tab and fix tabs overflow behavior ([650e1d8](https://github.com/Lftobs/dequel/commit/650e1d8a7c5f2bedbdec10c162e11368bdb207e4)) +- Add Biome linter + pre-commit hook, format codebase ([2ac7c3f](https://github.com/Lftobs/dequel/commit/2ac7c3f2b24ca146afcc6b9f206cf0ab9ffab7c8)) +- Update database creation engine selection UI ([faef669](https://github.com/Lftobs/dequel/commit/faef6693959dab303340f0245430c48504d08db2)) +- Improve SQL query editor ui ([f38b1a6](https://github.com/Lftobs/dequel/commit/f38b1a68fac5d9af7be44f124e6c879b68bf19ff)) +- Switch compose file to prebuilt images for api and web ([3e6f917](https://github.com/Lftobs/dequel/commit/3e6f917403b233dcc43d60068ed18ddaf77174b6)) +- Make dequel dashboard 100% mobile responsive ([905719f](https://github.com/Lftobs/dequel/commit/905719f336fd79c3818fe45cc023d0df7fdedc95)) +- Make all settings sections, tables, tabs and dialogs fully mobile responsive ([1049120](https://github.com/Lftobs/dequel/commit/1049120cf5c7243be5d75c4d9474a4ab326f9bb4)) + +### Bug Fixes +- Harden alerts and monitoring validation ([1a68e40](https://github.com/Lftobs/dequel/commit/1a68e4066b7d11004b622965aabf2a53767018dd)) +- Secure shared environment variable deletion by project ID ([e146487](https://github.com/Lftobs/dequel/commit/e1464876961f32b421800e708d328a20dc9233c2)) +- Resolve container name, storage type display, and S3 prefix ([3cb6610](https://github.com/Lftobs/dequel/commit/3cb66109d25c238f4011fadeaeb2094240602d9d)) +- Remove duplicate listRoutesByDeployment function ([80d43e5](https://github.com/Lftobs/dequel/commit/80d43e50f5a72dfac4b1908032c33d4b7b26ceed)) +- Add missing closing brace for getDatabaseTables function ([1666f13](https://github.com/Lftobs/dequel/commit/1666f13bf33a38d78ff914c14412b4c9d4d51a29)) +- Address CodeRabbit review comments on AI diagnosis PR ([0a1a724](https://github.com/Lftobs/dequel/commit/0a1a72441919cf528f83c0289c82a3375901b018)) +- Chown the sandbox job dir to bun before starting the runner ([bca4f14](https://github.com/Lftobs/dequel/commit/bca4f1470fb785e899be203e9f8769ec2b578deb)) +- Add MongoDB and Redis database studios ([5551981](https://github.com/Lftobs/dequel/commit/55519810bf8ae84387b4430188b39e1f42e1359b)) diff --git a/apps/docs/src/content/docs/ai-diagnosis.md b/apps/docs/src/content/docs/ai-diagnosis.md new file mode 100644 index 0000000..9ff25d2 --- /dev/null +++ b/apps/docs/src/content/docs/ai-diagnosis.md @@ -0,0 +1,50 @@ +--- +title: AI Diagnosis +category: Deployment +description: Diagnose failed builds with an AI agent, then apply one-click fix PRs or report Dequel bugs. +slug: ai-diagnosis +--- + +When a deployment fails, Dequel lets you hand the failure to an AI agent. The agent runs inside an isolated sandbox container holding a fresh checkout of Dequel's own source and your project's source. It looks up the files it needs itself, decides whether the fault is in your code or in Dequel itself, and proposes what to do next. Every write action requires your explicit click — the one exception is a Dequel bug report, which is posted to Slack automatically once the diagnosis finishes. + +The sandbox persists between diagnoses: Dequel's source is cloned once and updated to your running version on each run, while your project's source is pulled fresh per diagnosis and cleared afterwards. + +## Configure a provider key + +Before diagnosing anything, store an LLM provider key. Open **Settings → AI Diagnosis** and save a key for one of the supported providers (`openai`, `anthropic`, `gemini`, `groq`, `ollama`, or a custom OpenAI-compatible endpoint with a base URL). Keys are encrypted at rest and never shown again after saving. Leaving the key field empty keeps the stored key. + +## Diagnose a failed deployment + +Follow these steps to run a diagnosis. + +1. Open the project and switch to the **Deployments** tab. +2. Select a failed deployment to reveal its build logs. +3. Click **Diagnose** in the log header to open the diagnosis sheet. +4. Pick a provider and model, then click **Diagnose**. +5. Watch the agent work through four stages: reading logs, investigating in the sandbox, explaining the fix, and drafting the proposal. +6. Read the verdict and the explanation. + +The verdict is one of three outcomes. **Your code** means the fault is in your source. **Dequel** means the fault is in Dequel's builder or platform code. **Inconclusive** means the evidence was not sufficient, and no action buttons are shown. + +## Fix your code with one click + +When the verdict is **Your code** and the project deploys from Git, the sheet shows a **Create Fix Pull Request** button. Clicking it sends the agent back into the sandbox to edit the project source, still read-only everywhere else. Dequel takes the resulting diff, applies it to a new `dequel-fix/*` branch, and opens a pull request against your repository, so you only review and merge. If the agent cannot produce a change, or the diff does not apply cleanly, Dequel reports the problem and you apply the fix manually. This action is idempotent: repeated clicks return the same pull request. + +> **Note:** Fix PRs require a connected GitHub account with repository access. Connect it under **Settings → GitHub Integration** first. Without it, Dequel refuses to proceed. + +## Report a Dequel bug + +When the verdict is **Dequel**, Dequel posts the drafted report to the Dequel team's Slack channel automatically as soon as the diagnosis completes — no button, no approval step. The post carries the report in problem, Dequel version, root cause, proposed fix order, with the investigated source revision attached. If the channel is not configured, the run still completes and the sheet says so; copy the report text and share it manually. + +## Limits + +- Diagnosis runs only on deployments with `failed` status. +- One diagnosis exists per deployment and commit; starting it again returns the existing result. +- Fix PRs are available for Git projects only. Zip-upload projects still receive the written explanation. +- The sandbox agent pulls only the files it needs (about 2.4 KB per read during investigation, 8 KB when fixing, and 10 search hits at a time) and can keep working through large repositories across many steps. What bounds it instead is time and cost: a run times out after about 10 minutes, and every file read spends your provider's tokens. +- During investigation the agent has read-only file tools. When fixing, it can edit and create files inside the project source only. Neither mode can run shell commands or read anything outside the two source checkouts, so the provider key stored alongside the job stays out of the agent's reach. + +## Next steps + +- Read [Deployments](/docs/deployments) to understand build sources and rollback. +- Read [Configuration](/docs/configuration) for platform settings. diff --git a/apps/docs/src/content/docs/system-config.md b/apps/docs/src/content/docs/system-config.md index 5851c4f..e258942 100644 --- a/apps/docs/src/content/docs/system-config.md +++ b/apps/docs/src/content/docs/system-config.md @@ -50,29 +50,7 @@ On boot, these values seed the `github_integrations` table. You can also update ### SMTP -Set these to enable email alerts and notifications: - -| Variable | Default | Description | -|----------|---------|-------------| -| `SMTP_HOST` | `""` | SMTP server hostname | -| `SMTP_PORT` | `587` | SMTP server port | -| `SMTP_USER` | `""` | SMTP username | -| `SMTP_PASS` | `""` | SMTP password | -| `SMTP_FROM` | `dequel@localhost` | From address for outgoing emails | - -Config file equivalent: - -```json -{ - "smtpHost": "smtp.sendgrid.net", - "smtpPort": 587, - "smtpUser": "apikey", - "smtpPass": "...", - "smtpFrom": "dequel@example.com" -} -``` - -On boot, these values seed the `smtp_settings` table. The password is encrypted at rest using `ENV_ENCRYPTION_KEY`. You can also update these from the Settings page in the dashboard, and send a test email to verify the configuration. +SMTP settings are not read from environment variables or the config file. Configure them from the Settings page in the dashboard, where you can also send a test email to verify the setup. The password is encrypted at rest using `ENV_ENCRYPTION_KEY`. ### Ingress @@ -98,11 +76,6 @@ Config file equivalent: "githubClientSecret": "...", "githubAppName": "MyDequel", "githubWebhookSecret": "...", - "smtpHost": "smtp.sendgrid.net", - "smtpPort": 587, - "smtpUser": "apikey", - "smtpPass": "...", - "smtpFrom": "alerts@example.com", "envEncryptionKey": "your-secure-key-here" } ``` diff --git a/apps/web/package.json b/apps/web/package.json index f74681c..0ca3547 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -1,6 +1,6 @@ { "name": "dequel-web", - "version": "0.3.0", + "version": "0.4.0", "private": true, "type": "module", "scripts": { diff --git a/apps/web/src/api/__tests__/auth.test.ts b/apps/web/src/api/__tests__/auth.test.ts new file mode 100644 index 0000000..7207fe9 --- /dev/null +++ b/apps/web/src/api/__tests__/auth.test.ts @@ -0,0 +1,94 @@ +import { afterEach, beforeEach, describe, expect, it, mock } from "bun:test"; +import { cancelTokenRefresh, doRefreshToken, getMe, scheduleTokenRefresh } from "../auth"; +import { apiFetch } from "../http"; + +describe("Web Auth & Refresh Flow", () => { + const originalFetch = globalThis.fetch; + + beforeEach(() => { + cancelTokenRefresh(); + }); + + afterEach(() => { + cancelTokenRefresh(); + globalThis.fetch = originalFetch; + }); + + it("doRefreshToken deduplicates concurrent calls to a single network request", async () => { + let fetchCount = 0; + globalThis.fetch = mock(async (url: any) => { + if (String(url).endsWith("/auth/refresh")) { + fetchCount++; + await new Promise((r) => setTimeout(r, 10)); + return new Response(JSON.stringify({ status: "success", data: { username: "user", expiresIn: 900 } }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + } + return new Response("{}", { status: 404 }); + }); + + const [res1, res2, res3] = await Promise.all([doRefreshToken(), doRefreshToken(), doRefreshToken()]); + + expect(res1).toBe(true); + expect(res2).toBe(true); + expect(res3).toBe(true); + expect(fetchCount).toBe(1); + }); + + it("getMe schedules proactive refresh when authenticated", async () => { + globalThis.fetch = mock(async (url: any) => { + if (String(url).endsWith("/auth/me")) { + return new Response( + JSON.stringify({ status: "success", data: { authenticated: true, username: "user", expiresIn: 900 } }), + { + status: 200, + headers: { "Content-Type": "application/json" }, + }, + ); + } + return new Response("{}", { status: 404 }); + }); + + const res = await getMe(); + expect(res.authenticated).toBe(true); + expect(res.username).toBe("user"); + expect(res.expiresIn).toBe(900); + }); + + it("apiFetch automatically retries once upon receiving 401 on non-auth routes", async () => { + let attempts = 0; + let refreshCalled = false; + + globalThis.fetch = mock(async (url: any, opts: any) => { + const strUrl = String(url); + if (strUrl.endsWith("/auth/refresh")) { + refreshCalled = true; + return new Response(JSON.stringify({ status: "success", data: { username: "user", expiresIn: 900 } }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + } + if (strUrl.endsWith("/projects")) { + attempts++; + if (attempts === 1) { + return new Response(JSON.stringify({ status: "error", message: "Authentication required" }), { + status: 401, + headers: { "Content-Type": "application/json" }, + }); + } + return new Response(JSON.stringify({ status: "success", data: [{ id: "proj-1", name: "My App" }] }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + } + return new Response("{}", { status: 404 }); + }); + + const data = await apiFetch>("/projects"); + expect(refreshCalled).toBe(true); + expect(attempts).toBe(2); + expect(data).toHaveLength(1); + expect(data[0].name).toBe("My App"); + }); +}); diff --git a/apps/web/src/api/auth.ts b/apps/web/src/api/auth.ts new file mode 100644 index 0000000..69519aa --- /dev/null +++ b/apps/web/src/api/auth.ts @@ -0,0 +1,170 @@ +const BASE = "/api"; + +export class AuthError extends Error { + status: number; + constructor(msg: string, status: number) { + super(msg); + this.status = status; + } +} + +export interface MeResponse { + authenticated: boolean; + username?: string; + expiresIn?: number; +} + +export interface LoginResponse { + username: string; + expiresIn?: number; +} + +let refreshPromise: Promise | null = null; +let refreshTimer: ReturnType | null = null; +let tokenExpiresAt = 0; + +const authChannel = typeof BroadcastChannel !== "undefined" ? new BroadcastChannel("dequel_auth") : null; + +export const cancelTokenRefresh = () => { + if (refreshTimer) { + clearTimeout(refreshTimer); + refreshTimer = null; + } + tokenExpiresAt = 0; +}; + +export const scheduleTokenRefresh = (expiresInSeconds: number) => { + cancelTokenRefresh(); + if (expiresInSeconds <= 0) return; + + tokenExpiresAt = Date.now() + expiresInSeconds * 1000; + + const leadTimeSeconds = 120; + const delaySeconds = Math.max(expiresInSeconds - leadTimeSeconds, 5); + + refreshTimer = setTimeout(() => { + void doRefreshToken(); + }, delaySeconds * 1000); +}; + +export const doRefreshToken = async (): Promise => { + if (refreshPromise) { + return refreshPromise; + } + + refreshPromise = (async () => { + try { + const res = await fetch(`${BASE}/auth/refresh`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + }); + if (!res.ok) { + cancelTokenRefresh(); + return false; + } + const json = await res.json(); + const data = (json && typeof json === "object" && "data" in json ? json.data : json) as LoginResponse; + const expiresIn = data?.expiresIn ?? 900; + scheduleTokenRefresh(expiresIn); + authChannel?.postMessage({ type: "session_refreshed", expiresIn }); + return true; + } catch { + return false; + } finally { + refreshPromise = null; + } + })(); + + return refreshPromise; +}; + +const checkAndRefreshIfExpiring = () => { + if (tokenExpiresAt <= 0) return; + const remainingMs = tokenExpiresAt - Date.now(); + if (remainingMs <= 120_000) { + void doRefreshToken(); + } +}; + +if (typeof window !== "undefined") { + window.addEventListener("focus", checkAndRefreshIfExpiring); + document.addEventListener("visibilitychange", () => { + if (document.visibilityState === "visible") { + checkAndRefreshIfExpiring(); + } + }); +} + +if (authChannel) { + authChannel.onmessage = (event) => { + if (event.data?.type === "session_refreshed" && typeof event.data.expiresIn === "number") { + scheduleTokenRefresh(event.data.expiresIn); + } else if (event.data?.type === "session_logged_out") { + cancelTokenRefresh(); + } + }; +} + +export const login = async (username: string, password: string): Promise => { + const res = await fetch(`${BASE}/auth/login`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ username, password }), + }); + if (!res.ok) { + const body = await res.json().catch(() => ({ message: res.statusText })); + throw new AuthError(body.message ?? body.error ?? "Login failed", res.status); + } + const json = await res.json(); + const data = (json && typeof json === "object" && "data" in json ? json.data : json) as LoginResponse; + const expiresIn = data?.expiresIn ?? 900; + scheduleTokenRefresh(expiresIn); + authChannel?.postMessage({ type: "session_refreshed", expiresIn }); + return data; +}; + +export const logout = async (): Promise => { + cancelTokenRefresh(); + authChannel?.postMessage({ type: "session_logged_out" }); + await fetch(`${BASE}/auth/logout`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + }); +}; + +export const refreshSession = async (): Promise<{ username: string }> => { + const ok = await doRefreshToken(); + if (!ok) { + throw new AuthError("Failed to refresh session", 401); + } + return { username: "" }; +}; + +export const getMe = async (): Promise => { + const fetchMe = async (): Promise => { + const res = await fetch(`${BASE}/auth/me`); + if (!res.ok) { + return { authenticated: false }; + } + const json = await res.json(); + return (json && typeof json === "object" && "data" in json ? json.data : json) as MeResponse; + }; + + const initial = await fetchMe(); + if (initial.authenticated) { + scheduleTokenRefresh(initial.expiresIn ?? 900); + return initial; + } + + const refreshed = await doRefreshToken(); + if (refreshed) { + const second = await fetchMe(); + if (second.authenticated) { + scheduleTokenRefresh(second.expiresIn ?? 900); + } + return second; + } + + cancelTokenRefresh(); + return initial; +}; diff --git a/apps/web/src/api/client.ts b/apps/web/src/api/client.ts index 37c2a82..6598ca3 100644 --- a/apps/web/src/api/client.ts +++ b/apps/web/src/api/client.ts @@ -1,58 +1,28 @@ import type { + ActiveDiagRun, Alert, ApiKey, - BackupJob, - BackupStorageSettingsData, CreateProjectInput, - Database, Deployment, + DiagRun, + DiagStage, Domain, EnvironmentVariable, GithubIntegrationStatus, GithubRepo, + LlmKeyStatus, Log, Project, - QueryExecResult, ScalingPolicy, Server, SmtpSettingsStatus, - TableInfo, Volume, } from "../types"; +import { BASE, apiFetch } from "./http"; -const BASE = "/api"; - -class ApiError extends Error { - status: number; - constructor(msg: string, status: number) { - super(msg); - this.status = status; - } -} - -const apiFetch = async (path: string, opts?: RequestInit): Promise => { - const isFormData = opts?.body instanceof FormData; - const headers: Record = {}; - if (!isFormData) headers["Content-Type"] = "application/json"; - const res = await fetch(`${BASE}${path}`, { - ...opts, - headers: { - ...headers, - ...(opts?.headers as Record), - }, - }); - if (!res.ok) { - const body = await res.json().catch(() => ({ - message: res.statusText, - })); - throw new ApiError(body.message ?? body.error ?? "Request failed", res.status); - } - if (res.headers.get("content-type")?.includes("text/event-stream")) return res as unknown as T; - if (res.headers.get("content-type")?.includes("text/plain")) return res.text() as unknown as T; - const json = await res.json(); - if (json && typeof json === "object" && "status" in json && "data" in json) return json.data as T; - return json as T; -}; +export * from "./http"; +export * from "./auth"; +export * from "./databases"; // Projects export const listProjects = () => apiFetch("/projects"); @@ -173,85 +143,6 @@ export const deleteVolume = (id: string) => method: "DELETE", }); -// Databases -export const listAllDatabases = () => apiFetch("/databases"); -export const listDatabases = (projectId: string) => apiFetch(`/projects/${projectId}/databases`); -export const createDatabase = ( - projectId: string | null, - type: string, - options?: { - name?: string; - version?: string; - serverId?: string; - cpuLimit?: number | null; - memoryLimitMb?: number | null; - storageLimitMb?: number | null; - publicAccess?: boolean; - allowPublicAccessFromAnywhere?: boolean; - allowedCidrs?: string[]; - }, -) => - apiFetch(projectId ? `/projects/${projectId}/databases` : "/databases", { - method: "POST", - body: JSON.stringify({ type, projectId, ...options }), - }); -export const getDatabase = (id: string) => apiFetch(`/databases/${id}`); -export const deleteDatabase = (id: string) => apiFetch(`/databases/${id}`, { method: "DELETE" }); -export const getDatabaseCredentials = (id: string) => - apiFetch<{ - username: string; - password: string; - internalConnectionString: string; - externalConnectionString: string | null; - externalHost: string | null; - externalPort: number | null; - }>(`/databases/${id}/credentials`); -export const startDatabase = (id: string) => apiFetch(`/databases/${id}/start`, { method: "POST" }); -export const stopDatabase = (id: string) => apiFetch(`/databases/${id}/stop`, { method: "POST" }); -export const restartDatabase = (id: string) => apiFetch(`/databases/${id}/restart`, { method: "POST" }); -export const retryDatabase = (id: string) => apiFetch(`/databases/${id}/retry`, { method: "POST" }); -export const updateDatabaseSettings = ( - id: string, - data: { - name?: string; - cpuLimit?: number | null; - memoryLimitMb?: number | null; - storageLimitMb?: number | null; - publicAccess?: boolean; - allowPublicAccessFromAnywhere?: boolean; - allowedCidrs?: string[]; - backupEnabled?: boolean; - backupSchedule?: string; - backupRetention?: number; - }, -) => - apiFetch(`/databases/${id}`, { - method: "PATCH", - body: JSON.stringify(data), - }); -export const getDatabaseTables = (id: string) => apiFetch(`/databases/${id}/tables`); -export const queryDatabase = (id: string, query: string) => - apiFetch(`/databases/${id}/query`, { - method: "POST", - body: JSON.stringify({ query }), - }); - -// Backups -export const listBackups = () => apiFetch("/backups"); -export const triggerBackup = (targetId = "internal") => - apiFetch(targetId === "internal" ? "/backups" : `/backups/${targetId}/trigger`, { - method: "POST", - }); -export const restoreBackup = (id: string) => - apiFetch<{ restored: boolean }>(`/backups/${id}/restore`, { method: "POST" }); -export const deleteBackup = (id: string) => apiFetch<{ deleted: boolean }>(`/backups/${id}`, { method: "DELETE" }); -export const getBackupStorageSettings = () => apiFetch("/settings/backup"); -export const updateBackupStorageSettings = (data: BackupStorageSettingsData) => - apiFetch("/settings/backup", { - method: "PUT", - body: JSON.stringify(data), - }); - // Domains export const listDomains = (projectId: string) => apiFetch(`/projects/${projectId}/domains`); export const createDomain = ( @@ -299,28 +190,6 @@ export const deleteScalingPolicy = (projectId: string) => export const getServerIp = () => apiFetch<{ ip: string; baseDomain: string; resolves: boolean; url: string }>("/server/ip"); -// Auth -export const login = (username: string, password: string) => - apiFetch<{ username: string }>("/auth/login", { - method: "POST", - body: JSON.stringify({ username, password }), - }); - -export const logout = () => apiFetch("/auth/logout", { method: "POST" }); - -export const refreshSession = () => apiFetch<{ username: string }>("/auth/refresh", { method: "POST" }); - -export const getMe = async () => { - const res = await apiFetch<{ authenticated: boolean; username?: string }>("/auth/me"); - if (!res.authenticated) { - const refreshed = await apiFetch<{ username: string }>("/auth/refresh", { method: "POST" }).catch(() => null); - if (refreshed) { - return apiFetch<{ authenticated: boolean; username?: string }>("/auth/me"); - } - } - return res; -}; - // Prometheus export const queryPrometheus = (query: string) => apiFetch<{ @@ -463,7 +332,49 @@ export const testSmtpSettings = () => method: "POST", }); -// ─── GitHub Webhook ─────────────────────────────────────── +export const getLlmKeys = () => apiFetch("/settings/llm-keys"); + +export const getLlmDefaultModels = () => apiFetch>("/settings/llm-default-models"); + +export const syncLlmModels = (provider: string) => + apiFetch<{ provider: string; models: string[] }>(`/settings/llm-keys/${provider}/sync`, { + method: "POST", + }); + +export const getLlmModels = (provider: string, refresh = false) => + apiFetch<{ provider: string; models: string[]; cached: boolean }>( + `/settings/llm-keys/${provider}/models${refresh ? "?refresh=true" : ""}`, + ); + +export const setLlmKey = (data: { provider: string; apiKey?: string; baseURL?: string; models?: string[] }) => + apiFetch("/settings/llm-keys", { + method: "PUT", + body: JSON.stringify(data), + }); + +export const deleteLlmKey = (provider: string) => + apiFetch(`/settings/llm-keys/${provider}`, { + method: "DELETE", + }); + +export const startDiagnosis = (deploymentId: string, data: { provider: string; model: string }) => + apiFetch(`/deployments/${deploymentId}/diagnose`, { + method: "POST", + body: JSON.stringify(data), + }); + +export const getDiagnosis = (runId: string) => + apiFetch<{ run: DiagRun; stages: DiagStage[]; slackPosted: boolean }>(`/diagnoses/${runId}`); + +export const getActiveDiagnoses = () => apiFetch("/diagnoses/active"); + +export const streamDiagnosisUrl = (runId: string) => `${BASE}/diagnoses/${runId}/stream`; + +export const approveFixPr = (runId: string, idempotencyKey: string) => + apiFetch<{ prUrl: string }>(`/diagnoses/${runId}/approve-pr`, { + method: "POST", + body: JSON.stringify({ idempotencyKey }), + }); export const getRepoHooks = (owner: string, repo: string) => apiFetch>( diff --git a/apps/web/src/api/databases.ts b/apps/web/src/api/databases.ts new file mode 100644 index 0000000..354d2ec --- /dev/null +++ b/apps/web/src/api/databases.ts @@ -0,0 +1,79 @@ +import type { BackupJob, BackupStorageSettingsData, Database, QueryExecResult, TableInfo } from "../types"; +import { apiFetch } from "./http"; + +export const listAllDatabases = () => apiFetch("/databases"); +export const listDatabases = (projectId: string) => apiFetch(`/projects/${projectId}/databases`); +export const createDatabase = ( + projectId: string | null, + type: string, + options?: { + name?: string; + version?: string; + serverId?: string; + cpuLimit?: number | null; + memoryLimitMb?: number | null; + storageLimitMb?: number | null; + publicAccess?: boolean; + allowPublicAccessFromAnywhere?: boolean; + allowedCidrs?: string[]; + }, +) => + apiFetch(projectId ? `/projects/${projectId}/databases` : "/databases", { + method: "POST", + body: JSON.stringify({ type, projectId, ...options }), + }); +export const getDatabase = (id: string) => apiFetch(`/databases/${id}`); +export const deleteDatabase = (id: string) => apiFetch(`/databases/${id}`, { method: "DELETE" }); +export const getDatabaseCredentials = (id: string) => + apiFetch<{ + username: string; + password: string; + internalConnectionString: string; + externalConnectionString: string | null; + externalHost: string | null; + externalPort: number | null; + }>(`/databases/${id}/credentials`); +export const startDatabase = (id: string) => apiFetch(`/databases/${id}/start`, { method: "POST" }); +export const stopDatabase = (id: string) => apiFetch(`/databases/${id}/stop`, { method: "POST" }); +export const restartDatabase = (id: string) => apiFetch(`/databases/${id}/restart`, { method: "POST" }); +export const retryDatabase = (id: string) => apiFetch(`/databases/${id}/retry`, { method: "POST" }); +export const updateDatabaseSettings = ( + id: string, + data: { + name?: string; + cpuLimit?: number | null; + memoryLimitMb?: number | null; + storageLimitMb?: number | null; + publicAccess?: boolean; + allowPublicAccessFromAnywhere?: boolean; + allowedCidrs?: string[]; + backupEnabled?: boolean; + backupSchedule?: string; + backupRetention?: number; + }, +) => + apiFetch(`/databases/${id}`, { + method: "PATCH", + body: JSON.stringify(data), + }); +export const getDatabaseTables = (id: string) => apiFetch(`/databases/${id}/tables`); +export const queryDatabase = (id: string, query: string) => + apiFetch(`/databases/${id}/query`, { + method: "POST", + body: JSON.stringify({ query }), + }); + +export const listBackups = () => apiFetch("/backups"); +export const triggerBackup = (targetId = "internal") => + apiFetch(targetId === "internal" ? "/backups" : `/backups/${targetId}/trigger`, { + method: "POST", + }); +export const restoreBackup = (id: string) => + apiFetch<{ restored: boolean }>(`/backups/${id}/restore`, { method: "POST" }); +export const deleteBackup = (id: string) => apiFetch<{ deleted: boolean }>(`/backups/${id}`, { method: "DELETE" }); +export const getBackupStorageSettings = () => apiFetch("/settings/backup"); +export const updateBackupStorageSettings = (data: BackupStorageSettingsData) => + apiFetch("/settings/backup", { + method: "PUT", + body: JSON.stringify(data), + }); diff --git a/apps/web/src/api/http.ts b/apps/web/src/api/http.ts new file mode 100644 index 0000000..9b19f8b --- /dev/null +++ b/apps/web/src/api/http.ts @@ -0,0 +1,47 @@ +import { doRefreshToken } from "./auth"; + +export const BASE = "/api"; + +export class ApiError extends Error { + status: number; + constructor(msg: string, status: number) { + super(msg); + this.status = status; + } +} + +export const apiFetch = async (path: string, opts?: RequestInit): Promise => { + const isFormData = opts?.body instanceof FormData; + const headers: Record = {}; + if (!isFormData) headers["Content-Type"] = "application/json"; + + const execute = () => + fetch(`${BASE}${path}`, { + ...opts, + headers: { + ...headers, + ...(opts?.headers as Record), + }, + }); + + let res = await execute(); + + if (res.status === 401 && !path.startsWith("/auth/")) { + const refreshed = await doRefreshToken(); + if (refreshed) { + res = await execute(); + } + } + + if (!res.ok) { + const body = await res.json().catch(() => ({ + message: res.statusText, + })); + throw new ApiError(body.message ?? body.error ?? "Request failed", res.status); + } + if (res.headers.get("content-type")?.includes("text/event-stream")) return res as unknown as T; + if (res.headers.get("content-type")?.includes("text/plain")) return res.text() as unknown as T; + const json = await res.json(); + if (json && typeof json === "object" && "status" in json && "data" in json) return json.data as T; + return json as T; +}; diff --git a/apps/web/src/components/DiagNotifier.tsx b/apps/web/src/components/DiagNotifier.tsx new file mode 100644 index 0000000..22e11d2 --- /dev/null +++ b/apps/web/src/components/DiagNotifier.tsx @@ -0,0 +1,63 @@ +import { useQuery } from "@tanstack/react-query"; +import { useEffect, useRef } from "react"; +import * as api from "../api/client"; +import type { DiagRun } from "../types"; + +const causeLabel = (cause: DiagRun["cause"]): string => + cause === "user-source" ? "your code" : cause === "dequel-source" ? "Dequel" : "inconclusive"; + +const runLabel = (run: { projectName: string | null; deploymentId: string }): string => + `${run.projectName ?? "deployment"} ${run.deploymentId.slice(0, 8)}`; + +const notifyDone = (run: DiagRun & { projectName: string | null }) => { + window.dispatchEvent( + new CustomEvent("opencode:notification", { + detail: { + type: "success", + message: `Diagnosis complete for ${runLabel(run)}: ${causeLabel(run.report?.cause ?? run.cause)}`, + }, + }), + ); +}; + +const notifyError = (run: DiagRun & { projectName: string | null }) => { + const reason = (run.error ?? "unknown error").replace(/\s+/g, " ").slice(0, 140); + window.dispatchEvent( + new CustomEvent("opencode:notification", { + detail: { type: "error", message: `Diagnosis failed for ${runLabel(run)}: ${reason}` }, + }), + ); +}; + +export function DiagNotifier({ enabled }: { enabled: boolean }) { + const seen = useRef(new Map()); + const { data } = useQuery({ + queryKey: ["diagnoses", "active"], + queryFn: () => api.getActiveDiagnoses(), + refetchInterval: 10000, + enabled, + retry: false, + }); + + useEffect(() => { + if (!data) return; + const active = new Map(data.map((run) => [run.id, run.projectName])); + for (const run of data) { + if (!seen.current.has(run.id)) seen.current.set(run.id, run.projectName); + } + for (const [id, projectName] of [...seen.current]) { + if (active.has(id)) continue; + seen.current.delete(id); + void api + .getDiagnosis(id) + .then((full) => { + const run = { ...full.run, projectName }; + if (run.status === "done") notifyDone(run); + else if (run.status === "error") notifyError(run); + }) + .catch(() => {}); + } + }, [data]); + + return null; +} diff --git a/apps/web/src/components/Layout.tsx b/apps/web/src/components/Layout.tsx index 7e82556..7c97f11 100644 --- a/apps/web/src/components/Layout.tsx +++ b/apps/web/src/components/Layout.tsx @@ -7,6 +7,7 @@ import { parseMetrics } from "../lib/metrics"; import { Header } from "./layout/Header"; import { NotificationBanner } from "./layout/NotificationBanner"; import { Sidebar } from "./layout/Sidebar"; +import { DiagNotifier } from "./DiagNotifier"; export function Layout({ children }: { children: React.ReactNode }) { const location = useLocation(); @@ -16,8 +17,17 @@ export function Layout({ children }: { children: React.ReactNode }) { queryKey: ["auth", "me"], queryFn: () => api.getMe(), retry: false, + refetchInterval: (query) => (query.state.data?.authenticated ? 5 * 60 * 1000 : false), }); + useEffect(() => { + const handleUnauthorized = () => { + navigate({ to: "/login" }); + }; + window.addEventListener("dequel:unauthorized", handleUnauthorized); + return () => window.removeEventListener("dequel:unauthorized", handleUnauthorized); + }, [navigate]); + useEffect(() => { if (authLoading) return; if (location.pathname === "/login") { @@ -127,6 +137,7 @@ export function Layout({ children }: { children: React.ReactNode }) { /> setNotification(null)} /> +
    {children}
    diff --git a/apps/web/src/components/databases/CreateDatabaseDialog.tsx b/apps/web/src/components/databases/CreateDatabaseDialog.tsx index 1ca21e7..3d171b3 100644 --- a/apps/web/src/components/databases/CreateDatabaseDialog.tsx +++ b/apps/web/src/components/databases/CreateDatabaseDialog.tsx @@ -1,10 +1,11 @@ import { Cpu, Database, HardDrive, ShieldAlert } from "lucide-react"; -import { useEffect, useState } from "react"; +import { useState } from "react"; import { useQuery } from "@tanstack/react-query"; import * as api from "../../api/client"; import type { DatabaseType, Project } from "../../types"; import { Button } from "../ui/button"; -import { DATABASE_ENGINES, DatabaseSelect } from "../ui/DatabaseSelect"; +import { DATABASE_ENGINES } from "../ui/DatabaseSelect"; +import { getDatabaseLogo } from "../logos/DatabaseLogos"; import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle } from "../ui/dialog"; import { Input } from "../ui/input"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "../ui/select"; @@ -43,11 +44,6 @@ export function CreateDatabaseDialog({ queryFn: () => api.listServers().catch(() => []), }); - useEffect(() => { - const engine = DATABASE_ENGINES.find((item) => item.type === type); - if (engine) setVersion(engine.defaultVersion); - }, [type]); - const create = async () => { setIsCreating(true); setError(null); @@ -131,7 +127,25 @@ export function CreateDatabaseDialog({ - setType(val)} /> + @@ -139,14 +153,20 @@ export function CreateDatabaseDialog({
    - setVersion(event.target.value)} - className="bg-background/50 border-border/80 text-xs font-mono" - /> +
    diff --git a/apps/web/src/components/databases/DatabaseStudio.tsx b/apps/web/src/components/databases/DatabaseStudio.tsx index 382d600..e648a7f 100644 --- a/apps/web/src/components/databases/DatabaseStudio.tsx +++ b/apps/web/src/components/databases/DatabaseStudio.tsx @@ -1,273 +1,20 @@ -import { useQuery } from "@tanstack/react-query"; -import { Code2, Layers, ShieldAlert, Table as TableIcon } from "lucide-react"; -import { useEffect, useState } from "react"; -import * as api from "../../api/client"; -import type { Database, QueryExecResult } from "../../types"; -import { DataTableView } from "./studio/DataTableView"; -import { SqlQueryView } from "./studio/SqlQueryView"; -import { StructureView } from "./studio/StructureView"; -import { TableSidebar } from "./studio/TableSidebar"; +import type { Database } from "../../types"; +import { MongoStudio } from "./studio/mongo/MongoStudio"; +import { RedisStudio } from "./studio/redis/RedisStudio"; +import { SqlStudio } from "./studio/SqlStudio"; interface DatabaseStudioProps { database: Database; } export function DatabaseStudio({ database }: DatabaseStudioProps) { - const [activeTab, setActiveTab] = useState<"data" | "structure" | "sql">("data"); - const [selectedTable, setSelectedTable] = useState(null); - const [searchQuery, setSearchQuery] = useState(""); - const [page, setPage] = useState(1); - const pageSize = 10; + if (database.type === "mongodb") { + return ; + } - const [selectedRows, setSelectedRows] = useState([]); + if (database.type === "redis") { + return ; + } - // Custom SQL Query state - const [sqlQuery, setSqlQuery] = useState("SELECT * FROM information_schema.tables WHERE table_schema='public';"); - const [sqlResult, setSqlResult] = useState(null); - const [sqlError, setSqlError] = useState(null); - const [isExecutingSql, setIsExecutingSql] = useState(false); - - const [dataResult, setDataResult] = useState(null); - const [isLoadingData, setIsLoadingData] = useState(false); - const [dataError, setDataError] = useState(null); - - // Fetch Table List - const { - data: tables = [], - refetch: refetchTables, - isLoading: isLoadingTables, - } = useQuery({ - queryKey: ["database-tables", database.id], - queryFn: async () => { - const res = await api.getDatabaseTables(database.id).catch(() => []); - if (res.length > 0 && !selectedTable) { - setSelectedTable(res[0].name); - } - return res; - }, - }); - - // Auto-select first table if available - useEffect(() => { - if (tables.length > 0 && !selectedTable) { - setSelectedTable(tables[0].name); - } - }, [tables, selectedTable]); - - // Fetch Data for Selected Table - const loadTableData = async () => { - if (!selectedTable) return; - setIsLoadingData(true); - setDataError(null); - setSelectedRows([]); - - const offset = (page - 1) * pageSize; - let query = ""; - - if (database.type === "postgresql") { - query = `SELECT * FROM "${selectedTable}" LIMIT ${pageSize} OFFSET ${offset};`; - } else if (database.type === "mysql") { - query = `SELECT * FROM \`${selectedTable}\` LIMIT ${pageSize} OFFSET ${offset};`; - } else if (database.type === "mongodb") { - query = `db.${selectedTable}.find().skip(${offset}).limit(${pageSize})`; - } else { - query = `KEYS *`; - } - - try { - const res = await api.queryDatabase(database.id, query); - setDataResult(res); - } catch (err: any) { - setDataError(err.message || "Failed to load table records"); - setDataResult(null); - } finally { - setIsLoadingData(false); - } - }; - - useEffect(() => { - if (selectedTable && activeTab === "data") { - loadTableData(); - } - }, [selectedTable, page, activeTab]); - - const handleExecuteSql = async () => { - if (!sqlQuery.trim()) return; - setIsExecutingSql(true); - setSqlError(null); - try { - const res = await api.queryDatabase(database.id, sqlQuery); - setSqlResult(res); - } catch (err: any) { - setSqlError(err.message || "Query execution failed"); - setSqlResult(null); - } finally { - setIsExecutingSql(false); - } - }; - - const handleSaveInlineEdit = async (rowIndex: number, editedValues: Record) => { - if (!selectedTable || !dataResult) return; - const primaryKeyCol = dataResult.columns[0] || "id"; - const originalRow = dataResult.rows[rowIndex]; - const primaryKeyVal = originalRow[primaryKeyCol]; - - const setAssignments = dataResult.columns - .map((col) => { - const val = editedValues[col]; - if (val === undefined || val === "" || val.toUpperCase() === "NULL") return `"${col}" = NULL`; - if (!isNaN(Number(val)) && val.trim() !== "") return `"${col}" = ${val}`; - if (val.toLowerCase() === "true" || val.toLowerCase() === "false") return `"${col}" = ${val}`; - return `"${col}" = '${val.replace(/'/g, "''")}'`; - }) - .join(", "); - - const updateQuery = `UPDATE "${selectedTable}" SET ${setAssignments} WHERE "${primaryKeyCol}" = '${primaryKeyVal}';`; - - try { - await api.queryDatabase(database.id, updateQuery); - loadTableData(); - } catch (err: any) { - setDataError(`Save failed: ${err.message}`); - } - }; - - const handleSaveInlineAdd = async (newValues: Record) => { - if (!selectedTable || !dataResult) return; - const cols = dataResult.columns.filter((c) => newValues[c] !== undefined && newValues[c].trim() !== ""); - if (cols.length === 0) return; - - const colNames = cols.map((c) => `"${c}"`).join(", "); - const values = cols.map((c) => `'${newValues[c].replace(/'/g, "''")}'`).join(", "); - const insertQuery = `INSERT INTO "${selectedTable}" (${colNames}) VALUES (${values});`; - - try { - await api.queryDatabase(database.id, insertQuery); - loadTableData(); - } catch (err: any) { - setDataError(`Insert failed: ${err.message}`); - } - }; - - const handleDeleteSelected = async () => { - if (!selectedTable || !dataResult || selectedRows.length === 0) return; - const primaryKeyCol = dataResult.columns[0] || "id"; - const pks = selectedRows.map((idx) => dataResult.rows[idx][primaryKeyCol]); - const formattedPks = pks.map((pk) => `'${String(pk).replace(/'/g, "''")}'`).join(", "); - - const deleteQuery = `DELETE FROM "${selectedTable}" WHERE "${primaryKeyCol}" IN (${formattedPks});`; - - try { - await api.queryDatabase(database.id, deleteQuery); - setSelectedRows([]); - loadTableData(); - } catch (err: any) { - setDataError(`Delete failed: ${err.message}`); - } - }; - - return ( -
    - {/* Left Sidebar Schema Navigator */} - { - setSelectedTable(t); - setPage(1); - }} - onRefresh={refetchTables} - isLoading={isLoadingTables} - searchQuery={searchQuery} - onSearchChange={setSearchQuery} - /> - - {/* Main Data Studio Canvas */} -
    - {/* Top Mode Switcher Bar */} -
    -
    - - - -
    - -
    - Table: {selectedTable || "None"} -
    -
    - - {dataError && ( -
    - - {dataError} -
    - )} - - {activeTab === "data" && ( - - setSelectedRows(checked ? [...selectedRows, idx] : selectedRows.filter((i) => i !== idx)) - } - onSelectAllRows={(checked) => - setSelectedRows(checked && dataResult ? dataResult.rows.map((_, i) => i) : []) - } - onSaveInlineEdit={handleSaveInlineEdit} - onSaveInlineAdd={handleSaveInlineAdd} - onDeleteSelected={handleDeleteSelected} - /> - )} - - {activeTab === "structure" && } - - {activeTab === "sql" && ( - - )} -
    -
    - ); + return ; } diff --git a/apps/web/src/components/databases/studio/SqlStudio.tsx b/apps/web/src/components/databases/studio/SqlStudio.tsx new file mode 100644 index 0000000..64742b7 --- /dev/null +++ b/apps/web/src/components/databases/studio/SqlStudio.tsx @@ -0,0 +1,262 @@ +import { useQuery } from "@tanstack/react-query"; +import { Code2, Layers, ShieldAlert, Table as TableIcon } from "lucide-react"; +import { useEffect, useState } from "react"; +import * as api from "../../../api/client"; +import type { Database, QueryExecResult } from "../../../types"; +import { DataTableView } from "./DataTableView"; +import { SqlQueryView } from "./SqlQueryView"; +import { StructureView } from "./StructureView"; +import { TableSidebar } from "./TableSidebar"; + +interface SqlStudioProps { + database: Database; +} + +export function SqlStudio({ database }: SqlStudioProps) { + const [activeTab, setActiveTab] = useState<"data" | "structure" | "sql">("data"); + const [selectedTable, setSelectedTable] = useState(null); + const [searchQuery, setSearchQuery] = useState(""); + const [page, setPage] = useState(1); + const pageSize = 10; + + const [selectedRows, setSelectedRows] = useState([]); + + const [sqlQuery, setSqlQuery] = useState("SELECT * FROM information_schema.tables WHERE table_schema='public';"); + const [sqlResult, setSqlResult] = useState(null); + const [sqlError, setSqlError] = useState(null); + const [isExecutingSql, setIsExecutingSql] = useState(false); + + const [dataResult, setDataResult] = useState(null); + const [isLoadingData, setIsLoadingData] = useState(false); + const [dataError, setDataError] = useState(null); + + const { + data: tables = [], + refetch: refetchTables, + isLoading: isLoadingTables, + } = useQuery({ + queryKey: ["database-tables", database.id], + queryFn: async () => { + const res = await api.getDatabaseTables(database.id).catch(() => []); + if (res.length > 0 && !selectedTable) { + setSelectedTable(res[0].name); + } + return res; + }, + }); + + useEffect(() => { + if (tables.length > 0 && !selectedTable) { + setSelectedTable(tables[0].name); + } + }, [tables, selectedTable]); + + const loadTableData = async () => { + if (!selectedTable) return; + setIsLoadingData(true); + setDataError(null); + setSelectedRows([]); + + const offset = (page - 1) * pageSize; + let query = ""; + + if (database.type === "postgresql") { + query = `SELECT * FROM "${selectedTable}" LIMIT ${pageSize} OFFSET ${offset};`; + } else { + query = `SELECT * FROM \`${selectedTable}\` LIMIT ${pageSize} OFFSET ${offset};`; + } + + try { + const res = await api.queryDatabase(database.id, query); + setDataResult(res); + } catch (err: any) { + setDataError(err.message || "Failed to load table records"); + setDataResult(null); + } finally { + setIsLoadingData(false); + } + }; + + useEffect(() => { + if (selectedTable && activeTab === "data") { + loadTableData(); + } + }, [selectedTable, page, activeTab]); + + const handleExecuteSql = async () => { + if (!sqlQuery.trim()) return; + setIsExecutingSql(true); + setSqlError(null); + try { + const res = await api.queryDatabase(database.id, sqlQuery); + setSqlResult(res); + } catch (err: any) { + setSqlError(err.message || "Query execution failed"); + setSqlResult(null); + } finally { + setIsExecutingSql(false); + } + }; + + const handleSaveInlineEdit = async (rowIndex: number, editedValues: Record) => { + if (!selectedTable || !dataResult) return; + const primaryKeyCol = dataResult.columns[0] || "id"; + const originalRow = dataResult.rows[rowIndex]; + const primaryKeyVal = originalRow[primaryKeyCol]; + + const setAssignments = dataResult.columns + .map((col) => { + const val = editedValues[col]; + if (val === undefined || val === "" || val.toUpperCase() === "NULL") return `"${col}" = NULL`; + if (!isNaN(Number(val)) && val.trim() !== "") return `"${col}" = ${val}`; + if (val.toLowerCase() === "true" || val.toLowerCase() === "false") return `"${col}" = ${val}`; + return `"${col}" = '${val.replace(/'/g, "''")}'`; + }) + .join(", "); + + const updateQuery = `UPDATE "${selectedTable}" SET ${setAssignments} WHERE "${primaryKeyCol}" = '${primaryKeyVal}';`; + + try { + await api.queryDatabase(database.id, updateQuery); + loadTableData(); + } catch (err: any) { + setDataError(`Save failed: ${err.message}`); + } + }; + + const handleSaveInlineAdd = async (newValues: Record) => { + if (!selectedTable || !dataResult) return; + const cols = dataResult.columns.filter((c) => newValues[c] !== undefined && newValues[c].trim() !== ""); + if (cols.length === 0) return; + + const colNames = cols.map((c) => `"${c}"`).join(", "); + const values = cols.map((c) => `'${newValues[c].replace(/'/g, "''")}'`).join(", "); + const insertQuery = `INSERT INTO "${selectedTable}" (${colNames}) VALUES (${values});`; + + try { + await api.queryDatabase(database.id, insertQuery); + loadTableData(); + } catch (err: any) { + setDataError(`Insert failed: ${err.message}`); + } + }; + + const handleDeleteSelected = async () => { + if (!selectedTable || !dataResult || selectedRows.length === 0) return; + const primaryKeyCol = dataResult.columns[0] || "id"; + const pks = selectedRows.map((idx) => dataResult.rows[idx][primaryKeyCol]); + const formattedPks = pks.map((pk) => `'${String(pk).replace(/'/g, "''")}'`).join(", "); + + const deleteQuery = `DELETE FROM "${selectedTable}" WHERE "${primaryKeyCol}" IN (${formattedPks});`; + + try { + await api.queryDatabase(database.id, deleteQuery); + setSelectedRows([]); + loadTableData(); + } catch (err: any) { + setDataError(`Delete failed: ${err.message}`); + } + }; + + return ( +
    + { + setSelectedTable(t); + setPage(1); + }} + onRefresh={refetchTables} + isLoading={isLoadingTables} + searchQuery={searchQuery} + onSearchChange={setSearchQuery} + /> + +
    +
    +
    + + + +
    + +
    + Table: {selectedTable || "None"} +
    +
    + + {dataError && ( +
    + + {dataError} +
    + )} + + {activeTab === "data" && ( + + setSelectedRows(checked ? [...selectedRows, idx] : selectedRows.filter((i) => i !== idx)) + } + onSelectAllRows={(checked) => + setSelectedRows(checked && dataResult ? dataResult.rows.map((_, i) => i) : []) + } + onSaveInlineEdit={handleSaveInlineEdit} + onSaveInlineAdd={handleSaveInlineAdd} + onDeleteSelected={handleDeleteSelected} + /> + )} + + {activeTab === "structure" && } + + {activeTab === "sql" && ( + + )} +
    +
    + ); +} diff --git a/apps/web/src/components/databases/studio/mongo/MongoCollectionSidebar.tsx b/apps/web/src/components/databases/studio/mongo/MongoCollectionSidebar.tsx new file mode 100644 index 0000000..0db36aa --- /dev/null +++ b/apps/web/src/components/databases/studio/mongo/MongoCollectionSidebar.tsx @@ -0,0 +1,180 @@ +import { Boxes, Plus, RefreshCw, Search } from "lucide-react"; +import { useState } from "react"; +import type { TableInfo } from "../../../../types"; +import { Badge } from "../../../ui/badge"; +import { Button } from "../../../ui/button"; +import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from "../../../ui/dialog"; +import { Input } from "../../../ui/input"; + +interface MongoCollectionSidebarProps { + collections: TableInfo[]; + selectedCollection: string | null; + onSelectCollection: (name: string) => void; + onRefresh: () => void; + onCreateCollection: (name: string) => Promise; + isLoading: boolean; + databaseName: string; +} + +export function MongoCollectionSidebar({ + collections, + selectedCollection, + onSelectCollection, + onRefresh, + onCreateCollection, + isLoading, + databaseName, +}: MongoCollectionSidebarProps) { + const [searchQuery, setSearchQuery] = useState(""); + const [showCreateModal, setShowCreateModal] = useState(false); + const [newCollectionName, setNewCollectionName] = useState(""); + const [isCreating, setIsCreating] = useState(false); + const [createError, setCreateError] = useState(null); + + const filtered = collections.filter((c) => c.name.toLowerCase().includes(searchQuery.toLowerCase())); + + const handleCreate = async (e: React.FormEvent) => { + e.preventDefault(); + const name = newCollectionName.trim(); + if (!name) return; + setIsCreating(true); + setCreateError(null); + try { + await onCreateCollection(name); + setNewCollectionName(""); + setShowCreateModal(false); + } catch (err: any) { + setCreateError(err.message || "Failed to create collection"); + } finally { + setIsCreating(false); + } + }; + + return ( +
    +
    +
    + + + Collections + +
    + + +
    +
    + +
    + + setSearchQuery(e.target.value)} + placeholder="Search collections..." + className="w-full bg-transparent text-xs text-foreground focus:outline-none placeholder:text-muted-foreground" + /> +
    + +
    + {filtered.length === 0 ? ( +

    No collections found.

    + ) : ( + filtered.map((c) => { + const isSelected = selectedCollection === c.name; + return ( + + ); + }) + )} +
    +
    + +
    + DB: {databaseName} + {collections.length} Collections +
    + + {/* Create Collection Dialog */} + + +
    + + + Create Collection + + + Add a new document collection to the MongoDB database. + + + +
    + + setNewCollectionName(e.target.value)} + placeholder="e.g. orders, users, audit_logs" + className="bg-card border-border/80 font-mono text-xs" + autoFocus + /> +
    + + {createError &&

    {createError}

    } + + + + + +
    +     +     +
    + ); +} diff --git a/apps/web/src/components/databases/studio/mongo/MongoDocumentCard.tsx b/apps/web/src/components/databases/studio/mongo/MongoDocumentCard.tsx new file mode 100644 index 0000000..5ad45b3 --- /dev/null +++ b/apps/web/src/components/databases/studio/mongo/MongoDocumentCard.tsx @@ -0,0 +1,130 @@ +import { Check, Copy, Edit3, Trash2 } from "lucide-react"; +import { useState } from "react"; +import { Badge } from "../../../ui/badge"; +import { Button } from "../../../ui/button"; + +interface MongoDocumentCardProps { + doc: Record; + index: number; + onEdit: (doc: Record) => void; + onDelete: (id: string) => void; +} + +export function MongoDocumentCard({ doc, index, onEdit, onDelete }: MongoDocumentCardProps) { + const [copied, setCopied] = useState(false); + const docId = String(doc._id ?? doc.id ?? `doc-${index}`); + const jsonString = JSON.stringify(doc, null, 2); + + const handleCopy = () => { + navigator.clipboard.writeText(jsonString); + setCopied(true); + setTimeout(() => setCopied(false), 2000); + }; + + const renderValue = (val: unknown): React.ReactNode => { + if (val === null) return null; + if (val === undefined) return undefined; + if (typeof val === "boolean") return {String(val)}; + if (typeof val === "number") return {val}; + if (typeof val === "string") return "{val}"; + if (Array.isArray(val)) { + return ( + + [ + {val.map((item, i) => ( + + {i > 0 && ", "} + {renderValue(item)} + + ))} + ] + + ); + } + if (typeof val === "object") { + const entries = Object.entries(val as Record); + return ( + + {"{"} + + {entries.map(([k, v], i) => ( + + "{k}": {renderValue(v)} + {i < entries.length - 1 && ","} + + ))} + + {"}"} + + ); + } + return {String(val)}; + }; + + return ( +
    +
    +
    + + _id: {docId} + + + {Object.keys(doc).length} {Object.keys(doc).length === 1 ? "field" : "fields"} + +
    + +
    + + + +
    +
    + +
    + {"{"} +
    + {Object.entries(doc).map(([k, v], i, arr) => ( +
    + "{k}": +
    + {renderValue(v)} + {i < arr.length - 1 ? "," : ""} +
    +
    + ))} +
    + {"}"} +
    +
    + ); +} diff --git a/apps/web/src/components/databases/studio/mongo/MongoDocumentModal.tsx b/apps/web/src/components/databases/studio/mongo/MongoDocumentModal.tsx new file mode 100644 index 0000000..015b692 --- /dev/null +++ b/apps/web/src/components/databases/studio/mongo/MongoDocumentModal.tsx @@ -0,0 +1,129 @@ +import { Braces, Check, Copy, FileJson } from "lucide-react"; +import { useEffect, useState } from "react"; +import { Button } from "../../../ui/button"; +import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from "../../../ui/dialog"; + +interface MongoDocumentModalProps { + open: boolean; + onOpenChange: (open: boolean) => void; + documentToEdit: Record | null; + collectionName: string; + onSave: (doc: Record) => Promise; +} + +export function MongoDocumentModal({ + open, + onOpenChange, + documentToEdit, + collectionName, + onSave, +}: MongoDocumentModalProps) { + const isEdit = !!documentToEdit; + const [jsonText, setJsonText] = useState(""); + const [error, setError] = useState(null); + const [isSaving, setIsSaving] = useState(false); + + useEffect(() => { + if (open) { + setError(null); + if (documentToEdit) { + setJsonText(JSON.stringify(documentToEdit, null, 2)); + } else { + setJsonText("{\n \n}"); + } + } + }, [open, documentToEdit]); + + const handleFormat = () => { + try { + const parsed = JSON.parse(jsonText); + setJsonText(JSON.stringify(parsed, null, 2)); + setError(null); + } catch (err: any) { + setError(`Invalid JSON: ${err.message}`); + } + }; + + const handleSave = async () => { + setError(null); + let parsed: any; + try { + parsed = JSON.parse(jsonText); + } catch (err: any) { + setError(`Invalid JSON: ${err.message}`); + return; + } + + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + setError("Document must be a valid JSON object"); + return; + } + + setIsSaving(true); + try { + await onSave(parsed); + onOpenChange(false); + } catch (err: any) { + setError(err.message || "Failed to save document"); + } finally { + setIsSaving(false); + } + }; + + return ( + + + +
    + + + {isEdit ? "Edit Document" : "Insert Document"} —{" "} + {collectionName} + + +
    + + {isEdit + ? "Modify the document fields. Changes will replace the existing document." + : "Enter the document as a valid JSON object. A unique _id will be generated if omitted."} + +     + +
    +