diff --git a/apps/media-service/.env.example b/apps/media-service/.env.example index 558df8e..3f92d7e 100644 --- a/apps/media-service/.env.example +++ b/apps/media-service/.env.example @@ -30,3 +30,48 @@ MEDIA_DESCRIBE_TIMEOUT_MS=30000 ORCHESTRATOR_URL=http://localhost:4001 # Same value as orchestrator-service INTERNAL_SERVICE_SECRET (min 16 chars). INTERNAL_SERVICE_SECRET= + +# AI media: Site Blueprint + creative pipeline (docs/media-ai.md). Off by default. +# server, orchestrator-service and media-service all read this block; keep them identical. +MEDIA_ENABLED=false +# Credits per generated image, per video second and per Site Blueprint call. +# Required in production once MEDIA_ENABLED=true; 1 elsewhere when unset. +MEDIA_CREDIT_IMAGE= +MEDIA_CREDIT_VIDEO_PER_SECOND= +MEDIA_CREDIT_BLUEPRINT= +# images:videos per budget tier, plus hard caps. +MEDIA_BUDGET_IMAGES_ONLY=8:0 +MEDIA_BUDGET_LIGHT=4:1 +MEDIA_BUDGET_STANDARD=8:3 +MEDIA_BUDGET_RICH=14:6 +MEDIA_HARD_MAX_IMAGES=24 +MEDIA_HARD_MAX_VIDEOS=10 +MEDIA_HARD_MAX_GENERATED_ASSETS=60 +VIDEO_MIN_SECONDS=5 +VIDEO_MAX_SECONDS=10 +# Circuit breaker: provider spend over the last 24 h above this pauses AI media. Empty = no cap. +MEDIA_DAILY_COST_CAP_USD= +# Provider cost estimates in USD, used when a provider does not report cost. +MEDIA_COST_IMAGE_USD=0.25 +MEDIA_COST_VIDEO_USD_PER_SECOND=0.08 +MEDIA_COST_LLM_CALL_USD=0.05 +MEDIA_COST_BLUEPRINT_USD=0.4 + +# fake = no provider calls or spend. live needs OPENAI_API_KEY, RUNWARE_API_KEY and INTERNAL_SERVICE_SECRET. +MEDIA_AI_PROVIDER_MODE=fake +# OPENAI_API_KEY=replace-with-provider-key +OPENAI_IMAGE_MODEL=gpt-image-2.5-sunburst +OPENAI_IMAGE_QUALITY=high +VIDEO_PROVIDER=runware +# RUNWARE_API_KEY=replace-with-provider-key +MEDIA_VIDEO_MODEL=minimax:h3@max +MEDIA_VIDEO_RESOLUTION=768p +# Longest a single video render may take before the item falls back to a still. +MEDIA_VIDEO_TIMEOUT_S=600 +MEDIA_VIDEO_PROMPT_EXPANSION=disabled +MEDIA_VIDEO_PROMPT_MAX_CHARS=3000 +# Concurrent provider calls across all media-service replicas (Redis leases). +MEDIA_IMAGE_CONCURRENCY=3 +MEDIA_VIDEO_CONCURRENCY=2 +# https hosts provider result downloads may come from, comma-separated. +MEDIA_PROVIDER_HOST_ALLOWLIST=vm.runware.ai,im.runware.ai diff --git a/apps/media-service/package.json b/apps/media-service/package.json index 4538eb0..93b0e29 100644 --- a/apps/media-service/package.json +++ b/apps/media-service/package.json @@ -17,7 +17,9 @@ "dependencies": { "@aws-sdk/client-s3": "3.1144.0", "@aws-sdk/lib-storage": "3.1144.0", + "@aws-sdk/s3-request-presigner": "3.1144.0", "@repo/events": "workspace:*", + "@repo/media-ai": "workspace:*", "@repo/schemas": "workspace:*", "@useframe/db": "workspace:*", "bullmq": "^5.28.0", diff --git a/apps/media-service/src/config/schema.ts b/apps/media-service/src/config/schema.ts index bed5e06..5d864c8 100644 --- a/apps/media-service/src/config/schema.ts +++ b/apps/media-service/src/config/schema.ts @@ -41,8 +41,46 @@ export const envSchema = z ORCHESTRATOR_URL: z.string().url().default("http://localhost:4001"), INTERNAL_SERVICE_SECRET: optional(z.string().min(16, "INTERNAL_SERVICE_SECRET must be at least 16 chars")), MEDIA_DESCRIBE_TIMEOUT_MS: int(30_000), + + // AI creative pipeline. Budgets, caps and credit prices are read by @repo/media-ai. + MEDIA_AI_PROVIDER_MODE: z.enum(["fake", "live"]).default("fake"), + OPENAI_API_KEY: optional(z.string().min(1)), + OPENAI_IMAGE_MODEL: z.string().min(1).default("gpt-image-2.5-sunburst"), + OPENAI_IMAGE_QUALITY: z.enum(["low", "medium", "high", "xhigh", "max", "auto"]).default("high"), + VIDEO_PROVIDER: z.enum(["runware", "fal", "minimax"]).default("runware"), + RUNWARE_API_KEY: optional(z.string().min(1)), + MEDIA_VIDEO_MODEL: z.string().min(1).default("minimax:h3@max"), + MEDIA_VIDEO_RESOLUTION: z.enum(["480p", "768p"]).default("768p"), + MEDIA_VIDEO_TIMEOUT_S: int(600), + MEDIA_VIDEO_PROMPT_EXPANSION: z.enum(["disabled", "balanced", "quality"]).default("disabled"), + MEDIA_VIDEO_PROMPT_MAX_CHARS: int(3000), + MEDIA_IMAGE_CONCURRENCY: int(3), + MEDIA_VIDEO_CONCURRENCY: int(2), + // Hosts provider result downloads may come from (https only). + MEDIA_PROVIDER_HOST_ALLOWLIST: z + .string() + .default("vm.runware.ai,im.runware.ai") + .transform((v) => + v + .split(",") + .map((h) => h.trim().toLowerCase()) + .filter(Boolean), + ), }) .superRefine((value, ctx) => { + if (value.MEDIA_AI_PROVIDER_MODE === "live") { + if (!value.OPENAI_API_KEY) { + ctx.addIssue({ code: z.ZodIssueCode.custom, path: ["OPENAI_API_KEY"], message: "OPENAI_API_KEY is required in live mode" }) + } + if (value.VIDEO_PROVIDER !== "runware") { + ctx.addIssue({ code: z.ZodIssueCode.custom, path: ["VIDEO_PROVIDER"], message: "Only the runware video provider is built" }) + } else if (!value.RUNWARE_API_KEY) { + ctx.addIssue({ code: z.ZodIssueCode.custom, path: ["RUNWARE_API_KEY"], message: "RUNWARE_API_KEY is required in live mode" }) + } + if (!value.INTERNAL_SERVICE_SECRET) { + ctx.addIssue({ code: z.ZodIssueCode.custom, path: ["INTERNAL_SERVICE_SECRET"], message: "INTERNAL_SERVICE_SECRET is required in live mode" }) + } + } if (!value.MEDIA_R2_ENDPOINT && !value.CLOUDFLARE_ACCOUNT_ID) { ctx.addIssue({ code: z.ZodIssueCode.custom, diff --git a/apps/media-service/src/createAsset.ts b/apps/media-service/src/createAsset.ts index f22c279..a6e329b 100644 --- a/apps/media-service/src/createAsset.ts +++ b/apps/media-service/src/createAsset.ts @@ -92,3 +92,46 @@ export async function createAssetFromBytes(deps: Deps, input: CreateAssetInput): await deps.enqueue({ assetId: asset.id, ...(input.options ? { options: input.options } : {}) }, `${asset.id}-0`) return { assetId: asset.id } } + +// Direct provider delivery: the row and key exist first, the provider PUTs the +// original, then finishReservedAsset verifies it and starts processing. +export async function reserveGeneratedAsset( + deps: Deps, + input: { projectId: string; userId: string; kind: "IMAGE" | "VIDEO"; mime: string; purpose?: string; title?: string; generation?: Prisma.InputJsonValue }, +): Promise<{ assetId: string; key: string }> { + return deps.db.$transaction(async (tx) => { + const created = await tx.mediaAsset.create({ + data: { + projectId: input.projectId, + userId: input.userId, + kind: input.kind, + origin: "GENERATED", + status: "UPLOADING", + title: input.title?.slice(0, 120) ?? null, + purpose: input.purpose ?? null, + aiGenerated: true, + generation: input.generation, + originalKey: "", + originalMime: input.mime, + originalBytes: 0, + }, + select: { id: true }, + }) + const key = `${mediaPrefix(input.projectId, created.id)}orig` + await tx.mediaAsset.update({ where: { id: created.id }, data: { originalKey: key } }) + return { assetId: created.id, key } + }) +} + +export async function finishReservedAsset( + deps: Deps, + asset: { assetId: string; key: string; kind: "IMAGE" | "VIDEO" }, + options?: MediaProcessOptions, +): Promise { + const head = await deps.store.head(asset.key) + const max = asset.kind === "VIDEO" ? deps.config.maxVideoBytes : deps.config.maxImageBytes + if (!head || head.size <= 0 || head.size > max) return false + await deps.db.mediaAsset.update({ where: { id: asset.assetId }, data: { status: "PROCESSING", originalBytes: head.size } }) + await deps.enqueue({ assetId: asset.assetId, ...(options ? { options } : {}) }, `${asset.assetId}-0`) + return true +} diff --git a/apps/media-service/src/generate/assets.ts b/apps/media-service/src/generate/assets.ts new file mode 100644 index 0000000..b7963d2 --- /dev/null +++ b/apps/media-service/src/generate/assets.ts @@ -0,0 +1,118 @@ +import sharp from "sharp" +import type { Prisma } from "@useframe/db" +import { + deltaE76, + hexToRgb, + minDeltaE, + nearestStandardSize, + ProviderError, + requestSizeFor, + sha256, + type Rgb, + type Size, +} from "@repo/media-ai" +import { log } from "@/lib/logger.js" +import type { GenerateDeps } from "./deps.js" + +export type Analysis = { paletteDeltaE: number; textZoneLuminance: number; textZoneVariance: number } + +// Crops to the target ratio and resizes, so every frame matches the video canvas exactly. +export async function fitToSize(bytes: Buffer, target: Size): Promise { + return sharp(bytes).resize(target.width, target.height, { fit: "cover", position: "attention" }).png().toBuffer() +} + +type ImageCall = { + prompt: string + target: Size + requestKey: string + edit?: Buffer[] +} + +// Asks for the target size (or the smallest valid one above it); a rejected +// size falls back to the nearest standard size, then everything is cropped to the target. +export async function generateImage(deps: GenerateDeps, call: ImageCall): Promise<{ bytes: Buffer; costUsd: number; model: string; size: string }> { + return deps.semaphores.image.run(async () => { + const attempt = async (size: Size) => { + const s = `${size.width}x${size.height}` + const req = { prompt: call.prompt, size: s, quality: deps.config.imageQuality, format: "png" as const, requestKey: call.requestKey } + const result = call.edit?.length ? await deps.images.edit({ ...req, images: call.edit }) : await deps.images.generate(req) + return { ...result, size: s } + } + let result + try { + result = await attempt(requestSizeFor(call.target)) + } catch (err) { + if (!(err instanceof ProviderError) || err.kind !== "INVALID_SIZE") throw err + const fallback = nearestStandardSize(call.target) + log.warn("image size rejected; using a standard size", { requested: requestSizeFor(call.target), fallback }) + result = await attempt(fallback) + } + return { bytes: await fitToSize(result.bytes, call.target), costUsd: result.costUsd ?? deps.config.ai.providerCostUsd.image, model: result.model, size: result.size } + }) +} + +// Soft signals only: palette distance of the dominant colours and how calm the headline area is. +export async function analyzeImage(bytes: Buffer, palette: string[], zone: string): Promise { + const { data, info } = await sharp(bytes).resize(64, 64, { fit: "fill" }).removeAlpha().raw().toBuffer({ resolveWithObject: true }) + const buckets = new Map() + for (let i = 0; i < data.length; i += 3) { + const rgb: Rgb = [data[i]! & 0xe0, data[i + 1]! & 0xe0, data[i + 2]! & 0xe0] + const key = rgb.join(",") + const b = buckets.get(key) + if (b) b.count++ + else buckets.set(key, { rgb, count: 1 }) + } + const dominant = [...buckets.values()].sort((a, b) => b.count - a.count).slice(0, 5).map((b) => b.rgb) + const pal = palette.flatMap((h) => (hexToRgb(h) ? [hexToRgb(h)!] : [])) + const paletteDeltaE = pal.length ? Math.round(Math.min(...dominant.map((c) => minDeltaE(c, pal))) * 10) / 10 : 0 + + const third = Math.floor(info.width / 3) + const [x0, x1] = zone === "right" ? [info.width - third, info.width] : zone === "center" ? [third, 2 * third] : [0, third] + const lums: number[] = [] + for (let y = 0; y < info.height; y++) { + for (let x = x0; x < x1; x++) { + const i = (y * info.width + x) * 3 + lums.push((0.2126 * data[i]! + 0.7152 * data[i + 1]! + 0.0722 * data[i + 2]!) / 255) + } + } + const mean = lums.reduce((a, b) => a + b, 0) / lums.length + const variance = lums.reduce((a, b) => a + (b - mean) ** 2, 0) / lums.length + return { paletteDeltaE, textZoneLuminance: Math.round(mean * 1000) / 1000, textZoneVariance: Math.round(variance * 10000) / 10000 } +} + +// Scrim hint for background slots: needed when the headline area is busy or mid-grey. +export function overlayHint(analysis: Analysis, slotDefault: "none" | "light" | "dark" | undefined): "none" | "light" | "dark" { + const wanted = slotDefault ?? "dark" + if (wanted === "none") return "none" + const busy = analysis.textZoneVariance > 0.02 + if (!busy && wanted === "dark" && analysis.textZoneLuminance < 0.35) return "none" + if (!busy && wanted === "light" && analysis.textZoneLuminance > 0.7) return "none" + return wanted +} + +export async function waitForAsset(deps: GenerateDeps, assetId: string): Promise<"READY" | "FAILED"> { + const deadline = Date.now() + deps.config.assetTimeoutMs + let wait = 1000 + for (;;) { + const asset = await deps.db.mediaAsset.findUnique({ where: { id: assetId }, select: { status: true } }) + if (asset?.status === "READY" || asset?.status === "FAILED") return asset.status + if (!asset || Date.now() > deadline) return "FAILED" + await deps.sleep(wait) + wait = Math.min(wait * 1.5, 5000) + } +} + +export async function variantBytes(deps: GenerateDeps, assetId: string, role: string): Promise { + const asset = await deps.db.mediaAsset.findUnique({ where: { id: assetId }, select: { variants: true } }) + const variants = (Array.isArray(asset?.variants) ? asset.variants : []) as { role?: string; key?: string }[] + const key = variants.find((v) => v.role === role)?.key + return key ? deps.store.getBuffer(key, 200 * 1024 * 1024) : null +} + +export async function updateAsset(deps: GenerateDeps, assetId: string, data: Prisma.MediaAssetUpdateInput): Promise { + await deps.db.mediaAsset.update({ where: { id: assetId }, data }) +} + +export const promptHash = (prompt: string) => sha256(prompt).slice(0, 12) + +export { deltaE76 } diff --git a/apps/media-service/src/generate/context.ts b/apps/media-service/src/generate/context.ts new file mode 100644 index 0000000..e7f5dcb --- /dev/null +++ b/apps/media-service/src/generate/context.ts @@ -0,0 +1,99 @@ +import { randomUUID } from "node:crypto" +import { readFile, rm, writeFile } from "node:fs/promises" +import path from "node:path" +import { prisma } from "@useframe/db" +import { + FakeImageProvider, + FakeLlm, + FakeVideoProvider, + loadMediaAiConfig, + OpenAiImageProvider, + RunwareVideoProvider, +} from "@repo/media-ai" +import { env } from "@/config/env.js" +import { deps as processDeps } from "@/context.js" +import { createAssetFromBytes, finishReservedAsset, reserveGeneratedAsset } from "@/createAsset.js" +import { createOrchestratorLlm } from "@/lib/llmClient.js" +import { redis } from "@/lib/redis.js" +import { safeDownload } from "@/lib/safeDownload.js" +import { localSemaphore, redisSemaphore } from "@/lib/semaphore.js" +import { createS3MediaStore } from "@/lib/storage.js" +import { runTool } from "@/processing/ffmpeg.js" +import type { GenerateDeps } from "./deps.js" + +const live = env.MEDIA_AI_PROVIDER_MODE === "live" +const ffmpeg = processDeps.config.ffmpeg + +async function scratch(fn: (dir: (name: string) => string) => Promise): Promise { + const id = randomUUID() + const file = (name: string) => path.join(env.MEDIA_WORK_DIR, `${id}-${name}`) + try { + return await fn(file) + } finally { + await Promise.all(["in.mp4", "out.png", "fake.mp4"].map((n) => rm(file(n), { force: true }))) + } +} + +async function extractFrame(video: Buffer, atSec: number): Promise { + return scratch(async (file) => { + await writeFile(file("in.mp4"), video) + await runTool( + ffmpeg.ffmpegPath, + ["-nostdin", "-loglevel", "error", "-y", "-ss", String(atSec), "-protocol_whitelist", "file", "-i", file("in.mp4"), "-frames:v", "1", "-f", "image2", "-c:v", "png", file("out.png")], + ffmpeg.timeoutMs, + ) + return readFile(file("out.png")) + }) +} + +// Fake mode stands in for a provider download with a short, silent test clip. +async function fakeVideo(): Promise { + return scratch(async (file) => { + await runTool( + ffmpeg.ffmpegPath, + ["-nostdin", "-loglevel", "error", "-y", "-f", "lavfi", "-i", "testsrc2=size=1344x768:rate=24:duration=5", "-c:v", "libx264", "-pix_fmt", "yuv420p", file("fake.mp4")], + ffmpeg.timeoutMs, + ) + return readFile(file("fake.mp4")) + }) +} + +const store = createS3MediaStore({ + endpoint: env.MEDIA_R2_ENDPOINT, + accessKeyId: env.MEDIA_R2_ACCESS_KEY_ID, + secretAccessKey: env.MEDIA_R2_SECRET_ACCESS_KEY, + bucket: env.MEDIA_R2_BUCKET, +}) + +export const generateDeps: GenerateDeps = { + db: prisma, + store, + llm: live ? createOrchestratorLlm({ url: env.ORCHESTRATOR_URL, secret: env.INTERNAL_SERVICE_SECRET! }) : new FakeLlm(), + images: live ? new OpenAiImageProvider({ apiKey: env.OPENAI_API_KEY!, model: env.OPENAI_IMAGE_MODEL }) : new FakeImageProvider(), + video: live ? new RunwareVideoProvider({ apiKey: env.RUNWARE_API_KEY!, model: env.MEDIA_VIDEO_MODEL }) : new FakeVideoProvider(), + createAsset: (input) => createAssetFromBytes(processDeps, input), + reserveAsset: (input) => reserveGeneratedAsset(processDeps, input), + finishReserved: (asset) => finishReservedAsset(processDeps, { ...asset, kind: "VIDEO" }, { stripAudio: true, skipDescribe: true }), + download: (url) => + url.startsWith("fake://") && !live + ? fakeVideo() + : safeDownload(url, { allowedHosts: env.MEDIA_PROVIDER_HOST_ALLOWLIST, maxBytes: 200 * 1024 * 1024 }), + extractFrame, + semaphores: { + image: redisSemaphore(redis, "image", env.MEDIA_IMAGE_CONCURRENCY), + video: redisSemaphore(redis, "video", env.MEDIA_VIDEO_CONCURRENCY), + // Replaced per job by the runner: concepts are written one video at a time. + concepts: localSemaphore(1), + }, + config: { + ai: loadMediaAiConfig(process.env), + imageQuality: env.OPENAI_IMAGE_QUALITY, + videoResolution: env.MEDIA_VIDEO_RESOLUTION, + videoTimeoutS: env.MEDIA_VIDEO_TIMEOUT_S, + promptExpansion: env.MEDIA_VIDEO_PROMPT_EXPANSION, + promptMaxChars: env.MEDIA_VIDEO_PROMPT_MAX_CHARS, + assetTimeoutMs: env.MEDIA_FFMPEG_TIMEOUT_MS * 2 + 60_000, + directUpload: live, + }, + sleep: (ms) => new Promise((r) => setTimeout(r, ms)), +} diff --git a/apps/media-service/src/generate/deps.ts b/apps/media-service/src/generate/deps.ts new file mode 100644 index 0000000..1d5e11d --- /dev/null +++ b/apps/media-service/src/generate/deps.ts @@ -0,0 +1,51 @@ +import type { prisma } from "@useframe/db" +import type { ImageProvider, Llm, MediaAiConfig, VideoProvider } from "@repo/media-ai" +import type { CreateAssetInput } from "@/createAsset.js" +import type { MediaStore } from "@/lib/storage.js" +import type { Semaphore } from "@/lib/semaphore.js" + +export type GenerateDb = Pick< + typeof prisma, + | "mediaJob" + | "mediaGeneration" + | "mediaAsset" + | "creditBalance" + | "creditTransaction" + | "projectVersion" + | "project" + | "researchReport" + | "researchReportCitation" + | "styleTag" + | "projectBrief" + | "$transaction" +> + +export type GenerateConfig = { + ai: MediaAiConfig + imageQuality: string + videoResolution: "480p" | "768p" + videoTimeoutS: number + promptExpansion: "disabled" | "balanced" | "quality" + promptMaxChars: number + // How long to wait for Part A processing to turn an upload into variants. + assetTimeoutMs: number + directUpload: boolean +} + +export type ReservedAsset = { assetId: string; key: string } + +export type GenerateDeps = { + db: GenerateDb + store: Pick + llm: Llm + images: ImageProvider + video: VideoProvider + createAsset: (input: CreateAssetInput) => Promise<{ assetId: string }> + reserveAsset: (input: { projectId: string; userId: string; kind: "VIDEO"; mime: string; purpose: string; generation: object }) => Promise + finishReserved: (asset: ReservedAsset) => Promise + download: (url: string) => Promise + extractFrame: (video: Buffer, atSec: number) => Promise + semaphores: { image: Semaphore; video: Semaphore; concepts: Semaphore } + config: GenerateConfig + sleep: (ms: number) => Promise +} diff --git a/apps/media-service/src/generate/runner.ts b/apps/media-service/src/generate/runner.ts new file mode 100644 index 0000000..7dcfe55 --- /dev/null +++ b/apps/media-service/src/generate/runner.ts @@ -0,0 +1,1003 @@ +import { randomUUID } from "node:crypto" +import { briefStateOf, type Prisma } from "@useframe/db" +import type { SiteSpec } from "@repo/schemas" +import { + applyJobResult, + applyMediaBindings, + buildMediaBrief, + callJson, + checkConceptSet, + CHECKPOINT_TTL_MS, + conceptsPrompt, + ConceptSetSchema, + costItems, + estimateJobCost, + framesPrompt, + FramePromptsSchema, + imagePromptsPrompt, + ImagePromptBatchSchema, + imageQaPrompt, + ImageQaSchema, + isBlueprintFresh, + isPaidUser, + judgePrompt, + JudgeOutputSchema, + lintPrompt, + loadBlueprintInputs, + LlmRefusalError, + LlmValidationError, + MediaBriefSchema, + paletteOf, + planningSlots, + PromptRewriteSchema, + ProviderError, + readBlueprint, + readJobState, + refundAmount, + refundItem, + refundUnspent, + requestKey, + revisePrompt, + runSiteBlueprint, + scopeOf, + selectWinner, + softenPrompt, + stillSize, + videoFrameSize, + videoPromptPrompt, + VideoPromptSchema, + videoQaPrompt, + VideoQaSchema, + zodParse, + type Checkpoint, + type ItemStage, + type ItemState, + type JobPlan, + type JobState, + type Llm, + type LlmImage, + type MediaBindingOut, + type MediaBrief, + type PromptPair, + type RoutingKey, + type Substep, + type VisualSystem, +} from "@repo/media-ai" +import { errorMessage, log, metric } from "@/lib/logger.js" +import { localSemaphore } from "@/lib/semaphore.js" +import { analyzeImage, generateImage, overlayHint, updateAsset, variantBytes, waitForAsset } from "./assets.js" +import type { GenerateDeps } from "./deps.js" + +type JobRow = { + id: string + projectId: string + userId: string + mode: "AUTO" | "MANUAL" + status: string + versionId: string | null + scope: unknown + plan: unknown + mediaBrief: unknown + state: unknown + bindings: unknown + creditsReserved: number +} + +type SlotRef = ReturnType[number] + +type Ctx = { + deps: GenerateDeps + job: JobRow + state: JobState + brief: MediaBrief + visualSystem: VisualSystem + palette: string[] + slots: Map + forbidden: string[] + bindings: Record + save: () => Promise + stage: (substep: Substep) => void + cancelled: () => Promise +} + +const PHASE: Record = { CONCEPTS: 1, SELECT: 1, FRAMES: 2, IMG_QA: 2, VIDEO_PROMPT: 3, VIDEO: 3, VID_QA: 3 } +const CHECKPOINT_AFTER: [number, Checkpoint][] = [ + [1, "CONCEPT_PICK"], + [2, "IMAGES_APPROVE"], + [3, "VIDEO_APPROVE"], +] +const TERMINAL = new Set(["DONE", "FAILED", "DROPPED", "SKIPPED"]) +const MAX_IMAGE_ATTEMPTS = 3 + +class ItemFailure extends Error {} + +const json = (v: unknown) => v as Prisma.InputJsonValue + +// ---------------------------------------------------------------- LLM with an audit trail + +function auditedLlm(ctx: Pick, itemId: string, stage: string): Llm { + return { + async complete(req) { + const started = Date.now() + const row = await ctx.deps.db.mediaGeneration.create({ + data: { + jobId: ctx.job.id, + itemId, + stage, + kind: "llm", + provider: "anthropic", + model: req.routingKey, + requestKey: requestKey(ctx.job.id, stage, itemId, Date.now()) + randomUUID().slice(0, 8), + status: "RUNNING", + }, + select: { id: true }, + }) + try { + const res = await ctx.deps.llm.complete(req) + const cost = res.costUsd ?? ctx.deps.config.ai.providerCostUsd.llmCall + await ctx.deps.db.mediaGeneration.update({ + where: { id: row.id }, + data: { status: res.refused ? "REFUSED" : "SUCCEEDED", costUsd: cost, latencyMs: Date.now() - started }, + }) + await ctx.deps.db.mediaJob.update({ where: { id: ctx.job.id }, data: { costUsd: { increment: cost } } }) + return res + } catch (err) { + await ctx.deps.db.mediaGeneration.update({ where: { id: row.id }, data: { status: "FAILED", error: errorMessage(err).slice(0, 500), latencyMs: Date.now() - started } }) + throw err + } + }, + } +} + +function ask(ctx: Ctx, itemId: string, stage: string, purpose: string, key: RoutingKey, pair: PromptPair, parse: (raw: unknown) => { ok: true; value: T } | { ok: false; errors: string[] }, images?: LlmImage[]) { + return callJson(auditedLlm(ctx, itemId, stage), { routingKey: key, system: pair.system, prompt: pair.prompt, purpose, ...(images ? { images } : {}) }, parse).then((r) => r.value) +} + +const png = (bytes: Buffer): LlmImage => ({ mime: "image/png", base64: bytes.toString("base64") }) +const webp = (bytes: Buffer): LlmImage => ({ mime: "image/webp", base64: bytes.toString("base64") }) + +async function safePrompt(ctx: Ctx, itemId: string, prompt: string): Promise { + const issues = lintPrompt(prompt, ctx.forbidden) + if (issues.length === 0) return prompt + const { prompt: rewritten } = await ask(ctx, itemId, "LINT", "media.revise", "media-creative", revisePrompt({ prompt, fix: "Remove the flagged content.", reason: issues.join("; ") }), zodParse(PromptRewriteSchema)) + if (lintPrompt(rewritten, ctx.forbidden).length) throw new ItemFailure("The prompt could not be made safe") + return rewritten +} + +// ---------------------------------------------------------------- images + +async function generatedAsset( + ctx: Ctx, + s: ItemState, + stage: string, + attempt: number, + call: { prompt: string; target: { width: number; height: number }; edit?: Buffer[] }, + extra: { purpose: string; conceptId?: string; mechanism?: string; role: string }, +): Promise<{ assetId: string; bytes: Buffer }> { + const key = requestKey(ctx.job.id, stage, s.item.itemId, attempt) + const done = await ctx.deps.db.mediaGeneration.findUnique({ where: { requestKey: key }, select: { status: true, outputAssetId: true } }) + if (done?.status === "SUCCEEDED" && done.outputAssetId) { + const bytes = await originalBytes(ctx, done.outputAssetId) + if (bytes) return { assetId: done.outputAssetId, bytes } + } + const started = Date.now() + const row = done + ? await ctx.deps.db.mediaGeneration.update({ where: { requestKey: key }, data: { status: "RUNNING", error: null }, select: { id: true } }) + : await ctx.deps.db.mediaGeneration.create({ + data: { jobId: ctx.job.id, itemId: s.item.itemId, stage, kind: "image", provider: "openai", model: "image", attempt, requestKey: key, status: "RUNNING", promptText: call.prompt.slice(0, 8000) }, + select: { id: true }, + }) + try { + const image = await generateImage(ctx.deps, { prompt: call.prompt, target: call.target, requestKey: key, edit: call.edit }) + const { assetId } = await ctx.deps.createAsset({ + projectId: ctx.job.projectId, + userId: ctx.job.userId, + kind: "IMAGE", + origin: "GENERATED", + mime: "image/png", + purpose: extra.purpose, + title: `${s.item.role} (${s.item.itemId})`.slice(0, 120), + generation: json({ + jobId: ctx.job.id, + itemId: s.item.itemId, + conceptId: extra.conceptId ?? null, + mechanism: extra.mechanism ?? null, + provider: "openai", + model: image.model, + size: image.size, + summary: s.item.subjectDirection, + researchFindingIds: ctx.brief.researchFindingIds, + }), + options: { skipDescribe: true }, + bytes: image.bytes, + }) + if ((await waitForAsset(ctx.deps, assetId)) !== "READY") throw new ItemFailure("The generated image could not be processed") + await ctx.deps.db.mediaGeneration.update({ + where: { id: row.id }, + data: { status: "SUCCEEDED", outputAssetId: assetId, model: image.model, costUsd: image.costUsd, latencyMs: Date.now() - started }, + }) + await ctx.deps.db.mediaJob.update({ where: { id: ctx.job.id }, data: { costUsd: { increment: image.costUsd } } }) + return { assetId, bytes: image.bytes } + } catch (err) { + const refused = err instanceof ProviderError && err.kind === "REFUSED" + await ctx.deps.db.mediaGeneration.update({ where: { id: row.id }, data: { status: refused ? "REFUSED" : "FAILED", error: errorMessage(err).slice(0, 1000), latencyMs: Date.now() - started } }) + throw err + } +} + +async function originalBytes(ctx: Ctx, assetId: string): Promise { + const asset = await ctx.deps.db.mediaAsset.findUnique({ where: { id: assetId }, select: { originalKey: true } }) + return asset ? ctx.deps.store.getBuffer(asset.originalKey, 50 * 1024 * 1024) : null +} + +async function referenceImages(ctx: Ctx, s: ItemState): Promise { + const out: Buffer[] = [] + for (const id of s.item.referenceAssetIds ?? []) { + const bytes = await variantBytes(ctx.deps, id, "fallback") + if (bytes) out.push(bytes) + } + return out +} + +type QaVerdict = { pass: boolean; altText: string; fix?: string; reasons: string } + +async function qaImage(ctx: Ctx, s: ItemState, assetId: string, bytes: Buffer, opts: { intent: string; ending?: Buffer }): Promise { + const slot = ctx.slots.get(s.item.slotId)!.slot + const zone = s.item.textSafeZone ?? "left" + const analysis = await analyzeImage(bytes, ctx.palette, zone) + const view = (await variantBytes(ctx.deps, assetId, "w960")) ?? bytes + const images = [view.subarray(0, 4).toString("hex") === "89504e47" ? png(view) : webp(view)] + if (opts.ending) images.push(png(opts.ending)) + const qa = await ask( + ctx, + s.item.itemId, + "IMG_QA", + "media.image_qa", + "media-vision", + imageQaPrompt({ intent: opts.intent, palette: ctx.palette, analysis, isEnding: !!opts.ending, allowPeople: !!s.item.allowPeople, textSafeZone: zone }), + zodParse(ImageQaSchema), + images, + ) + const hero = slot.priority === "hero" + const reasons = [ + !qa.matchesIntent && "does not match the intent", + qa.hasReadableText && "has readable text", + qa.hasLogoOrBrand && "shows a logo or brand", + qa.hasDistortion && "has distortion", + !qa.safe && "is not safe", + qa.paletteAdherence < 3 && "ignores the palette", + qa.hasRealisticPerson && !s.item.allowPeople && "shows a realistic person", + hero && !qa.textZoneClear && "the headline area is busy", + opts.ending && (qa.identityMatchesOpening ?? 0) < 4 && "does not match the opening frame", + ].filter(Boolean) as string[] + await updateAsset(ctx.deps, assetId, { + generation: json({ ...(await currentGeneration(ctx, assetId)), analysis }), + }) + return { pass: reasons.length === 0, altText: qa.altText, fix: qa.promptFix, reasons: reasons.join(", "), overlay: overlayHint(analysis, slot.defaultOverlay) } +} + +async function currentGeneration(ctx: Ctx, assetId: string): Promise> { + const a = await ctx.deps.db.mediaAsset.findUnique({ where: { id: assetId }, select: { generation: true } }) + return (a?.generation as Record | null) ?? {} +} + +async function revise(ctx: Ctx, itemId: string, prompt: string, fix: string | undefined, reasons: string): Promise { + const { prompt: next } = await ask(ctx, itemId, "REVISE", "media.revise", "media-creative", revisePrompt({ prompt, fix: fix ?? "Fix the reported problems.", reason: reasons }), zodParse(PromptRewriteSchema)) + return safePrompt(ctx, itemId, next) +} + +async function discard(ctx: Ctx, assetId: string): Promise { + await ctx.deps.db.mediaAsset.updateMany({ where: { id: assetId, deletedAt: null }, data: { deletedAt: new Date() } }) +} + +function slotPurpose(ctx: Ctx, s: ItemState, asVideo: boolean): string { + const slot = ctx.slots.get(s.item.slotId)!.slot + if (asVideo) return slot.priority === "hero" ? "hero_video" : "section_video" + return slot.purpose +} + +// One batched call writes a prompt for every pending image item. +async function ensureImagePrompts(ctx: Ctx): Promise { + const pending = Object.values(ctx.state.items).filter((s) => s.item.kind === "image" && s.item.decision === "GENERATE" && !TERMINAL.has(s.status) && !s.imagePrompt && !s.image) + for (let i = 0; i < pending.length; i += 10) { + const chunk = pending.slice(i, i + 10) + ctx.stage("media.images") + const batch = await ask( + ctx, + "images", + "FRAMES", + "media.image_prompts", + "media-creative", + imagePromptsPrompt({ + items: chunk.map((s) => ({ + itemId: s.item.itemId, + role: s.item.role, + intent: s.item.intent, + subjectDirection: s.item.subjectDirection, + mood: s.item.mood, + aspect: s.item.aspect, + textSafeZone: s.item.textSafeZone ?? "left", + })), + visualSystem: ctx.visualSystem, + palette: ctx.palette, + brief: ctx.brief, + }), + zodParse(ImagePromptBatchSchema), + ) + for (const s of chunk) { + const p = batch.prompts.find((x) => x.itemId === s.item.itemId)?.prompt + if (p) s.imagePrompt = await safePrompt(ctx, s.item.itemId, p) + } + await ctx.save() + } +} + +async function imageItem(ctx: Ctx, s: ItemState): Promise { + let prompt = s.imagePrompt + if (!prompt) throw new ItemFailure("No prompt was written for this image") + if (s.note) prompt = await revise(ctx, s.item.itemId, prompt, s.note, "The user asked for a change") + const refs = await referenceImages(ctx, s) + if (refs.length) prompt += " Preserve the product in the reference images exactly." + const base = (s.done.length + 1) * 10 + for (let attempt = 1; attempt <= MAX_IMAGE_ATTEMPTS; attempt++) { + ctx.stage("media.images") + const { assetId, bytes } = await generatedAsset(ctx, s, "FRAMES", base + attempt, { prompt, target: stillSize(s.item.aspect), edit: refs.length ? refs : undefined }, { purpose: slotPurpose(ctx, s, false), role: s.item.role }) + ctx.stage("media.qa") + const qa = await qaImage(ctx, s, assetId, bytes, { intent: s.item.intent }) + if (qa.pass) { + await updateAsset(ctx.deps, assetId, { altText: qa.altText, description: s.item.intent.slice(0, 300), decorative: s.item.display === "background" }) + s.image = { prompt, assetId, attempts: attempt, altText: qa.altText } + s.overlay = qa.overlay + return + } + await discard(ctx, assetId) + if (attempt < MAX_IMAGE_ATTEMPTS) prompt = await revise(ctx, s.item.itemId, prompt, qa.fix, qa.reasons) + } + throw new ItemFailure("The images did not pass quality checks") +} + +// ---------------------------------------------------------------- video chain + +function usedByOthers(ctx: Ctx, self: ItemState): { mechanisms: string[]; motifs: string[] } { + const others = Object.values(ctx.state.items).filter((s) => s !== self && s.concept?.chosenId) + const chosen = others.map((s) => s.concept!.concepts.find((c) => c.id === s.concept!.chosenId)).filter(Boolean) + return { mechanisms: chosen.map((c) => c!.mechanism), motifs: chosen.map((c) => c!.title) } +} + +async function conceptsAndSelect(ctx: Ctx, s: ItemState): Promise<"ok" | "still"> { + const duration = s.item.video?.durationSec ?? 5 + return ctx.deps.semaphores.concepts.run(async () => { + for (let round = 1; round <= 2; round++) { + ctx.stage("media.concepts") + const used = usedByOthers(ctx, s) + let violation = s.note + let valid: ReturnType["valid"] = [] + for (let tries = 1; tries <= 2 && valid.length < 2; tries++) { + const set = await ask( + ctx, + s.item.itemId, + "CONCEPTS", + "media.concepts", + "media-creative", + conceptsPrompt({ brief: ctx.brief, item: s.item, visualSystem: ctx.visualSystem, usedMechanisms: used.mechanisms, usedMotifs: used.motifs, hasReference: !!s.item.referenceAssetIds?.length, violation }), + zodParse(ConceptSetSchema), + ) + const check = checkConceptSet(set.concepts, duration, ctx.palette) + valid = check.valid.filter((c) => !used.mechanisms.includes(c.mechanism)) + violation = check.errors.join("; ") || "two or more concepts reused a mechanism already used on this site" + } + if (valid.length < 2) return "still" + ctx.stage("media.select") + const judge = await ask(ctx, s.item.itemId, "SELECT", "media.judge", "media-creative", judgePrompt({ brief: ctx.brief, item: s.item, visualSystem: ctx.visualSystem, used: [...used.mechanisms, ...used.motifs], concepts: valid }), zodParse(JudgeOutputSchema)) + const { winnerId, ranked } = selectWinner(valid, judge) + if (!winnerId) continue + s.concept = { + concepts: valid, + judge, + ranked, + recommendedId: winnerId, + chosenId: winnerId, + directorNotes: judge.directorNotes, + regenerations: s.concept?.regenerations ?? 0, + } + s.note = undefined + return "ok" + } + return "still" + }) +} + +// Fewer than two good concepts, or every concept vetoed: a plain still image instead of a video. +async function downgradeToStill(ctx: Ctx, s: ItemState): Promise { + const slot = ctx.slots.get(s.item.slotId)!.slot + if (!slot.kinds.includes("image")) throw new ItemFailure("No usable video idea, and this spot can't take a still image") + const charge = ctx.state.charges[s.item.itemId] ?? 0 + await refundAmount(ctx.deps.db, ctx.job, ctx.state, `${s.item.itemId}:still`, charge - ctx.deps.config.ai.credits.image, "AI video replaced by a still image") + s.item = { ...s.item, kind: "image", video: undefined } + s.done = [] + await ensureImagePrompts(ctx) +} + +async function videoFrames(ctx: Ctx, s: ItemState): Promise { + const concept = s.concept!.concepts.find((c) => c.id === s.concept!.chosenId)! + ctx.stage("media.frames") + const prompts = await ask( + ctx, + s.item.itemId, + "FRAMES", + "media.frames", + "media-creative", + framesPrompt({ concept, directorNotes: [...s.concept!.directorNotes, ...(s.note ? [s.note] : [])], visualSystem: ctx.visualSystem, palette: ctx.palette, textSafeZone: s.item.textSafeZone ?? "left", aspect: s.item.aspect, wantEnding: true }), + zodParse(FramePromptsSchema), + ) + const target = videoFrameSize(s.item.aspect) + const refs = await referenceImages(ctx, s) + const meta = { purpose: slotPurpose(ctx, s, true), conceptId: concept.id, mechanism: concept.mechanism, role: s.item.role } + const base = (s.concept!.regenerations + 1) * 100 + (s.note ? 50 : 0) + + let openingPrompt = await safePrompt(ctx, s.item.itemId, prompts.openingPrompt + (refs.length ? " Preserve the product in the reference images exactly." : "")) + let opening: { assetId: string; bytes: Buffer; altText: string; overlay: "none" | "light" | "dark" } | null = null + for (let attempt = 1; attempt <= MAX_IMAGE_ATTEMPTS && !opening; attempt++) { + const out = await generatedAsset(ctx, s, "FRAMES:open", base + attempt, { prompt: openingPrompt, target, edit: refs.length ? refs : undefined }, meta) + ctx.stage("media.qa") + const qa = await qaImage(ctx, s, out.assetId, out.bytes, { intent: `${s.item.intent}. Opening frame: ${concept.openingFrame}` }) + if (qa.pass) opening = { ...out, altText: qa.altText, overlay: qa.overlay } + else { + await discard(ctx, out.assetId) + if (attempt < MAX_IMAGE_ATTEMPTS) openingPrompt = await revise(ctx, s.item.itemId, openingPrompt, qa.fix, qa.reasons) + } + } + if (!opening) throw new ItemFailure("The opening scene did not pass quality checks") + await updateAsset(ctx.deps, opening.assetId, { altText: opening.altText, description: concept.logline.slice(0, 300) }) + + let ending: { assetId: string } | null = null + if (prompts.endingPrompt) { + let endingPrompt = await safePrompt(ctx, s.item.itemId, prompts.endingPrompt) + for (let attempt = 1; attempt <= MAX_IMAGE_ATTEMPTS && !ending; attempt++) { + ctx.stage("media.frames") + const out = await generatedAsset(ctx, s, "FRAMES:end", base + attempt, { prompt: endingPrompt, target, edit: [opening.bytes] }, meta) + ctx.stage("media.qa") + const qa = await qaImage(ctx, s, out.assetId, out.bytes, { intent: `Ending frame: ${concept.endingFrame}`, ending: opening.bytes }) + if (qa.pass) ending = { assetId: out.assetId } + else { + await discard(ctx, out.assetId) + if (attempt < MAX_IMAGE_ATTEMPTS) endingPrompt = await revise(ctx, s.item.itemId, endingPrompt, qa.fix, qa.reasons) + } + } + } + s.frames = { + openingPrompt, + ...(prompts.endingPrompt ? { endingPrompt: prompts.endingPrompt } : {}), + openingAssetId: opening.assetId, + ...(ending ? { endingAssetId: ending.assetId } : {}), + attempts: 1, + altText: opening.altText, + } + s.overlay = opening.overlay + s.note = undefined +} + +async function videoPrompt(ctx: Ctx, s: ItemState): Promise { + const concept = s.concept!.concepts.find((c) => c.id === s.concept!.chosenId)! + const planned = s.item.video?.durationSec ?? ctx.deps.config.ai.videoMinSeconds + const opening = await originalBytes(ctx, s.frames!.openingAssetId) + const ending = s.frames!.endingAssetId ? await originalBytes(ctx, s.frames!.endingAssetId) : null + if (!opening) throw new ItemFailure("The opening scene is missing") + ctx.stage("media.video") + const out = await ask( + ctx, + s.item.itemId, + "VIDEO_PROMPT", + "media.video_prompt", + "media-vision", + videoPromptPrompt({ concept, directorNotes: s.concept!.directorNotes, durationSec: planned, minSeconds: ctx.deps.config.ai.videoMinSeconds, maxSeconds: ctx.deps.config.ai.videoMaxSeconds, hasEnding: !!ending, maxChars: ctx.deps.config.promptMaxChars }), + zodParse(VideoPromptSchema), + ending ? [png(opening), png(ending)] : [png(opening)], + ) + const { videoMinSeconds: min, videoMaxSeconds: max } = ctx.deps.config.ai + const durationSec = Math.round(Math.min(max, Math.max(min, planned - 2, Math.min(planned + 2, out.durationSec)))) + s.video = { + mode: ending && out.mode === "FIRST_AND_LAST" ? "FIRST_AND_LAST" : "FIRST_ONLY", + durationSec, + prompt: await safePrompt(ctx, s.item.itemId, out.prompt.slice(0, ctx.deps.config.promptMaxChars)), + loop: out.loop && !!s.item.video?.loopable, + candidateAssetIds: [], + attempts: 0, + } +} + +class VideoOutcome extends Error { + constructor(public readonly kind: "REFUSED" | "FAILED" | "TIMEOUT", message: string) { + super(message) + } +} + +async function renderVideo(ctx: Ctx, s: ItemState, prompt: string, keyAttempt: string): Promise { + const v = s.video! + const key = requestKey(ctx.job.id, "VIDEO", s.item.itemId, Number.parseInt(keyAttempt.replace(/\D/g, "") || "1", 10)) + keyAttempt + const provider = ctx.deps.video + const results = ctx.job.mode === "MANUAL" ? Math.min(3, Math.max(1, s.candidates ?? 1)) : 1 + let row = await ctx.deps.db.mediaGeneration.findUnique({ where: { requestKey: key } }) + if (row?.status === "SUCCEEDED") return v.candidateAssetIds + const direct = ctx.deps.config.directUpload && results === 1 + if (!row) { + // The task id is stored before submitting, so a crash resumes polling instead of paying twice. + row = await ctx.deps.db.mediaGeneration.create({ + data: { + jobId: ctx.job.id, + itemId: s.item.itemId, + stage: "VIDEO", + kind: "video", + provider: provider.name, + model: provider.model, + requestKey: key, + providerTaskId: provider.taskIdFor(key), + status: "PENDING", + promptText: prompt.slice(0, 8000), + inputAssetIds: json([s.frames!.openingAssetId, ...(v.mode === "FIRST_AND_LAST" && s.frames!.endingAssetId ? [s.frames!.endingAssetId] : [])]), + }, + }) + } + const taskId = row.providerTaskId! + const started = row.createdAt.getTime() + return ctx.deps.semaphores.video.run(async () => { + let reserved = row!.outputAssetId ? { assetId: row!.outputAssetId, key: "" } : null + const submit = async () => { + const first = await originalBytes(ctx, s.frames!.openingAssetId) + const last = v.mode === "FIRST_AND_LAST" && s.frames!.endingAssetId ? await originalBytes(ctx, s.frames!.endingAssetId) : null + if (!first) throw new ItemFailure("The opening scene is missing") + let deliverTo: { presignedPutUrl: string } | undefined + if (direct) { + reserved ??= await ctx.deps.reserveAsset({ + projectId: ctx.job.projectId, + userId: ctx.job.userId, + kind: "VIDEO", + mime: "video/mp4", + purpose: slotPurpose(ctx, s, true), + generation: { jobId: ctx.job.id, itemId: s.item.itemId, conceptId: s.concept!.chosenId, provider: provider.name, model: provider.model, summary: s.concept!.concepts.find((c) => c.id === s.concept!.chosenId)?.logline }, + }) + await ctx.deps.db.mediaGeneration.update({ where: { id: row!.id }, data: { outputAssetId: reserved.assetId } }) + const asset = await ctx.deps.db.mediaAsset.findUnique({ where: { id: reserved.assetId }, select: { originalKey: true } }) + deliverTo = { presignedPutUrl: await ctx.deps.store.presignPut(asset!.originalKey, "video/mp4", 3600) } + } + await provider.submit({ + prompt, + firstFrame: first, + ...(last ? { lastFrame: last } : {}), + durationSec: v.durationSec, + resolution: ctx.deps.config.videoResolution, + results, + promptExpansion: ctx.deps.config.promptExpansion, + requestKey: key, + taskId, + ...(deliverTo ? { deliverTo } : {}), + }) + await ctx.deps.db.mediaGeneration.update({ where: { id: row!.id }, data: { status: "RUNNING" } }) + } + if (row!.status === "PENDING") await submit() + let wait = 2000 + let beat = Date.now() + for (;;) { + if (Date.now() - started > ctx.deps.config.videoTimeoutS * 1000) { + await ctx.deps.db.mediaGeneration.update({ where: { id: row!.id }, data: { status: "FAILED", error: "timed out", latencyMs: Date.now() - started } }) + throw new VideoOutcome("TIMEOUT", "The video took too long") + } + const poll = await provider.poll(taskId) + if (poll.status === "FAILED" && poll.error === "TASK_NOT_FOUND") { + await submit() + } else if (poll.status === "SUCCEEDED") { + const assets: string[] = [] + for (const output of poll.outputs ?? []) { + if (output.delivered && reserved && (await ctx.deps.finishReserved(reserved))) assets.push(reserved.assetId) + else if (output.url) { + const bytes = await ctx.deps.download(output.url) + const { assetId } = await ctx.deps.createAsset({ + projectId: ctx.job.projectId, + userId: ctx.job.userId, + kind: "VIDEO", + origin: "GENERATED", + mime: "video/mp4", + purpose: slotPurpose(ctx, s, true), + title: `${s.item.role} video`.slice(0, 120), + generation: json({ jobId: ctx.job.id, itemId: s.item.itemId, conceptId: s.concept!.chosenId, mechanism: s.concept!.concepts.find((c) => c.id === s.concept!.chosenId)?.mechanism, provider: provider.name, model: provider.model, summary: s.concept!.concepts.find((c) => c.id === s.concept!.chosenId)?.logline, researchFindingIds: ctx.brief.researchFindingIds }), + options: { stripAudio: true, skipDescribe: true }, + bytes, + }) + assets.push(assetId) + } + } + const ready: string[] = [] + for (const id of assets) if ((await waitForAsset(ctx.deps, id)) === "READY") ready.push(id) + const cost = poll.costUsd ?? v.durationSec * results * ctx.deps.config.ai.providerCostUsd.videoPerSecond + await ctx.deps.db.mediaGeneration.update({ where: { id: row!.id }, data: { status: "SUCCEEDED", costUsd: cost, latencyMs: Date.now() - started, outputAssetId: ready[0] ?? null } }) + await ctx.deps.db.mediaJob.update({ where: { id: ctx.job.id }, data: { costUsd: { increment: cost } } }) + if (ready.length === 0) throw new VideoOutcome("FAILED", "The video could not be processed") + return ready + } else if (poll.status === "REFUSED" || poll.status === "FAILED") { + await ctx.deps.db.mediaGeneration.update({ where: { id: row!.id }, data: { status: poll.status, error: (poll.error ?? "").slice(0, 1000), latencyMs: Date.now() - started } }) + throw new VideoOutcome(poll.status, poll.error ?? "The video provider failed") + } + if (Date.now() - beat > 60_000) { + beat = Date.now() + await ctx.deps.db.mediaJob.update({ where: { id: ctx.job.id }, data: { stage: "media.video" } }) + } + await ctx.deps.sleep(wait) + wait = Math.min(wait * 1.5, 15_000) + } + }) +} + +async function qaVideo(ctx: Ctx, s: ItemState, assetId: string): Promise<{ pass: boolean; notes: string; fix?: string }> { + const concept = s.concept!.concepts.find((c) => c.id === s.concept!.chosenId)! + const [poster, last, mp4, opening, ending] = await Promise.all([ + variantBytes(ctx.deps, assetId, "poster"), + variantBytes(ctx.deps, assetId, "poster_last"), + variantBytes(ctx.deps, assetId, "mp4_720"), + variantBytes(ctx.deps, s.frames!.openingAssetId, "w960"), + s.frames!.endingAssetId ? variantBytes(ctx.deps, s.frames!.endingAssetId, "w960") : Promise.resolve(null), + ]) + if (!poster || !last || !mp4 || !opening) return { pass: false, notes: "frames could not be read" } + const middle = await ctx.deps.extractFrame(mp4, s.video!.durationSec / 2) + const qa = await ask( + ctx, + s.item.itemId, + "VID_QA", + "media.video_qa", + "media-vision", + videoQaPrompt({ concept, loopable: !!s.item.video?.loopable }), + zodParse(VideoQaSchema), + [webp(poster), png(middle), webp(last), webp(opening), ...(ending ? [webp(ending)] : [])], + ) + const pass = + qa.startMatchesOpening && + qa.consistentSubject && + qa.motionPlausible && + !qa.hasText && + !qa.hasLogo && + (s.video!.mode !== "FIRST_AND_LAST" || qa.endMatchesEnding) && + (!s.item.video?.loopable || qa.loopSeamOk !== false) + return { pass, notes: qa.notes, fix: qa.promptFix } +} + +async function videoStage(ctx: Ctx, s: ItemState): Promise { + const v = s.video! + let prompt = v.prompt + let softened = false + for (let attempt = 1; attempt <= 2; attempt++) { + ctx.stage("media.video") + let candidates: string[] + try { + candidates = await renderVideo(ctx, s, prompt, `${attempt}${softened ? "s" : ""}r${v.attempts}`) + } catch (err) { + if (err instanceof VideoOutcome && err.kind === "REFUSED" && !softened) { + softened = true + const { prompt: soft } = await ask(ctx, s.item.itemId, "VIDEO", "media.soften", "media-creative", softenPrompt({ prompt, reason: err.message }), zodParse(PromptRewriteSchema)) + prompt = await safePrompt(ctx, s.item.itemId, soft) + attempt-- + continue + } + throw err + } + ctx.stage("media.qa") + const qa: Record = {} + let fix: string | undefined + for (const id of candidates) { + const r = await qaVideo(ctx, s, id) + qa[id] = { pass: r.pass, notes: r.notes } + fix ??= r.fix + } + const passing = candidates.filter((id) => qa[id]!.pass) + if (passing.length) { + s.video = { ...v, prompt, candidateAssetIds: passing, chosenAssetId: passing[0], qa, attempts: attempt } + for (const id of candidates.filter((c) => !passing.includes(c))) await discard(ctx, id) + return + } + for (const id of candidates) await discard(ctx, id) + if (attempt === 1) prompt = await revise(ctx, s.item.itemId, prompt, fix, "The video failed review") + } + throw new VideoOutcome("FAILED", "The video failed review twice") +} + +// ---------------------------------------------------------------- item driver + +async function failItem(ctx: Ctx, s: ItemState, reason: string): Promise { + const slot = ctx.slots.get(s.item.slotId)?.slot + if (s.item.kind === "video" && s.frames?.openingAssetId && slot?.kinds.includes("image")) s.fallbackAssetId = s.frames.openingAssetId + s.status = "FAILED" + s.failureReason = reason + await refundItem(ctx.deps.db, ctx.job, ctx.state, s.item.itemId, "AI media item failed") + publish(ctx, s) + await ctx.save() + log.warn("media item failed", { jobId: ctx.job.id, itemId: s.item.itemId, reason }) +} + +function bindingFor(s: ItemState): MediaBindingOut | null { + const overlay = s.item.display === "background" && s.overlay ? { overlay: s.overlay } : {} + if (s.item.decision === "USE_UPLOADED" && s.item.assetId) { + return { assetId: s.item.assetId, ...(s.item.video ? { loop: s.item.video.loopable, playWhen: s.item.video.playWhen } : {}) } + } + if (s.status === "DONE" && s.item.kind === "image" && s.image?.assetId) return { assetId: s.image.assetId, ...overlay } + if (s.status === "DONE" && s.item.kind === "video" && s.video?.chosenAssetId) { + return { assetId: s.video.chosenAssetId, loop: s.video.loop, playWhen: s.item.video?.playWhen ?? "in_view", ...overlay } + } + if (s.fallbackAssetId) return { assetId: s.fallbackAssetId, ...overlay } + return null +} + +function publish(ctx: Ctx, s: ItemState): void { + const b = bindingFor(s) + if (b) ctx.bindings[s.item.slotId] = b + else delete ctx.bindings[s.item.slotId] +} + +function stagesOf(s: ItemState): ItemStage[] { + return s.item.kind === "video" ? ["CONCEPTS", "SELECT", "FRAMES", "IMG_QA", "VIDEO_PROMPT", "VIDEO", "VID_QA"] : ["FRAMES", "IMG_QA"] +} + +async function runItem(ctx: Ctx, s: ItemState, maxPhase: number): Promise { + if (TERMINAL.has(s.status) || s.item.decision !== "GENERATE") return + try { + for (;;) { + const next = stagesOf(s).find((stage) => !s.done.includes(stage)) + if (!next) break + if (PHASE[next] > maxPhase) return + if (await ctx.cancelled()) return + s.status = "RUNNING" + switch (next) { + case "CONCEPTS": + if ((await conceptsAndSelect(ctx, s)) === "still") { + await downgradeToStill(ctx, s) + continue + } + s.done.push("CONCEPTS", "SELECT") + break + case "SELECT": + s.done.push("SELECT") + break + case "FRAMES": + if (s.item.kind === "image") await imageItem(ctx, s) + else await videoFrames(ctx, s) + s.done.push("FRAMES", "IMG_QA") + break + case "IMG_QA": + s.done.push("IMG_QA") + break + case "VIDEO_PROMPT": + await videoPrompt(ctx, s) + s.done.push("VIDEO_PROMPT") + break + case "VIDEO": + await videoStage(ctx, s) + s.done.push("VIDEO", "VID_QA") + break + case "VID_QA": + s.done.push("VID_QA") + break + } + await ctx.save() + } + s.status = "DONE" + publish(ctx, s) + await ctx.save() + } catch (err) { + if (err instanceof ItemFailure || err instanceof VideoOutcome || err instanceof ProviderError || err instanceof LlmValidationError || err instanceof LlmRefusalError) { + await failItem(ctx, s, err.message) + return + } + log.error("media item error", { jobId: ctx.job.id, itemId: s.item.itemId, error: errorMessage(err) }) + await failItem(ctx, s, "Something went wrong while creating this item") + } +} + +// ---------------------------------------------------------------- job driver + +async function blueprintForRerun(deps: GenerateDeps, job: JobRow, state: JobState): Promise { + const scope = scopeOf(job.scope) + const plan = job.plan as { budget: JobPlan["budget"]; budgetTier: JobPlan["budgetTier"] } + const version = job.versionId ? await deps.db.projectVersion.findUnique({ where: { id: job.versionId }, select: { snapshot: true, blueprint: true } }) : null + const spec = version?.snapshot as Partial | null + if (!spec?.pages?.length) return null + const existing = readBlueprint(version!.blueprint) + const inScope = (slotId: string) => scope.kind === "site" || scope.slotIds.includes(slotId) + const inputs = await loadBlueprintInputs(deps.db, job.projectId, job.userId, true) + const brief = buildMediaBrief({ + brief: inputs.briefFacts, + project: inputs.project, + research: inputs.research, + imageryFindingIds: inputs.imageryFindingIds, + styleAvoid: inputs.styleAvoid, + mediaNotes: inputs.media.notes, + budget: plan.budget, + hasProductPhotoUploads: inputs.hasProductPhotoUploads, + }) + const keepBrief = (bp: NonNullable) => + deps.db.mediaJob.update({ where: { id: job.id }, data: { mediaBrief: json({ ...brief, blueprint: { narrative: bp.narrative.storyArc, visualSystem: bp.visualSystem } }) } }) + if (existing && isBlueprintFresh(existing, spec.pages, spec.copyFramework ?? "AIDA") && existing.mediaScript.some((i) => inScope(i.slotId))) { + await keepBrief(existing) + // The blueprint did not have to run again, so its reserved credits go back. + await refundItem(deps.db, job, state, "blueprint", "Existing site plan reused") + return { items: existing.mediaScript.filter((i) => inScope(i.slotId)), dropped: [], budget: plan.budget, budgetTier: plan.budgetTier } + } + const palette = [spec.designSystem?.primaryColor, spec.designSystem?.secondaryColor, spec.designSystem?.accentColor, ...paletteOf(brief)].filter((c): c is string => !!c) + const result = await runSiteBlueprint({ + mode: "media-only", + llm: auditedLlm({ deps, job }, "blueprint", "BLUEPRINT"), + site: { pages: spec.pages, copyFramework: spec.copyFramework ?? "AIDA", tokens: { colors: palette, fonts: [spec.designSystem?.fontPrimary ?? "", spec.designSystem?.fontSecondary ?? ""], styleTags: inputs.research?.styleTagKeys ?? [] } }, + palette, + mediaBrief: brief, + briefFacts: inputs.briefFacts, + research: inputs.research, + uploaded: inputs.uploaded, + budget: plan.budget, + config: deps.config.ai, + mediaNotes: inputs.media.notes, + ...(scope.kind === "slots" ? { slotIds: scope.slotIds } : {}), + }) + if (!result.ok) return null + await deps.db.projectVersion.update({ where: { id: job.versionId! }, data: { blueprint: json(result.blueprint) } }) + await keepBrief(result.blueprint) + state.blueprintDone = true + return { items: result.blueprint.mediaScript, dropped: result.blueprint.dropped, budget: plan.budget, budgetTier: plan.budgetTier } +} + +async function finish(deps: GenerateDeps, job: JobRow, data: Prisma.MediaJobUpdateInput): Promise { + await deps.db.mediaJob.update({ where: { id: job.id }, data: { finishedAt: new Date(), ...data } }) +} + +async function forbiddenTerms(deps: GenerateDeps, job: JobRow): Promise { + const brief = await deps.db.projectBrief.findFirst({ where: { projectId: job.projectId, userId: job.userId } }) + if (!brief) return [] + const data = briefStateOf(brief).data + return [ + ...(data.competitors ?? []).map((c) => c.name), + ...(data.testimonials ?? []).map((t) => t.personName), + ].filter((n): n is string => typeof n === "string" && n.trim().length >= 3) +} + +export async function runMediaJob(baseDeps: GenerateDeps, jobId: string): Promise { + const deps: GenerateDeps = { ...baseDeps, semaphores: { ...baseDeps.semaphores, concepts: localSemaphore(1) } } + const job = (await deps.db.mediaJob.findUnique({ where: { id: jobId } })) as JobRow | null + if (!job || ["DONE", "FAILED", "CANCELLED", "AWAITING_USER"].includes(job.status)) return + const state = readJobState(job.state) + const persist = async (data: Prisma.MediaJobUpdateInput = {}) => + deps.db.mediaJob.update({ where: { id: job.id }, data: { state: json(state), ...data } }) + + // Credits reserved at creation prove a paid account even if the balance is now 0. + if (job.creditsReserved <= 0 && !(await isPaidUser(deps.db, job.userId))) { + await refundUnspent(deps.db, { ...job, state }, "AI media is a paid feature") + await finish(deps, job, { status: "FAILED", failureReason: "PAID_FEATURE" }) + return + } + await deps.db.mediaJob.update({ where: { id: job.id }, data: { status: "RUNNING", startedAt: (job as { startedAt?: Date | null }).startedAt ?? new Date() } }) + + let plan = job.plan as Partial | null + if (!plan?.items) { + await deps.db.mediaJob.update({ where: { id: job.id }, data: { stage: "blueprint" } }) + const made = await blueprintForRerun(deps, job, state).catch((err: unknown) => { + log.error("media re-run blueprint failed", { jobId, error: errorMessage(err) }) + return null + }) + if (!made) { + const fresh = await refundUnspent(deps.db, { ...job, state: { ...state, blueprintDone: false } }, "AI media planning failed") + await finish(deps, job, { status: "FAILED", failureReason: "We couldn't plan AI media for this site. No credits were used.", state: json(fresh) }) + return + } + plan = made + await deps.db.mediaJob.update({ where: { id: job.id }, data: { plan: json(made), state: json(state) } }) + } + + const items = (plan.items ?? []) as JobPlan["items"] + if (!state.expanded) { + await deps.db.mediaJob.update({ where: { id: job.id }, data: { stage: "media.expand" } }) + for (const item of items) state.items[item.itemId] ??= { item, status: item.decision === "USE_UPLOADED" ? "DONE" : "PENDING", done: [] } + if (state.charges.unplanned !== undefined) { + const unplanned = state.charges.unplanned + delete state.charges.unplanned + const estimate = estimateJobCost(costItems(items), deps.config.ai) + Object.assign(state.charges, estimate.perItem) + await refundAmount(deps.db, job, state, "plan-adjust", unplanned - estimate.credits, "AI media plan was smaller than reserved") + } + state.expanded = true + await persist() + } + + const version = job.versionId ? await deps.db.projectVersion.findUnique({ where: { id: job.versionId }, select: { snapshot: true } }) : null + const pages = ((version?.snapshot as Partial | null)?.pages ?? []) as SiteSpec["pages"] + const briefParsed = MediaBriefSchema.safeParse(job.mediaBrief) + if (!briefParsed.success || !briefParsed.data.blueprint) { + await refundUnspent(deps.db, { ...job, state }, "AI media context missing") + await finish(deps, job, { status: "FAILED", failureReason: "AI media context is missing", state: json(state) }) + return + } + const brief = briefParsed.data + let saving: Promise = Promise.resolve() + let lastStage: string | null = null + const ctx: Ctx = { + deps, + job, + state, + brief, + visualSystem: brief.blueprint!.visualSystem, + palette: paletteOf(brief), + // Item slots are planned before the copy agent writes items, so slots come from the planning view. + slots: new Map(planningSlots(pages.length ? pages : []).map((r) => [r.slotId, r])), + forbidden: await forbiddenTerms(deps, job), + bindings: { ...((job.bindings as Record | null) ?? {}) }, + save: () => { + saving = saving.then(() => deps.db.mediaJob.update({ where: { id: job.id }, data: { state: json(state), bindings: json(ctx.bindings) } })) + return saving.then(() => undefined) + }, + cancelled: async () => { + const row = await deps.db.mediaJob.findUnique({ where: { id: job.id }, select: { status: true } }) + return !row || row.status === "CANCELLED" || row.status === "FAILED" + }, + stage: (substep) => { + if (substep === lastStage) return + lastStage = substep + void deps.db.mediaJob.update({ where: { id: job.id }, data: { stage: substep } }).catch(() => undefined) + }, + } + for (const s of Object.values(state.items)) { + if (!ctx.slots.has(s.item.slotId)) { + if (!TERMINAL.has(s.status)) await failItem(ctx, s, "Its spot no longer exists on the site") + continue + } + publish(ctx, s) + } + await ctx.save() + + const openCheckpoint = async (checkpoint: Checkpoint) => { + await saving + const opened = await deps.db.mediaJob.updateMany({ + where: { id: job.id, status: "RUNNING" }, + data: { state: json(state), status: "AWAITING_USER", checkpoint, checkpointExpiresAt: new Date(Date.now() + CHECKPOINT_TTL_MS), stage: "media.awaiting_user", bindings: json(ctx.bindings) }, + }) + if (opened.count) log.info("media job awaiting user", { jobId, checkpoint }) + } + const live = () => Object.values(state.items).filter((s) => s.item.decision === "GENERATE" && !["DROPPED", "SKIPPED", "FAILED"].includes(s.status)) + + if (job.mode === "MANUAL") { + if (!state.approved.includes("PLAN_APPROVE") && live().length > 0) return openCheckpoint("PLAN_APPROVE") + for (const [phase, checkpoint] of CHECKPOINT_AFTER) { + if (phase >= 2) await ensureImagePrompts(ctx).catch((err) => log.warn("image prompts failed", { jobId, error: errorMessage(err) })) + await Promise.all(Object.values(state.items).map((s) => runItem(ctx, s, phase))) + const relevant = + checkpoint === "CONCEPT_PICK" + ? live().filter((s) => s.item.kind === "video" && s.concept) + : checkpoint === "IMAGES_APPROVE" + ? live().filter((s) => s.frames || s.image) + : live().filter((s) => (s.video?.candidateAssetIds.length ?? 0) > 0) + if (!state.approved.includes(checkpoint) && relevant.length > 0) return openCheckpoint(checkpoint) + } + } else { + await ensureImagePrompts(ctx).catch((err) => log.warn("image prompts failed", { jobId, error: errorMessage(err) })) + await Promise.all(Object.values(state.items).map((s) => runItem(ctx, s, 3))) + } + + // BIND + ctx.stage("media.bind") + for (const s of Object.values(state.items)) { + if (s.status === "PENDING" && s.item.decision === "GENERATE") await failItem(ctx, s, "It could not be created") + if (s.status === "DONE" && s.item.decision === "GENERATE") state.settled[s.item.itemId] ??= "kept" + publish(ctx, s) + for (const id of s.video?.candidateAssetIds ?? []) if (id !== s.video?.chosenAssetId) await discard(ctx, id) + } + state.bound = true + await saving + const finished = await deps.db.mediaJob.updateMany({ + where: { id: job.id, status: "RUNNING" }, + data: { + state: json(state), + status: "DONE", + stage: "media.bind", + finishedAt: new Date(), + bindings: json(ctx.bindings), + concepts: json(Object.fromEntries(Object.values(state.items).filter((s) => s.concept).map((s) => [s.item.itemId, s.concept]))), + }, + }) + if (finished.count === 0) return + metric("media.job_done", 1, { jobId, items: Object.keys(state.items).length, bound: Object.keys(ctx.bindings).length }) + await applyJobResult(deps.db, job.id, (args) => applyMediaBindings(deps.db, args)).catch((err) => + log.error("applying media bindings failed", { jobId, error: errorMessage(err) }), + ) +} diff --git a/apps/media-service/src/generate/sweep.ts b/apps/media-service/src/generate/sweep.ts new file mode 100644 index 0000000..8679c8c --- /dev/null +++ b/apps/media-service/src/generate/sweep.ts @@ -0,0 +1,43 @@ +import type { Prisma } from "@useframe/db" +import { refundUnspent } from "@repo/media-ai" +import { log, metric } from "@/lib/logger.js" +import type { GenerateDb } from "./deps.js" + +const IDLE_MS = 30 * 60 * 1000 + +// Expired approvals cancel the job (the site keeps building without generated +// media); runs that stopped making progress fail. Both refund what was not spent. +export async function sweepMediaJobs(db: GenerateDb, now = new Date()): Promise<{ expired: number; reaped: number }> { + const expired = await db.mediaJob.findMany({ + where: { status: "AWAITING_USER", checkpointExpiresAt: { lt: now } }, + select: { id: true, userId: true, state: true }, + take: 100, + }) + for (const job of expired) { + const claimed = await db.mediaJob.updateMany({ where: { id: job.id, status: "AWAITING_USER" }, data: { status: "CANCELLED" } }) + if (claimed.count === 0) continue + const state = await refundUnspent(db, job, "AI media approval expired") + await db.mediaJob.update({ + where: { id: job.id }, + data: { state: state as unknown as Prisma.InputJsonValue, checkpoint: null, failureReason: "The approval window expired", finishedAt: now }, + }) + } + + const stuck = await db.mediaJob.findMany({ + where: { status: { in: ["RUNNING", "QUEUED"] }, updatedAt: { lt: new Date(now.getTime() - IDLE_MS) } }, + select: { id: true, userId: true, state: true }, + take: 100, + }) + for (const job of stuck) { + const claimed = await db.mediaJob.updateMany({ where: { id: job.id, status: { in: ["RUNNING", "QUEUED"] }, updatedAt: { lt: new Date(now.getTime() - IDLE_MS) } }, data: { status: "FAILED" } }) + if (claimed.count === 0) continue + const state = await refundUnspent(db, job, "AI media job stopped") + await db.mediaJob.update({ + where: { id: job.id }, + data: { state: state as unknown as Prisma.InputJsonValue, failureReason: "AI media stopped responding and was refunded", finishedAt: now }, + }) + } + if (expired.length || stuck.length) log.info("media jobs swept", { expired: expired.length, reaped: stuck.length }) + metric("media.jobs_swept", expired.length + stuck.length, { expired: expired.length, reaped: stuck.length }) + return { expired: expired.length, reaped: stuck.length } +} diff --git a/apps/media-service/src/lib/llmClient.ts b/apps/media-service/src/lib/llmClient.ts new file mode 100644 index 0000000..83bd44c --- /dev/null +++ b/apps/media-service/src/lib/llmClient.ts @@ -0,0 +1,19 @@ +import type { Llm, LlmRequest, LlmResponse } from "@repo/media-ai" + +// Model routing and provider keys live in the orchestrator; this service only sends the request. +export function createOrchestratorLlm(config: { url: string; secret: string; fetchImpl?: typeof fetch }): Llm { + const doFetch = config.fetchImpl ?? fetch + return { + async complete(req: LlmRequest): Promise { + const res = await doFetch(`${config.url.replace(/\/$/, "")}/internal/media-ai/llm`, { + method: "POST", + headers: { "content-type": "application/json", "x-internal-secret": config.secret }, + body: JSON.stringify(req), + signal: AbortSignal.timeout(req.routingKey === "blueprint" ? 270_000 : 120_000), + }) + const body = (await res.json().catch(() => null)) as { success?: boolean; data?: LlmResponse; message?: string } | null + if (!res.ok || !body?.success || !body.data) throw new Error(`Model call failed (${res.status}): ${body?.message ?? "no response"}`) + return body.data + }, + } +} diff --git a/apps/media-service/src/lib/safeDownload.ts b/apps/media-service/src/lib/safeDownload.ts new file mode 100644 index 0000000..1590805 --- /dev/null +++ b/apps/media-service/src/lib/safeDownload.ts @@ -0,0 +1,83 @@ +import { lookup } from "node:dns/promises" +import { isIP } from "node:net" + +export class UnsafeDownloadError extends Error { + constructor(message: string) { + super(message) + this.name = "UnsafeDownloadError" + } +} + +export function isBlockedAddress(ip: string): boolean { + if (isIP(ip) === 4) { + const [a, b] = ip.split(".").map(Number) as [number, number] + return ( + a === 0 || a === 10 || a === 127 || a >= 224 || + (a === 100 && b >= 64 && b <= 127) || + (a === 169 && b === 254) || + (a === 172 && b >= 16 && b <= 31) || + (a === 192 && (b === 168 || b === 0)) || + (a === 198 && (b === 18 || b === 19)) + ) + } + const v6 = ip.toLowerCase() + if (v6.startsWith("::ffff:")) return isBlockedAddress(v6.slice(7)) + return v6 === "::" || v6 === "::1" || v6.startsWith("fc") || v6.startsWith("fd") || v6.startsWith("fe8") || v6.startsWith("fe9") || v6.startsWith("fea") || v6.startsWith("feb") || v6.startsWith("ff") +} + +export type SafeDownloadOptions = { + allowedHosts: readonly string[] + maxBytes: number + timeoutMs?: number + maxRedirects?: number + fetchImpl?: typeof fetch + resolve?: (host: string) => Promise +} + +const defaultResolve = async (host: string) => (await lookup(host, { all: true })).map((r) => r.address) + +function hostAllowed(host: string, allowed: readonly string[]): boolean { + return allowed.some((h) => host === h || host.endsWith(`.${h}`)) +} + +// Provider results only: https, an allow-listed host that resolves to a public +// address on every hop, a size cap, and redirects checked one by one. +export async function safeDownload(rawUrl: string, options: SafeDownloadOptions): Promise { + const doFetch = options.fetchImpl ?? fetch + const resolve = options.resolve ?? defaultResolve + let url = rawUrl + for (let hop = 0; hop <= (options.maxRedirects ?? 3); hop++) { + let parsed: URL + try { + parsed = new URL(url) + } catch { + throw new UnsafeDownloadError("Invalid download URL") + } + if (parsed.protocol !== "https:") throw new UnsafeDownloadError("Downloads must use https") + if (parsed.username || parsed.password) throw new UnsafeDownloadError("Credentials in URL are not allowed") + const host = parsed.hostname.toLowerCase() + if (!hostAllowed(host, options.allowedHosts)) throw new UnsafeDownloadError(`Host ${host} is not an allowed provider host`) + const addresses = isIP(host) ? [host] : await resolve(host) + if (addresses.length === 0 || addresses.some(isBlockedAddress)) throw new UnsafeDownloadError(`Host ${host} resolves to a private address`) + + const res = await doFetch(parsed, { redirect: "manual", signal: AbortSignal.timeout(options.timeoutMs ?? 120_000) }) + if (res.status >= 300 && res.status < 400) { + const location = res.headers.get("location") + if (!location) throw new UnsafeDownloadError("Redirect without a location") + url = new URL(location, parsed).toString() + continue + } + if (!res.ok || !res.body) throw new Error(`Download failed with status ${res.status}`) + const declared = Number(res.headers.get("content-length") ?? "0") + if (declared > options.maxBytes) throw new UnsafeDownloadError("Download is larger than allowed") + const chunks: Buffer[] = [] + let size = 0 + for await (const chunk of res.body as unknown as AsyncIterable) { + size += chunk.length + if (size > options.maxBytes) throw new UnsafeDownloadError("Download is larger than allowed") + chunks.push(Buffer.from(chunk)) + } + return Buffer.concat(chunks) + } + throw new UnsafeDownloadError("Too many redirects") +} diff --git a/apps/media-service/src/lib/semaphore.ts b/apps/media-service/src/lib/semaphore.ts new file mode 100644 index 0000000..a0fcc48 --- /dev/null +++ b/apps/media-service/src/lib/semaphore.ts @@ -0,0 +1,74 @@ +import { randomUUID } from "node:crypto" +import type { Redis } from "ioredis" + +export interface Semaphore { + run(fn: () => Promise): Promise +} + +const ACQUIRE = ` +redis.call("ZREMRANGEBYSCORE", KEYS[1], "-inf", ARGV[1]) +if redis.call("ZCARD", KEYS[1]) < tonumber(ARGV[3]) then + redis.call("ZADD", KEYS[1], ARGV[2], ARGV[4]) + return 1 +end +return 0` + +// Global across worker replicas: each holder keeps a lease that expires on its own if the process dies. +export function redisSemaphore(redis: Redis, name: string, limit: number, leaseMs = 20 * 60_000): Semaphore { + const key = `media:sem:${name}` + return { + async run(fn) { + const token = randomUUID() + let wait = 500 + for (;;) { + const now = Date.now() + const ok = await redis.eval(ACQUIRE, 1, key, String(now), String(now + leaseMs), String(limit), token) + if (ok === 1) break + await new Promise((r) => setTimeout(r, wait)) + wait = Math.min(wait * 1.5, 5000) + } + const renew = setInterval(() => void redis.zadd(key, String(Date.now() + leaseMs), token).catch(() => undefined), leaseMs / 3) + try { + return await fn() + } finally { + clearInterval(renew) + await redis.zrem(key, token).catch(() => undefined) + } + }, + } +} + +export function localSemaphore(limit: number): Semaphore & { peak: number } { + let active = 0 + const queue: (() => void)[] = [] + const sem = { + peak: 0, + async run(fn: () => Promise): Promise { + if (active >= limit) await new Promise((r) => queue.push(r)) + active++ + sem.peak = Math.max(sem.peak, active) + try { + return await fn() + } finally { + active-- + queue.shift()?.() + } + }, + } + return sem +} + +// One runner per media job at a time. +export async function withJobLock(redis: Redis, jobId: string, fn: () => Promise): Promise { + const key = `media:job-lock:${jobId}` + const token = randomUUID() + if ((await redis.set(key, token, "PX", 15 * 60_000, "NX")) !== "OK") return null + const renew = setInterval(() => void redis.pexpire(key, 15 * 60_000).catch(() => undefined), 5 * 60_000) + try { + return await fn() + } finally { + clearInterval(renew) + const current = await redis.get(key).catch(() => null) + if (current === token) await redis.del(key).catch(() => undefined) + } +} diff --git a/apps/media-service/src/lib/storage.ts b/apps/media-service/src/lib/storage.ts index 88aecac..210bec5 100644 --- a/apps/media-service/src/lib/storage.ts +++ b/apps/media-service/src/lib/storage.ts @@ -11,6 +11,7 @@ import { S3Client, } from "@aws-sdk/client-s3" import { Upload } from "@aws-sdk/lib-storage" +import { getSignedUrl } from "@aws-sdk/s3-request-presigner" const MULTIPART_THRESHOLD = 5 * 1024 * 1024 @@ -28,6 +29,9 @@ export interface MediaStore { put(key: string, body: Buffer | Readable, size: number, contentType: string): Promise head(key: string): Promise<{ size: number } | null> deletePrefix(prefix: string): Promise + getBuffer(key: string, maxBytes: number): Promise + // A short-lived URL a provider can PUT a result to directly. + presignPut(key: string, contentType: string, expiresInSeconds: number): Promise } function isNotFound(err: unknown): boolean { @@ -109,6 +113,32 @@ export function createS3MediaStore(config: { put, + async getBuffer(key, maxBytes) { + let res + try { + res = await client.send(new GetObjectCommand({ Bucket, Key: key })) + } catch (err) { + if (isNotFound(err)) return null + throw err + } + if (!res.Body) return null + if ((res.ContentLength ?? 0) > maxBytes) throw new TooLargeError() + const chunks: Buffer[] = [] + let size = 0 + for await (const chunk of res.Body as Readable) { + size += (chunk as Buffer).length + if (size > maxBytes) throw new TooLargeError() + chunks.push(Buffer.from(chunk as Uint8Array)) + } + return Buffer.concat(chunks) + }, + + presignPut(key, contentType, expiresInSeconds) { + return getSignedUrl(client, new PutObjectCommand({ Bucket, Key: key, ContentType: contentType }), { + expiresIn: expiresInSeconds, + }) + }, + async head(key) { try { const res = await client.send(new HeadObjectCommand({ Bucket, Key: key })) diff --git a/apps/media-service/src/worker.ts b/apps/media-service/src/worker.ts index 40c4365..2b537b4 100644 --- a/apps/media-service/src/worker.ts +++ b/apps/media-service/src/worker.ts @@ -1,10 +1,14 @@ import { mkdir } from "node:fs/promises" import { Queue, Worker } from "bullmq" import { prisma } from "@useframe/db" -import { QUEUES, type MediaProcessJobPayload } from "@repo/events" +import { QUEUES, type MediaGenerateJobPayload, type MediaProcessJobPayload } from "@repo/events" import { env } from "@/config/env.js" import { closeContext, deps } from "@/context.js" +import { generateDeps } from "@/generate/context.js" +import { runMediaJob } from "@/generate/runner.js" +import { sweepMediaJobs } from "@/generate/sweep.js" import { runMediaGc } from "@/jobs/gc.js" +import { withJobLock } from "@/lib/semaphore.js" import { processAsset } from "@/jobs/process.js" import { startHealthServer } from "@/lib/health.js" import { errorMessage, log } from "@/lib/logger.js" @@ -16,6 +20,8 @@ const start = async () => { const gcQueue = new Queue(QUEUES.MEDIA_GC, { connection: redis }) await gcQueue.upsertJobScheduler("media-gc-daily", { pattern: "15 4 * * *" }, { name: "gc" }) + const sweepQueue = new Queue(QUEUES.MEDIA_JOBS_SWEEP, { connection: redis }) + await sweepQueue.upsertJobScheduler("media-jobs-sweep", { every: 5 * 60_000 }, { name: "sweep" }) const workers = [ new Worker( @@ -29,6 +35,13 @@ const start = async () => { }, ), new Worker(QUEUES.MEDIA_GC, () => runMediaGc(deps), { connection: redis }), + new Worker( + QUEUES.MEDIA_GENERATE, + (job) => withJobLock(redis, job.data.jobId, () => runMediaJob(generateDeps, job.data.jobId)), + // Video renders are polled inside the job for up to MEDIA_VIDEO_TIMEOUT_S; BullMQ renews the lock meanwhile. + { connection: redis, concurrency: 4, lockDuration: 120_000 }, + ), + new Worker(QUEUES.MEDIA_JOBS_SWEEP, () => sweepMediaJobs(generateDeps.db), { connection: redis }), ] for (const worker of workers) { worker.on("failed", (job, err) => @@ -54,7 +67,7 @@ const start = async () => { shuttingDown = true log.info("media worker shutting down; waiting for in-flight jobs") void Promise.all(workers.map((w) => w.close())) - .then(() => gcQueue.close()) + .then(() => Promise.all([gcQueue.close(), sweepQueue.close()])) .finally(() => closeContext().finally(() => process.exit(0))) } process.once("SIGTERM", shutdown) diff --git a/apps/media-service/test/fakes.ts b/apps/media-service/test/fakes.ts index cb0cfc1..a25bdec 100644 --- a/apps/media-service/test/fakes.ts +++ b/apps/media-service/test/fakes.ts @@ -102,6 +102,15 @@ export function createFakeStore(): MediaStore & { objects: Map; for (const key of [...objects.keys()]) if (key.startsWith(prefix) && objects.delete(key)) n++ return n }, + async getBuffer(key: string, maxBytes: number) { + const body = objects.get(key) + if (!body) return null + if (body.length > maxBytes) throw new TooLargeError() + return body + }, + async presignPut(key: string) { + return `https://upload.test/${key}` + }, } return store } diff --git a/apps/media-service/test/mediaAi.harness.ts b/apps/media-service/test/mediaAi.harness.ts new file mode 100644 index 0000000..fb0ce80 --- /dev/null +++ b/apps/media-service/test/mediaAi.harness.ts @@ -0,0 +1,197 @@ +import { + applyCheckpointAction, + createMediaJob, + FakeImageProvider, + FakeLlm, + FakeVideoProvider, + loadMediaAiConfig, + solidPng, + type CheckpointAction, + type Llm, + type MediaBrief, + type MediaItem, +} from "@repo/media-ai" +import type { SiteSpec } from "@repo/schemas" +import { localSemaphore } from "@/lib/semaphore.js" +import type { GenerateDb, GenerateDeps } from "@/generate/deps.js" +import { createMemoryDb } from "@repo/media-ai/testing" + +export const config = loadMediaAiConfig({ MEDIA_ENABLED: "true" }) +const USER = "user_1" +const PROJECT = "proj_1" +const VERSION = "ver_1" + +type SectionType = SiteSpec["pages"][number]["sections"][number]["type"] + +const section = (type: SectionType, index: number, items = 0) => ({ + type, + index, + content: { headline: `${type} headline`, body: "Body", items: Array.from({ length: items }, (_, i) => ({ title: `Item ${i}`, description: "Desc" })) }, +}) + +export const PAGES = [ + { + type: "HOME", + slug: "home", + title: "Home", + sections: [section("HERO", 0), section("FEATURES", 1, 3), section("HOW_IT_WORKS", 2, 3), section("CTA", 3), section("FOOTER", 4)], + }, +] as SiteSpec["pages"] + +const visualSystem = { + lighting: "soft window light", + lens: "50mm", + texture: "matte paper", + compositionRules: ["one subject"], + recurringMotif: "paper planes", + colorUsage: "primary on hero objects", + doNot: [], +} + +export const MEDIA_BRIEF: MediaBrief = { + subject: "Bookkeeping software for small studios", + audience: { segment: "B2B" }, + message: "Your books, finished without the busywork", + mood: ["calm"], + palette: { primary: "#2563eb", secondary: "#f1f5f9", accent: "#f97316", neutrals: [] }, + styleDirection: "Soft natural light, tactile paper objects", + researchFindingIds: [], + avoid: ["readable text"], + budget: { maxImages: 8, maxVideos: 3 }, + hasProductPhotoUploads: false, + blueprint: { narrative: "From scattered effort to one calm result.", visualSystem }, +} + +export function image(itemId: string, slotId: string, priority = 1): MediaItem { + return { + itemId, + slotId, + kind: "image", + decision: "GENERATE", + role: "hook", + intent: "Make the outcome feel effortless", + subjectDirection: "A paper plane over a desk", + mood: ["calm"], + aspect: "16:9", + display: slotId.endsWith("background") ? "background" : "inline", + textSafeZone: "left", + priority, + reason: "Sets the tone", + } +} + +export function video(itemId: string, slotId: string, priority = 1): MediaItem { + return { ...image(itemId, slotId, priority), kind: "video", video: { durationSec: 6, loopable: false, motionIntent: "slow glide", playWhen: "in_view" } } +} + +export type HarnessOptions = { + mode?: "AUTO" | "MANUAL" + items: MediaItem[] + balance?: number + llm?: Llm + images?: FakeImageProvider + video?: GenerateDeps["video"] + imageLimit?: number +} + +export async function harness(opts: HarnessOptions) { + const db = createMemoryDb() + const store = new Map() + await db.creditBalance.create({ data: { userId: USER, balance: opts.balance ?? 1000 } }) + await db.project.create({ data: { id: PROJECT, userId: USER, currentVersionId: VERSION } }) + await db.projectVersion.create({ + data: { + id: VERSION, + projectId: PROJECT, + versionNumber: 1, + siteType: "SINGLE_PAGE", + seo: {}, + snapshot: { pages: PAGES, copyFramework: "AIDA", media: {}, designSystem: { primaryColor: "#2563eb", secondaryColor: "#f1f5f9", accentColor: "#f97316" } }, + }, + }) + + const putAsset = async (kind: "IMAGE" | "VIDEO", bytes: Buffer, generation: unknown, status = "READY") => { + const row = await db.mediaAsset.create({ data: { projectId: PROJECT, userId: USER, kind, origin: "GENERATED", status, generation } }) + const key = `media/${row.id as string}` + store.set(`${key}/original`, bytes) + const still = kind === "IMAGE" ? bytes : solidPng(96, 54, [40, 90, 200]) + const roles = kind === "IMAGE" ? ["fallback", "w960", "w1600"] : ["poster", "poster_last", "mp4_720"] + for (const role of roles) store.set(`${key}/${role}`, role === "mp4_720" ? bytes : still) + await db.mediaAsset.update({ + where: { id: row.id }, + data: { originalKey: `${key}/original`, variants: roles.map((role) => ({ role, key: `${key}/${role}` })) }, + }) + return row.id as string + } + + const imageSemaphore = localSemaphore(opts.imageLimit ?? 2) + const videoSemaphore = localSemaphore(2) + const images = opts.images ?? new FakeImageProvider() + const videoProvider = opts.video ?? new FakeVideoProvider() + const deps: GenerateDeps = { + db: db as unknown as GenerateDb, + store: { + getBuffer: async (key: string) => store.get(key) ?? null, + presignPut: async (key: string) => `https://upload.test/${key}`, + } as GenerateDeps["store"], + llm: opts.llm ?? new FakeLlm(), + images, + video: videoProvider, + createAsset: async (input) => ({ assetId: await putAsset(input.kind, "bytes" in input ? input.bytes : Buffer.alloc(0), input.generation) }), + reserveAsset: async (input) => { + const assetId = await putAsset("VIDEO", Buffer.from("pending"), input.generation, "UPLOADING") + return { assetId, key: `media/${assetId}/original` } + }, + finishReserved: async (asset) => { + await db.mediaAsset.update({ where: { id: asset.assetId }, data: { status: "READY" } }) + return true + }, + download: async () => Buffer.from("fake-mp4"), + extractFrame: async () => solidPng(96, 54, [40, 90, 200]), + semaphores: { image: imageSemaphore, video: videoSemaphore, concepts: localSemaphore(1) }, + config: { + ai: config, + imageQuality: "high", + videoResolution: "768p", + videoTimeoutS: 600, + promptExpansion: "disabled", + promptMaxChars: 1500, + assetTimeoutMs: 1000, + directUpload: false, + }, + sleep: async () => undefined, + } + + const budget = { maxImages: 8, maxVideos: 3 } + const { jobId, reserved } = await createMediaJob(db as never, { + userId: USER, + projectId: PROJECT, + mode: opts.mode ?? "AUTO", + scope: { kind: "site" }, + versionId: VERSION, + plan: { items: opts.items, dropped: [], budget, budgetTier: "STANDARD" }, + budget, + budgetTier: "STANDARD", + mediaBrief: MEDIA_BRIEF, + chargeBlueprint: false, + config, + enqueue: async () => undefined, + baseMedia: {}, + }) + + const job = async () => (await db.mediaJob.findUnique({ where: { id: jobId } }))! + const balance = async () => ((await db.creditBalance.findUnique({ where: { userId: USER } }))!.balance as number) + const act = async (action: CheckpointAction) => applyCheckpointAction(db as never, (await job()) as never, action, config) + + return { db, deps, jobId, reserved, job, balance, act, images, videoProvider, imageSemaphore, videoSemaphore, store } +} + +export async function until(read: () => Promise, ms = 5000): Promise { + const end = Date.now() + ms + for (;;) { + const value = await read() + if (value) return value + if (Date.now() > end) throw new Error("timed out waiting") + await new Promise((r) => setTimeout(r, 5)) + } +} diff --git a/apps/media-service/test/runner.test.ts b/apps/media-service/test/runner.test.ts new file mode 100644 index 0000000..31222c2 --- /dev/null +++ b/apps/media-service/test/runner.test.ts @@ -0,0 +1,185 @@ +import { describe, expect, it } from "vitest" +import { FakeImageProvider, FakeLlm, FakeVideoProvider, readJobState, type VideoProvider } from "@repo/media-ai" +import { runMediaJob } from "@/generate/runner.js" +import { sweepMediaJobs } from "@/generate/sweep.js" +import { harness, image, until, video } from "./mediaAi.harness.js" + +const HERO = "home/hero-0/background" +const FEATURE = "home/features-1/item-0" +const STEP = "home/how_it_works-2/item-0" +const CTA = "home/cta-3/background" + +describe("runMediaJob (AUTO)", () => { + it("makes every item, binds them and adds a new version", async () => { + const h = await harness({ items: [image("img-1", FEATURE), video("vid-1", HERO)] }) + await runMediaJob(h.deps, h.jobId) + const job = await h.job() + expect(job.status).toBe("DONE") + expect(Object.keys(job.bindings as object).sort()).toEqual([FEATURE, HERO].sort()) + expect(h.videoProvider instanceof FakeVideoProvider && h.videoProvider.submits).toHaveLength(1) + + const versions = h.db.projectVersion.rows + expect(versions).toHaveLength(2) + expect(versions[1]!.label).toBe("Media added") + expect(job.resultVersionId).toBe(versions[1]!.id) + expect(h.db.project.rows[0]!.currentVersionId).toBe(versions[1]!.id) + expect(job.creditsRefunded).toBe(0) + expect(await h.balance()).toBe(1000 - h.reserved) + }) + + it("writes concepts one video at a time so each video gets a different mechanism", async () => { + const llm = new FakeLlm() + const h = await harness({ llm, items: [video("vid-1", HERO, 1), video("vid-2", STEP, 2)] }) + await runMediaJob(h.deps, h.jobId) + const state = readJobState((await h.job()).state) + const chosen = Object.values(state.items).map((s) => s.concept!.concepts.find((c) => c.id === s.concept!.chosenId)!.mechanism) + expect(new Set(chosen).size).toBe(2) + const conceptCalls = llm.calls.filter((c) => c.purpose === "media.concepts") + expect(conceptCalls).toHaveLength(2) + expect(chosen.some((m) => conceptCalls[1]!.system.includes(m))).toBe(true) + }) + + it("respects the image concurrency limit", async () => { + const h = await harness({ imageLimit: 1, items: [image("a", FEATURE), image("b", "home/features-1/item-1"), image("c", CTA)] }) + await runMediaJob(h.deps, h.jobId) + expect((await h.job()).status).toBe("DONE") + expect(h.imageSemaphore.peak).toBe(1) + }) + + it("refunds a refused image and keeps the rest of the site", async () => { + const images = new FakeImageProvider({ refuse: (p) => p.includes("for a") }) + const h = await harness({ images, items: [image("a", FEATURE), image("b", CTA)] }) + await runMediaJob(h.deps, h.jobId) + const job = await h.job() + const state = readJobState(job.state) + expect(job.status).toBe("DONE") + expect(state.items.a!.status).toBe("FAILED") + expect(state.items.b!.status).toBe("DONE") + expect(job.creditsRefunded).toBe(state.charges.a) + expect(Object.keys(job.bindings as object)).toEqual([CTA]) + }) + + it("falls back to the opening still when the video fails, and refunds the item", async () => { + const h = await harness({ video: new FakeVideoProvider({ outcome: () => "FAILED" }), items: [video("vid-1", HERO)] }) + await runMediaJob(h.deps, h.jobId) + const job = await h.job() + const s = readJobState(job.state).items["vid-1"]! + expect(s.status).toBe("FAILED") + expect(s.fallbackAssetId).toBe(s.frames!.openingAssetId) + expect((job.bindings as Record)[HERO]!.assetId).toBe(s.frames!.openingAssetId) + expect(job.creditsRefunded).toBe(h.reserved) + expect(await h.balance()).toBe(1000) + }) + + it("refunds an item only once even if refunds run twice", async () => { + const h = await harness({ video: new FakeVideoProvider({ outcome: () => "FAILED" }), items: [video("vid-1", HERO)] }) + await runMediaJob(h.deps, h.jobId) + await sweepMediaJobs(h.deps.db, new Date(Date.now() + 365 * 24 * 3600_000)) + expect(await h.balance()).toBe(1000) + expect(h.db.creditTransaction.rows.filter((t) => t.type === "REFUND")).toHaveLength(1) + }) +}) + +describe("runMediaJob resume", () => { + it("polls the stored task after a crash instead of submitting again", async () => { + const provider = new FakeVideoProvider() + let hang = true + const flaky: VideoProvider = { + name: provider.name, + model: provider.model, + taskIdFor: (k) => provider.taskIdFor(k), + submit: (req) => provider.submit(req), + poll: (id) => (hang ? new Promise(() => undefined) : provider.poll(id)), + } + const h = await harness({ video: flaky, items: [video("vid-1", HERO)] }) + void runMediaJob(h.deps, h.jobId) + await until(async () => h.db.mediaGeneration.rows.find((r) => r.stage === "VIDEO" && r.status === "RUNNING")) + + hang = false + await runMediaJob(h.deps, h.jobId) + expect((await h.job()).status).toBe("DONE") + expect(provider.submits).toHaveLength(1) + }) + + it("resubmits with the same task id when the provider lost the task", async () => { + const first = new FakeVideoProvider() + let hang = true + const h = await harness({ + video: { name: "fake", model: "fake-video", taskIdFor: (k) => first.taskIdFor(k), submit: (r) => first.submit(r), poll: (id) => (hang ? new Promise(() => undefined) : first.poll(id)) }, + items: [video("vid-1", HERO)], + }) + void runMediaJob(h.deps, h.jobId) + const row = await until(async () => h.db.mediaGeneration.rows.find((r) => r.stage === "VIDEO" && r.status === "RUNNING")) + + hang = false + const second = new FakeVideoProvider() + await runMediaJob({ ...h.deps, video: second }, h.jobId) + expect((await h.job()).status).toBe("DONE") + expect(second.submits.map((s) => s.taskId)).toEqual([row.providerTaskId]) + }) +}) + +describe("runMediaJob (MANUAL)", () => { + it("stops at each checkpoint and finishes after approvals", async () => { + const h = await harness({ mode: "MANUAL", items: [image("img-1", FEATURE), video("vid-1", HERO)] }) + const step = async () => { + await runMediaJob(h.deps, h.jobId) + return h.job() + } + + expect((await step()).checkpoint).toBe("PLAN_APPROVE") + await h.act({ checkpoint: "PLAN_APPROVE", action: "approve" }) + + let job = await step() + expect(job.checkpoint).toBe("CONCEPT_PICK") + expect(job.checkpointExpiresAt).toBeInstanceOf(Date) + await h.act({ checkpoint: "CONCEPT_PICK", action: "pick", itemId: "vid-1", conceptId: "B" }) + await expect(h.act({ checkpoint: "IMAGES_APPROVE", action: "approve" })).rejects.toMatchObject({ status: 409 }) + await h.act({ checkpoint: "CONCEPT_PICK", action: "approve" }) + + expect((await step()).checkpoint).toBe("IMAGES_APPROVE") + await h.act({ checkpoint: "IMAGES_APPROVE", action: "approve" }) + + expect((await step()).checkpoint).toBe("VIDEO_APPROVE") + await h.act({ checkpoint: "VIDEO_APPROVE", action: "approve" }) + + job = await step() + expect(job.status).toBe("DONE") + expect(readJobState(job.state).items["vid-1"]!.concept!.chosenId).toBe("B") + }) + + it("dropping an item at plan approval refunds it", async () => { + const h = await harness({ mode: "MANUAL", items: [image("img-1", FEATURE), image("img-2", CTA)] }) + await runMediaJob(h.deps, h.jobId) + await h.act({ checkpoint: "PLAN_APPROVE", action: "drop", itemId: "img-2" }) + const state = readJobState((await h.job()).state) + expect(state.items["img-2"]!.status).toBe("DROPPED") + expect(await h.balance()).toBe(1000 - h.reserved + state.charges["img-2"]!) + }) + + it("expired approvals cancel the job and refund everything unspent", async () => { + const h = await harness({ mode: "MANUAL", items: [image("img-1", FEATURE)] }) + await runMediaJob(h.deps, h.jobId) + const res = await sweepMediaJobs(h.deps.db, new Date(Date.now() + 73 * 3600_000)) + expect(res.expired).toBe(1) + expect((await h.job()).status).toBe("CANCELLED") + expect(await h.balance()).toBe(1000) + }) + + it("a cancelled job stops before the next stage", async () => { + const h = await harness({ items: [image("img-1", FEATURE)] }) + await h.db.mediaJob.update({ where: { id: h.jobId }, data: { status: "CANCELLED" } }) + await runMediaJob(h.deps, h.jobId) + expect(h.images.calls).toHaveLength(0) + }) +}) + +describe("runMediaJob paid check", () => { + it("still runs when the reservation used the last credits", async () => { + const probe = await harness({ items: [image("img-1", FEATURE)] }) + const h = await harness({ balance: probe.reserved, items: [image("img-1", FEATURE)] }) + expect(await h.balance()).toBe(0) + await runMediaJob(h.deps, h.jobId) + expect((await h.job()).status).toBe("DONE") + }) +}) diff --git a/apps/media-service/test/safeDownload.test.ts b/apps/media-service/test/safeDownload.test.ts new file mode 100644 index 0000000..99f91cb --- /dev/null +++ b/apps/media-service/test/safeDownload.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it } from "vitest" +import { isBlockedAddress, safeDownload, UnsafeDownloadError } from "@/lib/safeDownload.js" + +const ALLOWED = ["vm.runware.ai"] +const publicDns = async () => ["104.18.10.10"] + +function fetchOf(responses: Response[]) { + const urls: string[] = [] + const impl = (async (url: string | URL | Request) => { + urls.push(String(url)) + return responses.shift() ?? new Response(null, { status: 500 }) + }) as typeof fetch + return { urls, impl } +} + +describe("isBlockedAddress", () => { + it.each(["127.0.0.1", "10.0.0.5", "172.16.0.1", "192.168.1.1", "169.254.169.254", "100.64.0.1", "0.0.0.0", "::1", "fd00::1", "fe80::1", "::ffff:127.0.0.1"])( + "blocks %s", + (ip) => expect(isBlockedAddress(ip)).toBe(true), + ) + + it.each(["104.18.10.10", "8.8.8.8", "2606:4700::1111"])("allows %s", (ip) => expect(isBlockedAddress(ip)).toBe(false)) +}) + +describe("safeDownload", () => { + it("downloads from an allowed public host", async () => { + const f = fetchOf([new Response("video-bytes", { status: 200 })]) + const bytes = await safeDownload("https://vm.runware.ai/video/a.mp4", { allowedHosts: ALLOWED, maxBytes: 1000, fetchImpl: f.impl, resolve: publicDns }) + expect(bytes.toString()).toBe("video-bytes") + }) + + it("rejects http, credentials, other hosts and private addresses before fetching", async () => { + const f = fetchOf([]) + const opts = { allowedHosts: ALLOWED, maxBytes: 1000, fetchImpl: f.impl, resolve: publicDns } + await expect(safeDownload("http://vm.runware.ai/a.mp4", opts)).rejects.toThrow(UnsafeDownloadError) + await expect(safeDownload("https://u:p@vm.runware.ai/a.mp4", opts)).rejects.toThrow(/Credentials/) + await expect(safeDownload("https://evil.example/a.mp4", opts)).rejects.toThrow(/not an allowed/) + await expect(safeDownload("https://vm.runware.ai.evil.example/a.mp4", opts)).rejects.toThrow(/not an allowed/) + await expect(safeDownload("https://vm.runware.ai/a.mp4", { ...opts, resolve: async () => ["169.254.169.254"] })).rejects.toThrow(/private address/) + expect(f.urls).toHaveLength(0) + }) + + it("checks every redirect hop", async () => { + const f = fetchOf([new Response(null, { status: 302, headers: { location: "https://metadata.internal/latest" } })]) + await expect( + safeDownload("https://vm.runware.ai/a.mp4", { allowedHosts: ALLOWED, maxBytes: 1000, fetchImpl: f.impl, resolve: publicDns }), + ).rejects.toThrow(/not an allowed/) + expect(f.urls).toHaveLength(1) + }) + + it("stops oversized downloads", async () => { + const declared = fetchOf([new Response("x", { status: 200, headers: { "content-length": "5000" } })]) + await expect( + safeDownload("https://vm.runware.ai/a.mp4", { allowedHosts: ALLOWED, maxBytes: 1000, fetchImpl: declared.impl, resolve: publicDns }), + ).rejects.toThrow(/larger/) + const streamed = fetchOf([new Response("y".repeat(2000), { status: 200 })]) + await expect( + safeDownload("https://vm.runware.ai/a.mp4", { allowedHosts: ALLOWED, maxBytes: 1000, fetchImpl: streamed.impl, resolve: publicDns }), + ).rejects.toThrow(/larger/) + }) +}) diff --git a/apps/orchestrator-service/.env.example b/apps/orchestrator-service/.env.example index ff9c1c4..86e36a2 100644 --- a/apps/orchestrator-service/.env.example +++ b/apps/orchestrator-service/.env.example @@ -16,3 +16,38 @@ JEV_TIMEOUT_MS=1000 BRIEF_PREFILL_MAX_CHARS=20000 BRIEF_PREFILL_TIMEOUT_MS=25000 BRIEF_DOC_MAX_BYTES=10485760 + +# AI media: Site Blueprint + creative pipeline (docs/media-ai.md). Off by default. +# server, orchestrator-service and media-service all read this block; keep them identical. +MEDIA_ENABLED=false +# Credits per generated image, per video second and per Site Blueprint call. +# Required in production once MEDIA_ENABLED=true; 1 elsewhere when unset. +MEDIA_CREDIT_IMAGE= +MEDIA_CREDIT_VIDEO_PER_SECOND= +MEDIA_CREDIT_BLUEPRINT= +# images:videos per budget tier, plus hard caps. +MEDIA_BUDGET_IMAGES_ONLY=8:0 +MEDIA_BUDGET_LIGHT=4:1 +MEDIA_BUDGET_STANDARD=8:3 +MEDIA_BUDGET_RICH=14:6 +MEDIA_HARD_MAX_IMAGES=24 +MEDIA_HARD_MAX_VIDEOS=10 +MEDIA_HARD_MAX_GENERATED_ASSETS=60 +VIDEO_MIN_SECONDS=5 +VIDEO_MAX_SECONDS=10 +# Circuit breaker: provider spend over the last 24 h above this pauses AI media. Empty = no cap. +MEDIA_DAILY_COST_CAP_USD= +# Provider cost estimates in USD, used when a provider does not report cost. +MEDIA_COST_IMAGE_USD=0.25 +MEDIA_COST_VIDEO_USD_PER_SECOND=0.08 +MEDIA_COST_LLM_CALL_USD=0.05 +MEDIA_COST_BLUEPRINT_USD=0.4 + +# fake = deterministic stand-ins, no model calls. live needs ANTHROPIC_API_KEY. +MEDIA_AI_PROVIDER_MODE=fake +BLUEPRINT_MODEL_ID=claude-opus-5-5 +MEDIA_CREATIVE_MODEL_ID=claude-sonnet-5-5 +MEDIA_VISION_MODEL_ID=claude-sonnet-5-5 +# How long generation waits for AI images before writing copy, and for videos before compiling. +MEDIA_IMAGES_TIMEOUT_S=240 +MEDIA_VIDEO_TIMEOUT_S=600 diff --git a/apps/orchestrator-service/package.json b/apps/orchestrator-service/package.json index 8f1f333..6bda0cc 100644 --- a/apps/orchestrator-service/package.json +++ b/apps/orchestrator-service/package.json @@ -16,8 +16,10 @@ "dependencies": { "@ai-sdk/anthropic": "^1.0.0", "@ai-sdk/openai": "^1.0.0", + "@anthropic-ai/sdk": "0.133.0", "@repo/events": "workspace:*", "@repo/jev": "workspace:*", + "@repo/media-ai": "workspace:*", "@repo/rag": "workspace:*", "@repo/schemas": "workspace:*", "@repo/site-builder": "workspace:*", diff --git a/apps/orchestrator-service/src/agents/copy.agent.ts b/apps/orchestrator-service/src/agents/copy.agent.ts index ce778ff..e5a704c 100644 --- a/apps/orchestrator-service/src/agents/copy.agent.ts +++ b/apps/orchestrator-service/src/agents/copy.agent.ts @@ -19,6 +19,7 @@ export async function runCopyAgent( providerOptions?: ReturnType, promptFn: (vars: CopyPromptVars) => string = DEFAULT_COPY_PROMPT, briefCtx: ProjectBriefContext | null = null, + mediaContextFor?: (pageSlug: string, section: Section) => string | undefined, ): Promise> { if (!spec.pages) return spec const briefFacts = briefCtx ? briefFactsBlock(briefCtx) : undefined @@ -53,6 +54,7 @@ export async function runCopyAgent( brandTone: spec.designBrief?.brand.tone, frameworkRationale: spec.designBrief?.frameworkRationale, briefFacts, + mediaContext: mediaContextFor?.(page.slug, section), }) const result = streamText({ diff --git a/apps/orchestrator-service/src/agents/orchestrator.ts b/apps/orchestrator-service/src/agents/orchestrator.ts index 6d4e231..fd4a31f 100644 --- a/apps/orchestrator-service/src/agents/orchestrator.ts +++ b/apps/orchestrator-service/src/agents/orchestrator.ts @@ -13,6 +13,17 @@ import { prisma, type Prisma } from "@useframe/db"; import type { GenerateRequest, SiteSpec } from "@repo/schemas"; import { applyBriefToSpec, loadBriefContext } from "@/lib/briefPipeline.js"; import { loadAssetMenu, runMediaPlacement } from "./mediaPlacement.agent.js"; +import { parseStructureForBlueprint, structurePromptVars } from "./structure.agent.js"; +import { DEFAULT_STRUCTURE_PROMPT } from "@/prompts/structure.prompt.js"; +import { + bindingsOf, + copyMediaContext, + planSiteMedia, + waitForAllMedia, + waitForImages, + type SiteMediaPlan, +} from "@/lib/siteMedia.js"; +import { applyJobResult, applyMediaBindings, markCompiled } from "@repo/media-ai"; async function ensureProject( request: GenerateRequest, @@ -131,14 +142,48 @@ export async function runOrchestrator( stage: "GENERATE", message: "Building page structure...", }); - spec = await runStructureAgent(res, spec, request, model, providerOptions, undefined, briefCtx); + // Media projects: the Site Blueprint. With an approved design brief it + // runs over the brief's fixed sections; without one it replaces the + // structure call (full mode). Every other project skips it entirely. + const mediaInput = { res, userId, projectId: projectId!, versionId: versionId!, tier: request.tier } + let media: SiteMediaPlan + if (spec.designBrief) { + spec = await runStructureAgent(res, spec, request, model, providerOptions, undefined, briefCtx); + media = await planSiteMedia({ ...mediaInput, spec, briefCtx }); + } else { + media = await planSiteMedia({ + ...mediaInput, + spec, + briefCtx, + full: { + prompt: DEFAULT_STRUCTURE_PROMPT(structurePromptVars(request, briefCtx)), + parse: (raw) => parseStructureForBlueprint(raw, briefCtx), + }, + }); + spec = media.structure + ? { ...spec, ...media.structure, citations: [] } + : await runStructureAgent(res, spec, request, model, providerOptions, undefined, briefCtx); + } + const imagesReady = media.jobId ? await waitForImages(res, media.jobId) : null; sseWrite(res, { type: "stage", stage: "COPY", message: "Writing conversion copy...", }); - spec = await runCopyAgent(res, spec, request, model, providerOptions, undefined, briefCtx); + const blueprint = media.blueprint; + spec = await runCopyAgent( + res, + spec, + request, + model, + providerOptions, + undefined, + briefCtx, + blueprint + ? (pageSlug, section) => copyMediaContext(blueprint, imagesReady, pageSlug, `${section.type.toLowerCase()}-${section.index}`) + : undefined, + ); sseWrite(res, { type: "stage", @@ -162,18 +207,23 @@ export async function runOrchestrator( spec = await runCritiqueAgent(spec, request, model, providerOptions); spec = applyBriefToSpec(spec, briefCtx); - // Placed last, once sections and their items are final. - const menu = await loadAssetMenu(projectId, userId); - if (menu.items.length > 0 && spec.pages) { - sseWrite(res, { - type: "stage", - stage: "GENERATE", - message: "Placing your media...", - }); - spec = { - ...spec, - media: await runMediaPlacement({ spec: { pages: spec.pages, media: spec.media }, menu, model, providerOptions }), - }; + if (media.jobId) { + // The job owns placement: compile with whatever it has bound so far. + spec = { ...spec, media: bindingsOf(await waitForAllMedia(res, media.jobId)) }; + } else { + // Placed last, once sections and their items are final. + const menu = await loadAssetMenu(projectId, userId); + if (menu.items.length > 0 && spec.pages) { + sseWrite(res, { + type: "stage", + stage: "GENERATE", + message: "Placing your media...", + }); + spec = { + ...spec, + media: await runMediaPlacement({ spec: { pages: spec.pages, media: spec.media }, menu, model, providerOptions }), + }; + } } const snapshot = toSnapshot(spec as SiteSpec); @@ -187,6 +237,14 @@ export async function runOrchestrator( }, }); + if (media.jobId) { + const jobId = media.jobId; + await markCompiled(prisma, jobId, (snapshot.media ?? {}) as Record); + await applyJobResult(prisma, jobId, (args) => applyMediaBindings(prisma, args)).catch((err) => + console.error("[media] applying finished media failed", err instanceof Error ? err.name : err), + ); + } + await prisma.project.update({ where: { id: projectId }, data: { status: "READY" }, diff --git a/apps/orchestrator-service/src/agents/structure.agent.ts b/apps/orchestrator-service/src/agents/structure.agent.ts index 74a1dd4..d9d5ce7 100644 --- a/apps/orchestrator-service/src/agents/structure.agent.ts +++ b/apps/orchestrator-service/src/agents/structure.agent.ts @@ -44,6 +44,54 @@ function sectionsFromBrief( }) } +type ModelPage = { type: string; slug: string; title: string; sections?: Array<{ type: string; index: number }> } + +// The one reading of a structure answer, shared with the Site Blueprint's full mode. +export function structureFromModelOutput( + parsed: { siteType?: SiteSpec["siteType"]; copyFramework?: SiteSpec["copyFramework"]; pages?: ModelPage[] }, + briefCtx: ProjectBriefContext | null, + preferredSiteType?: SiteSpec["siteType"], +): Pick { + return { + siteType: preferredSiteType ?? parsed.siteType ?? "SINGLE_PAGE", + copyFramework: parsed.copyFramework ?? "AIDA", + pages: (parsed.pages ?? []).map((p) => ({ + type: p.type as SiteSpec["pages"][number]["type"], + slug: p.slug, + title: p.title, + sections: filterSections( + (p.sections ?? []).filter((s) => VALID_SECTION_TYPES.has(s.type)).map((s) => ({ type: s.type as SectionType, index: s.index })), + briefCtx, + ).map((s, index) => ({ ...s, index })), + })), + } +} + +export function structurePromptVars(request: GenerateRequest, briefCtx: ProjectBriefContext | null): StructurePromptVars { + return { + startupIdea: request.startupIdea, + niche: request.niche, + targetAudience: request.targetAudience, + inputType: request.inputType, + ...(briefCtx ? { allowedSections: allowedSectionTypes(briefCtx), briefFacts: briefFactsBlock(briefCtx) } : {}), + } +} + +// Validation for the blueprint's full mode: the same reading, plus no empty pages. +export function parseStructureForBlueprint(raw: unknown, briefCtx: ProjectBriefContext | null): + | { ok: true; value: Pick } + | { ok: false; errors: string[] } { + const value = structureFromModelOutput(raw as Parameters[0], briefCtx, structurePreferredSiteType(briefCtx)) + const errors: string[] = [] + if (value.pages.length === 0) errors.push("pages: at least one page is required") + for (const page of value.pages) if (page.sections.length === 0) errors.push(`pages.${page.slug}: no allowed sections left`) + return errors.length ? { ok: false, errors } : { ok: true, value } +} + +export function structurePreferredSiteType(briefCtx: ProjectBriefContext | null): SiteSpec["siteType"] | undefined { + return briefCtx?.data.siteType === "SINGLE_PAGE" || briefCtx?.data.siteType === "MULTI_PAGE" ? briefCtx.data.siteType : undefined +} + export async function runStructureAgent( res: Response, spec: Partial, @@ -53,10 +101,7 @@ export async function runStructureAgent( promptFn: (vars: StructurePromptVars) => string = DEFAULT_STRUCTURE_PROMPT, briefCtx: ProjectBriefContext | null = null, ): Promise> { - const reindex = (sections: T[]): T[] => - filterSections(sections, briefCtx).map((s, index) => ({ ...s, index })) - const preferredSiteType = - briefCtx?.data.siteType === "SINGLE_PAGE" || briefCtx?.data.siteType === "MULTI_PAGE" ? briefCtx.data.siteType : undefined + const preferredSiteType = structurePreferredSiteType(briefCtx) // When a DesignBrief was approved via the Research step, its layout and // copyFramework decisions are already grounded in retrieved research — @@ -85,13 +130,7 @@ export async function runStructureAgent( } } - const prompt = promptFn({ - startupIdea: request.startupIdea, - niche: request.niche, - targetAudience: request.targetAudience, - inputType: request.inputType, - ...(briefCtx ? { allowedSections: allowedSectionTypes(briefCtx), briefFacts: briefFactsBlock(briefCtx) } : {}), - }) + const prompt = promptFn(structurePromptVars(request, briefCtx)) const result = streamText({ model, @@ -110,32 +149,8 @@ export async function runStructureAgent( try { const jsonMatch = fullText.match(/\{[\s\S]*\}/) if (!jsonMatch) throw new Error("No JSON in structure response") - const parsed = JSON.parse(jsonMatch[0]) - - return { - ...spec, - siteType: preferredSiteType ?? parsed.siteType ?? "SINGLE_PAGE", - copyFramework: parsed.copyFramework ?? "AIDA", - pages: (parsed.pages ?? []).map( - (p: { - type: string - slug: string - title: string - sections: Array<{ type: string; index: number }> - }) => ({ - type: p.type, - slug: p.slug, - title: p.title, - sections: reindex( - (p.sections ?? []).map((s: { type: string; index: number }) => ({ - type: s.type, - index: s.index, - })), - ), - }), - ), - citations: [], - } + const parsed = structureFromModelOutput(JSON.parse(jsonMatch[0]), briefCtx, preferredSiteType) + return { ...spec, ...parsed, citations: [] } } catch { return { ...spec, diff --git a/apps/orchestrator-service/src/config/env.ts b/apps/orchestrator-service/src/config/env.ts index 86f8044..fec3edd 100644 --- a/apps/orchestrator-service/src/config/env.ts +++ b/apps/orchestrator-service/src/config/env.ts @@ -1,6 +1,7 @@ import { z } from "zod"; import dotenv from "dotenv"; import { JevEnvSchema } from "@repo/jev/config"; +import { loadMediaAiConfig, type MediaAiConfig } from "@repo/media-ai"; dotenv.config(); @@ -45,6 +46,14 @@ const envSchema = z BRIEF_PREFILL_MAX_CHARS: z.coerce.number().int().positive().default(20000), BRIEF_PREFILL_TIMEOUT_MS: z.coerce.number().int().positive().default(25000), BRIEF_DOC_MAX_BYTES: z.coerce.number().int().positive().default(10485760), + + // AI media (Site Blueprint + creative pipeline). fake = no real model or provider calls. + MEDIA_AI_PROVIDER_MODE: z.enum(["fake", "live"]).default("fake"), + BLUEPRINT_MODEL_ID: z.string().min(1).default("claude-opus-5-5"), + MEDIA_CREATIVE_MODEL_ID: z.string().min(1).default("claude-sonnet-5-5"), + MEDIA_VISION_MODEL_ID: z.string().min(1).default("claude-sonnet-5-5"), + MEDIA_IMAGES_TIMEOUT_S: z.coerce.number().int().min(0).default(240), + MEDIA_VIDEO_TIMEOUT_S: z.coerce.number().int().min(0).default(600), }) .refine( (d) => @@ -65,6 +74,16 @@ if (!parsed.success) { export const env = parsed.data; +let media: MediaAiConfig; +try { + media = loadMediaAiConfig(process.env); +} catch (err) { + console.error("Invalid environment variables:"); + console.error(err instanceof Error ? err.message : err); + process.exit(1); +} +export const mediaAiConfig = media; + export const CONFIGURED_PROVIDERS = { anthropic: env.ANTHROPIC_API_KEY !== "", deepseek: env.DEEPSEEK_API_KEY !== "", diff --git a/apps/orchestrator-service/src/index.ts b/apps/orchestrator-service/src/index.ts index ec7e5f9..4818fb3 100644 --- a/apps/orchestrator-service/src/index.ts +++ b/apps/orchestrator-service/src/index.ts @@ -17,6 +17,7 @@ import planRoute from "@/routes/plan.route.js"; import extractRoute from "@/routes/extract.route.js"; import briefRoute from "@/routes/brief.route.js"; import mediaRoute from "@/routes/media.route.js"; +import mediaAiRoute from "@/routes/mediaAi.route.js"; import { MODELS } from "@/llm/providers.js"; import { prisma } from "@useframe/db"; @@ -31,6 +32,7 @@ app.use( ); // Carries its own larger body parser (an image), so it sits ahead of the 1mb one. app.use("/", mediaRoute); +app.use("/", mediaAiRoute); app.use(express.json({ limit: "1mb" })); if (env.NODE_ENV === "development") { diff --git a/apps/orchestrator-service/src/lib/siteMedia.test.ts b/apps/orchestrator-service/src/lib/siteMedia.test.ts new file mode 100644 index 0000000..3632410 --- /dev/null +++ b/apps/orchestrator-service/src/lib/siteMedia.test.ts @@ -0,0 +1,210 @@ +import { beforeEach, describe, expect, it, vi } from "vitest" +import type { Response } from "express" +import type { SiteSpec } from "@repo/schemas" +import { DATA_PARAGRAPH, FakeLlm, loadMediaAiConfig, readBlueprint, readBlueprintNotice, type Llm, type LlmRequest } from "@repo/media-ai" +import { createMemoryDb } from "@repo/media-ai/testing" + +const m = vi.hoisted(() => ({ + db: null as ReturnType | null, + llm: null as Llm | null, + enabled: true, + add: vi.fn(), +})) + +vi.mock("@/config/env.js", () => ({ + env: { MEDIA_IMAGES_TIMEOUT_S: 0, MEDIA_VIDEO_TIMEOUT_S: 0 }, + get mediaAiConfig() { + return { ...loadMediaAiConfig({ MEDIA_ENABLED: "true" }), enabled: m.enabled } + }, +})) +vi.mock("@useframe/db", async (importOriginal) => ({ + ...(await importOriginal()), + get prisma() { + return m.db! + }, + briefStateOf: (row: { data: unknown }) => ({ data: row.data }), +})) +vi.mock("@/llm/claude.js", () => ({ + get mediaLlm() { + return m.llm! + }, +})) +vi.mock("@/lib/redis.js", () => ({ redis: {} })) +vi.mock("bullmq", () => ({ Queue: class { add = m.add } })) + +const { planSiteMedia, copyMediaContext } = await import("./siteMedia.js") +const { DEFAULT_COPY_PROMPT } = await import("@/prompts/copy.prompt.js") + +const USER = "user_1" +const PROJECT = "proj_1" +const VERSION = "ver_1" + +const pages = () => + [ + { + type: "HOME", + slug: "home", + title: "Home", + sections: [ + { type: "HERO", index: 0, content: { headline: "Books, done" } }, + { type: "FEATURES", index: 1, content: { headline: "Why", items: [] } }, + { type: "CTA", index: 2, content: { headline: "Start" } }, + ], + }, + ] as SiteSpec["pages"] + +const designBrief = { + colors: { primary: "#2563eb", secondary: "#f1f5f9", accent: "#f97316" }, + typography: { primary: "Inter", secondary: "Inter" }, + avoid: [], +} as unknown as SiteSpec["designBrief"] + +function counting(inner: Llm = new FakeLlm()): Llm & { calls: LlmRequest[] } { + const calls: LlmRequest[] = [] + return { calls, complete: (req) => (calls.push(req), inner.complete(req)) } +} + +function res() { + const writes: string[] = [] + return { writes, res: { destroyed: false, writableEnded: false, write: (s: string) => writes.push(s) } as unknown as Response } +} + +async function seed(opts: { balance?: number; brief?: Record; blueprint?: unknown } = {}) { + const db = m.db! + await db.project.create({ data: { id: PROJECT, userId: USER, startupIdea: "Bookkeeping for studios", targetAudience: "Studio owners", niche: null } }) + await db.projectVersion.create({ data: { id: VERSION, projectId: PROJECT, versionNumber: 1, blueprint: opts.blueprint ?? null } }) + await db.creditBalance.create({ data: { userId: USER, balance: opts.balance ?? 1000 } }) + await db.researchReport.create({ + data: { + projectId: PROJECT, + primaryColor: "#2563eb", + secondaryColor: "#f1f5f9", + accentColor: "#f97316", + colorPalette: [], + imageStyle: "Soft natural light, tactile paper objects", + imageDirection: null, + styleTagKeys: [], + }, + }) + if (opts.brief) await db.projectBrief.create({ data: { projectId: PROJECT, userId: USER, status: "APPROVED", data: opts.brief } }) +} + +const input = (r: Response, tier: "free" | "paid" = "paid") => ({ + res: r, + userId: USER, + projectId: PROJECT, + versionId: VERSION, + tier, + spec: { pages: pages(), copyFramework: "AIDA" as const, designBrief }, + briefCtx: null, +}) + +beforeEach(() => { + m.db = createMemoryDb() + m.enabled = true + m.add.mockReset() +}) + +describe("planSiteMedia", () => { + it("makes no blueprint call for free projects, disabled media or media turned off", async () => { + await seed({ brief: { mediaMode: "OFF" } }) + const llm = counting() + m.llm = llm + const r = res() + + expect((await planSiteMedia(input(r.res, "free"))).jobId).toBeNull() + m.enabled = false + expect((await planSiteMedia(input(r.res))).jobId).toBeNull() + m.enabled = true + expect((await planSiteMedia(input(r.res))).jobId).toBeNull() + + expect(llm.calls).toHaveLength(0) + expect(r.writes).toHaveLength(0) + expect(m.db!.mediaJob.rows).toHaveLength(0) + expect(m.db!.projectVersion.rows[0]!.blueprint).toBeNull() + }) + + it("runs the blueprint over the brief's fixed sections and starts a job", async () => { + await seed() + const llm = counting() + m.llm = llm + const r = res() + const spec = input(r.res).spec + const before = structuredClone(spec.pages) + + const plan = await planSiteMedia({ ...input(r.res), spec }) + expect(plan.jobId).not.toBeNull() + expect(plan.structure).toBeUndefined() + expect(spec.pages).toEqual(before) + expect(llm.calls).toHaveLength(1) + expect(llm.calls[0]!.routingKey).toBe("blueprint") + expect(r.writes.join("")).toContain('"substep":"blueprint"') + expect(readBlueprint(m.db!.projectVersion.rows[0]!.blueprint)?.mode).toBe("brief") + + const job = m.db!.mediaJob.rows[0]! + expect((job.state as { charges: Record }).charges.blueprint).toBe(1) + expect(m.add).toHaveBeenCalledTimes(1) + }) + + it("skips media with a notice when the blueprint fails twice, and charges nothing", async () => { + await seed() + const llm = counting({ complete: async () => ({ text: "not json" }) }) + m.llm = llm + const plan = await planSiteMedia(input(res().res)) + expect(plan).toEqual({ jobId: null, blueprint: null }) + expect(llm.calls).toHaveLength(2) + expect(readBlueprintNotice(m.db!.projectVersion.rows[0]!.blueprint)?.notice).toMatch(/couldn't plan AI media/) + expect(m.db!.mediaJob.rows).toHaveLength(0) + expect((await m.db!.creditBalance.findUnique({ where: { userId: USER } }))!.balance).toBe(1000) + }) + + it("re-runs a stale blueprint and reuses a fresh one without charging for it", async () => { + await seed() + m.llm = counting() + await planSiteMedia(input(res().res)) + const fresh = m.db!.projectVersion.rows[0]!.blueprint + + m.db = createMemoryDb() + await seed({ blueprint: fresh }) + const reuse = counting() + m.llm = reuse + await planSiteMedia(input(res().res)) + expect(reuse.calls).toHaveLength(0) + expect((m.db!.mediaJob.rows[0]!.state as { charges: Record }).charges.blueprint).toBeUndefined() + + m.db = createMemoryDb() + await seed({ blueprint: { ...(fresh as object), briefHash: "stale" } }) + const rerun = counting() + m.llm = rerun + await planSiteMedia(input(res().res)) + expect(rerun.calls).toHaveLength(1) + }) +}) + +describe("copy context (Call 2)", () => { + it("gives the copy agent the narrative and planned media as read-only DATA", async () => { + await seed() + m.llm = counting() + const plan = await planSiteMedia(input(res().res)) + const blueprint = plan.blueprint! + const sectionWithMedia = blueprint.mediaScript[0]!.slotId.split("/")[1]! + const context = copyMediaContext(blueprint, null, "home", sectionWithMedia) + expect(context).toContain(blueprint.narrative.storyArc) + expect(copyMediaContext(blueprint, null, "home", "footer-9")).toBeUndefined() + + const prompt = DEFAULT_COPY_PROMPT({ + copyFramework: "AIDA", + startupIdea: "x", + niche: "y", + targetAudience: "z", + pageTitle: "Home", + pageType: "HOME", + sectionType: "HERO", + sectionIndex: 0, + mediaContext: context, + }) + expect(prompt).toContain(DATA_PARAGRAPH) + expect(prompt).toMatch(/[\s\S]*<\/DATA>/) + expect(prompt).toContain("AIDA framework still decides structure and tone") + }) +}) diff --git a/apps/orchestrator-service/src/lib/siteMedia.ts b/apps/orchestrator-service/src/lib/siteMedia.ts new file mode 100644 index 0000000..c515e45 --- /dev/null +++ b/apps/orchestrator-service/src/lib/siteMedia.ts @@ -0,0 +1,264 @@ +import type { Response } from "express" +import { Queue } from "bullmq" +import { prisma, type Prisma } from "@useframe/db" +import { QUEUES, type MediaGenerateJobPayload } from "@repo/events" +import type { SiteSpec } from "@repo/schemas" +import { + buildMediaBrief, + checkEligibility, + createMediaJob, + fitPlanToBalance, + InsufficientCreditsError, + isBlueprintFresh, + loadBlueprintInputs, + MediaBriefError, + readBlueprint, + readJobState, + runSiteBlueprint, + SUBSTEP_LABELS, + type MediaItem, + type PersistedBlueprint, + type StructureResult, + type Substep, +} from "@repo/media-ai" +import { env, mediaAiConfig } from "@/config/env.js" +import { mediaLlm } from "@/llm/claude.js" +import { sseWrite } from "@/llm/stream.js" +import { redis } from "@/lib/redis.js" +import type { ProjectBriefContext } from "@/lib/briefPipeline.js" + +const queue = new Queue(QUEUES.MEDIA_GENERATE, { connection: redis }) + +export async function enqueueMediaJob(jobId: string): Promise { + await queue.add("generate", { jobId }, { jobId: `${jobId}-${Date.now()}`, attempts: 1, removeOnComplete: 500, removeOnFail: 500 }) +} + +export type SiteMediaPlan = { + jobId: string | null + blueprint: PersistedBlueprint | null + structure?: StructureResult +} + +const NONE: SiteMediaPlan = { jobId: null, blueprint: null } + +function emitSubstep(res: Response, substep: Substep): void { + sseWrite(res, { type: "stage", stage: "GENERATE", message: SUBSTEP_LABELS[substep], substep }) +} + +async function storeNotice(versionId: string, notice: string): Promise { + await prisma.projectVersion + .update({ + where: { id: versionId }, + data: { blueprint: { status: "skipped", notice, createdAt: new Date().toISOString() } as Prisma.InputJsonValue }, + }) + .catch(() => undefined) +} + +type PlanInput = { + res: Response + userId: string + projectId: string + versionId: string + tier: "free" | "paid" + spec: Partial + briefCtx: ProjectBriefContext | null + // full: no approved design brief, so the blueprint also designs the pages. + full?: { prompt: string; parse: (raw: unknown) => { ok: true; value: StructureResult } | { ok: false; errors: string[] } } +} + +// Runs the Site Blueprint and starts the media job for eligible paid projects. +// For every other project it returns at once: no extra call, no new fields. +export async function planSiteMedia(input: PlanInput): Promise { + if (input.tier !== "paid" || !mediaAiConfig.enabled) return NONE + const inputs = await loadBlueprintInputs(prisma, input.projectId, input.userId, true).catch(() => null) + if (!inputs) return NONE + const eligibility = await checkEligibility(prisma, { + userId: input.userId, + projectId: input.projectId, + mediaMode: inputs.media.mode, + budgetTier: inputs.media.budgetTier, + config: mediaAiConfig, + }) + if (!eligibility.ok) { + console.log(`[media] skipped for project ${input.projectId}: ${eligibility.reason}`) + if (eligibility.reason === "circuit_open") await storeNotice(input.versionId, "AI media is temporarily unavailable, so this version was built without it.") + return NONE + } + + const balance = (await prisma.creditBalance.findUnique({ where: { userId: input.userId }, select: { balance: true } }))?.balance ?? 0 + if (balance < mediaAiConfig.credits.blueprint + mediaAiConfig.credits.image) { + await storeNotice(input.versionId, "There weren't enough credits for AI media, so this version was built without it.") + return NONE + } + + const designBrief = input.spec.designBrief + const palette = [ + ...(designBrief ? [designBrief.colors.primary, designBrief.colors.secondary, designBrief.colors.accent] : []), + ] + let mediaBrief + try { + mediaBrief = buildMediaBrief({ + brief: inputs.briefFacts, + project: inputs.project, + research: inputs.research, + imageryFindingIds: inputs.imageryFindingIds, + styleAvoid: [...inputs.styleAvoid, ...(designBrief?.avoid ?? [])], + mediaNotes: inputs.media.notes, + budget: eligibility.budget, + hasProductPhotoUploads: inputs.hasProductPhotoUploads, + }) + } catch (err) { + if (err instanceof MediaBriefError) { + console.log(`[media] skipped for project ${input.projectId}: ${err.message}`) + return NONE + } + throw err + } + palette.push(mediaBrief.palette.primary, mediaBrief.palette.secondary, mediaBrief.palette.accent, ...mediaBrief.palette.neutrals) + + const generated = await prisma.mediaAsset.count({ where: { projectId: input.projectId, origin: "GENERATED", deletedAt: null } }) + const existing = readBlueprint( + (await prisma.projectVersion.findUnique({ where: { id: input.versionId }, select: { blueprint: true } }))?.blueprint, + ) + + emitSubstep(input.res, "blueprint") + let blueprint: PersistedBlueprint + let structure: StructureResult | undefined + const reused = !input.full && !!existing && !!input.spec.pages && isBlueprintFresh(existing, input.spec.pages, input.spec.copyFramework ?? "AIDA") + if (reused) { + blueprint = existing! + } else { + const result = await runSiteBlueprint({ + mode: input.full ? "full" : "brief", + llm: mediaLlm, + ...(input.full + ? { structure: input.full } + : { + site: { + pages: input.spec.pages ?? [], + copyFramework: input.spec.copyFramework ?? "AIDA", + tokens: { + colors: palette, + fonts: designBrief ? [designBrief.typography.primary, designBrief.typography.secondary] : [], + styleTags: inputs.research?.styleTagKeys ?? [], + }, + }, + }), + palette, + mediaBrief, + briefFacts: inputs.briefFacts, + research: inputs.research, + uploaded: inputs.uploaded, + budget: eligibility.budget, + config: mediaAiConfig, + mediaNotes: inputs.media.notes, + generatedAllowance: mediaAiConfig.hardMaxGeneratedAssets - generated, + }).catch((err: unknown) => { + console.error(`[media] blueprint call failed for project ${input.projectId}:`, err instanceof Error ? err.name : "unknown") + return { ok: false as const, errors: ["blueprint call failed"], attempts: 2, costUsd: 0 } + }) + if (!result.ok) { + console.log(`[media] blueprint failed twice for project ${input.projectId}`) + await storeNotice(input.versionId, "We couldn't plan AI media for this version, so it was built without it. You can try again from the Media tab.") + return NONE + } + blueprint = result.blueprint + structure = result.structure + } + + await prisma.projectVersion.update({ + where: { id: input.versionId }, + data: { blueprint: blueprint as unknown as Prisma.InputJsonValue }, + }) + + const fitted = fitPlanToBalance(blueprint.mediaScript, balance, mediaAiConfig, { blueprint: !reused }) + try { + const { jobId } = await createMediaJob(prisma, { + userId: input.userId, + projectId: input.projectId, + mode: inputs.media.mode === "MANUAL" ? "MANUAL" : "AUTO", + scope: { kind: "site" }, + versionId: input.versionId, + plan: { + items: fitted.items, + dropped: [...blueprint.dropped, ...fitted.dropped], + budget: eligibility.budget, + budgetTier: inputs.media.budgetTier, + }, + budget: eligibility.budget, + budgetTier: inputs.media.budgetTier, + mediaBrief: { ...mediaBrief, blueprint: { narrative: blueprint.narrative.storyArc, visualSystem: blueprint.visualSystem } }, + chargeBlueprint: !reused, + config: mediaAiConfig, + enqueue: enqueueMediaJob, + }) + return { jobId, blueprint, ...(structure ? { structure } : {}) } + } catch (err) { + if (err instanceof InsufficientCreditsError) { + await storeNotice(input.versionId, "There weren't enough credits for AI media, so this version was built without it.") + return { jobId: null, blueprint, ...(structure ? { structure } : {}) } + } + throw err + } +} + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) + +type JobSnapshot = { status: string; stage: string | null; state: unknown; bindings: unknown } + +// Polls the job until `done` holds, it settles, it waits on the user, or time runs out. +async function waitForJob(res: Response, jobId: string, timeoutS: number, done: (job: JobSnapshot) => boolean): Promise { + const deadline = Date.now() + timeoutS * 1000 + let lastStage: string | null = null + for (;;) { + const job = await prisma.mediaJob.findUnique({ where: { id: jobId }, select: { status: true, stage: true, state: true, bindings: true } }) + if (!job) return null + if (job.stage && job.stage !== lastStage && (job.stage as Substep) in SUBSTEP_LABELS) { + lastStage = job.stage + emitSubstep(res, job.stage as Substep) + } + if (job.status === "AWAITING_USER") emitSubstep(res, "media.awaiting_user") + if (done(job) || ["DONE", "FAILED", "CANCELLED", "AWAITING_USER"].includes(job.status) || Date.now() >= deadline) return job + await sleep(2000) + } +} + +const settled = (status: string) => ["DONE", "FAILED", "DROPPED", "SKIPPED"].includes(status) + +export async function waitForImages(res: Response, jobId: string): Promise { + return waitForJob(res, jobId, env.MEDIA_IMAGES_TIMEOUT_S, (job) => + Object.values(readJobState(job.state).items).every((i) => i.item.kind !== "image" || settled(i.status)), + ) +} + +export async function waitForAllMedia(res: Response, jobId: string): Promise { + return waitForJob(res, jobId, env.MEDIA_VIDEO_TIMEOUT_S, (job) => job.status === "DONE") +} + +export function bindingsOf(job: JobSnapshot | null): NonNullable { + const b = job?.bindings + return b && typeof b === "object" ? (b as NonNullable) : {} +} + +// Read-only context for the copy agent: what each section will show, so copy and visuals match. +export function copyMediaContext(blueprint: PersistedBlueprint, job: JobSnapshot | null, pageSlug: string, sectionId: string): string | undefined { + const state = job ? readJobState(job.state) : null + const items = blueprint.mediaScript.filter((i: MediaItem) => i.slotId.startsWith(`${pageSlug}/${sectionId}/`)) + if (items.length === 0) return undefined + const lines = items.map((i) => { + const s = state?.items[i.itemId] + const alt = s?.image?.altText ?? s?.frames?.altText + const concept = s?.concept?.concepts.find((c) => c.id === s.concept?.chosenId) + return `- ${i.slotId.split("/")[2]}: ${i.kind} that should make the visitor feel: ${i.intent}${concept ? `; it shows: ${concept.logline}` : ""}${alt ? `; alt text: ${alt}` : ""}` + }) + const itemSlots = items.map((i) => /item-(\d+)$/.exec(i.slotId)?.[1]).filter(Boolean).map(Number) + const need = itemSlots.length ? Math.max(...itemSlots) + 1 : 0 + return [ + `Story of the whole site: ${blueprint.narrative.storyArc}`, + `Media planned in this section:\n${lines.join("\n")}`, + need ? `Write at least ${need} items so every planned visual has its item.` : "", + ] + .filter(Boolean) + .join("\n") +} + diff --git a/apps/orchestrator-service/src/llm/claude.ts b/apps/orchestrator-service/src/llm/claude.ts new file mode 100644 index 0000000..027077a --- /dev/null +++ b/apps/orchestrator-service/src/llm/claude.ts @@ -0,0 +1,59 @@ +import Anthropic from "@anthropic-ai/sdk" +import { FakeLlm, type Llm, type LlmRequest, type LlmResponse, type RoutingKey } from "@repo/media-ai" +import { env } from "@/config/env.js" + +// USD per million tokens (input, output) for the routing-key models. +const PRICES: Record = { + "claude-opus-5-5": [4, 20], + "claude-sonnet-5-5": [2, 10], +} + +export function modelForRoutingKey(key: RoutingKey): string { + if (key === "blueprint") return env.BLUEPRINT_MODEL_ID + return key === "media-vision" ? env.MEDIA_VISION_MODEL_ID : env.MEDIA_CREATIVE_MODEL_ID +} + +// Blueprint calls on Opus at high effort take longer than the 90 s budget of the other media calls. +const timeoutFor = (key: RoutingKey) => (key === "blueprint" ? 240_000 : 90_000) + +// The routing keys run on Claude 5.5 models, which take effort through +// output_config (the AI SDK adapter used elsewhere cannot send it) and reject +// sampling parameters, so no temperature is ever sent. +export function createClaudeLlm(client = new Anthropic({ apiKey: env.ANTHROPIC_API_KEY, maxRetries: 1 })): Llm { + return { + async complete(req: LlmRequest): Promise { + const model = modelForRoutingKey(req.routingKey) + const stream = client.beta.messages.stream( + { + model, + max_tokens: req.maxTokens ?? 16_000, + betas: ["server-side-fallback-2026-07-01"], + fallbacks: "default", + output_config: { effort: "high" }, + system: req.system, + messages: [ + { + role: "user", + content: [ + ...(req.images ?? []).map((img) => ({ + type: "image" as const, + source: { type: "base64" as const, media_type: img.mime, data: img.base64 }, + })), + { type: "text" as const, text: req.prompt }, + ], + }, + ], + }, + { signal: AbortSignal.timeout(timeoutFor(req.routingKey)) }, + ) + const message = await stream.finalMessage() + const [inPrice, outPrice] = PRICES[model] ?? [0, 0] + const costUsd = (message.usage.input_tokens * inPrice + message.usage.output_tokens * outPrice) / 1e6 + if (message.stop_reason === "refusal") return { text: "", refused: true, costUsd } + const text = message.content.flatMap((b) => (b.type === "text" ? [b.text] : [])).join("") + return { text, costUsd } + }, + } +} + +export const mediaLlm: Llm = env.MEDIA_AI_PROVIDER_MODE === "live" ? createClaudeLlm() : new FakeLlm() diff --git a/apps/orchestrator-service/src/prompts/copy.prompt.ts b/apps/orchestrator-service/src/prompts/copy.prompt.ts index d970191..16284af 100644 --- a/apps/orchestrator-service/src/prompts/copy.prompt.ts +++ b/apps/orchestrator-service/src/prompts/copy.prompt.ts @@ -1,3 +1,5 @@ +import { DATA_PARAGRAPH, dataBlock } from "@repo/media-ai" + export type CopyPromptVars = { copyFramework: string startupIdea: string @@ -10,6 +12,7 @@ export type CopyPromptVars = { brandTone?: string frameworkRationale?: string briefFacts?: string + mediaContext?: string } export const DEFAULT_COPY_PROMPT = (v: CopyPromptVars): string => ` @@ -20,7 +23,7 @@ Niche: ${v.niche} Target Audience: ${v.targetAudience} ${v.brandTone ? `Brand tone to match: ${v.brandTone}\n` : ""}Page: ${v.pageTitle} (${v.pageType}) Section: ${v.sectionType} (position ${v.sectionIndex}) -${v.briefFacts ? `\n${v.briefFacts}\n` : ""} +${v.briefFacts ? `\n${v.briefFacts}\n` : ""}${v.mediaContext ? `\nVISUALS: read-only context so the copy matches the images; the ${v.copyFramework} framework still decides structure and tone.\n${DATA_PARAGRAPH}\n${dataBlock("VISUALS", v.mediaContext)}\n` : ""} Write compelling copy for this section. Respond ONLY with valid JSON: { "headline": "string", diff --git a/apps/orchestrator-service/src/routes/mediaAi.route.ts b/apps/orchestrator-service/src/routes/mediaAi.route.ts new file mode 100644 index 0000000..0ac04c2 --- /dev/null +++ b/apps/orchestrator-service/src/routes/mediaAi.route.ts @@ -0,0 +1,41 @@ +import express, { Router, type Request, type Response } from "express" +import { z } from "zod" +import { ROUTING_KEYS } from "@repo/media-ai" +import { isValidationWorker as isInternalService } from "@/lib/internalValidationAuth.js" +import { mediaLlm } from "@/llm/claude.js" + +const router: Router = Router() + +const LlmRequestSchema = z.object({ + routingKey: z.enum(ROUTING_KEYS), + system: z.string().min(1).max(60_000), + prompt: z.string().min(1).max(200_000), + images: z + .array(z.object({ mime: z.enum(["image/png", "image/webp", "image/jpeg"]), base64: z.string().min(1).max(12_000_000) })) + .max(5) + .optional(), + maxTokens: z.number().int().min(256).max(64_000).optional(), + purpose: z.string().regex(/^[a-z0-9._-]{1,40}$/), +}) + +// apps/media-service only: model routing and provider keys stay in this service. +router.post("/internal/media-ai/llm", express.json({ limit: "40mb" }), (req: Request, res: Response): void => { + if (!isInternalService(req)) { + res.status(401).json({ success: false, message: "Unauthorized" }) + return + } + const parsed = LlmRequestSchema.safeParse(req.body) + if (!parsed.success) { + res.status(400).json({ success: false, message: "Validation failed" }) + return + } + void mediaLlm + .complete(parsed.data) + .then((data) => res.json({ success: true, data })) + .catch((err: unknown) => { + console.error("[media-ai] llm call failed:", parsed.data.purpose, err instanceof Error ? err.name : "unknown") + if (!res.headersSent) res.status(502).json({ success: false, message: "Model call failed" }) + }) +}) + +export default router diff --git a/apps/orchestrator-service/vitest.config.ts b/apps/orchestrator-service/vitest.config.ts index df2cbb3..1a354f2 100644 --- a/apps/orchestrator-service/vitest.config.ts +++ b/apps/orchestrator-service/vitest.config.ts @@ -13,6 +13,7 @@ export default defineConfig({ "src/tools/safeFetch.test.ts", "src/lib/docExtract.test.ts", "src/agents/mediaPlacement.test.ts", + "src/lib/siteMedia.test.ts", ], }, }); diff --git a/apps/server/.env.example b/apps/server/.env.example index 0f364c4..51cee76 100644 --- a/apps/server/.env.example +++ b/apps/server/.env.example @@ -41,3 +41,29 @@ MEDIA_PAID_MAX_VIDEOS=5 MEDIA_PAID_MAX_VIDEO_BYTES=157286400 MEDIA_PAID_TOTAL_BYTES=5368709120 MEDIA_MAX_AUTOPLAY_VIDEOS_PER_PAGE=3 + +# AI media: Site Blueprint + creative pipeline (docs/media-ai.md). Off by default. +# server, orchestrator-service and media-service all read this block; keep them identical. +MEDIA_ENABLED=false +# Credits per generated image, per video second and per Site Blueprint call. +# Required in production once MEDIA_ENABLED=true; 1 elsewhere when unset. +MEDIA_CREDIT_IMAGE= +MEDIA_CREDIT_VIDEO_PER_SECOND= +MEDIA_CREDIT_BLUEPRINT= +# images:videos per budget tier, plus hard caps. +MEDIA_BUDGET_IMAGES_ONLY=8:0 +MEDIA_BUDGET_LIGHT=4:1 +MEDIA_BUDGET_STANDARD=8:3 +MEDIA_BUDGET_RICH=14:6 +MEDIA_HARD_MAX_IMAGES=24 +MEDIA_HARD_MAX_VIDEOS=10 +MEDIA_HARD_MAX_GENERATED_ASSETS=60 +VIDEO_MIN_SECONDS=5 +VIDEO_MAX_SECONDS=10 +# Circuit breaker: provider spend over the last 24 h above this pauses AI media. Empty = no cap. +MEDIA_DAILY_COST_CAP_USD= +# Provider cost estimates in USD, used when a provider does not report cost. +MEDIA_COST_IMAGE_USD=0.25 +MEDIA_COST_VIDEO_USD_PER_SECOND=0.08 +MEDIA_COST_LLM_CALL_USD=0.05 +MEDIA_COST_BLUEPRINT_USD=0.4 diff --git a/apps/server/package.json b/apps/server/package.json index c72c023..7690fb4 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -18,6 +18,7 @@ "@aws-sdk/client-s3": "3.1144.0", "@aws-sdk/s3-request-presigner": "3.1144.0", "@repo/events": "workspace:*", + "@repo/media-ai": "workspace:*", "@repo/schemas": "workspace:*", "@repo/site-builder": "workspace:*", "@useframe/db": "workspace:*", diff --git a/apps/server/src/config/env.ts b/apps/server/src/config/env.ts index d1392a1..8ef06dc 100644 --- a/apps/server/src/config/env.ts +++ b/apps/server/src/config/env.ts @@ -1,4 +1,5 @@ import { z } from "zod"; +import { loadMediaAiConfig, type MediaAiConfig } from "@repo/media-ai"; import dotenv from "dotenv"; import type { SignOptions } from "jsonwebtoken"; @@ -118,3 +119,14 @@ if (!parsed.success) { } export const env = parsed.data; + +// AI media budgets, caps and credit prices (MEDIA_ENABLED, MEDIA_BUDGET_*, MEDIA_CREDIT_*, ...). +let media: MediaAiConfig; +try { + media = loadMediaAiConfig(process.env); +} catch (err) { + console.error("Invalid environment variables:"); + console.error(err instanceof Error ? err.message : err); + process.exit(1); +} +export const mediaAiConfig = media; diff --git a/apps/server/src/modules/briefs/briefs.service.ts b/apps/server/src/modules/briefs/briefs.service.ts index 7864856..1af62e3 100644 --- a/apps/server/src/modules/briefs/briefs.service.ts +++ b/apps/server/src/modules/briefs/briefs.service.ts @@ -44,6 +44,13 @@ import { callBriefPrefill, callBriefResolve } from "@/lib/orchestrator.js" import { AppError } from "@/middleware/errorHandler.js" import { ProjectsService } from "@/modules/projects/projects.service.js" import { BriefUploadsService } from "./briefUploads.service.js" +import { CreditsService } from "@/modules/credits/credits.service.js" + +// AI media is paid-only: a free account's brief always stores OFF. +async function mediaFieldsForTier | undefined>(userId: string, set: T): Promise { + if (!set || set.mediaMode === undefined || set.mediaMode === "OFF") return set + return (await CreditsService.getBalance(userId)) > 0 ? set : ({ ...set, mediaMode: "OFF" } as T) +} type AuthUser = { id: string; email: string; plan: string } @@ -258,7 +265,7 @@ export const BriefsService = { briefId, userId, expectedVersion: input.expectedVersion, - changes: { set: input.set, unset: input.unset, aiDecide: input.aiDecide }, + changes: { set: await mediaFieldsForTier(userId, input.set), unset: input.unset, aiDecide: input.aiDecide }, source: "user", columns: { ...(input.mode ? { mode: input.mode } : {}), @@ -469,7 +476,7 @@ export const BriefsService = { briefId: brief.id, userId: user.id, expectedVersion: input.expectedVersion, - changes: { set: input.set, unset: input.unset, aiDecide: input.aiDecide }, + changes: { set: await mediaFieldsForTier(user.id, input.set), unset: input.unset, aiDecide: input.aiDecide }, source: "user", mode: "submit", strictProof: true, diff --git a/apps/server/src/modules/media/media.routes.ts b/apps/server/src/modules/media/media.routes.ts index 0da85d0..f348a08 100644 --- a/apps/server/src/modules/media/media.routes.ts +++ b/apps/server/src/modules/media/media.routes.ts @@ -3,6 +3,7 @@ import { authenticate } from "@/middleware/authenticate.js" import { hourlyRateLimit } from "@/middleware/rateLimit.js" import { validate } from "@/middleware/validator.js" import { MediaController } from "./media.controller.js" +import { MediaJobsController } from "@/modules/mediaJobs/mediaJobs.controller.js" import { PatchMediaSchema, ResolveMediaSchema, StartUploadSchema } from "./media.schema.js" // Mounted under /api/projects/:slug/media. @@ -25,3 +26,4 @@ export default router export const mediaAccountRouter: Router = Router() mediaAccountRouter.use(authenticate) mediaAccountRouter.get("/quota", MediaController.quota) +mediaAccountRouter.get("/estimate", MediaJobsController.accountEstimate) diff --git a/apps/server/src/modules/media/media.service.test.ts b/apps/server/src/modules/media/media.service.test.ts index dfaa9c6..f992ba3 100644 --- a/apps/server/src/modules/media/media.service.test.ts +++ b/apps/server/src/modules/media/media.service.test.ts @@ -215,6 +215,46 @@ describe("list, get and patch", () => { expect(detail.warnings[0]).toMatch(/over 8 MB/) }) + it("explains how an AI asset was made without naming providers or models", async () => { + m.fake!.jobs.push({ + id: "job1", + projectId: "projA", + userId: A, + state: { items: { "vid-1": { item: { itemId: "vid-1" }, status: "DONE", done: [], concept: { chosenId: "A", concepts: [{ id: "A", title: "Paper becomes a plane" }] } } } }, + }) + m.fake!.findings.push({ id: "f1", title: "Motion that shows a process builds trust" }) + const asset = await readyAsset({ + aiGenerated: true, + generation: { + jobId: "job1", + itemId: "vid-1", + conceptId: "A", + mechanism: "MATERIAL_TRANSFORM", + provider: "runware", + model: "minimax-h3-max", + size: "1344x768", + summary: "Loose papers fold into one paper plane", + researchFindingIds: ["f1"], + analysis: { paletteDeltaE: 3 }, + }, + }) + const detail = await MediaService.get(A, "proj-a", asset.id as string) + expect(detail.generation).toEqual({ + concept: "Paper becomes a plane", + approach: "Material transformation", + summary: "Loose papers fold into one paper plane", + research: [{ id: "f1", title: "Motion that shows a process builds trust" }], + }) + expect(JSON.stringify(detail)).not.toMatch(/runware|minimax|1344x768|paletteDeltaE/) + }) + + it("does not read another user's job for provenance", async () => { + m.fake!.jobs.push({ id: "jobB", projectId: "projA", userId: B, state: { items: { x: { item: { itemId: "x" }, status: "DONE", done: [], concept: { chosenId: "A", concepts: [{ id: "A", title: "Secret" }] } } } } }) + const asset = await readyAsset({ aiGenerated: true, generation: { jobId: "jobB", itemId: "x", conceptId: "A" } }) + const detail = await MediaService.get(A, "proj-a", asset.id as string) + expect(detail.generation?.concept).toBeNull() + }) + it("updates alt text, caption and the decorative flag", async () => { const asset = await readyAsset() const res = await MediaService.patch(A, "proj-a", asset.id as string, { altText: "A team", decorative: true }) diff --git a/apps/server/src/modules/media/media.service.ts b/apps/server/src/modules/media/media.service.ts index 47ece1a..a3ced1c 100644 --- a/apps/server/src/modules/media/media.service.ts +++ b/apps/server/src/modules/media/media.service.ts @@ -11,6 +11,7 @@ import { type ResolvedMediaAsset, } from "@repo/schemas" import { listMediaSlots } from "@repo/site-builder" +import { MECHANISM_LABELS, readJobState, type Mechanism } from "@repo/media-ai" import { env } from "@/config/env.js" import { redis } from "@/lib/redis.js" import { getMediaStorage, type MediaStorage } from "@/lib/mediaStorage.js" @@ -183,6 +184,40 @@ function serialize(asset: AssetRow) { } } +export type MediaProvenance = { + concept: string | null + approach: string | null + summary: string | null + research: { id: string; title: string }[] +} + +// Only user-facing provenance leaves the API: no provider, model, size or QA internals. +async function provenanceOf(projectId: string, userId: string, generation: unknown): Promise { + if (!generation || typeof generation !== "object") return null + const g = generation as Record + const str = (v: unknown) => (typeof v === "string" && v ? v : null) + const jobId = str(g.jobId) + const itemId = str(g.itemId) + const conceptId = str(g.conceptId) + const mechanism = str(g.mechanism) + const findingIds = Array.isArray(g.researchFindingIds) ? g.researchFindingIds.filter((x): x is string => typeof x === "string").slice(0, 20) : [] + const [job, findings] = await Promise.all([ + jobId && itemId && conceptId + ? prisma.mediaJob.findFirst({ where: { id: jobId, projectId, userId }, select: { state: true } }) + : null, + findingIds.length + ? prisma.researchFinding.findMany({ where: { id: { in: findingIds } }, select: { id: true, title: true }, take: 20 }) + : [], + ]) + const concept = job ? readJobState(job.state).items[itemId!]?.concept?.concepts.find((c) => c.id === conceptId) : undefined + return { + concept: concept?.title ?? null, + approach: mechanism && mechanism in MECHANISM_LABELS ? MECHANISM_LABELS[mechanism as Mechanism] : null, + summary: str(g.summary)?.slice(0, 300) ?? null, + research: findings, + } +} + export type MediaUsage = { slotId: string; pageSlug: string; pageTitle: string; sectionType: string; slotKey: string } async function usageOf(project: { currentVersionId: string | null }, assetIds: string[]): Promise> { @@ -389,7 +424,7 @@ export const MediaService = { ...serialize(asset), usage, warnings, - generation: asset.aiGenerated ? asset.generation : null, + generation: asset.aiGenerated ? await provenanceOf(project.id, userId, asset.generation) : null, preview: { image: sign(asset, variants, "w1600") ?? sign(asset, variants, "fallback"), video: sign(asset, variants, "mp4_720"), diff --git a/apps/server/src/modules/media/media.testkit.ts b/apps/server/src/modules/media/media.testkit.ts index f3b538d..f712e8a 100644 --- a/apps/server/src/modules/media/media.testkit.ts +++ b/apps/server/src/modules/media/media.testkit.ts @@ -44,6 +44,8 @@ export function createFakeDb() { const projects: Row[] = [] const versions: Row[] = [] const assets: Row[] = [] + const jobs: Row[] = [] + const findings: Row[] = [] let seq = 0 const apply = (row: Row, data: Row) => { @@ -103,6 +105,10 @@ export function createFakeDb() { project: { findFirst: async ({ where }: { where?: Row }) => projects.find((p) => matches(p, where)) ?? null }, projectVersion: { findUnique: async ({ where }: { where: Row }) => versions.find((v) => v.id === where.id) ?? null }, mediaAsset, + mediaJob: { findFirst: async ({ where }: { where?: Row }) => jobs.find((j) => matches(j, where)) ?? null }, + researchFinding: { + findMany: async ({ where }: { where: { id: { in: string[] } } }) => findings.filter((f) => where.id.in.includes(f.id as string)), + }, $transaction: async (fn: (tx: unknown) => Promise) => fn(db), $queryRaw: async (_strings: TemplateStringsArray, userId: string) => { const used = assets @@ -111,7 +117,7 @@ export function createFakeDb() { return [{ used }] }, } - return { db, projects, versions, assets } + return { db, projects, versions, assets, jobs, findings } } export function createFakeStorage(): MediaStorage & { objects: Map } { diff --git a/apps/server/src/modules/mediaJobs/mediaJobs.controller.ts b/apps/server/src/modules/mediaJobs/mediaJobs.controller.ts new file mode 100644 index 0000000..6b702d5 --- /dev/null +++ b/apps/server/src/modules/mediaJobs/mediaJobs.controller.ts @@ -0,0 +1,39 @@ +import type { Response, NextFunction } from "express" +import type { AuthenticatedRequest } from "@/types/index.js" +import { MediaJobsService } from "./mediaJobs.service.js" +import { AccountEstimateQuerySchema, type CheckpointAction, type StartJobInput } from "./mediaJobs.schema.js" + +type Handler = (req: AuthenticatedRequest) => Promise + +const handle = + (fn: Handler, status = 200) => + async (req: AuthenticatedRequest, res: Response, next: NextFunction): Promise => { + try { + res.status(status).json({ success: true, data: await fn(req) }) + } catch (err) { + next(err) + } + } + +export const MediaJobsController = { + estimate: handle((req) => MediaJobsService.estimate(req.user!.id, req.params.slug!)), + start: handle((req) => MediaJobsService.start(req.user!.id, req.params.slug!, req.body as StartJobInput), 201), + latest: handle((req) => MediaJobsService.latest(req.user!.id, req.params.slug!)), + get: handle((req) => MediaJobsService.get(req.user!.id, req.params.slug!, req.params.jobId!)), + checkpoint: handle((req) => + MediaJobsService.checkpoint(req.user!.id, req.params.slug!, req.params.jobId!, req.body as CheckpointAction), + ), + cancel: handle((req) => MediaJobsService.cancel(req.user!.id, req.params.slug!, req.params.jobId!)), + accountEstimate: async (req: AuthenticatedRequest, res: Response, next: NextFunction): Promise => { + const query = AccountEstimateQuerySchema.safeParse(req.query) + if (!query.success) { + res.status(422).json({ success: false, message: "Validation failed", errors: query.error.flatten().fieldErrors }) + return + } + try { + res.json({ success: true, data: await MediaJobsService.accountEstimate(req.user!.id, query.data.budget) }) + } catch (err) { + next(err) + } + }, +} diff --git a/apps/server/src/modules/mediaJobs/mediaJobs.routes.ts b/apps/server/src/modules/mediaJobs/mediaJobs.routes.ts new file mode 100644 index 0000000..e421380 --- /dev/null +++ b/apps/server/src/modules/mediaJobs/mediaJobs.routes.ts @@ -0,0 +1,20 @@ +import { Router } from "express" +import { authenticate } from "@/middleware/authenticate.js" +import { hourlyRateLimit } from "@/middleware/rateLimit.js" +import { validate } from "@/middleware/validator.js" +import { MediaJobsController } from "./mediaJobs.controller.js" +import { CheckpointActionSchema, StartJobSchema } from "./mediaJobs.schema.js" + +// Mounted under /api/projects/:slug/media, ahead of the media library routes. +const router: Router = Router({ mergeParams: true }) + +router.use(authenticate) + +router.get("/estimate", MediaJobsController.estimate) +router.post("/jobs", hourlyRateLimit("media-job-start", 10), validate(StartJobSchema), MediaJobsController.start) +router.get("/jobs/latest", MediaJobsController.latest) +router.get("/jobs/:jobId", MediaJobsController.get) +router.post("/jobs/:jobId/checkpoint", hourlyRateLimit("media-checkpoint", 60), validate(CheckpointActionSchema), MediaJobsController.checkpoint) +router.post("/jobs/:jobId/cancel", hourlyRateLimit("media-job-cancel", 30), MediaJobsController.cancel) + +export default router diff --git a/apps/server/src/modules/mediaJobs/mediaJobs.schema.ts b/apps/server/src/modules/mediaJobs/mediaJobs.schema.ts new file mode 100644 index 0000000..d25bfff --- /dev/null +++ b/apps/server/src/modules/mediaJobs/mediaJobs.schema.ts @@ -0,0 +1,7 @@ +import { z } from "zod" +import { CheckpointActionSchema, MEDIA_BUDGET_TIERS, StartJobSchema } from "@repo/media-ai" + +export { CheckpointActionSchema, StartJobSchema } +export type { CheckpointAction, StartJobInput } from "@repo/media-ai" + +export const AccountEstimateQuerySchema = z.object({ budget: z.enum(MEDIA_BUDGET_TIERS).default("STANDARD") }) diff --git a/apps/server/src/modules/mediaJobs/mediaJobs.service.test.ts b/apps/server/src/modules/mediaJobs/mediaJobs.service.test.ts new file mode 100644 index 0000000..1b06ce6 --- /dev/null +++ b/apps/server/src/modules/mediaJobs/mediaJobs.service.test.ts @@ -0,0 +1,97 @@ +import { beforeEach, describe, expect, it, vi } from "vitest" +import { loadMediaAiConfig } from "@repo/media-ai" +import { TEST_ENV } from "../media/media.testkit.js" +import { createMemoryDb } from "@repo/media-ai/testing" + +const m = vi.hoisted(() => ({ db: null as ReturnType | null, add: vi.fn() })) + +vi.mock("@/config/env.js", () => ({ env: TEST_ENV, mediaAiConfig: loadMediaAiConfig({ MEDIA_ENABLED: "true" }) })) +vi.mock("@useframe/db", async (importOriginal) => ({ + ...(await importOriginal()), + get prisma() { + return m.db! + }, + briefStateOf: () => ({ data: {} }), +})) +vi.mock("@/lib/redis.js", () => ({ redis: {} })) +vi.mock("bullmq", () => ({ Queue: class { add = m.add } })) + +const { MediaJobsService } = await import("./mediaJobs.service.js") + +const A = "userA" +const B = "userB" +const PAGES = [{ type: "HOME", slug: "home", title: "Home", sections: [{ type: "HERO", index: 0, content: { headline: "Hi" } }] }] + +async function seed(balanceA: number) { + const db = m.db! + await db.project.create({ data: { id: "projA", slug: "proj-a", userId: A, currentVersionId: "verA" } }) + await db.project.create({ data: { id: "projB", slug: "proj-b", userId: B, currentVersionId: null } }) + await db.projectVersion.create({ data: { id: "verA", projectId: "projA", versionNumber: 1, snapshot: { pages: PAGES, copyFramework: "AIDA", media: {} } } }) + await db.creditBalance.create({ data: { userId: A, balance: balanceA } }) + await db.creditBalance.create({ data: { userId: B, balance: 500 } }) +} + +beforeEach(() => { + m.db = createMemoryDb() + m.add.mockReset() +}) + +describe("MediaJobsService", () => { + it("is paid-only for estimates and starts", async () => { + await seed(0) + await expect(MediaJobsService.estimate(A, "proj-a")).rejects.toMatchObject({ statusCode: 403, code: "PAID_FEATURE" }) + await expect(MediaJobsService.start(A, "proj-a", { mode: "AUTO", scope: { kind: "site" } })).rejects.toMatchObject({ statusCode: 403 }) + await expect(MediaJobsService.accountEstimate(A, "STANDARD")).rejects.toMatchObject({ statusCode: 403 }) + }) + + it("starts a job, reserves credits once and enqueues it", async () => { + await seed(1000) + const job = await MediaJobsService.start(A, "proj-a", { mode: "AUTO", scope: { kind: "site" } }) + expect(job.status).toBe("QUEUED") + expect(job.creditsReserved).toBeGreaterThan(0) + expect(m.add).toHaveBeenCalledTimes(1) + expect((await m.db!.creditBalance.findUnique({ where: { userId: A } }))!.balance).toBe(1000 - job.creditsReserved) + expect(JSON.stringify(job)).not.toMatch(/openai|runware|minimax|claude|gpt/i) + await expect(MediaJobsService.start(A, "proj-a", { mode: "AUTO", scope: { kind: "site" } })).rejects.toMatchObject({ statusCode: 409 }) + }) + + it("answers 402 with the shortfall when credits do not cover the run", async () => { + await seed(3) + await expect(MediaJobsService.start(A, "proj-a", { mode: "AUTO", scope: { kind: "site" } })).rejects.toMatchObject({ + statusCode: 402, + code: "INSUFFICIENT_CREDITS", + }) + expect(m.db!.mediaJob.rows).toHaveLength(0) + expect((await m.db!.creditBalance.findUnique({ where: { userId: A } }))!.balance).toBe(3) + }) + + it("hides other users' projects and jobs behind 404", async () => { + await seed(1000) + const job = await MediaJobsService.start(A, "proj-a", { mode: "AUTO", scope: { kind: "site" } }) + await expect(MediaJobsService.get(B, "proj-a", job.id)).rejects.toMatchObject({ statusCode: 404 }) + await expect(MediaJobsService.get(B, "proj-b", job.id)).rejects.toMatchObject({ statusCode: 404 }) + await expect(MediaJobsService.checkpoint(B, "proj-b", job.id, { checkpoint: "PLAN_APPROVE", action: "approve" })).rejects.toMatchObject({ statusCode: 404 }) + await expect(MediaJobsService.cancel(B, "proj-b", job.id)).rejects.toMatchObject({ statusCode: 404 }) + }) + + it("rejects an answer to a checkpoint that is not open", async () => { + await seed(1000) + const job = await MediaJobsService.start(A, "proj-a", { mode: "MANUAL", scope: { kind: "site" } }) + await expect(MediaJobsService.checkpoint(A, "proj-a", job.id, { checkpoint: "PLAN_APPROVE", action: "approve" })).rejects.toMatchObject({ + statusCode: 409, + code: "MEDIA_CHECKPOINT_STALE", + }) + }) + + it("lets the owner see and cancel a run that used their last credits, refunding it", async () => { + await seed(1000) + const started = await MediaJobsService.start(A, "proj-a", { mode: "AUTO", scope: { kind: "site" } }) + await m.db!.creditBalance.update({ where: { userId: A }, data: { balance: 0 } }) + const latest = await MediaJobsService.latest(A, "proj-a") + expect(latest.job?.id).toBe(started.id) + const cancelled = await MediaJobsService.cancel(A, "proj-a", started.id) + expect(cancelled.status).toBe("CANCELLED") + expect(cancelled.creditsRefunded).toBe(started.creditsReserved) + await expect(MediaJobsService.cancel(A, "proj-a", started.id)).rejects.toMatchObject({ statusCode: 409 }) + }) +}) diff --git a/apps/server/src/modules/mediaJobs/mediaJobs.service.ts b/apps/server/src/modules/mediaJobs/mediaJobs.service.ts new file mode 100644 index 0000000..a19914c --- /dev/null +++ b/apps/server/src/modules/mediaJobs/mediaJobs.service.ts @@ -0,0 +1,317 @@ +import { Queue } from "bullmq" +import { briefStateOf, prisma } from "@useframe/db" +import { QUEUES, type MediaGenerateJobPayload } from "@repo/events" +import type { SiteSpec } from "@repo/schemas" +import { + ACTIVE_JOB_STATUSES, + applyCheckpointAction, + budgetFor, + checkEligibility, + CheckpointError, + costItems, + createMediaJob, + estimateJobCost, + InsufficientCreditsError, + isBlueprintFresh, + isPaidUser, + maxCreditsForBudget, + MECHANISM_LABELS, + mediaSettingsOf, + planningSlots, + readBlueprint, + readBlueprintNotice, + readJobState, + refundUnspent, + SUBSTEP_LABELS, + type CheckpointAction, + type IneligibleReason, + type ItemState, + type MediaBudgetTier, + type StartJobInput, + type Substep, +} from "@repo/media-ai" +import { env, mediaAiConfig } from "@/config/env.js" +import { redis } from "@/lib/redis.js" +import { AppError } from "@/middleware/errorHandler.js" +import { signMediaUrl } from "@/modules/media/media.signing.js" + +let queue: Queue | null = null + +async function enqueue(jobId: string): Promise { + queue ??= new Queue(QUEUES.MEDIA_GENERATE, { connection: redis }) + await queue.add("generate", { jobId }, { jobId: `${jobId}-${Date.now()}`, attempts: 1, removeOnComplete: 500, removeOnFail: 500 }) +} + +async function requirePaid(userId: string): Promise { + if (!(await isPaidUser(prisma, userId))) { + throw new AppError("AI images and videos are available on paid plans", 403, "PAID_FEATURE") + } +} + +async function loadProject(userId: string, slug: string) { + const project = await prisma.project.findFirst({ + where: { slug, userId, deletedAt: null }, + select: { id: true, currentVersionId: true }, + }) + if (!project) throw new AppError("Project not found", 404) + return project +} + +async function briefSettings(projectId: string, userId: string) { + const brief = await prisma.projectBrief.findFirst({ where: { projectId, userId } }) + return mediaSettingsOf(brief ? (briefStateOf(brief).data as Record) : null, true) +} + +async function currentVersion(projectId: string, versionId: string | null) { + if (!versionId) return null + const version = await prisma.projectVersion.findFirst({ + where: { id: versionId, projectId }, + select: { id: true, snapshot: true, blueprint: true }, + }) + const spec = version?.snapshot as Partial | null + return version && spec?.pages?.length ? { ...version, spec: spec as SiteSpec } : null +} + +const REASONS: Record = { + disabled: ["AI media isn't available right now", 503], + free: ["AI images and videos are available on paid plans", 403], + mode_off: ["AI media is turned off for this project", 409], + no_budget: ["Choose how much AI media you want first", 409], + circuit_open: ["AI media is temporarily unavailable. Please try again later.", 503], + project_busy: ["AI media is already being made for this project", 409], + project_daily_limit: ["You've reached today's limit of AI media runs for this project", 429], + user_busy: ["You already have two AI media runs in progress", 409], +} + +const sign = (assetId: string | undefined, role: "w960" | "mp4_720" | "poster") => { + if (!assetId || !env.MEDIA_SIGNING_SECRET) return null + try { + return signMediaUrl(assetId, role) + } catch { + return null + } +} + +function itemView(s: ItemState, charges: Record) { + const scores = new Map(s.concept?.ranked.map((r) => [r.id, r.score]) ?? []) + return { + itemId: s.item.itemId, + slotId: s.item.slotId, + kind: s.item.kind, + decision: s.item.decision, + role: s.item.role, + intent: s.item.intent, + reason: s.item.reason, + aspect: s.item.aspect, + durationSec: s.video?.durationSec ?? s.item.video?.durationSec ?? null, + status: s.status, + failureReason: s.failureReason ?? null, + credits: charges[s.item.itemId] ?? 0, + concepts: + s.concept?.concepts.map((c) => ({ + id: c.id, + title: c.title, + logline: c.logline, + whyItFits: c.whyItFits, + mechanism: MECHANISM_LABELS[c.mechanism], + beats: c.beats, + score: scores.get(c.id) ?? null, + })) ?? [], + recommendedId: s.concept?.recommendedId ?? null, + chosenId: s.concept?.chosenId ?? null, + conceptRegenerationsLeft: Math.max(0, 2 - (s.concept?.regenerations ?? 0)), + frames: s.frames ? { opening: sign(s.frames.openingAssetId, "w960"), ending: sign(s.frames.endingAssetId, "w960") } : null, + image: s.image ? sign(s.image.assetId, "w960") : null, + candidates: + s.video?.candidateAssetIds.map((id) => ({ + id, + video: sign(id, "mp4_720"), + poster: sign(id, "poster"), + notes: s.video?.qa?.[id]?.notes ?? null, + })) ?? [], + chosenCandidateId: s.video?.chosenAssetId ?? null, + usesStill: !!s.fallbackAssetId, + } +} + +type JobRow = NonNullable>> + +function jobView(job: JobRow) { + const state = readJobState(job.state) + const plan = (job.plan ?? {}) as { dropped?: { itemId: string; reason: string }[] } + return { + id: job.id, + mode: job.mode, + status: job.status, + stage: job.stage, + stageLabel: job.stage && job.stage in SUBSTEP_LABELS ? SUBSTEP_LABELS[job.stage as Substep] : null, + checkpoint: job.checkpoint, + checkpointExpiresAt: job.checkpointExpiresAt, + failureReason: job.failureReason, + creditsReserved: job.creditsReserved, + creditsRefunded: job.creditsRefunded, + resultVersionId: job.resultVersionId, + createdAt: job.createdAt, + finishedAt: job.finishedAt, + dropped: plan.dropped ?? [], + items: Object.values(state.items).map((s) => itemView(s, state.charges)), + } +} + +async function loadJob(userId: string, slug: string, jobId: string) { + const project = await loadProject(userId, slug) + const job = await prisma.mediaJob.findFirst({ where: { id: jobId, projectId: project.id, userId } }) + if (!job) throw new AppError("Media job not found", 404) + return { project, job } +} + +export const MediaJobsService = { + // An upper bound for a budget tier, for the brief before any plan exists. + async accountEstimate(userId: string, tier: MediaBudgetTier) { + await requirePaid(userId) + const budget = budgetFor(mediaAiConfig, tier) + return { credits: maxCreditsForBudget(budget, mediaAiConfig, { blueprint: true }), budget, maximum: true } + }, + + async estimate(userId: string, slug: string) { + await requirePaid(userId) + const project = await loadProject(userId, slug) + const settings = await briefSettings(project.id, userId) + const budget = budgetFor(mediaAiConfig, settings.budgetTier) + const version = await currentVersion(project.id, project.currentVersionId) + const blueprint = version ? readBlueprint(version.blueprint) : null + const balance = (await prisma.creditBalance.findUnique({ where: { userId }, select: { balance: true } }))?.balance ?? 0 + if (version && blueprint && isBlueprintFresh(blueprint, version.spec.pages, version.spec.copyFramework)) { + const estimate = estimateJobCost(costItems(blueprint.mediaScript), mediaAiConfig) + return { credits: estimate.credits, items: estimate.items, balance, budgetTier: settings.budgetTier, maximum: false } + } + const credits = maxCreditsForBudget(budget, mediaAiConfig, { blueprint: true }) + return { + credits, + items: [ + { kind: "image" as const, count: budget.maxImages, credits: budget.maxImages * mediaAiConfig.credits.image }, + { kind: "video" as const, count: budget.maxVideos, credits: credits - budget.maxImages * mediaAiConfig.credits.image - mediaAiConfig.credits.blueprint }, + { kind: "blueprint" as const, count: 1, credits: mediaAiConfig.credits.blueprint }, + ], + balance, + budgetTier: settings.budgetTier, + maximum: true, + } + }, + + async start(userId: string, slug: string, input: StartJobInput) { + await requirePaid(userId) + const project = await loadProject(userId, slug) + const version = await currentVersion(project.id, project.currentVersionId) + if (!version) throw new AppError("Generate your site first, then add AI media", 409, "MEDIA_NO_SITE") + const settings = await briefSettings(project.id, userId) + const eligibility = await checkEligibility(prisma, { + userId, + projectId: project.id, + mediaMode: input.mode, + budgetTier: settings.budgetTier, + config: mediaAiConfig, + }) + if (!eligibility.ok) { + const [message, status] = REASONS[eligibility.reason] + throw new AppError(message, status, eligibility.reason === "free" ? "PAID_FEATURE" : `MEDIA_${eligibility.reason.toUpperCase()}`) + } + if (input.scope.kind === "slots") { + const known = new Set(planningSlots(version.spec.pages).map((r) => r.slotId)) + const unknown = input.scope.slotIds.filter((id) => !known.has(id)) + if (unknown.length) throw new AppError("Some of those spots don't exist on this site", 422, "MEDIA_UNKNOWN_SLOT") + } + const blueprint = readBlueprint(version.blueprint) + const inScope = (slotId: string) => input.scope.kind === "site" || input.scope.slotIds.includes(slotId) + const fresh = blueprint && isBlueprintFresh(blueprint, version.spec.pages, version.spec.copyFramework) + const planned = fresh ? blueprint.mediaScript.filter((i) => inScope(i.slotId)) : [] + const mediaBrief = planned.length && blueprint ? await mediaBriefFor(project.id, userId, blueprint) : undefined + // Without an earlier media brief the job plans for itself, which may reuse the fresh blueprint. + const items = mediaBrief ? planned : [] + try { + const { jobId } = await createMediaJob(prisma, { + userId, + projectId: project.id, + mode: input.mode, + scope: input.scope, + versionId: version.id, + plan: items.length ? { items, dropped: [], budget: eligibility.budget, budgetTier: settings.budgetTier } : null, + budget: eligibility.budget, + budgetTier: settings.budgetTier, + ...(mediaBrief ? { mediaBrief } : {}), + chargeBlueprint: !items.length, + config: mediaAiConfig, + enqueue, + baseMedia: (version.spec.media ?? {}) as Record, + }) + const job = await prisma.mediaJob.findUniqueOrThrow({ where: { id: jobId } }) + return jobView(job) + } catch (err) { + if (err instanceof InsufficientCreditsError) { + throw new AppError(`You need ${err.needed} credits for this; you have ${err.balance}`, 402, "INSUFFICIENT_CREDITS", { + data: { needed: err.needed, balance: err.balance }, + }) + } + throw err + } + }, + + // Viewing, answering or cancelling an existing run needs ownership, not a balance: + // a run may have reserved the user's last credits. + async latest(userId: string, slug: string) { + const project = await loadProject(userId, slug) + const [job, version] = await Promise.all([ + prisma.mediaJob.findFirst({ where: { projectId: project.id, userId }, orderBy: { createdAt: "desc" } }), + project.currentVersionId + ? prisma.projectVersion.findFirst({ where: { id: project.currentVersionId, projectId: project.id }, select: { blueprint: true } }) + : null, + ]) + const blueprint = readBlueprint(version?.blueprint) + return { + job: job ? jobView(job) : null, + notice: readBlueprintNotice(version?.blueprint)?.notice ?? null, + narrative: blueprint?.narrative ?? null, + script: blueprint?.mediaScript.map((i) => ({ itemId: i.itemId, slotId: i.slotId, kind: i.kind, role: i.role, reason: i.reason, intent: i.intent })) ?? [], + } + }, + + async get(userId: string, slug: string, jobId: string) { + const { job } = await loadJob(userId, slug, jobId) + return jobView(job) + }, + + async checkpoint(userId: string, slug: string, jobId: string, action: CheckpointAction) { + const { job } = await loadJob(userId, slug, jobId) + try { + const { resume } = await applyCheckpointAction(prisma, job, action, mediaAiConfig) + if (resume) await enqueue(job.id) + } catch (err) { + if (err instanceof CheckpointError) throw new AppError(err.message, err.status, err.code) + throw err + } + return jobView(await prisma.mediaJob.findUniqueOrThrow({ where: { id: job.id } })) + }, + + async cancel(userId: string, slug: string, jobId: string) { + const { job } = await loadJob(userId, slug, jobId) + const claimed = await prisma.mediaJob.updateMany({ + where: { id: job.id, status: { in: [...ACTIVE_JOB_STATUSES] } }, + data: { status: "CANCELLED", checkpoint: null, failureReason: "Cancelled", finishedAt: new Date() }, + }) + if (claimed.count === 0) throw new AppError("This media run has already finished", 409, "MEDIA_JOB_FINISHED") + const state = await refundUnspent(prisma, job, "AI media cancelled") + await prisma.mediaJob.update({ where: { id: job.id }, data: { state: state as object } }) + return jobView(await prisma.mediaJob.findUniqueOrThrow({ where: { id: job.id } })) + }, +} + +async function mediaBriefFor(projectId: string, userId: string, blueprint: NonNullable>) { + const recent = await prisma.mediaJob.findMany({ + where: { projectId, userId }, + orderBy: { createdAt: "desc" }, + select: { mediaBrief: true }, + take: 10, + }) + const brief = (recent.find((j) => j.mediaBrief && typeof j.mediaBrief === "object")?.mediaBrief ?? null) as Record | null + return brief ? { ...brief, blueprint: { narrative: blueprint.narrative.storyArc, visualSystem: blueprint.visualSystem } } : undefined +} diff --git a/apps/server/src/modules/projects/projects.routes.ts b/apps/server/src/modules/projects/projects.routes.ts index 45938d4..5afd1a6 100644 --- a/apps/server/src/modules/projects/projects.routes.ts +++ b/apps/server/src/modules/projects/projects.routes.ts @@ -13,6 +13,7 @@ import pipelineRoutes from "@/modules/pipeline/pipeline.routes.js" import domainsRoutes from "@/modules/domains/domains.routes.js" import { projectBriefRouter } from "@/modules/briefs/briefs.routes.js" import mediaRoutes from "@/modules/media/media.routes.js" +import mediaJobsRoutes from "@/modules/mediaJobs/mediaJobs.routes.js" const router: Router = Router() @@ -36,6 +37,7 @@ router.use("/:slug/seo-step", seoStepRoutes) router.use("/:slug", deployRoutes) router.use("/:slug/pipeline", pipelineRoutes) router.use("/:slug/domains", domainsRoutes) +router.use("/:slug/media", mediaJobsRoutes) router.use("/:slug/media", mediaRoutes) export default router diff --git a/apps/web/src/components/brief/BriefFieldInput.tsx b/apps/web/src/components/brief/BriefFieldInput.tsx index af814eb..445c5f9 100644 --- a/apps/web/src/components/brief/BriefFieldInput.tsx +++ b/apps/web/src/components/brief/BriefFieldInput.tsx @@ -4,6 +4,8 @@ import { useId, useState, type ReactNode } from "react" import { AUDIENCE_SEGMENTS, AVAILABILITY, + BRIEF_MEDIA_BUDGETS, + BRIEF_MEDIA_MODES, BRIEF_OPTION_LABELS, BRIEF_SITE_TYPES, BRIEF_UI_COPY, @@ -257,6 +259,7 @@ export function BriefFieldInput(props: FieldRenderProps) { case "problem": case "visualNotes": case "objections": + case "mediaNotes": return ( onChange(v)} onBlur={onBlur} disabled={disabled} describedBy={describedBy} invalid={invalid} /> ) @@ -283,6 +286,10 @@ export function BriefFieldInput(props: FieldRenderProps) { return case "trafficSources": return + case "mediaMode": + return + case "mediaBudget": + return case "language": return case "niche": diff --git a/apps/web/src/components/brief/BriefModal.tsx b/apps/web/src/components/brief/BriefModal.tsx index 0326151..ead68c0 100644 --- a/apps/web/src/components/brief/BriefModal.tsx +++ b/apps/web/src/components/brief/BriefModal.tsx @@ -1,6 +1,7 @@ "use client" import { useCallback, useEffect, useMemo, useRef, useState } from "react" +import Link from "next/link" import { useRouter } from "next/navigation" import { useQuery, useQueryClient } from "@tanstack/react-query" import { toast } from "sonner" @@ -26,6 +27,8 @@ import { useAuth } from "@/lib/authContext" import { isFreeTierExhausted } from "@/lib/freeTierError" import { useProjectModalStore, type BriefModalRequest } from "@/store/projectModalStore" import { briefAssetUrl, briefsApi, type BriefApiError } from "@/lib/api/services/briefs.service" +import { mediaApi } from "@/lib/api/services/media.service" +import { mediaJobsApi, type MediaBudgetTier } from "@/lib/api/services/mediaJobs.service" import { BriefFieldInput } from "./BriefFieldInput" import { PrefillPanel } from "./PrefillPanel" import { useBriefDraft } from "./useBriefDraft" @@ -176,7 +179,9 @@ function BriefModalSession({ request, onClose }: { request: BriefModalRequest; o const steps = stepsFor(view) const stepIndex = Math.max(0, steps.indexOf(step)) const isLastStep = stepIndex === steps.length - 1 - const fields = useMemo(() => fieldsFor(step, view, draft.data), [step, view, draft.data]) + const quota = useQuery({ queryKey: ["media-quota"], queryFn: () => mediaApi.quota(), enabled: !!user, staleTime: 60_000 }) + const paid = quota.data?.tier === "paid" + const fields = useMemo(() => fieldsFor(step, view, draft.data, paid), [step, view, draft.data, paid]) const errors: BriefFieldErrors = { ...draft.errors, ...clientErrors } const goTo = useCallback( @@ -551,6 +556,9 @@ function BriefModalSession({ request, onClose }: { request: BriefModalRequest; o accountEmail={user?.email} /> ))} + {step === "brand" && view === "FULL" && quota.data && ( + + )} )} @@ -687,3 +695,29 @@ function FieldBlock({ ) } + +function MediaBriefNote({ paid, mode, budget }: { paid: boolean; mode: string | undefined; budget: MediaBudgetTier }) { + const on = paid && mode !== "OFF" + const estimate = useQuery({ + queryKey: ["media-estimate", budget], + queryFn: () => mediaJobsApi.accountEstimate(budget), + enabled: on, + staleTime: 5 * 60_000, + }) + if (!paid) { + return ( +

+ Paid plans can add AI-made images and short videos that tell your site's story.{" "} + + See plans + +

+ ) + } + if (!on || !estimate.data) return null + return ( +

+ AI media costs up to {estimate.data.credits} credits. Anything we don't make is refunded. +

+ ) +} diff --git a/apps/web/src/components/brief/briefFields.ts b/apps/web/src/components/brief/briefFields.ts index d091ad6..a61d860 100644 --- a/apps/web/src/components/brief/briefFields.ts +++ b/apps/web/src/components/brief/briefFields.ts @@ -19,8 +19,11 @@ export type BriefViewMode = "QUICK" | "FULL" // "Start faster" panel rather than the brand step. const PANEL_ONLY: FieldId[] = ["sourceDocument"] -export function isFieldVisible(id: FieldId, data: BriefData): boolean { +export const MEDIA_FIELDS: FieldId[] = ["mediaMode", "mediaBudget", "mediaNotes"] + +export function isFieldVisible(id: FieldId, data: BriefData, paid = false): boolean { if (PANEL_ONLY.includes(id)) return false + if (MEDIA_FIELDS.includes(id)) return paid && (id === "mediaMode" || data.mediaMode !== "OFF") if (id === "language") return BUILDER_CAPABILITIES.languages.length > 1 if (id === "launchDate") return data.availability === "LAUNCHING_SOON" return true @@ -30,9 +33,9 @@ export function stepsFor(view: BriefViewMode): StepId[] { return view === "QUICK" ? [...QUICK_STEP_IDS] : [...STEP_IDS] } -export function fieldsFor(step: StepId, view: BriefViewMode, data: BriefData): FieldId[] { +export function fieldsFor(step: StepId, view: BriefViewMode, data: BriefData, paid = false): FieldId[] { return fieldsForStep(step).filter( - (id) => isFieldVisible(id, data) && (view === "FULL" || REQUIRED_FIELD_IDS.includes(id)), + (id) => isFieldVisible(id, data, paid) && (view === "FULL" || REQUIRED_FIELD_IDS.includes(id)), ) } @@ -83,6 +86,10 @@ export function summarize(id: FieldId, data: BriefData): string { return label("audienceSegment", v as string) case "siteType": return label("siteType", v as string) + case "mediaMode": + return label("mediaMode", v as string) + case "mediaBudget": + return label("mediaBudget", v as string) case "tone": return (v as string[]).map((t) => label("tone", t)).join(", ") case "trafficSources": diff --git a/apps/web/src/components/media/MediaAiPanel.tsx b/apps/web/src/components/media/MediaAiPanel.tsx new file mode 100644 index 0000000..3de3a5c --- /dev/null +++ b/apps/web/src/components/media/MediaAiPanel.tsx @@ -0,0 +1,203 @@ +"use client" + +import Link from "next/link" +import { useEffect, useRef, useState } from "react" +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query" +import { toast } from "sonner" +import { + ACTIVE_MEDIA_JOB, + mediaJobsApi, + type CheckpointAction, + type MediaJob, + type MediaJobOverview, +} from "@/lib/api/services/mediaJobs.service" +import { MediaCheckpointCard } from "./MediaCheckpointCard" + +type Props = { projectSlug: string; paid: boolean; onMediaChanged: () => void } + +const ITEM_STATUS: Record = { + PENDING: "Waiting", + RUNNING: "In progress", + DONE: "Done", + FAILED: "Kept your current media", + DROPPED: "Removed", + SKIPPED: "Using the still", +} + +const button = + "rounded-xl px-4 py-2 text-[12.5px] font-semibold transition-opacity focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-indigo-500 disabled:opacity-50 cursor-pointer" + +function finishedLine(job: MediaJob): string { + const refunded = job.creditsRefunded > 0 ? ` ${job.creditsRefunded} unused credits were refunded.` : "" + if (job.status === "DONE") return `AI media is ready and has been added to your site as a new version.${refunded}` + if (job.status === "CANCELLED") return `${job.failureReason ?? "Cancelled"}.${refunded}` + return `${job.failureReason ?? "AI media couldn't be finished"}. Your site is unchanged.${refunded}` +} + +function WhyHere({ overview }: { overview: MediaJobOverview }) { + if (!overview.narrative && overview.script.length === 0) return null + return ( +
+ Why these images and videos are here + {overview.narrative &&

{overview.narrative.storyArc}

} + {overview.script.length > 0 && ( +
    + {overview.script.map((i) => ( +
  • + + {i.kind === "video" ? "Video" : "Image"} · {i.role}: + {" "} + {i.reason} +
    • + ))} +
+ )} +
+ ) +} + +export function MediaAiPanel({ projectSlug, paid, onMediaChanged }: Props) { + const queryClient = useQueryClient() + const [mode, setMode] = useState<"AUTO" | "MANUAL">("AUTO") + const key = ["media-job", projectSlug] + + const overview = useQuery({ + queryKey: key, + queryFn: () => mediaJobsApi.latest(projectSlug), + refetchInterval: (q) => { + const status = q.state.data?.job?.status + if (status === "AWAITING_USER") return 15_000 + return status && ACTIVE_MEDIA_JOB.includes(status) ? 3000 : false + }, + }) + const job = overview.data?.job ?? null + const running = !!job && ACTIVE_MEDIA_JOB.includes(job.status) + + const estimate = useQuery({ + queryKey: ["media-job-estimate", projectSlug], + queryFn: () => mediaJobsApi.estimate(projectSlug), + enabled: paid && !running, + staleTime: 60_000, + }) + + // Refresh the library when generated files arrive or the job settles. + const lastSeen = useRef(null) + useEffect(() => { + if (!job) return + const signature = `${job.id}:${job.status}:${job.items.filter((i) => i.status === "DONE").length}` + if (lastSeen.current && lastSeen.current !== signature) onMediaChanged() + lastSeen.current = signature + }, [job, onMediaChanged]) + + const setJob = (next: MediaJob) => queryClient.setQueryData(key, (old) => (old ? { ...old, job: next } : old)) + const onError = (err: Error) => toast.error(err.message) + + const start = useMutation({ + mutationFn: () => mediaJobsApi.start(projectSlug, { mode, scope: { kind: "site" } }), + onSuccess: (next) => { + setJob(next) + void queryClient.invalidateQueries({ queryKey: key }) + }, + onError, + }) + const act = useMutation({ + mutationFn: (action: CheckpointAction) => mediaJobsApi.checkpoint(projectSlug, job!.id, action), + onSuccess: setJob, + onError: (err: Error & { code?: string }) => { + onError(err) + if (err.code === "MEDIA_CHECKPOINT_STALE") void queryClient.invalidateQueries({ queryKey: key }) + }, + }) + const cancel = useMutation({ + mutationFn: () => mediaJobsApi.cancel(projectSlug, job!.id), + onSuccess: setJob, + onError, + }) + + if (!paid && !job) { + return ( +
+

+ Generate images and short videos made for your site's story. Available on paid plans. +

+ + See plans + +
+ ) + } + + const data = overview.data + return ( +
+
+
+

AI images and videos

+

+ {running + ? job!.status === "AWAITING_USER" + ? "Waiting for your answer below" + : job!.stageLabel ?? "Getting started…" + : estimate.data + ? `${estimate.data.maximum ? "Up to " : ""}${estimate.data.credits} credits · you have ${estimate.data.balance}` + : "Made for this site, from your brief and research"} +

+
+ {running ? ( + + ) : !paid ? ( + + Add credits + + ) : ( +
+
+ {(["AUTO", "MANUAL"] as const).map((m) => ( + + ))} +
+ +
+ )} +
+ + {data?.notice &&

{data.notice}

} + + {job && running && job.status !== "AWAITING_USER" && job.items.length > 0 && ( +
    + {job.items.map((i) => ( +
  • + + {i.kind === "video" ? "Video" : "Image"} · {i.role} + + {ITEM_STATUS[i.status] ?? i.status} +
    • + ))} +
+ )} + + {job?.status === "AWAITING_USER" && ( + act.mutate({ ...a, checkpoint: job.checkpoint! })} /> + )} + + {job && !running &&

{finishedLine(job)}

} + + {data && } +
+ ) +} diff --git a/apps/web/src/components/media/MediaCheckpointCard.tsx b/apps/web/src/components/media/MediaCheckpointCard.tsx new file mode 100644 index 0000000..d39ed27 --- /dev/null +++ b/apps/web/src/components/media/MediaCheckpointCard.tsx @@ -0,0 +1,256 @@ +"use client" + +import { useId, useState } from "react" +import type { CheckpointAction, MediaJob, MediaJobItem } from "@/lib/api/services/mediaJobs.service" + +type Props = { + job: MediaJob + pending: boolean + onAction: (action: Omit) => void +} + +const TITLES: Record, { title: string; help: string }> = { + PLAN_APPROVE: { title: "Approve the media plan", help: "Here's what we plan to make and why. Remove anything you don't want." }, + CONCEPT_PICK: { title: "Pick an idea for each video", help: "We wrote three ideas per video. Our pick is marked; choose another if you prefer." }, + IMAGES_APPROVE: { title: "Check the images", help: "These are the stills, and the first and last scenes of each video." }, + VIDEO_APPROVE: { title: "Choose your videos", help: "Pick the take you like best for each video." }, +} + +const button = + "rounded-xl px-3 py-1.5 text-[12px] font-semibold transition-colors focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-indigo-500 disabled:opacity-50 cursor-pointer" +const primary = `${button} bg-inv text-inv hover:opacity-90` +const secondary = `${button} border border-base text-sec hover:bg-tertiary` + +function expiresIn(iso: string | null): string { + if (!iso) return "" + const hours = Math.max(0, Math.round((new Date(iso).getTime() - Date.now()) / 3_600_000)) + return hours >= 1 ? `${hours} hour${hours === 1 ? "" : "s"}` : "less than an hour" +} + +const active = (i: MediaJobItem) => i.decision === "GENERATE" && !["DROPPED", "SKIPPED", "FAILED"].includes(i.status) + +function Thumb({ src, label }: { src: string | null; label: string }) { + if (!src) return null + return ( +
+ {/* eslint-disable-next-line @next/next/no-img-element -- short-lived signed URL */} + {label} +
{label}
+
+ ) +} + +function ConceptPicker({ item, pending, onAction }: { item: MediaJobItem; pending: boolean; onAction: Props["onAction"] }) { + const name = useId() + const noteId = useId() + const [note, setNote] = useState("") + const chosen = item.chosenId ?? item.recommendedId + return ( +
+ {item.role} + {item.concepts.map((c) => ( + + ))} + +