Feat/media library

.github/workflows/helm-ci.yml

@@ -66,7 +66,7 @@ jobs:
         run: |
           set -euo pipefail
           crds="https://raw.githubusercontent.com/datreeio/CRDs-catalog/${CRDS_CATALOG_REF}/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json"
-          all_on="--set services.deploy-service-worker.enabled=true --set services.billing-service.enabled=true --set services.scoring-service.enabled=true --set networkPolicy.enabled=true --set migration.enabled=true"
+          all_on="--set services.deploy-service-worker.enabled=true --set services.media-service.enabled=true --set services.billing-service.enabled=true --set services.scoring-service.enabled=true --set networkPolicy.enabled=true --set migration.enabled=true"
           render() {
             echo "::group::$1"
             shift

AGENTS.md

@@ -42,6 +42,7 @@ Read the implementation and package manifests as the source of truth. See `READM
 | `apps/worker`                                          | BullMQ processors for scans, SEO, scoring, domains, PDFs, billing jobs, and cache expiry                      |
 | `apps/deploy-service`                                  | Site builds, R2 uploads, KV publishing (`syncSiteToKvs`), rollback, GC, reaper, reconcile; see `RUNBOOK.md`   |
 | `apps/site-edge`                                       | Cloudflare Worker serving generated sites from KV + R2; deployed with Wrangler, never from CI                 |
+| `apps/media-service`                                   | BullMQ worker for the media library: upload processing (sharp/ffmpeg), vision descriptions, media GC          |
 | `infra/terraform`, `infra/k8s`                         | Cloudflare sites-hosting Terraform module and GKE manifests (applied by a human, never by agents)             |
 | `infra/helm/useframe`                                  | Helm chart for the backend workloads on GKE; agents run only offline checks (lint/template/unittest)          |
 | `packages/db`                                          | Shared Prisma client, schema, migrations, seed data, and scan cache helpers                                   |
@@ -155,7 +156,7 @@ Run these from the repository root unless otherwise noted:
 - There is no root `test` script. Run relevant existing tests/evaluations where available and report coverage gaps instead of inventing a passing test command.
 - Shared packages such as db, schemas, events, and site-builder export `dist`; rebuild them after changes before restarting consumers (`@useframe/db` runs `prisma generate` then `tsc`). A running watch process does not prove shared output is current.
 - Backend services build with `tsc && tsc-alias` (the `@/` alias becomes relative paths) and run in production with plain `node dist/...`; never ship `tsx` in an image. Runtime workspace packages declare `files` (normally `dist`) because `pnpm deploy` only copies published files.
-- Service default ports: API 4000, orchestrator 4001, billing 4002, scoring 4003, research 4004, deploy 4005. Confirm actual configured ports and probe their `/health` routes. Check the frontend HTTP response and worker startup/Redis connectivity separately; the worker has no HTTP health endpoint.
+- Service default ports: API 4000, orchestrator 4001, billing 4002, scoring 4003, research 4004, deploy 4005. Confirm actual configured ports and probe their `/health` routes. Check the frontend HTTP response and worker startup/Redis connectivity separately; the worker and media-service have no HTTP endpoint beyond an optional `HEALTH_PORT` listener (media-service).
 - Identify project-owned processes before restarting them. Do not kill every Node process. Leave the updated development services running after verification; on Windows, launch background helpers without visible windows.
 - Check required PostgreSQL, Redis, and external-service configuration before starting dependent flows. If configuration or infrastructure prevents startup, report the exact blocker and what could be verified.
 

CLAUDE.md

@@ -42,6 +42,7 @@ Read the implementation and package manifests as the source of truth. See `READM
 | `apps/worker`                                          | BullMQ processors for scans, SEO, scoring, domains, PDFs, billing jobs, and cache expiry                      |
 | `apps/deploy-service`                                  | Site builds, R2 uploads, KV publishing (`syncSiteToKvs`), rollback, GC, reaper, reconcile; see `RUNBOOK.md`   |
 | `apps/site-edge`                                       | Cloudflare Worker serving generated sites from KV + R2; deployed with Wrangler, never from CI                 |
+| `apps/media-service`                                   | BullMQ worker for the media library: upload processing (sharp/ffmpeg), vision descriptions, media GC          |
 | `infra/terraform`, `infra/k8s`                         | Cloudflare sites-hosting Terraform module and GKE manifests (applied by a human, never by agents)             |
 | `infra/helm/useframe`                                  | Helm chart for the backend workloads on GKE; agents run only offline checks (lint/template/unittest)          |
 | `packages/db`                                          | Shared Prisma client, schema, migrations, seed data, and scan cache helpers                                   |
@@ -155,7 +156,7 @@ Run these from the repository root unless otherwise noted:
 - There is no root `test` script. Run relevant existing tests/evaluations where available and report coverage gaps instead of inventing a passing test command.
 - Shared packages such as db, schemas, events, and site-builder export `dist`; rebuild them after changes before restarting consumers (`@useframe/db` runs `prisma generate` then `tsc`). A running watch process does not prove shared output is current.
 - Backend services build with `tsc && tsc-alias` (the `@/` alias becomes relative paths) and run in production with plain `node dist/...`; never ship `tsx` in an image. Runtime workspace packages declare `files` (normally `dist`) because `pnpm deploy` only copies published files.
-- Service default ports: API 4000, orchestrator 4001, billing 4002, scoring 4003, research 4004, deploy 4005. Confirm actual configured ports and probe their `/health` routes. Check the frontend HTTP response and worker startup/Redis connectivity separately; the worker has no HTTP health endpoint.
+- Service default ports: API 4000, orchestrator 4001, billing 4002, scoring 4003, research 4004, deploy 4005. Confirm actual configured ports and probe their `/health` routes. Check the frontend HTTP response and worker startup/Redis connectivity separately; the worker and media-service have no HTTP endpoint beyond an optional `HEALTH_PORT` listener (media-service).
 - Identify project-owned processes before restarting them. Do not kill every Node process. Leave the updated development services running after verification; on Windows, launch background helpers without visible windows.
 - Check required PostgreSQL, Redis, and external-service configuration before starting dependent flows. If configuration or infrastructure prevents startup, report the exact blocker and what could be verified.
 

GEMINI.md

@@ -42,6 +42,7 @@ Read the implementation and package manifests as the source of truth. See `READM
 | `apps/worker`                                          | BullMQ processors for scans, SEO, scoring, domains, PDFs, billing jobs, and cache expiry                      |
 | `apps/deploy-service`                                  | Site builds, R2 uploads, KV publishing (`syncSiteToKvs`), rollback, GC, reaper, reconcile; see `RUNBOOK.md`   |
 | `apps/site-edge`                                       | Cloudflare Worker serving generated sites from KV + R2; deployed with Wrangler, never from CI                 |
+| `apps/media-service`                                   | BullMQ worker for the media library: upload processing (sharp/ffmpeg), vision descriptions, media GC          |
 | `infra/terraform`, `infra/k8s`                         | Cloudflare sites-hosting Terraform module and GKE manifests (applied by a human, never by agents)             |
 | `infra/helm/useframe`                                  | Helm chart for the backend workloads on GKE; agents run only offline checks (lint/template/unittest)          |
 | `packages/db`                                          | Shared Prisma client, schema, migrations, seed data, and scan cache helpers                                   |
@@ -155,7 +156,7 @@ Run these from the repository root unless otherwise noted:
 - There is no root `test` script. Run relevant existing tests/evaluations where available and report coverage gaps instead of inventing a passing test command.
 - Shared packages such as db, schemas, events, and site-builder export `dist`; rebuild them after changes before restarting consumers (`@useframe/db` runs `prisma generate` then `tsc`). A running watch process does not prove shared output is current.
 - Backend services build with `tsc && tsc-alias` (the `@/` alias becomes relative paths) and run in production with plain `node dist/...`; never ship `tsx` in an image. Runtime workspace packages declare `files` (normally `dist`) because `pnpm deploy` only copies published files.
-- Service default ports: API 4000, orchestrator 4001, billing 4002, scoring 4003, research 4004, deploy 4005. Confirm actual configured ports and probe their `/health` routes. Check the frontend HTTP response and worker startup/Redis connectivity separately; the worker has no HTTP health endpoint.
+- Service default ports: API 4000, orchestrator 4001, billing 4002, scoring 4003, research 4004, deploy 4005. Confirm actual configured ports and probe their `/health` routes. Check the frontend HTTP response and worker startup/Redis connectivity separately; the worker and media-service have no HTTP endpoint beyond an optional `HEALTH_PORT` listener (media-service).
 - Identify project-owned processes before restarting them. Do not kill every Node process. Leave the updated development services running after verification; on Windows, launch background helpers without visible windows.
 - Check required PostgreSQL, Redis, and external-service configuration before starting dependent flows. If configuration or infrastructure prevents startup, report the exact blocker and what could be verified.
 

apps/deploy-service/.env.example

@@ -26,6 +26,14 @@ SITES_R2_SECRET_ACCESS_KEY=
 # Defaults to https://<CLOUDFLARE_ACCOUNT_ID>.r2.cloudflarestorage.com
 SITES_R2_ENDPOINT=
 
+# Media library (read-only token on useframe-media). Needed only to deploy sites that use media;
+# such deploys fail with a clear message while these are unset.
+MEDIA_R2_BUCKET=useframe-media
+MEDIA_R2_READ_ACCESS_KEY_ID=
+MEDIA_R2_READ_SECRET_ACCESS_KEY=
+# Defaults to https://<CLOUDFLARE_ACCOUNT_ID>.r2.cloudflarestorage.com
+MEDIA_R2_ENDPOINT=
+
 DEPLOY_RETAIN_COUNT=10
 DEPLOY_MAX_FILES=5000
 DEPLOY_MAX_TOTAL_MB=100

apps/deploy-service/src/config/schema.ts

@@ -43,6 +43,13 @@ export const envSchema = z
     SITES_R2_SECRET_ACCESS_KEY: z.string().min(1, "SITES_R2_SECRET_ACCESS_KEY is required"),
     SITES_R2_ENDPOINT: optional(z.string().url()),
 
+    // Read-only access to the private media library, used to copy referenced
+    // media into each deployment. Deploys of sites without media work without it.
+    MEDIA_R2_BUCKET: z.string().min(1).default("useframe-media"),
+    MEDIA_R2_READ_ACCESS_KEY_ID: optional(z.string().min(1)),
+    MEDIA_R2_READ_SECRET_ACCESS_KEY: optional(z.string().min(1)),
+    MEDIA_R2_ENDPOINT: optional(z.string().url()),
+
     DEPLOY_RETAIN_COUNT: int(10),
     DEPLOY_MAX_FILES: int(5000),
     DEPLOY_MAX_TOTAL_MB: int(100),
@@ -88,6 +95,8 @@ export const envSchema = z
     ...value,
     SITES_R2_ENDPOINT:
       value.SITES_R2_ENDPOINT ?? `https://${value.CLOUDFLARE_ACCOUNT_ID}.r2.cloudflarestorage.com`,
+    MEDIA_R2_ENDPOINT:
+      value.MEDIA_R2_ENDPOINT ?? `https://${value.CLOUDFLARE_ACCOUNT_ID}.r2.cloudflarestorage.com`,
     SITES_EDGE_CNAME_TARGET: value.SITES_EDGE_CNAME_TARGET ?? `cname.${value.SITES_BASE_DOMAIN}`,
     DEPLOY_WORK_DIR:
       value.DEPLOY_WORK_DIR ??

apps/deploy-service/src/context.ts

@@ -31,6 +31,15 @@ export const deps: Deps = {
     secretAccessKey: env.SITES_R2_SECRET_ACCESS_KEY,
     bucket: env.SITES_R2_BUCKET,
   }),
+  mediaSource:
+    env.MEDIA_R2_READ_ACCESS_KEY_ID && env.MEDIA_R2_READ_SECRET_ACCESS_KEY
+      ? createR2({
+          endpoint: env.MEDIA_R2_ENDPOINT,
+          accessKeyId: env.MEDIA_R2_READ_ACCESS_KEY_ID,
+          secretAccessKey: env.MEDIA_R2_READ_SECRET_ACCESS_KEY,
+          bucket: env.MEDIA_R2_BUCKET,
+        })
+      : null,
   redis,
   runQueue,
   validationQueue: env.DEPLOY_ENQUEUE_VALIDATION

apps/deploy-service/src/deps.ts

@@ -37,6 +37,8 @@ export type Deps = {
   db: Db;
   kv: KvClient;
   r2: R2;
+  // Read-only client for the private media library; absent until configured.
+  mediaSource?: R2 | null;
   redis: LockClient;
   runQueue: Pick<Queue<DeployRunJobPayload>, "add" | "getJob">;
   validationQueue?: Pick<Queue, "add">;

apps/deploy-service/src/pipeline/files.ts

@@ -1,26 +1,42 @@
 import { z } from "zod";
-import { SiteSpecSchema } from "@repo/schemas";
+import { parseMediaVariants, SiteSpecSchema, type ResolvedMediaAsset } from "@repo/schemas";
 import {
   buildRobotsTxt,
   buildSitemapXml,
-  buildSiteFiles,
+  buildSite,
   withScaffold,
   type GeneratedFile,
 } from "@repo/site-builder";
 import type { Db } from "@/site/sync.js";
 import { DeployError } from "./errors.js";
+import type { MediaCopyItem } from "./media.js";
 
 const NextFilesSchema = z.array(z.object({ path: z.string().min(1), content: z.string() })).min(1);
 
 export type VersionSource = { snapshot: unknown; nextFiles: unknown };
 
+// Media as loaded from the library: resolved assets for the builder plus the
+// source key of every variant, so the copy step can find what the markup uses.
+export type VersionMedia = {
+  assets: ResolvedMediaAsset[];
+  sources: Map<string, { key: string; title: string }>;
+};
+
+export type BuildableVersion = { files: GeneratedFile[]; media: MediaCopyItem[]; mediaWarnings: string[] };
+
+const NO_MEDIA: VersionMedia = { assets: [], sources: new Map() };
+
 // Replicate versions carry a raw Next.js tree; generated versions carry a SiteSpec.
-export function filesFromVersion(version: VersionSource, siteUrl: string): GeneratedFile[] {
+export function filesFromVersion(
+  version: VersionSource,
+  siteUrl: string,
+  media: VersionMedia = NO_MEDIA,
+): BuildableVersion {
   if (version.nextFiles !== null && version.nextFiles !== undefined) {
     const parsed = NextFilesSchema.safeParse(version.nextFiles);
     if (!parsed.success) throw new DeployError("This version's files are not in a deployable format.");
     try {
-      return withScaffold(parsed.data);
+      return { files: withScaffold(parsed.data), media: [], mediaWarnings: [] };
     } catch (err) {
       throw new DeployError(
         "This version's files are not in a deployable format.",
@@ -31,20 +47,92 @@ export function filesFromVersion(version: VersionSource, siteUrl: string): Gener
 
   const spec = SiteSpecSchema.safeParse(version.snapshot);
   if (!spec.success) throw new DeployError("This version's site data is not in a deployable format.");
-  return buildSiteFiles(spec.data, {
-    robotsTxt: buildRobotsTxt(siteUrl),
-    sitemapXml: buildSitemapXml(spec.data.pages, siteUrl),
+  const built = buildSite(
+    spec.data,
+    { robotsTxt: buildRobotsTxt(siteUrl), sitemapXml: buildSitemapXml(spec.data.pages, siteUrl) },
+    { media: { target: "export", assets: media.assets } },
+  );
+  const items = built.mediaFiles.map((file) => {
+    const source = media.sources.get(`${file.assetId}/${file.role}`);
+    if (!source) throw new DeployError("A media file on this site could not be found.", `${file.assetId}/${file.role}`);
+    const variant = media.assets.find((a) => a.id === file.assetId)!.variants.find((v) => v.role === file.role)!;
+    return {
+      assetId: file.assetId,
+      title: source.title,
+      sourceKey: source.key,
+      destRelPath: file.path,
+      bytes: file.bytes,
+      mime: variant.mime,
+    };
+  });
+  return { files: built.files, media: items, mediaWarnings: built.mediaWarnings };
+}
+
+export function referencedAssetIds(snapshot: unknown): string[] {
+  const media = (snapshot as { media?: unknown } | null)?.media;
+  if (!media || typeof media !== "object") return [];
+  const ids = Object.values(media as Record<string, { assetId?: unknown }>).flatMap((b) =>
+    typeof b?.assetId === "string" ? [b.assetId] : [],
+  );
+  return [...new Set(ids)];
+}
+
+export async function loadVersionMedia(db: Db, projectId: string, assetIds: string[]): Promise<VersionMedia> {
+  if (assetIds.length === 0) return NO_MEDIA;
+  // Scoped to the project, so a binding can never pull another project's media.
+  const rows = await db.mediaAsset.findMany({
+    where: { id: { in: assetIds }, projectId },
+    select: {
+      id: true,
+      kind: true,
+      origin: true,
+      status: true,
+      deletedAt: true,
+      title: true,
+      width: true,
+      height: true,
+      durationMs: true,
+      dominantColor: true,
+      lqip: true,
+      altText: true,
+      decorative: true,
+      variants: true,
+    },
+  });
+  const sources = new Map<string, { key: string; title: string }>();
+  const assets = rows.map((row): ResolvedMediaAsset => {
+    const variants = parseMediaVariants(row.variants);
+    for (const v of variants) sources.set(`${row.id}/${v.role}`, { key: v.key, title: row.title ?? "Untitled" });
+    return {
+      id: row.id,
+      kind: row.kind,
+      origin: row.origin,
+      status: row.status,
+      deleted: row.deletedAt !== null,
+      title: row.title,
+      width: row.width,
+      height: row.height,
+      durationMs: row.durationMs,
+      dominantColor: row.dominantColor,
+      lqip: row.lqip,
+      altText: row.altText,
+      decorative: row.decorative,
+      variants: variants.map(({ role, mime, width, height, bytes, sha256 }) => ({ role, mime, width, height, bytes, sha256 })),
+    };
   });
+  return { assets, sources };
 }
 
 export async function getBuildableFiles(
   db: Db,
   versionId: string,
   siteUrl: string,
-): Promise<GeneratedFile[]> {
+  projectId: string,
+): Promise<BuildableVersion> {
   const version = await db.projectVersion.findUniqueOrThrow({
     where: { id: versionId },
     select: { snapshot: true, nextFiles: true },
   });
-  return filesFromVersion(version, siteUrl);
+  const media = await loadVersionMedia(db, projectId, referencedAssetIds(version.snapshot));
+  return filesFromVersion(version, siteUrl, media);
 }