From 68231adeec66c74c82c768192f22800d334e32bb Mon Sep 17 00:00:00 2001 From: Forrest Brown Date: Sat, 19 Sep 2026 19:27:03 +0800 Subject: [PATCH 1/7] feat(providers,worker): place video input images by their staged role The reference-image roles users pick in the console were dropped at the queue boundary, so video plugins always received images in array order and a first/last frame pair could silently land in the wrong slot. MediaInputImage now carries the stored role, the worker forwards it as extra.imageRoles only when every image has a role the plugin can place (a partial list, an untagged legacy row, or a prompt_image/source_video keeps today's positional behaviour instead of being rejected upstream), and Veo resolves the list into image/lastFrame/referenceImages. --- apps/worker/src/jobs/index.ts | 13 +++- packages/providers/src/core/types.ts | 1 + .../providers/src/plugins/veo-video/index.ts | 33 ++++++++++ .../src/plugins/veo-video/veo-video.test.ts | 61 +++++++++++++++++++ 4 files changed, 107 insertions(+), 1 deletion(-) diff --git a/apps/worker/src/jobs/index.ts b/apps/worker/src/jobs/index.ts index 1038e6b..6a9d778 100644 --- a/apps/worker/src/jobs/index.ts +++ b/apps/worker/src/jobs/index.ts @@ -232,7 +232,7 @@ async function resolveMediaInputImages(jobId: string): Promise 0 ? images : undefined } @@ -425,6 +425,9 @@ async function resolveSnapshot(job: Record): Promise, prompt: string, inputImages: MediaInputImage[] | undefined, revision: ModelConfigRevisionEntity): MediaRequest { const mediaKind = job.media_kind === 'video' ? 'video' : 'image' const normalized = (job.normalized_request as Record | null) || null @@ -442,6 +445,14 @@ function buildMediaRequest(job: Record, prompt: string, inputIm if (params[key] !== undefined) extra[key] = params[key] } if (size && mediaKind === 'video' && !extra.aspectRatio) extra.aspectRatio = size + const storedRoles = inputImages?.map(image => image.role) + // Only a complete list of placement roles is trustworthy: a partial list, an + // untagged legacy row, or a role a plugin cannot place (prompt_image, + // source_video) must keep today's positional behaviour instead of being + // rejected upstream or misplacing frames. + if (storedRoles && storedRoles.length > 0 && storedRoles.every(role => !!role && FORWARDABLE_INPUT_ROLES.includes(role))) { + extra.imageRoles = storedRoles + } return { modality: mediaKind, vendorModelId, diff --git a/packages/providers/src/core/types.ts b/packages/providers/src/core/types.ts index b0e49f7..44b1ea4 100644 --- a/packages/providers/src/core/types.ts +++ b/packages/providers/src/core/types.ts @@ -93,6 +93,7 @@ export type MediaInputImage = { width?: number height?: number sizeBytes?: number + role?: string } /** diff --git a/packages/providers/src/plugins/veo-video/index.ts b/packages/providers/src/plugins/veo-video/index.ts index 8417e71..d5abb12 100644 --- a/packages/providers/src/plugins/veo-video/index.ts +++ b/packages/providers/src/plugins/veo-video/index.ts @@ -52,6 +52,8 @@ type VeoImagePayload = { mimeType: string } +type VeoImageRole = 'first_frame' | 'last_frame' | 'reference_image' + type VeoInstance = { prompt: string image?: VeoImagePayload @@ -568,6 +570,17 @@ export class VeoVideoPlugin implements MediaProviderPlugin { buildInstance(request: MediaRequest): VeoInstance { const images = (request.inputImages ?? []).map(img => this.toImagePayload(img)) const instance: VeoInstance = { prompt: request.prompt } + const roles = this.resolveExplicitImageRoles(request, images.length) + if (roles) { + const referenceImages: Array<{ image: VeoImagePayload }> = [] + roles.forEach((role, index) => { + if (role === 'first_frame') instance.image = images[index] + else if (role === 'last_frame') instance.lastFrame = images[index] + else referenceImages.push({ image: images[index] }) + }) + if (referenceImages.length > 0) instance.referenceImages = referenceImages + return instance + } if (images.length >= 1) { instance.image = images[0] } @@ -580,6 +593,26 @@ export class VeoVideoPlugin implements MediaProviderPlugin { return instance } + /** + * Stored roles win only when the list is complete, aligned with the image + * count, and describes a request Veo accepts: an unknown or duplicated role, + * or a last frame without its first frame, is treated as invalid so the + * caller falls back to the positional contract instead of losing bytes. + */ + private resolveExplicitImageRoles(request: MediaRequest, count: number): VeoImageRole[] | null { + const explicit = request.extra?.['imageRoles'] + if (!Array.isArray(explicit) || explicit.length === 0 || explicit.length !== count) return null + const values = explicit as unknown[] + for (const value of values) { + if (value !== 'first_frame' && value !== 'last_frame' && value !== 'reference_image') return null + } + const roles = values as VeoImageRole[] + if (roles.filter(role => role === 'first_frame').length !== 1) return null + const lastFrames = roles.filter(role => role === 'last_frame').length + if (lastFrames > 1) return null + return roles + } + resolveVideoParameters(request: MediaRequest, config?: ProviderConfig): VeoParameters { const extra = { ...(request.extra ?? {}), ...(config ? readExtra(config) : {}) } // Request-level fields win over shared config extras for per-call overrides. diff --git a/packages/providers/src/plugins/veo-video/veo-video.test.ts b/packages/providers/src/plugins/veo-video/veo-video.test.ts index c815045..70beb8c 100644 --- a/packages/providers/src/plugins/veo-video/veo-video.test.ts +++ b/packages/providers/src/plugins/veo-video/veo-video.test.ts @@ -133,6 +133,67 @@ test('Veo submit maps input images to image/lastFrame/referenceImages roles', as assert.ok(instance.image.bytesBase64Encoded.length > 0) }) +const imageBytes = (byte: number) => Buffer.from([byte, 0x21, 0x42]) +const inputImage = (byte: number) => { + const data = imageBytes(byte) + return { data, mimeType: 'image/png' as const, sizeBytes: data.length } +} +const encoded = (byte: number) => imageBytes(byte).toString('base64') + +test('Veo buildInstance places input images by explicit imageRoles', () => { + const framed = veoVideoPlugin.buildInstance( + mockRequest({ inputImages: [inputImage(1), inputImage(2)], extra: { imageRoles: ['first_frame', 'last_frame'] } }), + ) + assert.equal(framed.image?.bytesBase64Encoded, encoded(1)) + assert.equal(framed.lastFrame?.bytesBase64Encoded, encoded(2)) + assert.equal(framed.referenceImages, undefined) + + const mixed = veoVideoPlugin.buildInstance( + mockRequest({ + inputImages: [inputImage(1), inputImage(2), inputImage(3)], + extra: { imageRoles: ['first_frame', 'reference_image', 'last_frame'] }, + }), + ) + assert.equal(mixed.image?.bytesBase64Encoded, encoded(1)) + assert.equal(mixed.lastFrame?.bytesBase64Encoded, encoded(3)) + assert.deepEqual(mixed.referenceImages?.map(entry => entry.image.bytesBase64Encoded), [encoded(2)]) + + // Roles, never array order, decide the slot each image lands in. + const swapped = veoVideoPlugin.buildInstance( + mockRequest({ inputImages: [inputImage(1), inputImage(2)], extra: { imageRoles: ['last_frame', 'first_frame'] } }), + ) + assert.equal(swapped.image?.bytesBase64Encoded, encoded(2)) + assert.equal(swapped.lastFrame?.bytesBase64Encoded, encoded(1)) +}) + +test('Veo buildInstance keeps positional placement for a missing or unusable imageRoles list', () => { + const images = [inputImage(1), inputImage(2), inputImage(3), inputImage(4)] + const positional = veoVideoPlugin.buildInstance(mockRequest({ inputImages: images })) + assert.equal(positional.image?.bytesBase64Encoded, encoded(1)) + assert.equal(positional.lastFrame?.bytesBase64Encoded, encoded(2)) + assert.deepEqual( + positional.referenceImages?.map(entry => entry.image.bytesBase64Encoded), + [encoded(3), encoded(4)], + ) + + const unusable: Array | undefined> = [ + undefined, + {}, + { imageRoles: 'first_frame' }, + { imageRoles: ['first_frame'] }, + { imageRoles: ['first_frame', 'last_frame', 'reference_image', 'prompt_image'] }, + { imageRoles: ['reference_image', 'reference_image', 'reference_image', 'reference_image'] }, + { imageRoles: ['last_frame', 'last_frame', 'reference_image', 'reference_image'] }, + ] + for (const extra of unusable) { + assert.deepEqual( + veoVideoPlugin.buildInstance(mockRequest({ inputImages: images, extra })), + positional, + 'position must win for ' + JSON.stringify(extra), + ) + } +}) + test('Veo validateRequest enforces Veo 3.1 bounds', async () => { const config = makeConfig() const context = makeContext(globalThis.fetch) From 4f6f06f0e417cb6d6156570201698263ff894170 Mon Sep 17 00:00:00 2001 From: Forrest Brown Date: Sat, 19 Sep 2026 19:27:32 +0800 Subject: [PATCH 2/7] feat(api): expose the sanitised provider failure detail on the job DTO A rejected generation only surfaced an error code, so users saw PROVIDER_REJECTED with no way to tell what to fix. jobDto now returns the already-trimmed provider_error.detail as errorMessage, capped at 300 chars, and reads that one field instead of spreading the object: HTTP status, provider reference ids and any other transport metadata stay server-side. --- apps/api/src/backend.test.ts | 20 ++++++++++++++++++++ apps/api/src/shared/dto.ts | 6 ++++++ 2 files changed, 26 insertions(+) diff --git a/apps/api/src/backend.test.ts b/apps/api/src/backend.test.ts index 4085bff..8ee1a53 100644 --- a/apps/api/src/backend.test.ts +++ b/apps/api/src/backend.test.ts @@ -106,6 +106,26 @@ test('user job DTO applies final prompt visibility without changing the original assert.equal(visible.finalPrompt, 'optimized'); assert.equal(visible.prompt, 'original') }) +test('user job DTO exposes only the truncated provider failure detail', async () => { + const base = { id: 'job', created_by: 'user', model_id: 'model', model_name: 'Video', prompt: 'p', size: '16:9', count: 1, status: 'failed', error_code: 'PROVIDER_REJECTED', created_at: new Date() } + const rejected = await jobDto({ ...base, provider_error: { status: 400, detail: ' Input image resolution is unsupported ', providerReferenceId: 'nested-ref' } }) + assert.equal(rejected.errorCode, 'PROVIDER_REJECTED') + assert.equal(rejected.errorMessage, 'Input image resolution is unsupported') + assert.equal(JSON.stringify(rejected).includes('nested-ref'), false) + assert.equal(JSON.stringify(rejected).includes('"status":400'), false) + const truncated = await jobDto({ ...base, provider_error: { detail: 'x'.repeat(400) } }) + assert.equal(truncated.errorMessage?.length, 300) + for (const row of [ + base, + { ...base, provider_error: null }, + { ...base, provider_error: { status: 502 } }, + { ...base, provider_error: { detail: ' ' } }, + { ...base, provider_error: 'not-an-object' }, + ]) { + assert.equal((await jobDto(row)).errorMessage, undefined) + } +}) + test('manual retry resumes from the correct generation phase', () => { assert.deepEqual(retryPreparation({ optimization_mode: 'disabled' }), { phase: 'image_generating', resetOptimization: false }) assert.deepEqual(retryPreparation({ optimization_mode: 'enabled', prompt_optimization_id: 'po', final_prompt: 'optimized prompt', template_instruction_snapshot: 'template' }), { phase: 'image_generating', resetOptimization: false }) diff --git a/apps/api/src/shared/dto.ts b/apps/api/src/shared/dto.ts index 998677e..b2a9f9d 100644 --- a/apps/api/src/shared/dto.ts +++ b/apps/api/src/shared/dto.ts @@ -209,6 +209,11 @@ export async function jobDto(row: Record, outputs: Record } + // Only the already-sanitised provider detail is user-visible; never spread the object. + const providerError = row.provider_error && typeof row.provider_error === 'object' + ? row.provider_error as { detail?: unknown } + : undefined + const providerDetail = typeof providerError?.detail === 'string' ? providerError.detail.trim() : '' return { id: row.id, createdBy: row.created_by, @@ -240,6 +245,7 @@ export async function jobDto(row: Record, outputs: Record Date: Sat, 19 Sep 2026 19:28:27 +0800 Subject: [PATCH 3/7] feat(web-next): drive the create console from model parameter descriptors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Video controls used to hardcode one dropdown set, which broke as soon as two plugins spelled the same parameter differently (Veo takes string enums, Seedance integer seconds). generation-params now resolves every field through the selected model's declared descriptors with an alias table, so wire keys and value types come from the descriptor itself and the panel renders only what a model actually supports; the store holds a videoParams map instead of four fixed fields, so switching models needs no reset and the model choice is remembered per 图像/视频 tab. Reference images are staged under the role the chosen model reserves for them (首帧 / 尾帧 / 参考图) and reconcileStagedRoles re-assigns roles on model change, blocking submit rather than silently dropping an overflow. An active-jobs board reports queue position, phase, elapsed time, cancel-in flight and the new job errorMessage; media-frame renders image and video outputs behind one surface. --- .../src/app/(workspace)/generate/page.tsx | 11 +- .../generate/components/active-jobs-board.tsx | 361 ++++++++ .../generate/components/generate-console.tsx | 865 +++++++++++------- .../components/generate-mode-tabs.tsx | 86 ++ .../components/reference-images-dialog.tsx | 92 +- .../components/reference-images-trigger.tsx | 27 +- .../components/video-parameter-controls.tsx | 210 +++++ .../shared/components/job-status-badge.tsx | 22 + .../src/shared/components/media-frame.tsx | 145 +++ apps/web-next/src/shared/hooks/useJobs.ts | 11 +- .../src/shared/lib/generation-params.ts | 403 +++++--- apps/web-next/src/shared/lib/job-status.ts | 65 ++ .../src/shared/lib/reference-upload.ts | 48 +- .../src/shared/stores/generate-ui-store.ts | 59 +- apps/web-next/src/shared/types/index.ts | 23 +- 15 files changed, 1930 insertions(+), 498 deletions(-) create mode 100644 apps/web-next/src/features/generate/components/active-jobs-board.tsx create mode 100644 apps/web-next/src/features/generate/components/generate-mode-tabs.tsx create mode 100644 apps/web-next/src/features/generate/components/video-parameter-controls.tsx create mode 100644 apps/web-next/src/shared/components/job-status-badge.tsx create mode 100644 apps/web-next/src/shared/components/media-frame.tsx create mode 100644 apps/web-next/src/shared/lib/job-status.ts diff --git a/apps/web-next/src/app/(workspace)/generate/page.tsx b/apps/web-next/src/app/(workspace)/generate/page.tsx index 0e1a514..e8dabb2 100644 --- a/apps/web-next/src/app/(workspace)/generate/page.tsx +++ b/apps/web-next/src/app/(workspace)/generate/page.tsx @@ -1,12 +1,19 @@ import { GenerateConsole } from '@/features/generate/components/generate-console' +import { Suspense } from 'react' export const dynamic = 'force-dynamic' export const metadata = { title: '创作工作台 - MuseCanvas', - description: 'AI 图像生成工作室与控制台', + description: 'AI 图像与视频生成工作室与控制台', } export default function GeneratePage() { - return + // The console mirrors its 图像/视频 tab into `?tab=`, so it reads search params + // and needs a boundary like every other `useSearchParams` consumer. + return ( + 正在加载创作台...}> + + + ) } diff --git a/apps/web-next/src/features/generate/components/active-jobs-board.tsx b/apps/web-next/src/features/generate/components/active-jobs-board.tsx new file mode 100644 index 0000000..05fa116 --- /dev/null +++ b/apps/web-next/src/features/generate/components/active-jobs-board.tsx @@ -0,0 +1,361 @@ +'use client' + +import { useEffect, useRef, useState } from 'react' +import { ChevronRight, RefreshCw, XCircle } from 'lucide-react' +import { JobStatusBadge } from '@/shared/components/job-status-badge' +import { MediaFrame } from '@/shared/components/media-frame' +import { formatElapsed, isJobActive, phaseLabel } from '@/shared/lib/job-status' +import { isVideoOutput, outputUrl } from '@/shared/types' +import type { GenerationJob } from '@/shared/types' + +const LINGER_MS = 6000 +const FADE_MS = 500 +const TICK_MS = 500 + +export interface ActiveJobsBoardProps { + jobs: GenerationJob[] + isLoading: boolean + isError: boolean + onReload: () => void + selectedJobId: string | null + onSelectJob: (jobId: string) => void + onCancel: (jobId: string) => void + onRetry: (jobId: string) => void + pendingCancelId?: string | null + pendingRetryId?: string | null + /** `rail` collapses; `inline` is the <768px copy and stays open. */ + variant?: 'rail' | 'inline' + /** The mobile copy would otherwise park an empty card under the prompt box. */ + hideWhenEmpty?: boolean + open?: boolean + onToggle?: (open: boolean) => void +} + +export function ActiveJobsBoard({ + jobs, + isLoading, + isError, + onReload, + selectedJobId, + onSelectJob, + onCancel, + onRetry, + pendingCancelId, + pendingRetryId, + variant = 'rail', + hideWhenEmpty = false, + open = true, + onToggle, +}: ActiveJobsBoardProps) { + const [now, setNow] = useState(() => Date.now()) + const [lingering, setLingering] = useState>({}) + const [announcement, setAnnouncement] = useState('') + const previousActiveIds = useRef | null>(null) + const headingRef = useRef(null) + const latestJobs = useRef(jobs) + latestJobs.current = jobs + + const activeJobs = jobs.filter(isJobActive) + const activeSignature = activeJobs.map((job) => job.id).join(',') + const lingeringIds = Object.keys(lingering) + const isRail = variant === 'rail' + const listId = `${variant}-active-jobs-list` + const headingId = `${variant}-active-jobs-heading` + const count = activeJobs.length + + const alive = count > 0 || lingeringIds.length > 0 + + useEffect(() => { + if (!alive) return + setNow(Date.now()) + const timer = window.setInterval(() => setNow(Date.now()), TICK_MS) + return () => window.clearInterval(timer) + }, [alive]) + + useEffect(() => { + const ids = new Set(activeSignature ? activeSignature.split(',') : []) + const finished = previousActiveIds.current + ? [...previousActiveIds.current].filter((id) => !ids.has(id)) + : [] + + if (finished.length > 0) { + const expireAt = Date.now() + LINGER_MS + setLingering((current) => { + const next = { ...current } + for (const id of finished) next[id] = expireAt + return next + }) + if (ids.size === 0) { + const settled = latestJobs.current.find((job) => job.id === finished[0]) + const outcome = + settled?.status === 'succeeded' + ? '已完成' + : settled?.status === 'failed' + ? '生成失败' + : '已结束' + setAnnouncement( + finished.length === 1 && settled + ? `任务${outcome}：${settled.prompt.slice(0, 16)}，详情见历史记录` + : `有 ${finished.length} 个任务已结束，详情见历史记录`, + ) + } + } + previousActiveIds.current = ids + }, [activeSignature]) + + useEffect(() => { + const expired = lingeringIds.filter((id) => lingering[id] <= now) + if (expired.length === 0) return + const holdsFocus = expired.some((id) => document.querySelector(`[data-job-row="${id}"]:focus-within`)) + setLingering((current) => { + const next = { ...current } + for (const id of expired) delete next[id] + return next + }) + if (holdsFocus) headingRef.current?.focus() + }, [now, lingering]) + + const jobsById = new Map(jobs.map((job) => [job.id, job])) + const finishedRows = lingeringIds + .map((id) => jobsById.get(id)) + .filter((job): job is GenerationJob => Boolean(job) && !isJobActive(job!)) + const rows = [...activeJobs, ...finishedRows] + + if (hideWhenEmpty && rows.length === 0 && !isLoading && !isError) return null + + return ( +
+ {isRail ? ( +

+ +

+ ) : ( +

+ 进行中 + {count > 0 && ( + {count} + )} +

+ )} + + {isError ? ( +
+

任务列表获取失败

+ +
+ ) : rows.length === 0 && !open ? null : rows.length === 0 ? ( +
+ {isLoading ? ( +
+

加载任务中…

+ + +
+ ) : ( +

当前没有进行中的任务

+ )} +
+ ) : ( +
    + {rows.map((job) => ( + + ))} +
+ )} + +

+ {announcement} +

+
+ ) +} + +interface ActiveJobRowProps { + job: GenerationJob + now: number + expiringAt?: number + selected: boolean + onSelectJob: (jobId: string) => void + onCancel: (jobId: string) => void + onRetry: (jobId: string) => void + pendingCancelId?: string | null + pendingRetryId?: string | null +} + +function ActiveJobRow({ + job, + now, + expiringAt, + selected, + onSelectJob, + onCancel, + onRetry, + pendingCancelId, + pendingRetryId, +}: ActiveJobRowProps) { + const active = isJobActive(job) + const firstOutput = job.outputs?.[0] + const cancelling = Boolean(job.cancelRequested) || pendingCancelId === job.id + // `progress` is binary in the worker, so the ticking clock is the only liveness + // signal between 2.5s polls; it stops at `completedAt` once the job settles. + const clockEnd = job.completedAt ? Date.parse(job.completedAt) : now + const unit = job.mediaKind === 'video' ? '段' : '张' + + const stage = active + ? job.status === 'queued' + ? '已提交，等待服务返回' + : job.status === 'retry_wait' + ? '上次未成功，等待重试' + : phaseLabel(job.phase) + : job.status === 'succeeded' + ? `${job.outputs?.length ?? 0} ${unit}产物` + : '' + + const meta = [job.modelName, job.size, Number.isFinite(job.count) ? `${job.count} ${unit}` : null] + .filter(Boolean) + .join(' · ') + + return ( +
  • + {active && ( +
    • + ) +} diff --git a/apps/web-next/src/features/generate/components/generate-console.tsx b/apps/web-next/src/features/generate/components/generate-console.tsx index 3365a32..5ae56e0 100644 --- a/apps/web-next/src/features/generate/components/generate-console.tsx +++ b/apps/web-next/src/features/generate/components/generate-console.tsx @@ -1,69 +1,205 @@ 'use client' -import { useState } from 'react' +import { useEffect, useMemo, useRef, useState } from 'react' +import { useRouter, useSearchParams } from 'next/navigation' import { useGenerateUiStore } from '@/shared/stores/generate-ui-store' import { useModelsQuery } from '@/shared/hooks/useModels' import { useJobsQuery, useCreateJob, useCancelJob, useRetryJob } from '@/shared/hooks/useJobs' -import { buildGenerationInputs } from '@/shared/lib/generation-params' -import { clearReferenceImages } from '@/shared/lib/reference-upload' -import type { GenerationJob } from '@/shared/types' +import { + buildGenerationInputs, + buildVideoParameters, + inputPlanViolations, + resolveImageInputPlan, +} from '@/shared/lib/generation-params' +import { clearReferenceImages, reconcileStagedRoles } from '@/shared/lib/reference-upload' +import { MediaFrame } from '@/shared/components/media-frame' +import { JobStatusBadge } from '@/shared/components/job-status-badge' +import { isJobActive } from '@/shared/lib/job-status' +import { isVideoOutput, modelMediaKind, outputUrl } from '@/shared/types' +import type { + CreateGenerationRequest, + GenerateModeTab, + GenerationJob, + MediaKind, +} from '@/shared/types' +import { GenerateModeTabs } from './generate-mode-tabs' +import { VideoParameterControls } from './video-parameter-controls' import { ReferenceImagesTrigger } from './reference-images-trigger' +import { ActiveJobsBoard } from './active-jobs-board' import { - Brush, + ChevronRight, Clock, Download, + Film, + Image, Loader2, - Maximize2, PanelRightClose, PanelRightOpen, RefreshCw, Sparkles, + Video, XCircle, } from 'lucide-react' +const SELECT_CLASS = + 'rounded-[var(--radius-control)] border border-border bg-surface-subtle px-2.5 py-1.5 text-xs font-medium text-foreground outline-none hover:bg-surface-subtle-strong' + +/** `?tab=` accepts only the two kinds; anything else (typo, stale bookmark) is image. */ +function readTabParam(value: string | null): GenerateModeTab { + return value === 'video' ? 'video' : 'image' +} + +/** The DTO has no `modelKind` on a job row, so the kind comes from `mediaKind` + * first and from the emitted outputs as a fallback. */ +function jobMediaKind(job: GenerationJob): MediaKind { + if (job.mediaKind === 'video') return 'video' + if (job.mediaKind === 'image') return 'image' + return (job.outputs ?? []).some((output) => isVideoOutput(output)) ? 'video' : 'image' +} + export function GenerateConsole() { const prompt = useGenerateUiStore((s) => s.prompt) - const selectedModelId = useGenerateUiStore((s) => s.selectedModelId) + const activeTab = useGenerateUiStore((s) => s.activeTab) + const selectedModelIdByKind = useGenerateUiStore((s) => s.selectedModelIdByKind) const count = useGenerateUiStore((s) => s.count) const size = useGenerateUiStore((s) => s.size) const quality = useGenerateUiStore((s) => s.quality) + const videoParams = useGenerateUiStore((s) => s.videoParams) const selectedJobId = useGenerateUiStore((s) => s.selectedJobId) + const stagedOrder = useGenerateUiStore((s) => + s.stagedImages.map((image) => image.localId).join('|'), + ) const setPrompt = useGenerateUiStore((s) => s.setPrompt) + const setActiveTab = useGenerateUiStore((s) => s.setActiveTab) const setSelectedModelId = useGenerateUiStore((s) => s.setSelectedModelId) const setCount = useGenerateUiStore((s) => s.setCount) const setSize = useGenerateUiStore((s) => s.setSize) + const setVideoParam = useGenerateUiStore((s) => s.setVideoParam) const setSelectedJobId = useGenerateUiStore((s) => s.setSelectedJobId) // Derived primitives, so upload progress ticks never re-render this console. const isReferenceUploadBusy = useGenerateUiStore((s) => s.stagedImages.some((image) => image.status === 'pending' || image.status === 'uploading' || image.status === 'processing'), ) - const { data: models = [], isLoading: modelsLoading } = useModelsQuery() - const { data: jobs = [], isLoading: jobsLoading } = useJobsQuery(30) + const router = useRouter() + const searchParams = useSearchParams() + + const { data: models, isLoading: modelsLoading } = useModelsQuery() + const { + data: jobs = [], + isLoading: jobsLoading, + isError: jobsError, + refetch: refetchJobs, + } = useJobsQuery(30) const createJobMutation = useCreateJob() const cancelJobMutation = useCancelJob() const retryJobMutation = useRetryJob() - const [historyOpen, setHistoryOpen] = useState(true) + const railOpen = useGenerateUiStore((s) => s.railOpen) + const activeBoardOpen = useGenerateUiStore((s) => s.activeBoardOpen) + const historyOpen = useGenerateUiStore((s) => s.historyOpen) + const setRailOpen = useGenerateUiStore((s) => s.setRailOpen) + const setActiveBoardOpen = useGenerateUiStore((s) => s.setActiveBoardOpen) + const setHistoryOpen = useGenerateUiStore((s) => s.setHistoryOpen) + const [errorMessage, setErrorMessage] = useState('') + // `null` until the first URL read, then the last tab we pushed. It keeps the + // mount effect from pushing over the incoming `?tab=` and keeps a re-run + // (caused by `searchParams` identity) from pushing the same URL again. + const mirroredTabRef = useRef(null) - // Set default model if none selected - const activeModelId = selectedModelId || (models.length > 0 ? models[0].id : '') - const currentModel = models.find((m) => m.id === activeModelId) + // GET /api/models returns image AND video models: each tab may only drive its + // own kind, otherwise a video model gets auto-selected into the image flow. + const imageModels = useMemo( + () => (models ?? []).filter((model) => modelMediaKind(model) !== 'video'), + [models], + ) + const videoModels = useMemo( + () => (models ?? []).filter((model) => modelMediaKind(model) === 'video'), + [models], + ) + const isVideoTab = activeTab === 'video' + const tabModels = isVideoTab ? videoModels : imageModels + const videoModelMissing = !modelsLoading && isVideoTab && videoModels.length === 0 + + const storedModelId = selectedModelIdByKind[activeTab] + // Same id and still offered -> keep it; otherwise fall back to the first model + // of *this* kind. Never across kinds. + const activeModelId = tabModels.some((model) => model.id === storedModelId) + ? storedModelId + : tabModels[0]?.id ?? '' + const currentModel = tabModels.find((model) => model.id === activeModelId) + const inputPlan = useMemo(() => resolveImageInputPlan(currentModel), [currentModel]) + // One noun for the staged-image surface: frame slots stop calling them references. + const inputNoun = inputPlan.slots.some( + (slot) => slot.role === 'first_frame' || slot.role === 'last_frame', + ) + ? '输入画面' + : '参考图' + + useEffect(() => { + if (mirroredTabRef.current === null) { + mirroredTabRef.current = activeTab + const fromUrl = readTabParam(searchParams.get('tab')) + if (fromUrl !== activeTab) setActiveTab(fromUrl) + return + } + if (mirroredTabRef.current === activeTab) return + mirroredTabRef.current = activeTab + router.push(`/generate?tab=${activeTab}`, { scroll: false }) + }, [activeTab, router, searchParams, setActiveTab]) + + // Persist the resolved fallback so an admin rename/removal cannot silently + // jump the selection mid-session. Writing it is what ends the loop: the next + // run sees `storedModelId === activeModelId` and returns. + useEffect(() => { + if (!activeModelId) return + if (storedModelId === activeModelId) return + setSelectedModelId(activeTab, activeModelId) + }, [activeTab, activeModelId, storedModelId, setSelectedModelId]) + + // Frame roles are positional, so every staging change (add / remove / reorder / + // model or tab switch) re-derives them. `stagedOrder` is a joined string and the + // reconciliation itself is a no-op once the roles match, so this cannot cycle. + useEffect(() => { + if (!stagedOrder) return + void reconcileStagedRoles(currentModel) + }, [stagedOrder, currentModel]) // Find currently selected or running job const selectedJob = jobs.find((j) => j.id === selectedJobId) || jobs[0] || null - const isJobActive = selectedJob && ['queued', 'running', 'retry_wait'].includes(selectedJob.status) + const selectedJobActive = selectedJob ? isJobActive(selectedJob) : false + const stageOutputs = selectedJob?.outputs ?? [] + const stageIsVideo = selectedJob ? jobMediaKind(selectedJob) === 'video' : false + const showVideoEmptyState = videoModelMissing && !stageIsVideo + + const boardProps = { + jobs, + isLoading: jobsLoading, + isError: jobsError, + onReload: () => void refetchJobs(), + selectedJobId, + onSelectJob: setSelectedJobId, + onCancel: (jobId: string) => cancelJobMutation.mutate(jobId), + onRetry: (jobId: string) => retryJobMutation.mutate(jobId), + pendingCancelId: cancelJobMutation.isPending ? (cancelJobMutation.variables ?? null) : null, + pendingRetryId: retryJobMutation.isPending ? (retryJobMutation.variables ?? null) : null, + } async function handleGenerate(e?: React.FormEvent) { if (e) e.preventDefault() + const video = activeTab === 'video' if (!prompt.trim()) { - setErrorMessage('请输入生图提示词') + setErrorMessage(video ? '请输入视频生成提示词' : '请输入生图提示词') return } if (!activeModelId) { - setErrorMessage('请选择生成模型') + setErrorMessage(video ? '请选择视频生成模型' : '请选择生成模型') + return + } + if (video && videoModels.length === 0) { + setErrorMessage('尚未配置可用的视频模型，请先在模型管理中启用 Seedance 或 Veo 预设') return } @@ -71,31 +207,48 @@ export function GenerateConsole() { // stale render of an upload that finished between render and click. const stagedImages = useGenerateUiStore.getState().stagedImages if (stagedImages.some((image) => image.status !== 'ready' && image.status !== 'error')) { - setErrorMessage('参考图正在上传，请等待完成后再生成') + setErrorMessage(`${inputNoun}正在上传，请等待完成后再生成`) return } if (stagedImages.some((image) => image.status === 'error')) { - setErrorMessage('存在上传失败的参考图，请重试或先移除') + setErrorMessage(`存在上传失败的${inputNoun}，请重试或先移除`) return } const inputs = buildGenerationInputs(stagedImages) if (stagedImages.length > 0 && inputs.length === 0) { - setErrorMessage('没有可用的参考图，请重新上传') + setErrorMessage(`没有可用的${inputNoun}，请重新上传`) return } + if (video) { + // A lone 尾帧 (or an overflow past the frame capacity) is not representable + // in either provider request, so it must never leave the browser. + const violations = inputPlanViolations(stagedImages, inputPlan) + if (violations.length > 0) { + setErrorMessage(violations[0]) + return + } + } setErrorMessage('') try { - const job = await createJobMutation.mutateAsync({ - modelId: activeModelId, - prompt: prompt.trim(), - parameters: { - count, - size, - quality, - }, - ...(inputs.length > 0 ? { inputs } : {}), - }) + const payload: CreateGenerationRequest = video + ? { + modelId: activeModelId, + prompt: prompt.trim(), + parameters: buildVideoParameters(currentModel, videoParams), + ...(inputs.length > 0 ? { inputs } : {}), + } + : { + modelId: activeModelId, + prompt: prompt.trim(), + parameters: { + count, + size, + quality, + }, + ...(inputs.length > 0 ? { inputs } : {}), + } + const job = await createJobMutation.mutateAsync(payload) setSelectedJobId(job.id) // The uploads are now attached to the job; only release local object URLs. await clearReferenceImages({ deleteRemote: false }) @@ -112,316 +265,404 @@ export function GenerateConsole() { } return ( -
    - {/* Left: Interactive Generation Studio */} -
    -
    - {/* Prompt & Input Box */} -
    -