diff --git a/.env.example b/.env.example index 308b7f209..bd08b8384 100644 --- a/.env.example +++ b/.env.example @@ -64,6 +64,13 @@ FAL_KEY= FAL_IMAGE_MODEL= FAL_VIDEO_MODEL= +# Optional DreamAPI provider. Users register and top up at the DreamAPI dashboard. +# The key stays in this machine's server process and is never sent to the browser. +DREAMAPI_API_KEY= +DREAMAPI_IMAGE_MODEL= +DREAMAPI_VIDEO_MODEL= +DREAMAPI_VOICE_MODEL= + WAVESPEED_BASE_URL=https://api.wavespeed.ai WAVESPEED_API_KEY= WAVESPEED_IMAGE_MODEL=wavespeed-ai/flux-dev diff --git a/assets/agent/openchatcut-tool-schemas.json b/assets/agent/openchatcut-tool-schemas.json index b0c9af445..0a38fd80c 100644 --- a/assets/agent/openchatcut-tool-schemas.json +++ b/assets/agent/openchatcut-tool-schemas.json @@ -1175,9 +1175,10 @@ "wavespeed", "byteplus", "grok-imagine", - "fal" + "fal", + "dreamapi" ], - "description": "gpt-image-2 is the default; nano-banana is best for reference-heavy work; image-01 is MiniMax (at most 9 outputs; one subject reference when R2 is configured); wavespeed is WaveSpeed AI (fast generic image models, no references); byteplus is BytePlus ModelArk Seedream (no references yet); grok-imagine is xAI Grok Imagine (text-to-image only, no references, at most 4 outputs, 1K/2K)." + "description": "gpt-image-2 is the default; nano-banana is best for reference-heavy work; image-01 is MiniMax (at most 9 outputs; one subject reference when R2 is configured); wavespeed is WaveSpeed AI (fast generic image models, no references); byteplus is BytePlus ModelArk Seedream (no references yet); grok-imagine is xAI Grok Imagine (text-to-image only, no references, at most 4 outputs, 1K/2K). dreamapi is DreamAPI; pair it with dreamapiModel." }, "falModel": { "type": "string", @@ -1194,6 +1195,33 @@ ], "description": "With model=fal, select a curated Fal image model; omitted uses the saved Fal image default. nano-banana-2: Nano Banana 2 — count 1–4, resolutions 0.5K/1K/2K/4K, up to 14 image refs, image editing, default 1K/auto; nano-banana-pro: Nano Banana Pro — count 1–4, resolutions 1K/2K/4K, up to 14 image refs, image editing, default 1K/auto; gpt-image-2: GPT Image 2 — count 1–4, resolutions 1K, up to 16 image refs, image editing, default 1K/4:3; flux-2: FLUX.2 — count 1–4, resolutions 1K, default 1K/4:3; flux-2-pro: FLUX.2 Pro — count 1–1, resolutions 1K, up to 4 image refs, image editing, default 1K/4:3; seedream-5-pro: Seedream 5.0 Pro — count 1–1, resolutions 1K/2K, up to 10 image refs, image editing, default 2K/auto; ideogram-4: Ideogram 4 — count 1–4, resolutions 1K, default 1K/1:1; qwen-image-3: Qwen Image 3 — count 1–4, resolutions 1K, default 1K/1:1; recraft-v3: Recraft V3 — count 1–1, resolutions 1K, up to 1 image refs, image editing, default 1K/1:1. Model-specific limits are validated before submission." }, + "dreamapiModel": { + "type": "string", + "enum": [ + "dreamimage-2", + "flux-text", + "flux-image", + "seedream", + "nano-banana-2", + "nano-banana-pro", + "gpt-image-2", + "gpt-image-2.5-flare-text", + "gpt-image-2.5-flare-image", + "gpt-image-2.5-sunburst-text", + "gpt-image-2.5-sunburst-image", + "colorize", + "enhance", + "outpainting", + "swap-face", + "remove-background", + "try-on" + ], + "description": "With model=dreamapi, select a DreamAPI image model. dreamimage-2: 必须提供输入图和提示词。; flux-text: 宽高为 16 的倍数，最大 1600。; flux-image: 限 1 张参考图。; seedream: 用 seedreamModel 选择 4.0 / 4.5 / 5.0 Lite / 5.0 Pro。; nano-banana-2: 分辨率 512/1K/2K/4K。; nano-banana-pro: 分辨率只有 1K/2K。; gpt-image-2: 参考图最多 4 张。; gpt-image-2.5-flare-text: 质量含 xhigh 与 max。; gpt-image-2.5-flare-image: 参考图 1–4 张。; gpt-image-2.5-sunburst-text: 与 Flare 文生图同一套限制。; gpt-image-2.5-sunburst-image: 参考图 1–4 张。; colorize: 一张含人脸的图片。; enhance: 一张输入图。; outpainting: 按像素向四周扩展。; swap-face: 目标图加人脸图。; remove-background: 一张输入图。; try-on: 模特图必填，上装、下装、连体衣至少一张。" + }, + "dreamapi": { + "type": "object", + "description": "DreamAPI model inputs such as image, images, url, face, width, height, seedreamModel, size, text. Keys are validated against the selected dreamapiModel before any request is sent." + }, "prompt": { "type": "string", "description": "Detailed description of the image to generate." @@ -1351,9 +1379,24 @@ "openai", "gemini", "mistral", - "cartesia" + "cartesia", + "dreamapi" + ], + "description": "Required opt-in provider. Use only a provider listed as configured in capabilities. Voice IDs and catalogs are never interchangeable. dreamapi is DreamAPI TTS or voice clone." + }, + "dreamapiModel": { + "type": "string", + "enum": [ + "voice-clone", + "tts-clone", + "tts-common", + "tts-pro" ], - "description": "Required opt-in provider. Use only a provider listed as configured in capabilities. Voice IDs and catalogs are never interchangeable." + "description": "With provider=dreamapi, select tts-common, tts-pro, tts-clone, or voice-clone." + }, + "dreamapi": { + "type": "object", + "description": "DreamAPI voice inputs: text, lang, audioId, cloneId, or voiceUrl." }, "text": { "type": "string", @@ -1940,9 +1983,10 @@ "byteplus", "grok-imagine-video", "ofox", - "fal" + "fal", + "dreamapi" ], - "description": "hailuo is MiniMax: 6 or 10s; firstFrame optional; lastFrame allowed with firstFrame; no multi-ref or multi-shot. 1080p is 6s only. byteplus is BytePlus ModelArk Seedance — same request shape/limits as seedance2. grok-imagine-video is xAI Grok Imagine: text-to-video only, 1–15s, audio track included, no references/frames. ofox is the OFox multi-model gateway (Seedance/Wan and more behind one key): 2–30s with per-model limits enforced by the API; supports firstFrame (and optional lastFrame), or up to 9 refImages (frames and refImages are mutually exclusive); no refVideos/refAudios yet." + "description": "hailuo is MiniMax: 6 or 10s; firstFrame optional; lastFrame allowed with firstFrame; no multi-ref or multi-shot. 1080p is 6s only. byteplus is BytePlus ModelArk Seedance — same request shape/limits as seedance2. grok-imagine-video is xAI Grok Imagine: text-to-video only, 1–15s, audio track included, no references/frames. ofox is the OFox multi-model gateway (Seedance/Wan and more behind one key): 2–30s with per-model limits enforced by the API; supports firstFrame (and optional lastFrame), or up to 9 refImages (frames and refImages are mutually exclusive); no refVideos/refAudios yet. dreamapi is DreamAPI; pair it with dreamapiModel." }, "falModel": { "type": "string", @@ -1961,6 +2005,34 @@ ], "description": "With model=fal, select a curated Fal video model; omitted uses the saved Fal video default. seedance-2.5: Seedance 2.5 — audio toggle supported, resolutions 480p/720p, durations 4–30s, up to 30 image refs, up to 10 video refs, up to 10 audio refs, first frame/last frame/reference mode, default 720p/5s/auto; seedance-2.0: Seedance 2.0 — audio toggle supported, resolutions 480p/720p/1080p/4k, durations 4–15s, up to 9 image refs, up to 3 video refs, up to 3 audio refs, first frame/last frame/reference mode, default 720p/5s/auto; kling-v3-standard: Kling 3.0 Standard — audio toggle supported, resolutions 720p, durations 3–15s, first frame/last frame, default 720p/5s/16:9; kling-v3-pro: Kling 3.0 Pro — audio toggle supported, resolutions 1080p, durations 3–15s, first frame/last frame, default 1080p/5s/16:9; kling-o3-standard: Kling O3 Standard — audio toggle supported, resolutions 720p, durations 3–15s, up to 4 image refs, reference mode, default 720p/5s/16:9; kling-o3-pro: Kling O3 Pro — audio toggle supported, resolutions 1080p, durations 3–15s, up to 4 image refs, reference mode, default 1080p/5s/16:9; veo-3.1: Veo 3.1 — audio toggle supported, resolutions 720p/1080p/4k, durations 4/6/8s, first frame, default 720p/8s/16:9; veo-3.1-fast: Veo 3.1 Fast — audio toggle supported, resolutions 720p/1080p/4k, durations 4/6/8s, first frame, default 720p/8s/16:9; wan-3.0: Wan 3.0 — audio toggle supported, resolutions 480p/720p/1080p, durations 5/10s, first frame/last frame, default 1080p/5s/adaptive; minimax-h3-max: MiniMax H3 Max — no audio toggle, resolutions 480p/768p/1080p, durations 5–15s, up to 9 image refs, up to 3 video refs, up to 3 audio refs, reference mode, default 768p/5s/16:9; pixverse-v6: PixVerse V6 — audio toggle supported, resolutions 360p/540p/720p/1080p, durations 1–15s, first frame, default 720p/5s/16:9. Omit duration/resolution to use model defaults." }, + "dreamapiModel": { + "type": "string", + "enum": [ + "dreamvideo-3-text", + "dreamvideo-3-image", + "wan-2.1-text", + "wan-2.1-image", + "wan-2.1-head-tail", + "seedance-2.5", + "seedance-2.0", + "seedance-2.0-mini", + "lipsync", + "lipsync-2", + "dreamavatar-3-fast", + "dreamact", + "video-translate-2", + "swap-face-video", + "video-matting", + "matting-composite", + "watermark-remover", + "video-enhance" + ], + "description": "With model=dreamapi, select a DreamAPI video model. dreamvideo-3-text: 时长 3–15 秒，分辨率 480P/720P。; dreamvideo-3-image: 1 张图为图生视频，2 张图为首尾帧。; wan-2.1-text: 提示词最长 1500 字。; wan-2.1-image: 需要一张参考图。; wan-2.1-head-tail: 首帧和尾帧都必填。; seedance-2.5: 时长 4–30 秒。带参考视频时可指定 reference/edit/extend。; seedance-2.0: 时长 4–15 秒，分辨率可到 4k。; seedance-2.0-mini: 时长 4–15 秒，分辨率只有 480p/720p。; lipsync: 原视频和驱动音频。; lipsync-2: 可另传干声音频。设置里的默认视频模型建议用这个。; dreamavatar-3-fast: 人像、音频和提示词。音频请控制在 3 分钟内。; dreamact: 驱动视频最长 1 分钟。; video-translate-2: 语言只允许 en、zh、es。; swap-face-video: 原视频请控制在 5 分钟内。; video-matting: 只接受 MP4。成片和 alpha 都会返回。; matting-composite: 背景类型为 color、image 或 video。; watermark-remover: 视频最长 120 秒、最高 2K。; video-enhance: 一条原视频。" + }, + "dreamapi": { + "type": "object", + "description": "DreamAPI video inputs such as prompt, image, images, srcVideo, audio, resolution, duration, sourceLanguage, and targetLanguage. Keys are validated against dreamapiModel before any request is sent." + }, "prompt": { "type": "string", "description": "Required for normal generation and Kling intelligence; omit for Kling customize." diff --git a/assets/readme-pic/dreamapi-image-settings.png b/assets/readme-pic/dreamapi-image-settings.png new file mode 100644 index 000000000..a430cc281 Binary files /dev/null and b/assets/readme-pic/dreamapi-image-settings.png differ diff --git a/assets/readme-pic/dreamapi-media-pool.png b/assets/readme-pic/dreamapi-media-pool.png new file mode 100644 index 000000000..9277c02bc Binary files /dev/null and b/assets/readme-pic/dreamapi-media-pool.png differ diff --git a/assets/readme-pic/dreamapi-video-settings.png b/assets/readme-pic/dreamapi-video-settings.png new file mode 100644 index 000000000..cf5cd38b6 Binary files /dev/null and b/assets/readme-pic/dreamapi-video-settings.png differ diff --git a/assets/readme-pic/dreamapi-voice-settings.png b/assets/readme-pic/dreamapi-voice-settings.png new file mode 100644 index 000000000..f15458bf9 Binary files /dev/null and b/assets/readme-pic/dreamapi-voice-settings.png differ diff --git a/config/vite.config.ts b/config/vite.config.ts index 7dab72963..c3627fb02 100644 --- a/config/vite.config.ts +++ b/config/vite.config.ts @@ -110,6 +110,7 @@ export default defineConfig(({ mode }) => { const imageKey = env.IMAGE_API_KEY || env.OPENAI_API_KEY || ''; const geminiKey = env.GEMINI_API_KEY || ''; const falKey = env.FAL_KEY || ''; + const dreamApiKey = env.DREAMAPI_API_KEY || ''; const elevenKey = env.ELEVENLABS_API_KEY || ''; const doubaoKey = env.DOUBAO_TTS_API_KEY || ''; const murekaKey = env.MUREKA_API_KEY || ''; @@ -133,9 +134,9 @@ export default defineConfig(({ mode }) => { define: { __APP_VERSION__: JSON.stringify(appPackage.version), __CONFIGURED_CAPS__: JSON.stringify({ - image: Boolean(imageKey || geminiKey || minimaxKey || falKey), - voice: Boolean(doubaoKey || elevenKey || minimaxKey), - video: Boolean(seedanceKey || klingKey || minimaxKey || falKey), + image: Boolean(imageKey || geminiKey || minimaxKey || falKey || dreamApiKey), + voice: Boolean(doubaoKey || elevenKey || minimaxKey || dreamApiKey), + video: Boolean(seedanceKey || klingKey || minimaxKey || falKey || dreamApiKey), music: Boolean(murekaKey || minimaxKey), sound: Boolean(elevenKey), stock: Boolean(pexelsKey || pixabayKey || unsplashKey || freesoundKey), diff --git a/package.json b/package.json index 5e3721641..93bf126c7 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,7 @@ "build:cli": "node scripts/build-cli.mjs", "verify:cli": "tsx cli/cli.verify.ts", "sync:shader-sources": "node scripts/sync-shader-sources.mjs", - "test:serial": "npm run verify:fal && node remotion/performance.verify.mjs && tsx src/transcript/local-asr-warmup.verify.ts && tsx src/transcript/assemblyai.verify.ts && tsx src/transcript/assemblyai-resume.verify.ts && tsx src/transcript/transcribe-jobs.verify.ts && tsx src/persist/mediaIntelligenceStores.verify.ts && tsx src/editor/rateStretch.verify.ts && tsx server/media-acceleration.verify.ts && tsx server/plugins/scene-detection.verify.ts && tsx server/plugins/stock.verify.ts && tsx src/agent/proposal.verify.ts && tsx src/agent/external-edit-session.verify.ts && tsx src/agent/serverRunToolExecutor.race.verify.ts && tsx src/components/chat/widget-parse.verify.ts && tsx src/agent/tools/generation-tool-input.verify.ts && tsx server/plugins/image.verify.ts && tsx server/plugins/video.verify.ts && tsx server/plugins/ofox-video-provider.verify.ts && tsx server/plugins/music.verify.ts && tsx server/plugins/music-sonilo.verify.ts && tsx server/plugins/music-media.verify.ts && tsx server/plugins/sound.verify.ts && tsx server/plugins/voice.verify.ts && tsx server/plugins/voice-media.verify.ts && tsx src/agent/tools/followup-tools.verify.ts && tsx src/agent/tools/clean-script-rules.verify.ts && tsx src/agent/tools/mg-export-params.verify.ts && tsx src/agent/tools/scene-detection-tools.verify.ts && tsx src/agent/tools/isolate-voice-tools.verify.ts && tsx src/agent/tools/template-tools.verify.ts && tsx src/agent/tools/script-tools.verify.ts && tsx src/agent/tools/timeline-target-tools.verify.ts && tsx src/agent/tools/track-tools.verify.ts && tsx server/plugins/generation-jobs.verify.ts && tsx server/plugins/export-runtime.verify.ts && tsx server/plugins/export-params.verify.ts && tsx src/export/browserExport.verify.ts && tsx src/agent/ai-sdk.verify.ts && tsx server/llm-proxy.verify.ts && tsx server/external-agent/broker.verify.ts && tsx server/plugins/project-store.verify.ts && tsx server/plugins/extension-store.verify.ts && tsx server/plugins/normalize-media.verify.ts && tsx src/persist/projectStore.verify.ts && tsx src/persist/design-style-metadata.verify.ts && tsx src/persist/designStyleTransfer.verify.ts && tsx src/captions/manualCaptions.verify.ts && tsx src/captions/captionPreviewTarget.verify.ts && tsx src/agent/tools/captions-track-order.verify.ts && tsx src/skins.verify.ts && tsx src/theme-tokenization.verify.ts && tsx server/mobile-upload-service.verify.ts && tsx src/media/mobileUploadApi.verify.ts && tsx src/tracking/tracking.verify.ts && tsx src/media/semantic-search/semanticSearch.verify.ts && tsx src/color/autoGrade.verify.ts && tsx server/plugins/auto-grade.verify.ts && tsx src/transcript/phrases.verify.ts && tsx src/agent/tools/transcript-read.verify.ts && tsx src/export/quality.verify.ts && tsx src/export/autoQa.verify.ts && tsx server/plugins/export-qa.verify.ts && tsx src/agent/tools/export-qa-tools.verify.ts && tsx src/persist/migrations/migrations.verify.ts && tsx server/keystore.verify.ts && tsx src/components/timeline/transition-context-menu.verify.ts && tsx shared/model-capabilities.verify.ts && tsx desktop/agent-path-import.verify.ts && tsx desktop/agent-path-import.e2e.verify.ts && tsx src/agent/tools/agent-path-import-tools.verify.ts && tsx server/key-probes.verify.ts && tsx src/agent/settings/agent-settings.verify.ts && tsx src/editor/layouts.verify.ts && tsx src/audio/silence.verify.ts && tsx src/editor/silenceRebuild.verify.ts && tsx src/color/scopes.verify.ts && tsx src/audio/beats.verify.ts && tsx src/editor/keyframes.verify.ts && tsx src/editor/camSwitch.verify.ts && tsx src/export/fcpxml.verify.ts && tsx src/agent/timelineDelta.verify.ts && tsx server/plugins/extract-frames.verify.ts && tsx src/agent/tools/undo-tools.verify.ts && tsx src/editor/clipFit.verify.ts && tsx src/editor/sourceLimit.verify.ts && tsx src/editor/rippleChain.verify.ts && tsx src/editor/snap.verify.ts && tsx src/transcript/cutPad.verify.ts && tsx src/agent/tools/track-gaps.verify.ts && tsx src/persist/uploadRefs.verify.ts && tsx server/plugins/result-download.verify.ts && tsx src/agent/systemPromptOrder.verify.ts && tsx src/review/reviewModel.verify.ts && tsx src/editor/historyGesture.verify.ts && tsx src/agent/abortedTurn.verify.ts && tsx src/agent/changeLog.verify.ts && tsx server/plugins/reference-mime.verify.ts && tsx src/components/inspector/inspector-controls.verify.ts && tsx --tsconfig tsconfig.app.json src/components/timeline/timelineFitRange.verify.ts && tsx --tsconfig tsconfig.app.json src/components/timeline/timeline-clip-duration.verify.ts && tsx --tsconfig tsconfig.app.json src/components/timeline/trackOverlap.verify.ts && node src/components/timeline/playhead-watchdog.verify.mjs && tsx --tsconfig tsconfig.app.json src/components/timeline/timelineSeek.verify.ts && tsx --tsconfig tsconfig.app.json src/components/timeline/trackDelete.verify.ts && tsx --tsconfig tsconfig.app.json src/components/timeline/track-mute-visual.verify.ts && npm run test:contrib-timeline-preview && tsx server/external-agent/client-connect.verify.ts && tsx config/vite.config.verify.ts && tsx desktop/embedded-project-store-http.verify.ts && tsx desktop/embedded-server.verify.ts && node remotion/direct-hardware.verify.mjs && node remotion/aac-encoder-delay.verify.mjs && node remotion/video-decoder.verify.mjs && node remotion/video-decoder-render.verify.mjs && tsx server/agent-runs/acceptance-loop.verify.ts && tsx server/agent-runs/llm-retry.verify.ts && tsx server/agent-runs/store-business-failure.verify.ts && tsx server/error-scrub.verify.ts && tsx server/external-agent/broker-poll-refresh.verify.ts && tsx server/external-agent/capcut-command.verify.ts && tsx server/external-agent/jianying-export.verify.ts && tsx server/external-agent/jianying-export-commands.verify.ts &&tsx src/export/jianyingDraftRequest.verify.ts && tsx server/external-agent/offline-mcp-process-recovery.verify.ts && tsx server/outbound-proxy.verify.ts && tsx server/plugins/mobile-upload.verify.ts && tsx server/plugins/read-path-auth.verify.ts && tsx server/plugins/skill-exec.verify.ts && tsx server/plugins/sonilo-media.verify.ts && tsx server/plugins/upload-post.verify.ts && tsx src/agent/tools/publish-tools.verify.ts && tsx server/plugins/sound-sonilo-job.verify.ts && tsx server/storage/fulltext-search.verify.ts && tsx server/storage/hybrid-search.verify.ts && tsx server/storage/semantic-vectors.verify.ts && tsx server/xai-oauth-session.verify.ts && tsx shared/transcript-window.verify.ts && tsx src/agent/approval-mode.verify.ts && tsx src/agent/codex/runtime-tool-boundary.verify.ts && tsx src/agent/execution-modes.verify.ts && tsx src/agent/execution-policy-args.verify.ts && tsx src/agent/external-bridge-attempt-error.verify.ts && tsx src/agent/serverRunDraftStore.verify.ts && tsx src/agent/serverRunToolBusinessFailure.verify.ts && tsx src/agent/tools/agent-export-media-pool.verify.ts && tsx src/agent/tools/core-data-tools.verify.ts && tsx src/agent/tools/livestream-sequence-workflow.verify.ts && tsx src/agent/tools/search-tools.verify.ts && tsx src/audio/silero-vad.verify.ts && tsx src/captions/captionFallback.verify.ts && tsx src/captions/wordAlignment.verify.ts && tsx src/components/chat/chatRunStatus.verify.ts && tsx src/components/timeline/usePlayheadPaint.verify.ts && tsx src/editor/reducerGuards.verify.ts && tsx src/editor/sharedVisual.verify.ts && tsx src/editor/trackCreateBatching.verify.ts && tsx src/export/downloadBlob.verify.ts && tsx src/export/export-reachability.verify.ts && tsx src/generate/asset-reference.verify.ts && tsx src/gl/clipFxPreview.verify.ts && tsx --tsconfig tsconfig.app.json src/editor/serverVideoDecoder.verify.ts && tsx src/i18n/dictRegistry.verify.ts && tsx src/media/previewMedia.verify.ts && tsx src/media/semantic-search/vectorStore.verify.ts && tsx src/media/transcriptParagraphs.verify.ts && tsx src/multicam/professionalMulticamPersistence.verify.ts && tsx src/multicam/professionalMulticamSwitch.verify.ts && tsx src/multicam/professionalMulticamSync.verify.ts && tsx src/persist/multicamPreservation.verify.ts && tsx src/persist/unreadablePreservation.verify.ts && tsx src/transcript/auto-transcribe-ingest.verify.ts && tsx src/transcript/local-asr-silence.verify.ts && tsx src/components/timeline/frameCoalescer.verify.ts && node scripts/run-affected-verifies.verify.mjs && node scripts/sync-whisper-cli.verify.mjs && node scripts/verify-registration.verify.mjs && node scripts/issue-template-version.verify.mjs && tsx src/media/drag.verify.ts && tsx shared/file-name.verify.ts && tsx server/external-agent/external-skill.verify.mjs && node src/gl/clipFxExport.verify.mjs && node src/gl/clipFxExport.offthread.verify.mjs && tsx src/components/settings/mcpGuideLoader.verify.ts && tsx shared/onnxruntime-version.verify.ts && node shared/media-dependencies.verify.mjs && tsx server/plugins/import-url-errors.verify.ts && tsx server/plugins/probe-media.verify.ts && tsx server/product-assets.verify.ts && tsx server/plugins/import-probe.verify.ts && tsx server/media-binaries.verify.ts && node scripts/verify-gate-coverage.verify.mjs && node scripts/media-binary-spawn.verify.mjs && node scripts/shader-sources.verify.mjs && tsx src/editor/storeCommandBuilder.verify.ts && tsx cli/cli.verify.ts && tsx src/components/settings/asr-model-tier-selection.verify.ts", + "test:serial": "npm run verify:fal && npm run verify:dreamapi && node remotion/performance.verify.mjs && tsx src/transcript/local-asr-warmup.verify.ts && tsx src/transcript/assemblyai.verify.ts && tsx src/transcript/assemblyai-resume.verify.ts && tsx src/transcript/transcribe-jobs.verify.ts && tsx src/persist/mediaIntelligenceStores.verify.ts && tsx src/editor/rateStretch.verify.ts && tsx server/media-acceleration.verify.ts && tsx server/plugins/scene-detection.verify.ts && tsx server/plugins/stock.verify.ts && tsx src/agent/proposal.verify.ts && tsx src/agent/external-edit-session.verify.ts && tsx src/agent/serverRunToolExecutor.race.verify.ts && tsx src/components/chat/widget-parse.verify.ts && tsx src/agent/tools/generation-tool-input.verify.ts && tsx server/plugins/image.verify.ts && tsx server/plugins/video.verify.ts && tsx server/plugins/ofox-video-provider.verify.ts && tsx server/plugins/music.verify.ts && tsx server/plugins/music-sonilo.verify.ts && tsx server/plugins/music-media.verify.ts && tsx server/plugins/sound.verify.ts && tsx server/plugins/voice.verify.ts && tsx server/plugins/voice-media.verify.ts && tsx src/agent/tools/followup-tools.verify.ts && tsx src/agent/tools/clean-script-rules.verify.ts && tsx src/agent/tools/mg-export-params.verify.ts && tsx src/agent/tools/scene-detection-tools.verify.ts && tsx src/agent/tools/isolate-voice-tools.verify.ts && tsx src/agent/tools/template-tools.verify.ts && tsx src/agent/tools/script-tools.verify.ts && tsx src/agent/tools/timeline-target-tools.verify.ts && tsx src/agent/tools/track-tools.verify.ts && tsx server/plugins/generation-jobs.verify.ts && tsx server/plugins/export-runtime.verify.ts && tsx server/plugins/export-params.verify.ts && tsx src/export/browserExport.verify.ts && tsx src/agent/ai-sdk.verify.ts && tsx server/llm-proxy.verify.ts && tsx server/external-agent/broker.verify.ts && tsx server/plugins/project-store.verify.ts && tsx server/plugins/extension-store.verify.ts && tsx server/plugins/normalize-media.verify.ts && tsx src/persist/projectStore.verify.ts && tsx src/persist/design-style-metadata.verify.ts && tsx src/persist/designStyleTransfer.verify.ts && tsx src/captions/manualCaptions.verify.ts && tsx src/captions/captionPreviewTarget.verify.ts && tsx src/agent/tools/captions-track-order.verify.ts && tsx src/skins.verify.ts && tsx src/theme-tokenization.verify.ts && tsx server/mobile-upload-service.verify.ts && tsx src/media/mobileUploadApi.verify.ts && tsx src/tracking/tracking.verify.ts && tsx src/media/semantic-search/semanticSearch.verify.ts && tsx src/color/autoGrade.verify.ts && tsx server/plugins/auto-grade.verify.ts && tsx src/transcript/phrases.verify.ts && tsx src/agent/tools/transcript-read.verify.ts && tsx src/export/quality.verify.ts && tsx src/export/autoQa.verify.ts && tsx server/plugins/export-qa.verify.ts && tsx src/agent/tools/export-qa-tools.verify.ts && tsx src/persist/migrations/migrations.verify.ts && tsx server/keystore.verify.ts && tsx src/components/timeline/transition-context-menu.verify.ts && tsx shared/model-capabilities.verify.ts && tsx desktop/agent-path-import.verify.ts && tsx desktop/agent-path-import.e2e.verify.ts && tsx src/agent/tools/agent-path-import-tools.verify.ts && tsx server/key-probes.verify.ts && tsx src/agent/settings/agent-settings.verify.ts && tsx src/editor/layouts.verify.ts && tsx src/audio/silence.verify.ts && tsx src/editor/silenceRebuild.verify.ts && tsx src/color/scopes.verify.ts && tsx src/audio/beats.verify.ts && tsx src/editor/keyframes.verify.ts && tsx src/editor/camSwitch.verify.ts && tsx src/export/fcpxml.verify.ts && tsx src/agent/timelineDelta.verify.ts && tsx server/plugins/extract-frames.verify.ts && tsx src/agent/tools/undo-tools.verify.ts && tsx src/editor/clipFit.verify.ts && tsx src/editor/sourceLimit.verify.ts && tsx src/editor/rippleChain.verify.ts && tsx src/editor/snap.verify.ts && tsx src/transcript/cutPad.verify.ts && tsx src/agent/tools/track-gaps.verify.ts && tsx src/persist/uploadRefs.verify.ts && tsx server/plugins/result-download.verify.ts && tsx src/agent/systemPromptOrder.verify.ts && tsx src/review/reviewModel.verify.ts && tsx src/editor/historyGesture.verify.ts && tsx src/agent/abortedTurn.verify.ts && tsx src/agent/changeLog.verify.ts && tsx server/plugins/reference-mime.verify.ts && tsx src/components/inspector/inspector-controls.verify.ts && tsx --tsconfig tsconfig.app.json src/components/timeline/timelineFitRange.verify.ts && tsx --tsconfig tsconfig.app.json src/components/timeline/timeline-clip-duration.verify.ts && tsx --tsconfig tsconfig.app.json src/components/timeline/trackOverlap.verify.ts && node src/components/timeline/playhead-watchdog.verify.mjs && tsx --tsconfig tsconfig.app.json src/components/timeline/timelineSeek.verify.ts && tsx --tsconfig tsconfig.app.json src/components/timeline/trackDelete.verify.ts && tsx --tsconfig tsconfig.app.json src/components/timeline/track-mute-visual.verify.ts && npm run test:contrib-timeline-preview && tsx server/external-agent/client-connect.verify.ts && tsx config/vite.config.verify.ts && tsx desktop/embedded-project-store-http.verify.ts && tsx desktop/embedded-server.verify.ts && node remotion/direct-hardware.verify.mjs && node remotion/aac-encoder-delay.verify.mjs && node remotion/video-decoder.verify.mjs && node remotion/video-decoder-render.verify.mjs && tsx server/agent-runs/acceptance-loop.verify.ts && tsx server/agent-runs/llm-retry.verify.ts && tsx server/agent-runs/store-business-failure.verify.ts && tsx server/error-scrub.verify.ts && tsx server/external-agent/broker-poll-refresh.verify.ts && tsx server/external-agent/capcut-command.verify.ts && tsx server/external-agent/jianying-export.verify.ts && tsx server/external-agent/jianying-export-commands.verify.ts &&tsx src/export/jianyingDraftRequest.verify.ts && tsx server/external-agent/offline-mcp-process-recovery.verify.ts && tsx server/outbound-proxy.verify.ts && tsx server/plugins/mobile-upload.verify.ts && tsx server/plugins/read-path-auth.verify.ts && tsx server/plugins/skill-exec.verify.ts && tsx server/plugins/sonilo-media.verify.ts && tsx server/plugins/upload-post.verify.ts && tsx src/agent/tools/publish-tools.verify.ts && tsx server/plugins/sound-sonilo-job.verify.ts && tsx server/storage/fulltext-search.verify.ts && tsx server/storage/hybrid-search.verify.ts && tsx server/storage/semantic-vectors.verify.ts && tsx server/xai-oauth-session.verify.ts && tsx shared/transcript-window.verify.ts && tsx src/agent/approval-mode.verify.ts && tsx src/agent/codex/runtime-tool-boundary.verify.ts && tsx src/agent/execution-modes.verify.ts && tsx src/agent/execution-policy-args.verify.ts && tsx src/agent/external-bridge-attempt-error.verify.ts && tsx src/agent/serverRunDraftStore.verify.ts && tsx src/agent/serverRunToolBusinessFailure.verify.ts && tsx src/agent/tools/agent-export-media-pool.verify.ts && tsx src/agent/tools/core-data-tools.verify.ts && tsx src/agent/tools/livestream-sequence-workflow.verify.ts && tsx src/agent/tools/search-tools.verify.ts && tsx src/audio/silero-vad.verify.ts && tsx src/captions/captionFallback.verify.ts && tsx src/captions/wordAlignment.verify.ts && tsx src/components/chat/chatRunStatus.verify.ts && tsx src/components/timeline/usePlayheadPaint.verify.ts && tsx src/editor/reducerGuards.verify.ts && tsx src/editor/sharedVisual.verify.ts && tsx src/editor/trackCreateBatching.verify.ts && tsx src/export/downloadBlob.verify.ts && tsx src/export/export-reachability.verify.ts && tsx src/generate/asset-reference.verify.ts && tsx src/gl/clipFxPreview.verify.ts && tsx --tsconfig tsconfig.app.json src/editor/serverVideoDecoder.verify.ts && tsx src/i18n/dictRegistry.verify.ts && tsx src/media/previewMedia.verify.ts && tsx src/media/semantic-search/vectorStore.verify.ts && tsx src/media/transcriptParagraphs.verify.ts && tsx src/multicam/professionalMulticamPersistence.verify.ts && tsx src/multicam/professionalMulticamSwitch.verify.ts && tsx src/multicam/professionalMulticamSync.verify.ts && tsx src/persist/multicamPreservation.verify.ts && tsx src/persist/unreadablePreservation.verify.ts && tsx src/transcript/auto-transcribe-ingest.verify.ts && tsx src/transcript/local-asr-silence.verify.ts && tsx src/components/timeline/frameCoalescer.verify.ts && node scripts/run-affected-verifies.verify.mjs && node scripts/sync-whisper-cli.verify.mjs && node scripts/verify-registration.verify.mjs && node scripts/issue-template-version.verify.mjs && tsx src/media/drag.verify.ts && tsx shared/file-name.verify.ts && tsx server/external-agent/external-skill.verify.mjs && node src/gl/clipFxExport.verify.mjs && node src/gl/clipFxExport.offthread.verify.mjs && tsx src/components/settings/mcpGuideLoader.verify.ts && tsx shared/onnxruntime-version.verify.ts && node shared/media-dependencies.verify.mjs && tsx server/plugins/import-url-errors.verify.ts && tsx server/plugins/probe-media.verify.ts && tsx server/product-assets.verify.ts && tsx server/plugins/import-probe.verify.ts && tsx server/media-binaries.verify.ts && node scripts/verify-gate-coverage.verify.mjs && node scripts/media-binary-spawn.verify.mjs && node scripts/shader-sources.verify.mjs && tsx src/editor/storeCommandBuilder.verify.ts && tsx cli/cli.verify.ts && tsx src/components/settings/asr-model-tier-selection.verify.ts", "posttest": "npm run verify:background-fill && npm run verify:speech-providers && npm run verify:desktop-worker-lifecycle && npm run verify:server-runs && npm run verify:extract-audio", "verify:server-runs": "tsx server/agent-runs/model.verify.ts && tsx server/agent-runs/tool-catalog-generation.verify.ts && tsx server/agent-runs/codex-turn.verify.ts && tsx server/copilot/copilot-agent.verify.ts && tsx server/agent-runs/copilot-turn.verify.ts && tsx server/claude-code/claude-code-agent.verify.ts && tsx server/agent-runs/claude-code-turn.verify.ts && tsx server/agent-runs/executor.verify.ts && tsx server/agent-runs/tool-policy.verify.ts && tsx server/agent-runs/store.verify.ts && tsx server/agent-runs/store-tool-requests.verify.ts && tsx server/agent-runs/store-event-window.verify.ts && tsx server/agent-runs/store-approval.verify.ts && tsx server/agent-runs/store-capability.verify.ts && tsx server/agent-runs/routes.verify.ts && tsx src/agent/useServerRun.verify.ts && tsx src/agent/serverRunFetchEventStream.verify.ts && tsx src/agent/serverRunSessionStorage.verify.ts && tsx src/agent/serverRunModelHistory.verify.ts && tsx src/agent/serverRunOwnership.verify.ts && tsx src/agent/serverRunProtocol.verify.ts && tsx src/agent/serverRunToolExecutor.verify.ts && tsx src/agent/serverRunToolRecovery.verify.ts && tsx src/agent/useAgentHistoryActions.verify.ts && tsx src/components/chat/serverRunInspector.verify.ts", "verify:background-fill": "tsx src/editor/backgroundFill.verify.ts && tsx --tsconfig tsconfig.app.json src/components/preview/previewTransform.verify.ts && tsx --tsconfig tsconfig.app.json src/components/inspector/background-fill-control.verify.tsx && tsx src/agent/tools/edit-item-background-fill.verify.ts && tsx src/export/fcpxml-background-fill.verify.ts", @@ -89,6 +89,7 @@ "verify:sqlite-migration": "tsx server/storage/sqlite-store.verify.ts && tsx server/storage/sqlite-migration-http.verify.ts && tsx server/plugins/project-store-export-recovery-sqlite.verify.ts && tsx src/export/serverExportRecovery.verify.ts", "verify:extract-audio": "tsx server/plugins/extract-audio.verify.ts", "verify:fal": "node --import tsx server/plugins/fal-catalog-input.verify.ts && node --import tsx server/plugins/fal-provider.verify.ts && node --import tsx server/plugins/fal-client.verify.ts && node --import tsx server/plugins/fal-routing.verify.ts", + "verify:dreamapi": "node --import tsx server/plugins/dreamapi-catalog.verify.ts", "verify:system-fonts": "tsx server/plugins/system-fonts.verify.ts && tsx --tsconfig tsconfig.app.json src/fonts/systemFonts.verify.ts && tsx --tsconfig tsconfig.app.json src/editor/TimelineGraphicLayers.verify.ts" }, "dependencies": { diff --git a/server/keystore-names.ts b/server/keystore-names.ts index 35668dea1..c832a7314 100644 --- a/server/keystore-names.ts +++ b/server/keystore-names.ts @@ -83,6 +83,7 @@ export const KEY_NAMES = [ "GEMINI_API_KEY", "GEMINI_BASE_URL", "FAL_KEY", + "DREAMAPI_API_KEY", "WAVESPEED_API_KEY", "WAVESPEED_BASE_URL", "BYTEPLUS_API_KEY", @@ -144,6 +145,9 @@ export const KEY_NAMES = [ MODEL_CAPABILITY_OVERRIDES_KEY, "FAL_IMAGE_MODEL", "FAL_VIDEO_MODEL", + "DREAMAPI_IMAGE_MODEL", + "DREAMAPI_VIDEO_MODEL", + "DREAMAPI_VOICE_MODEL", "GEMINI_IMAGE_MODEL", "MINIMAX_IMAGE_MODEL", "WAVESPEED_IMAGE_MODEL", @@ -207,6 +211,9 @@ export const NON_SECRET_NAMES: ReadonlySet = new Set([ MODEL_CAPABILITY_OVERRIDES_KEY, "FAL_IMAGE_MODEL", "FAL_VIDEO_MODEL", + "DREAMAPI_IMAGE_MODEL", + "DREAMAPI_VIDEO_MODEL", + "DREAMAPI_VOICE_MODEL", "GEMINI_IMAGE_MODEL", "IMAGE_BASE_URL", "GEMINI_BASE_URL", diff --git a/server/keystore.ts b/server/keystore.ts index c1799b1da..07fe78b5b 100644 --- a/server/keystore.ts +++ b/server/keystore.ts @@ -153,7 +153,8 @@ export function computeCaps(): Caps { has("MINIMAX_API_KEY") || has("WAVESPEED_API_KEY") || has("BYTEPLUS_API_KEY") || - has("FAL_KEY"), + has("FAL_KEY") || + has("DREAMAPI_API_KEY"), voice: has("DOUBAO_TTS_API_KEY") || has("ELEVENLABS_API_KEY") || @@ -164,10 +165,11 @@ export function computeCaps(): Caps { (getKey("PREFERRED_VOICE_VENDOR") === "openai" && has("OPENAI_API_KEY")) || (getKey("PREFERRED_VOICE_VENDOR") === "gemini" && has("GEMINI_API_KEY")) || (getKey("PREFERRED_VOICE_VENDOR") === "mistral" && has("LLM_MISTRAL_API_KEY")) || - (getKey("PREFERRED_VOICE_VENDOR") === "cartesia" && has("CARTESIA_API_KEY")), + (getKey("PREFERRED_VOICE_VENDOR") === "cartesia" && has("CARTESIA_API_KEY")) || + has("DREAMAPI_API_KEY"), video: has("SEEDANCE_API_KEY") || has("KLING_API_KEY") || has("MINIMAX_API_KEY") || has("BYTEPLUS_API_KEY") - || has("LLM_OFOX_API_KEY") || has("FAL_KEY"), + || has("LLM_OFOX_API_KEY") || has("FAL_KEY") || has("DREAMAPI_API_KEY"), music: has("MUREKA_API_KEY") || has("MINIMAX_API_KEY") || has("ATLASCLOUD_API_KEY") || has("SONILO_API_KEY"), sound: has("ELEVENLABS_API_KEY") || has("SONILO_API_KEY"), stock: diff --git a/server/keystore.verify.ts b/server/keystore.verify.ts index fd97d46bf..96ac873a0 100644 --- a/server/keystore.verify.ts +++ b/server/keystore.verify.ts @@ -149,6 +149,7 @@ const MODEL_ROUTING_NAMES = [ 'COPILOT_MODEL', 'COPILOT_REASONING_EFFORT', 'CLAUDE_CODE_MODEL', MODEL_CAPABILITY_OVERRIDES_KEY, 'FAL_IMAGE_MODEL', 'FAL_VIDEO_MODEL', + 'DREAMAPI_IMAGE_MODEL', 'DREAMAPI_VIDEO_MODEL', 'DREAMAPI_VOICE_MODEL', 'GEMINI_IMAGE_MODEL', 'IMAGE_BASE_URL', 'GEMINI_BASE_URL', 'ELEVENLABS_TTS_MODEL', 'ELEVENLABS_SOUND_MODEL', 'OPENAI_TTS_MODEL', 'GEMINI_TTS_MODEL', 'MISTRAL_TTS_MODEL', 'CARTESIA_TTS_MODEL', diff --git a/server/plugins/dreamapi-catalog.verify.ts b/server/plugins/dreamapi-catalog.verify.ts new file mode 100644 index 000000000..6ecbb9ff3 --- /dev/null +++ b/server/plugins/dreamapi-catalog.verify.ts @@ -0,0 +1,51 @@ +import assert from 'node:assert/strict'; +import { buildDreamApiCall, DREAMAPI_MODELS, DREAMAPI_MODEL_IDS } from '../../shared/dreamapi-models.ts'; +import { DREAMAPI_BASE_URL, runDreamApi } from './dreamapi-provider.ts'; + +assert.equal(new Set(DREAMAPI_MODEL_IDS).size, DREAMAPI_MODELS.length, 'DreamAPI model ids are unique'); +assert.ok(DREAMAPI_MODELS.length >= 36, 'catalog includes the DreamAPI skill models'); +assert.ok(DREAMAPI_MODELS.every((model) => model.label.startsWith('DreamAPI ·')), 'every model is labeled DreamAPI'); +assert.ok(DREAMAPI_MODELS.every((model) => model.endpoint.startsWith('/api/')), 'endpoints stay on the DreamAPI path prefix'); + +assert.throws(() => buildDreamApiCall('missing', {}), /Choose a DreamAPI model/); +assert.throws(() => buildDreamApiCall('dreamvideo-3-text', { prompt: 'a', duration: 2 }), /duration must be at least 3/); +assert.throws(() => buildDreamApiCall('dreamimage-2', { prompt: 'edit' }), /image is required/); +assert.throws(() => buildDreamApiCall('lipsync', { srcVideo: 'https://example.com/a.mp4' }), /audio is required/); +assert.throws(() => buildDreamApiCall('video-translate-2', { video: 'https://example.com/a.mp4', sourceLanguage: 'fr', targetLanguage: 'zh' }), /sourceLanguage/); +assert.throws(() => buildDreamApiCall('try-on', { modelImage: 'https://example.com/m.png' }), /upper, lower, or overall/); +assert.throws(() => buildDreamApiCall('seedream', { prompt: 'cat', seedreamModel: 'seedream-4.5', size: '1024x1024' }), /pixel range/); +assert.throws(() => buildDreamApiCall('flux-text', { prompt: 'cat', width: 100 }), /multiple of 16/); +assert.throws(() => buildDreamApiCall('lipsync', { srcVideo: 'https://example.com/a.mp4', audio: 'https://example.com/a.mp3', prompt: 'no' }), /prompt is not supported/); + +const lipsync = buildDreamApiCall('lipsync-2', { + srcVideo: 'https://example.com/a.mp4', audio: 'https://example.com/a.mp3', vocalAudio: 'https://example.com/v.mp3', +}); +assert.equal(lipsync.body.vocalAudioUrl, 'https://example.com/v.mp3'); +assert.equal((lipsync.body.videoParams as { video_enhance: number }).video_enhance, 1); + +let submitCount = 0; +const fetchImpl: typeof fetch = async (url, init) => { + const href = String(url); + const body = JSON.parse(String(init?.body ?? '{}')) as { taskId?: string }; + if (href === `${DREAMAPI_BASE_URL}/api/getAsyncResult`) { + return Response.json({ code: 0, data: { task: { status: 3, taskId: body.taskId }, videos: [{ videoUrl: 'https://cdn.example/out.mp4' }] } }); + } + submitCount += 1; + return Response.json({ code: 0, data: { taskId: 'task-1' } }); +}; + +const resumed = await runDreamApi( + buildDreamApiCall('wan-2.1-text', { prompt: 'city' }), + { apiKey: 'test-key', fetchImpl, existingTaskId: 'dreamapi:task-1', pollIntervalMs: 0 }, +); +assert.equal(submitCount, 0, 'resuming a stored task does not submit another paid request'); +assert.deepEqual(resumed.urls, ['https://cdn.example/out.mp4']); + +const fresh = await runDreamApi( + buildDreamApiCall('wan-2.1-text', { prompt: 'city' }), + { apiKey: 'test-key', fetchImpl, pollIntervalMs: 0 }, +); +assert.equal(submitCount, 1); +assert.equal(fresh.taskId, 'task-1'); + +console.log('dreamapi catalog ok'); diff --git a/server/plugins/dreamapi-provider.ts b/server/plugins/dreamapi-provider.ts new file mode 100644 index 000000000..6c625dbb8 --- /dev/null +++ b/server/plugins/dreamapi-provider.ts @@ -0,0 +1,183 @@ +import { readFile } from 'node:fs/promises'; +import { basename, extname } from 'node:path'; +import { getKey } from '../keystore.ts'; +import { resolveUploadFile } from '../media-dir.ts'; +import { buildDreamApiCall, type DreamApiCall, type DreamApiModel } from '../../shared/dreamapi-models.ts'; + +export const DREAMAPI_BASE_URL = 'https://api.newportai.com'; + +type FetchImpl = (url: string, init?: RequestInit) => Promise; + +export interface DreamApiRunOptions { + apiKey: string; + fetchImpl?: FetchImpl; + existingTaskId?: string; + registerTask?: (taskId: string) => Promise; + pollIntervalMs?: number; +} + +export interface DreamApiRunResult { + taskId: string; + urls: string[]; + cloneId?: string; +} + +function apiKey(): string { + const key = getKey('DREAMAPI_API_KEY').trim(); + if (!key) throw new Error('DreamAPI is not configured. Set DREAMAPI_API_KEY in Settings. The key stays on this machine.'); + return key; +} + +function isHttp(value: string): boolean { + return /^https?:\/\//i.test(value); +} + +async function localFile(value: string): Promise<{ path: string; name: string } | undefined> { + if (isHttp(value)) return undefined; + if (value.startsWith('/media/uploads/')) { + const name = basename(value); + const path = resolveUploadFile(name); + if (!path) throw new Error(`DreamAPI media file not found: ${value}`); + return { path, name }; + } + if (value.startsWith('/')) return { path: value, name: basename(value) }; + return undefined; +} + +async function dreamApiJson(fetchImpl: FetchImpl, key: string, path: string, body: unknown): Promise> { + const response = await fetchImpl(`${DREAMAPI_BASE_URL}${path}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${key}` }, + body: JSON.stringify(body), + }); + const payload = await response.json() as { code?: number; message?: string; data?: Record }; + if (!response.ok || payload.code !== 0) { + throw new Error(payload.message || `DreamAPI request failed (${response.status})`); + } + return payload.data ?? {}; +} + +async function uploadLocal(fetchImpl: FetchImpl, key: string, filePath: string, fileName: string): Promise { + const policy = await dreamApiJson(fetchImpl, key, '/api/file/v1/get_policy', { scene: 'Dream-CN' }); + const host = String(policy.host ?? ''); + const accessId = String(policy.accessId ?? policy.OSSAccessKeyId ?? ''); + const policyText = String(policy.policy ?? ''); + const signature = String(policy.signature ?? ''); + const fullKey = String(policy.key ?? ''); + const uploadDir = String(policy.dir ?? ''); + const objectKey = fullKey || (uploadDir ? `${uploadDir}${fileName}` : ''); + if (!host || !objectKey || !accessId || !policyText || !signature) throw new Error('DreamAPI upload policy is incomplete'); + const bytes = await readFile(filePath); + const form = new FormData(); + form.set('key', objectKey); + form.set('policy', policyText); + form.set('OSSAccessKeyId', accessId); + form.set('signature', signature); + form.set('callback', String(policy.callback ?? '')); + form.set('success_action_status', '200'); + form.set('file', new Blob([bytes]), fileName); + const uploaded = await fetchImpl(host, { method: 'POST', body: form }); + if (!uploaded.ok) throw new Error(`DreamAPI upload failed (${uploaded.status})`); + const uploadedJson = await uploaded.json() as { data?: { reqId?: string } }; + const reqId = uploadedJson.data?.reqId || String(policy.reqId ?? ''); + if (!reqId) throw new Error('DreamAPI upload did not return reqId'); + const finished = await dreamApiJson(fetchImpl, key, '/api/file/v1/policy_upload_finish', { reqId }); + const url = String(finished.url ?? ''); + if (!url) throw new Error('DreamAPI upload did not return a URL'); + return url; +} + +async function resolveMediaValue(fetchImpl: FetchImpl, key: string, value: string): Promise { + const file = await localFile(value); + if (!file) return value; + return uploadLocal(fetchImpl, key, file.path, file.name || `upload${extname(file.path)}`); +} + +export async function prepareDreamApiCall(modelId: string | undefined, input: Record, key: string, fetchImpl: FetchImpl): Promise { + const draft = buildDreamApiCall(modelId, input); + const resolved: Record = { ...input }; + for (const field of draft.model.fields) { + const value = input[field.key]; + if (field.kind === 'media' && typeof value === 'string') { + resolved[field.key] = await resolveMediaValue(fetchImpl, key, value); + } + if (field.kind === 'mediaList' && Array.isArray(value)) { + const urls: string[] = []; + for (const item of value) { + if (typeof item !== 'string') continue; + urls.push(await resolveMediaValue(fetchImpl, key, item)); + } + resolved[field.key] = urls; + } + } + return buildDreamApiCall(modelId, resolved); +} + +function collectUrls(data: Record): { urls: string[]; cloneId?: string } { + const urls: string[] = []; + const push = (value: unknown, key: string) => { + if (!Array.isArray(value)) return; + for (const item of value) { + if (item && typeof item === 'object' && typeof (item as Record)[key] === 'string') { + urls.push((item as Record)[key]); + } + } + }; + push(data.images, 'imageUrl'); + push(data.videos, 'videoUrl'); + push(data.audios, 'audioUrl'); + const matting = data.mattingResult; + if (matting && typeof matting === 'object') { + const record = matting as Record; + for (const key of ['mattingFileUrl', 'mattingAlphaFileUrl', 'compositeFileUrl']) { + if (typeof record[key] === 'string') urls.push(record[key]); + } + } + const cloneId = typeof data.cloneId === 'string' ? data.cloneId : undefined; + return { urls: [...new Set(urls)], cloneId }; +} + +export async function runDreamApi(call: DreamApiCall, options: DreamApiRunOptions): Promise { + const fetchImpl = options.fetchImpl ?? fetch; + const key = options.apiKey; + let taskId = options.existingTaskId?.startsWith('dreamapi:') ? options.existingTaskId.slice('dreamapi:'.length) : ''; + if (!taskId) { + const submitted = await dreamApiJson(fetchImpl, key, call.model.endpoint, call.body); + taskId = String(submitted.taskId ?? ''); + if (!taskId) throw new Error('DreamAPI did not return a taskId'); + if (options.registerTask) await options.registerTask(`dreamapi:${taskId}`); + } + const deadline = Date.now() + 15 * 60_000; + while (Date.now() < deadline) { + const data = await dreamApiJson(fetchImpl, key, '/api/getAsyncResult', { taskId }); + const task = data.task as { status?: number; reason?: string } | undefined; + const status = task?.status; + if (status === 3) { + const found = collectUrls(data); + if (!found.urls.length && !found.cloneId) throw new Error('DreamAPI succeeded without a media URL'); + return { taskId, ...found }; + } + if (status === 4) throw new Error(task?.reason || 'DreamAPI generation failed'); + await new Promise((resolve) => setTimeout(resolve, options.pollIntervalMs ?? 3_000)); + } + throw new Error('DreamAPI generation timed out'); +} + +export async function generateDreamApi(modelId: string | undefined, input: Record, options: Omit & { apiKey?: string }): Promise { + const key = options.apiKey ?? apiKey(); + const fetchImpl = options.fetchImpl ?? fetch; + if (options.existingTaskId?.startsWith('dreamapi:')) { + const model = buildDreamApiCall(modelId, input).model; + return runDreamApi({ model, body: {} }, { ...options, apiKey: key, fetchImpl }); + } + const call = await prepareDreamApiCall(modelId, input, key, fetchImpl); + return runDreamApi(call, { ...options, apiKey: key, fetchImpl }); +} + +export function dreamApiMediaInput(model: DreamApiModel, input: Record): Record { + const next: Record = {}; + for (const field of model.fields) { + if (input[field.key] !== undefined) next[field.key] = input[field.key]; + } + return next; +} diff --git a/server/plugins/image.ts b/server/plugins/image.ts index 87c0051d1..14da73c65 100644 --- a/server/plugins/image.ts +++ b/server/plugins/image.ts @@ -1,4 +1,6 @@ import { generateFalCatalogImage } from './fal-client.ts'; +import { buildDreamApiCall } from '../../shared/dreamapi-models.ts'; +import { generateDreamApi } from './dreamapi-provider.ts'; import { getKey } from '../keystore.ts'; import { buildFalCatalogImageRequest, type FalCatalogInput } from './fal-catalog-input.ts'; import { proxyDispatcher } from '../outbound-proxy.ts'; @@ -60,6 +62,8 @@ interface ImagePluginOptions { interface ImageRequest { model?: string; falModel?: string; + dreamapiModel?: string; + dreamapi?: Record; prompt?: string; aspectRatio?: string; imageSize?: string; @@ -80,9 +84,12 @@ interface ImageRequest { } export interface ValidImageRequest { - model: 'gpt-image-2' | 'nano-banana' | 'image-01' | 'wavespeed' | 'byteplus' | 'grok-imagine' | 'fal'; + model: 'gpt-image-2' | 'nano-banana' | 'image-01' | 'wavespeed' | 'byteplus' | 'grok-imagine' | 'fal' | 'dreamapi'; falModel?: string; falInput?: FalCatalogInput; + dreamapiModel?: string; + dreamapi?: Record; + dreamapiCall?: ReturnType; prompt: string; aspectRatio?: string; imageSize: string; @@ -164,6 +171,15 @@ export function validateImageRequest(input: ImageRequest): ValidImageRequest { referencePaths: input.referencePaths ?? [], aspectRatio: input.aspectRatio ?? '16:9', imageSize: input.imageSize ?? '1K', quality: 'high', outputFormat: 'png' }; } + if (model === 'dreamapi') { + const dreamapiCall = buildDreamApiCall(input.dreamapiModel, input.dreamapi ?? {}); + if (dreamapiCall.model.kind !== 'image') throw new Error(`${dreamapiCall.model.id} is not a DreamAPI image model`); + return { + model, dreamapiModel: dreamapiCall.model.id, dreamapi: input.dreamapi ?? {}, dreamapiCall, prompt: String(input.prompt ?? input.dreamapi?.prompt ?? input.dreamapi?.text ?? ''), + count: 1, referencePaths: input.referencePaths ?? [], aspectRatio: input.aspectRatio ?? '16:9', + imageSize: input.imageSize ?? '1K', quality: 'high', outputFormat: 'png', + }; + } if (model !== 'gpt-image-2' && model !== 'nano-banana' && model !== 'image-01' && model !== 'wavespeed' && model !== 'byteplus' && model !== 'grok-imagine') { throw new Error(`unsupported model ${model}`); } @@ -385,6 +401,7 @@ export function imageGenerationPlugin(options: ImagePluginOptions): Plugin { try { const raw = await readJson(req); if (raw.model === 'fal' && !raw.falModel) raw.falModel = getKey('FAL_IMAGE_MODEL').trim() || undefined; + if (raw.model === 'dreamapi' && !raw.dreamapiModel) raw.dreamapiModel = getKey('DREAMAPI_IMAGE_MODEL').trim() || undefined; const input = validateImageRequest(raw); const { model, prompt, aspectRatio, imageSize, quality, count, referencePaths, maskPath, @@ -393,10 +410,14 @@ export function imageGenerationPlugin(options: ImagePluginOptions): Plugin { } = input; const [width, height] = input.width != null && input.height != null ? [input.width, input.height] - : model === 'fal' ? [0, 0] : dimensions(aspectRatio!, imageSize); + : model === 'fal' || model === 'dreamapi' ? [0, 0] : dimensions(aspectRatio!, imageSize); let images: ProviderImage[]; if (model === 'fal') { images = await generateFalCatalogImage(input.falInput!); + } else if (model === 'dreamapi') { + const generated = await generateDreamApi(input.dreamapiModel, input.dreamapi ?? {}, {}); + if (!generated.urls.length) throw new Error('DreamAPI image generation returned no URL'); + images = generated.urls.map((url) => ({ url })); } else if (model === 'nano-banana') { if (!options.geminiApiKey) throw new Error('Nano Banana is not configured. Set GEMINI_API_KEY in .env.local.'); if (!aspectRatio) throw new Error('Nano Banana requires aspectRatio'); @@ -434,7 +455,7 @@ export function imageGenerationPlugin(options: ImagePluginOptions): Plugin { } const paths = await Promise.all(images.map((image) => saveImage(image, model === 'gpt-image-2' ? outputFormat : 'png'))); const reportDimensions = model !== 'image-01' || input.width != null; - const imageDimensions = model === 'fal' + const imageDimensions = model === 'fal' || model === 'dreamapi' ? (images[0].width && images[0].height ? { width: images[0].width, height: images[0].height } : {}) : reportDimensions ? { width, height } : {}; sendJson(res, 200, { paths, ...imageDimensions }); diff --git a/server/plugins/video-validation.ts b/server/plugins/video-validation.ts index 32fd2f6de..adcbf666c 100644 --- a/server/plugins/video-validation.ts +++ b/server/plugins/video-validation.ts @@ -1,11 +1,14 @@ import { buildFalCatalogVideoRequest } from './fal-catalog-input.ts'; +import { buildDreamApiCall } from '../../shared/dreamapi-models.ts'; export interface MultiPrompt { prompt: string; duration: number | string; index: number } export type VideoResolution = '360p' | '480p' | '512p' | '540p' | '720p' | '768p' | '1080p' | '480P' | '768P' | '1080P' | '4k'; export type KlingVideoReferType = 'feature' | 'base'; export interface VideoRequest { operationId?: string; - model?: 'seedance2' | 'kling' | 'hailuo' | 'byteplus' | 'grok-imagine-video' | 'ofox' | 'fal'; + model?: 'seedance2' | 'kling' | 'hailuo' | 'byteplus' | 'grok-imagine-video' | 'ofox' | 'fal' | 'dreamapi'; + dreamapiModel?: string; + dreamapi?: Record; /** Explicit curated Fal model ID; omitted uses the saved Fal default. */ falModel?: string; prompt?: string; @@ -37,7 +40,9 @@ export interface VideoRequest { } export interface ValidVideoRequest extends Omit { - model: 'seedance2' | 'kling' | 'hailuo' | 'byteplus' | 'grok-imagine-video' | 'ofox' | 'fal'; + model: 'seedance2' | 'kling' | 'hailuo' | 'byteplus' | 'grok-imagine-video' | 'ofox' | 'fal' | 'dreamapi'; + dreamapiModel?: string; + dreamapi?: Record; /** Explicit curated Fal model ID; omitted uses the saved Fal default. */ falModel?: string; prompt: string; @@ -237,6 +242,18 @@ function validateOfox(input: ValidVideoRequest): ValidVideoRequest { } export function validateVideoRequest(input: VideoRequest): ValidVideoRequest { + if (input.model === 'dreamapi') { + const call = buildDreamApiCall(input.dreamapiModel, input.dreamapi ?? {}); + if (call.model.kind !== 'video') throw new Error(`${call.model.id} is not a DreamAPI video model`); + return { + ...input, model: 'dreamapi', dreamapiModel: call.model.id, dreamapi: input.dreamapi ?? {}, + prompt: String(input.prompt ?? input.dreamapi?.prompt ?? ''), + durationSeconds: typeof input.dreamapi?.duration === 'number' ? input.dreamapi.duration : 5, + durationSpecified: input.dreamapi?.duration !== undefined, + ratio: String(input.dreamapi?.ratio ?? input.dreamapi?.aspectRatio ?? '16:9'), + refImagePaths: [], refVideoPaths: [], refAudioPaths: [], + }; + } if (input.model === 'fal') { if (!input.falModel?.trim()) throw new Error('Choose a Fal video model in Settings or specify falModel'); for (const key of ['mode', 'refVideoMode', 'promptOptimizer', 'fastPretreatment', 'seed', 'cameraFixed', 'watermark', 'returnLastFrame', 'executionExpiresAfter', 'priority', 'multiPrompts', 'shotType'] as const) { @@ -247,7 +264,7 @@ export function validateVideoRequest(input: VideoRequest): ValidVideoRequest { return normalized; } if (input.model !== 'seedance2' && input.model !== 'kling' && input.model !== 'hailuo' && input.model !== 'byteplus' && input.model !== 'grok-imagine-video' && input.model !== 'ofox') { - throw new Error('model must be seedance2, kling, hailuo, byteplus, grok-imagine-video, ofox, or fal'); + throw new Error('model must be seedance2, kling, hailuo, byteplus, grok-imagine-video, ofox, fal, or dreamapi'); } if (input.model === 'hailuo' && input.ratio !== undefined) throw new Error('hailuo does not accept ratio; framing follows the first frame when present'); const normalized = common(input, input.model); diff --git a/server/plugins/video.ts b/server/plugins/video.ts index 8cba55422..483693871 100644 --- a/server/plugins/video.ts +++ b/server/plugins/video.ts @@ -1,4 +1,5 @@ import { generateFalVideo } from './fal-client.ts'; +import { generateDreamApi } from './dreamapi-provider.ts'; import { getKey } from '../keystore.ts'; import { proxyDispatcher } from '../outbound-proxy.ts'; import type { IncomingMessage, ServerResponse } from 'node:http'; @@ -18,6 +19,7 @@ import { materializeVideoReferences, mediaDataUrl, providerMediaUrl, + saveVideo, ServerReferencePreflightError, } from './video-media.ts'; import { generateGrokVideo } from './grok-video-provider.ts'; @@ -342,6 +344,28 @@ async function runVideoOperation( const checkpoint = generationResultCheckpoint(storedResultUrls, expectedResultCount, providerTaskId); let urls = checkpoint.urls; if (!checkpoint.complete) { + if (providerTaskId?.startsWith('dreamapi:') || (!providerTaskId && input.model === 'dreamapi')) { + const generated = await generateDreamApi(input.dreamapiModel, input.dreamapi ?? {}, { + existingTaskId: providerTaskId, + registerTask: (taskId) => registerProviderTask('dreamapi', taskId), + }); + if (!generated.urls.length) throw new Error('DreamAPI video generation returned no URL'); + const downloadDream = async () => { + const items = []; + for (const [index, url] of generated.urls.entries()) { + const saved = await saveVideo(url); + items.push({ + assetId: index === 0 ? operationId : `${operationId}:${index}`, + kind: 'video' as const, + name: index === 0 ? name : `${name} · ${index + 1}`, + ...saved, + }); + } + return items.length === 1 ? items[0] : items; + }; + for (const [index, url] of generated.urls.entries()) await registerDownload(url, downloadDream, index); + return downloadDream(); + } if (providerTaskId?.startsWith('fal:') || (!providerTaskId && input.model === 'fal')) { urls = requireGenerationResultUrls([await generateFalVideo(input, registerProviderTask, providerTaskId)], expectedResultCount); } else if (input.model === 'seedance2' || input.model === 'byteplus') { @@ -377,7 +401,7 @@ async function runVideoOperation( return download(); } export function videoGenerationPlugin(options: VideoOptions): Plugin { - for (const provider of ['seedance2', 'kling', 'hailuo', 'byteplus', 'grok-imagine-video', 'ofox', 'fal'] as const) { + for (const provider of ['seedance2', 'kling', 'hailuo', 'byteplus', 'grok-imagine-video', 'ofox', 'fal', 'dreamapi'] as const) { registerGenerationJobResumer('submit_video', provider, async ( snapshot: GenerationJobSnapshot, _update, @@ -408,6 +432,7 @@ export function videoGenerationPlugin(options: VideoOptions): Plugin { try { const raw = await readJson(req); if (raw.model === 'fal' && !raw.falModel) raw.falModel = getKey('FAL_VIDEO_MODEL').trim() || undefined; + if (raw.model === 'dreamapi' && !raw.dreamapiModel) raw.dreamapiModel = getKey('DREAMAPI_VIDEO_MODEL').trim() || undefined; const input = validateVideoRequest(await materializeVideoReferences(raw)); // Persist the resolved choice so settings changes cannot redirect a retry. if (input.model === 'fal') raw.falModel = input.falModel; diff --git a/server/plugins/voice-types.ts b/server/plugins/voice-types.ts index fad4a2009..8c5677ec8 100644 --- a/server/plugins/voice-types.ts +++ b/server/plugins/voice-types.ts @@ -2,7 +2,7 @@ import type { MinimaxLanguageBoost } from '../../shared/media-provider-params.ts export type VoiceProvider = | 'elevenlabs' | 'doubao' | 'minimax' | 'inworld' | 'fishaudio' | 'speechify' - | 'openai' | 'gemini' | 'mistral' | 'cartesia'; + | 'openai' | 'gemini' | 'mistral' | 'cartesia' | 'dreamapi'; export type MinimaxVoiceFormat = 'mp3' | 'pcm' | 'flac' | 'wav' | 'pcmu_raw' | 'pcmu_wav' | 'opus'; export type VoiceEffect = 'spacious_echo' | 'auditorium_echo' | 'lofi_telephone' | 'robotic'; @@ -46,6 +46,8 @@ export interface VoiceOptions { export interface VoiceRequest { provider?: string; + dreamapiModel?: string; + dreamapi?: Record; text?: string; voiceId?: string; modelId?: string; diff --git a/server/plugins/voice-validation.ts b/server/plugins/voice-validation.ts index 837c50fbb..7940d67ff 100644 --- a/server/plugins/voice-validation.ts +++ b/server/plugins/voice-validation.ts @@ -1,4 +1,5 @@ import type { ValidVoiceRequest, VoiceRequest } from './voice-types.ts'; +import { buildDreamApiCall } from '../../shared/dreamapi-models.ts'; import { MINIMAX_LANGUAGE_BOOSTS } from '../../shared/media-provider-params.ts'; const EMOTIONS = new Set(['happy', 'sad', 'angry', 'fearful', 'disgusted', 'surprised', 'calm', 'fluent', 'whisper']); @@ -127,6 +128,16 @@ function validateAiProvider(input: VoiceRequest, provider: 'openai' | 'gemini' | export function validateVoiceRequest(input: VoiceRequest): ValidVoiceRequest { const provider = input.provider; + if (provider === 'dreamapi') { + const call = buildDreamApiCall(input.dreamapiModel, input.dreamapi ?? {}); + if (call.model.kind !== 'voice') throw new Error(`${call.model.id} is not a DreamAPI voice model`); + const text = String(input.dreamapi?.text ?? input.text ?? '').trim() || 'DreamAPI'; + const voiceId = String(input.dreamapi?.audioId ?? input.dreamapi?.cloneId ?? input.voiceId ?? 'dreamapi'); + return { + ...input, provider, dreamapiModel: call.model.id, dreamapi: input.dreamapi ?? {}, + text, voiceId, outputFormat: 'mp3', sampleRate: 24_000, audioFormat: 'mp3', channel: 1, + }; + } if (provider !== 'elevenlabs' && provider !== 'doubao' && provider !== 'minimax' && provider !== 'inworld' && provider !== 'fishaudio' && provider !== 'speechify' && !isAiProvider(provider)) { throw new Error('unsupported voice provider'); diff --git a/server/plugins/voice.ts b/server/plugins/voice.ts index d294bea51..0bb00289a 100644 --- a/server/plugins/voice.ts +++ b/server/plugins/voice.ts @@ -6,6 +6,8 @@ import { saveVoiceAudio, saveVoiceSubtitle } from './voice-media.ts'; import { doubaoVoice, elevenLabsVoice, fishAudioVoice, inworldVoice, minimaxVoice, speechifyVoice } from './voice-providers.ts'; import type { VoiceOptions, VoiceProvider, VoiceRequest } from './voice-types.ts'; import { validateVoiceRequest } from './voice-validation.ts'; +import { generateDreamApi } from './dreamapi-provider.ts'; +import { getKey } from '../keystore.ts'; export { validateVoiceRequest }; @@ -43,7 +45,22 @@ export function voiceGenerationPlugin(options: VoiceOptions): Plugin { server.middlewares.use('/generate/voice', async (req, res) => { if (req.method !== 'POST') { sendJson(res, 405, { error: 'method not allowed — use POST' }); return; } try { - const input = validateVoiceRequest(await readJson(req)); + const raw = await readJson(req); + if (raw.provider === 'dreamapi' && !raw.dreamapiModel) raw.dreamapiModel = getKey('DREAMAPI_VOICE_MODEL').trim() || undefined; + const input = validateVoiceRequest(raw); + if (input.provider === 'dreamapi') { + const generated = await generateDreamApi(input.dreamapiModel, input.dreamapi ?? {}, {}); + if (generated.cloneId && generated.urls.length === 0) { + sendJson(res, 200, { cloneId: generated.cloneId }); + return; + } + if (!generated.urls[0]) throw new Error('DreamAPI voice generation returned no audio URL'); + const audioResponse = await fetch(generated.urls[0]); + if (!audioResponse.ok) throw new Error(`DreamAPI audio download failed (${audioResponse.status})`); + const saved = await saveVoiceAudio(Buffer.from(await audioResponse.arrayBuffer()), 'mp3', 24_000); + sendJson(res, 200, { ...saved, cloneId: generated.cloneId }); + return; + } if (isAiVoiceProvider(input.provider)) { const audio = await generateAiVoice(options, input); const saved = await saveVoiceAudio(audio.bytes, audio.codec, audio.sampleRate); diff --git a/shared/dreamapi-models.ts b/shared/dreamapi-models.ts new file mode 100644 index 000000000..9f0024c14 --- /dev/null +++ b/shared/dreamapi-models.ts @@ -0,0 +1,386 @@ +// DreamAPI catalog. Endpoints and limits follow dreamfaceapp/skill scripts. +// A model id is the only way to reach an endpoint; unknown ids never receive the key. + +export type DreamApiKind = 'image' | 'video' | 'voice'; +export type DreamApiFieldKind = 'string' | 'number' | 'boolean' | 'strings' | 'media' | 'mediaList'; + +export interface DreamApiField { + key: string; + json?: string; + kind: DreamApiFieldKind; + required?: boolean; + choices?: readonly string[]; + min?: number; + max?: number; + maxItems?: number; + maxLength?: number; + multipleOf?: number; +} + +export interface DreamApiModel { + id: string; + label: string; + kind: DreamApiKind; + endpoint: string; + note: string; + fields: readonly DreamApiField[]; + constants?: Record; + shape?: 'outpaint' | 'lipsync' | 'lipsync2'; +} + +const prompt = (max = 4000): DreamApiField => ({ key: 'prompt', kind: 'string', required: true, maxLength: max }); +const media = (key: string, json?: string, required = false): DreamApiField => ({ key, json, kind: 'media', required }); +const mediaList = (key: string, json: string | undefined, maxItems: number, required = false): DreamApiField => ({ + key, json, kind: 'mediaList', maxItems, required, +}); + +const SEEDANCE_SHARED: DreamApiField[] = [ + mediaList('images', 'images', 9), + media('imageUrl', 'imageUrl'), + media('endImageUrl', 'endImageUrl'), + { key: 'ratio', kind: 'string' }, + { key: 'seed', kind: 'number' }, + { key: 'generateAudio', kind: 'boolean' }, +]; + +export const DREAMAPI_MODELS: readonly DreamApiModel[] = [ + { + id: 'dreamvideo-3-text', label: 'DreamAPI · DreamVideo 3.0 文生视频', kind: 'video', + endpoint: '/api/async/dreamvideo_3.0/text_to_video', note: '时长 3–15 秒，分辨率 480P/720P。', + fields: [prompt(), { key: 'duration', kind: 'number', min: 3, max: 15 }, { key: 'resolution', kind: 'string', choices: ['480P', '720P'] }, { key: 'aspectRatio', kind: 'string', choices: ['auto', '9:16', '3:4', '1:1', '4:3', '16:9'] }, { key: 'seed', kind: 'number' }], + }, + { + id: 'dreamvideo-3-image', label: 'DreamAPI · DreamVideo 3.0 图生视频', kind: 'video', + endpoint: '/api/async/dreamvideo_3.0/image_to_video', note: '1 张图为图生视频，2 张图为首尾帧。', + fields: [prompt(), mediaList('images', 'images', 9, true), { key: 'duration', kind: 'number', min: 3, max: 15 }, { key: 'resolution', kind: 'string', choices: ['480P', '720P'] }, { key: 'aspectRatio', kind: 'string', choices: ['auto', '9:16', '3:4', '1:1', '4:3', '16:9'] }, { key: 'seed', kind: 'number' }], + }, + { + id: 'wan-2.1-text', label: 'DreamAPI · Wan2.1 文生视频', kind: 'video', + endpoint: '/api/async/wan/text_to_video/2.1', note: '提示词最长 1500 字。', + fields: [prompt(1500), { key: 'resolution', kind: 'string', choices: ['480p', '720p'] }], + }, + { + id: 'wan-2.1-image', label: 'DreamAPI · Wan2.1 图生视频', kind: 'video', + endpoint: '/api/async/wan/image_to_video/2.1', note: '需要一张参考图。', + fields: [prompt(1500), media('image', 'image', true), { key: 'resolution', kind: 'string', choices: ['480p', '720p'] }], + }, + { + id: 'wan-2.1-head-tail', label: 'DreamAPI · Wan2.1 首尾帧', kind: 'video', + endpoint: '/api/async/wan/head_tail_to_video/2.1', note: '首帧和尾帧都必填。', + fields: [prompt(1500), media('first', 'firstImage', true), media('last', 'lastImage', true)], + }, + { + id: 'seedance-2.5', label: 'DreamAPI · Seedance 2.5', kind: 'video', + endpoint: '/api/async/seedance_2.5', note: '时长 4–30 秒。带参考视频时可指定 reference/edit/extend。', + constants: { model: 'seedance-2.5' }, + fields: [prompt(1500), { key: 'resolution', kind: 'string', required: true, choices: ['480p', '720p', '1080p'] }, { key: 'duration', kind: 'number', required: true, min: 4, max: 30 }, ...SEEDANCE_SHARED, mediaList('videos', 'videos', 10), mediaList('audios', 'audios', 3), { key: 'omniReferenceTaskType', kind: 'string', choices: ['reference', 'edit', 'extend'] }], + }, + { + id: 'seedance-2.0', label: 'DreamAPI · Seedance 2.0', kind: 'video', + endpoint: '/api/async/seedance_2.0', note: '时长 4–15 秒，分辨率可到 4k。', + constants: { model: 'seedance-2.0' }, + fields: [prompt(1500), { key: 'resolution', kind: 'string', required: true, choices: ['480p', '720p', '1080p', '4k'] }, { key: 'duration', kind: 'number', required: true, min: 4, max: 15 }, ...SEEDANCE_SHARED, mediaList('videos', 'videos', 3), mediaList('audios', 'audios', 3)], + }, + { + id: 'seedance-2.0-mini', label: 'DreamAPI · Seedance 2.0 Mini', kind: 'video', + endpoint: '/api/async/seedance_2.0', note: '时长 4–15 秒，分辨率只有 480p/720p。', + constants: { model: 'seedance-2.0-mini' }, + fields: [prompt(1500), { key: 'resolution', kind: 'string', required: true, choices: ['480p', '720p'] }, { key: 'duration', kind: 'number', required: true, min: 4, max: 15 }, ...SEEDANCE_SHARED, mediaList('videos', 'videos', 3), mediaList('audios', 'audios', 3)], + }, + { + id: 'dreamimage-2', label: 'DreamAPI · DreamImage 2.0', kind: 'image', + endpoint: '/api/async/dreamimage_2.0', note: '必须提供输入图和提示词。', + fields: [prompt(), media('image', 'image', true)], + }, + { + id: 'flux-text', label: 'DreamAPI · Flux 文生图', kind: 'image', + endpoint: '/api/async/flux_text2image', note: '宽高为 16 的倍数，最大 1600。', + fields: [prompt(), { key: 'width', kind: 'number', min: 16, max: 1600, multipleOf: 16 }, { key: 'height', kind: 'number', min: 16, max: 1600, multipleOf: 16 }, { key: 'num', kind: 'number', min: 1, max: 10 }, { key: 'seed', kind: 'number' }], + }, + { + id: 'flux-image', label: 'DreamAPI · Flux 图生图', kind: 'image', + endpoint: '/api/async/flux_image2image', note: '限 1 张参考图。', + fields: [prompt(), media('image', 'imageUrl', true), { key: 'width', kind: 'number', min: 16, max: 1600, multipleOf: 16 }, { key: 'height', kind: 'number', min: 16, max: 1600, multipleOf: 16 }, { key: 'num', kind: 'number', min: 1, max: 10 }, { key: 'seed', kind: 'number' }], + }, + { + id: 'seedream', label: 'DreamAPI · Seedream', kind: 'image', + endpoint: '/api/async/seedream', note: '用 seedreamModel 选择 4.0 / 4.5 / 5.0 Lite / 5.0 Pro。', + fields: [{ key: 'seedreamModel', json: 'model', kind: 'string', choices: ['seedream-5.0-pro', 'seedream-5.0-lite', 'seedream-4.5', 'seedream-4.0'] }, prompt(), mediaList('images', 'image', 10), { key: 'size', kind: 'string' }, { key: 'seed', kind: 'number' }], + }, + { + id: 'nano-banana-2', label: 'DreamAPI · Nano Banana 2', kind: 'image', + endpoint: '/api/async/google_gemini_image', note: '分辨率 512/1K/2K/4K。', + constants: { model: 'gemini-3.1-flash-image-preview' }, + fields: [{ key: 'text', kind: 'string', required: true }, { key: 'imageSize', kind: 'string', choices: ['512', '1K', '2K', '4K'] }, { key: 'aspectRatio', kind: 'string' }, mediaList('images', 'images', 14)], + }, + { + id: 'nano-banana-pro', label: 'DreamAPI · Nano Banana Pro', kind: 'image', + endpoint: '/api/async/google_gemini_image', note: '分辨率只有 1K/2K。', + constants: { model: 'gemini-3-pro-image-preview' }, + fields: [{ key: 'text', kind: 'string', required: true }, { key: 'imageSize', kind: 'string', choices: ['1K', '2K'] }, { key: 'aspectRatio', kind: 'string' }, mediaList('images', 'images', 14)], + }, + { + id: 'gpt-image-2', label: 'DreamAPI · GPT Image 2', kind: 'image', + endpoint: '/api/async/gpt_image', note: '参考图最多 4 张。', + fields: [prompt(), { key: 'width', kind: 'number', min: 16, max: 3840, multipleOf: 16 }, { key: 'height', kind: 'number', min: 16, max: 3840, multipleOf: 16 }, { key: 'quality', kind: 'string', choices: ['low', 'medium', 'high'] }, { key: 'n', kind: 'number', min: 1, max: 10 }, mediaList('images', 'images', 4)], + }, + { + id: 'gpt-image-2.5-flare-text', label: 'DreamAPI · GPT Image 2.5 Flare 文生图', kind: 'image', + endpoint: '/api/async/gpt_image_2.5/flare/text_to_image', note: '质量含 xhigh 与 max。', + fields: [prompt(), { key: 'width', kind: 'number', min: 16, max: 3840, multipleOf: 16 }, { key: 'height', kind: 'number', min: 16, max: 3840, multipleOf: 16 }, { key: 'quality', kind: 'string', choices: ['low', 'medium', 'high', 'xhigh', 'max'] }, { key: 'n', kind: 'number', min: 1, max: 10 }], + }, + { + id: 'gpt-image-2.5-flare-image', label: 'DreamAPI · GPT Image 2.5 Flare 图生图', kind: 'image', + endpoint: '/api/async/gpt_image_2.5/flare/image_to_image', note: '参考图 1–4 张。', + fields: [prompt(), { key: 'width', kind: 'number', min: 16, max: 3840, multipleOf: 16 }, { key: 'height', kind: 'number', min: 16, max: 3840, multipleOf: 16 }, { key: 'quality', kind: 'string', choices: ['low', 'medium', 'high', 'xhigh', 'max'] }, { key: 'n', kind: 'number', min: 1, max: 10 }, mediaList('images', 'images', 4, true)], + }, + { + id: 'gpt-image-2.5-sunburst-text', label: 'DreamAPI · GPT Image 2.5 Sunburst 文生图', kind: 'image', + endpoint: '/api/async/gpt_image_2.5/sunburst/text_to_image', note: '与 Flare 文生图同一套限制。', + fields: [prompt(), { key: 'width', kind: 'number', min: 16, max: 3840, multipleOf: 16 }, { key: 'height', kind: 'number', min: 16, max: 3840, multipleOf: 16 }, { key: 'quality', kind: 'string', choices: ['low', 'medium', 'high', 'xhigh', 'max'] }, { key: 'n', kind: 'number', min: 1, max: 10 }], + }, + { + id: 'gpt-image-2.5-sunburst-image', label: 'DreamAPI · GPT Image 2.5 Sunburst 图生图', kind: 'image', + endpoint: '/api/async/gpt_image_2.5/sunburst/image_to_image', note: '参考图 1–4 张。', + fields: [prompt(), { key: 'width', kind: 'number', min: 16, max: 3840, multipleOf: 16 }, { key: 'height', kind: 'number', min: 16, max: 3840, multipleOf: 16 }, { key: 'quality', kind: 'string', choices: ['low', 'medium', 'high', 'xhigh', 'max'] }, { key: 'n', kind: 'number', min: 1, max: 10 }, mediaList('images', 'images', 4, true)], + }, + { + id: 'colorize', label: 'DreamAPI · 黑白上色', kind: 'image', + endpoint: '/api/async/colorize', note: '一张含人脸的图片。', + fields: [media('url', 'url', true)], + }, + { + id: 'enhance', label: 'DreamAPI · 图像增强', kind: 'image', + endpoint: '/api/async/enhance', note: '一张输入图。', + fields: [media('image', 'imageUrl', true)], + }, + { + id: 'outpainting', label: 'DreamAPI · 扩图', kind: 'image', + endpoint: '/api/async/outpainting', note: '按像素向四周扩展。', shape: 'outpaint', + fields: [media('url', 'url', true), { key: 'left', kind: 'number', min: 0 }, { key: 'right', kind: 'number', min: 0 }, { key: 'top', kind: 'number', min: 0 }, { key: 'bottom', kind: 'number', min: 0 }], + }, + { + id: 'swap-face', label: 'DreamAPI · 图片换脸', kind: 'image', + endpoint: '/api/async/swap_face', note: '目标图加人脸图。', + fields: [media('url', 'url', true), media('face', 'faceImgUrl', true)], + }, + { + id: 'remove-background', label: 'DreamAPI · 去背景', kind: 'image', + endpoint: '/api/async/remove_background', note: '一张输入图。', + fields: [media('url', 'url', true)], + }, + { + id: 'try-on', label: 'DreamAPI · 虚拟试衣', kind: 'image', + endpoint: '/api/async/tryon_clothes', note: '模特图必填，上装、下装、连体衣至少一张。', + fields: [media('modelImage', 'modelImage', true), media('upper', 'upperClothes'), media('lower', 'lowerClothes'), media('overall', 'overallClothes'), { key: 'prompt', kind: 'string' }, { key: 'enableShoes', kind: 'boolean' }, { key: 'width', kind: 'number', min: 1 }, { key: 'height', kind: 'number', min: 1 }], + }, + { + id: 'lipsync', label: 'DreamAPI · LipSync', kind: 'video', + endpoint: '/api/async/lipsync', note: '原视频和驱动音频。', shape: 'lipsync', + fields: [media('srcVideo', 'srcVideoUrl', true), media('audio', 'audioUrl', true), { key: 'videoWidth', kind: 'number', min: 0 }, { key: 'videoHeight', kind: 'number', min: 0 }, { key: 'videoEnhance', kind: 'boolean' }], + }, + { + id: 'lipsync-2', label: 'DreamAPI · LipSync 2.0', kind: 'video', + endpoint: '/api/async/lipsync/2.0', note: '可另传干声音频。设置里的默认视频模型建议用这个。', shape: 'lipsync2', + fields: [media('srcVideo', 'srcVideoUrl', true), media('audio', 'audioUrl', true), media('vocalAudio', 'vocalAudioUrl'), { key: 'videoWidth', kind: 'number', min: 0 }, { key: 'videoHeight', kind: 'number', min: 0 }, { key: 'videoEnhance', kind: 'number', min: 0, max: 1 }, { key: 'fps', kind: 'string' }], + }, + { + id: 'dreamavatar-3-fast', label: 'DreamAPI · DreamAvatar 3.0 Fast', kind: 'video', + endpoint: '/api/async/dreamavatar/image_to_video/3.0fast', note: '人像、音频和提示词。音频请控制在 3 分钟内。', + fields: [media('image', 'image', true), media('audio', 'audio', true), prompt(), { key: 'resolution', kind: 'string', choices: ['480p', '720p'] }], + }, + { + id: 'dreamact', label: 'DreamAPI · Dreamact', kind: 'video', + endpoint: '/api/async/wan/dreamact/2.1', note: '驱动视频最长 1 分钟。', + fields: [media('video', 'video', true), mediaList('images', 'images', 8, true), { key: 'seed', kind: 'number' }, { key: 'resolution', kind: 'string', choices: ['480P', '720P'] }, { key: 'taskMode', kind: 'string', choices: ['replace_body', 'pose2v'] }, { key: 'qualityMode', kind: 'string', choices: ['high', 'base'] }], + }, + { + id: 'video-translate-2', label: 'DreamAPI · 视频翻译 2.0', kind: 'video', + endpoint: '/api/async/video_translate/2.0', note: '语言只允许 en、zh、es。', + fields: [media('video', 'videoUrl', true), { key: 'sourceLanguage', kind: 'string', required: true, choices: ['en', 'zh', 'es'] }, { key: 'targetLanguage', kind: 'string', required: true, choices: ['en', 'zh', 'es'] }, { key: 'enableLipSync', kind: 'boolean' }], + }, + { + id: 'swap-face-video', label: 'DreamAPI · 视频换脸', kind: 'video', + endpoint: '/api/async/swap_face_for_video', note: '原视频请控制在 5 分钟内。', + fields: [media('srcVideo', 'srcVideoUrl', true), media('face', 'srcImgUrl', true)], + }, + { + id: 'video-matting', label: 'DreamAPI · 视频抠像', kind: 'video', + endpoint: '/api/async/image_matting_process_video', note: '只接受 MP4。成片和 alpha 都会返回。', + fields: [media('srcFile', 'srcFileUrl', true), { key: 'bitRate', kind: 'string' }, { key: 'removeColorSpill', kind: 'boolean' }], + }, + { + id: 'matting-composite', label: 'DreamAPI · 抠像合成', kind: 'video', + endpoint: '/api/async/image_matting_composite_video', note: '背景类型为 color、image 或 video。', + fields: [media('srcFile', 'srcFileUrl', true), media('alpha', 'mattingAlphaFileUrl', true), { key: 'backgroundType', kind: 'string', required: true, choices: ['color', 'image', 'video'] }, { key: 'backgroundColor', kind: 'string' }, media('backgroundUrl', 'backgroundUrl')], + }, + { + id: 'watermark-remover', label: 'DreamAPI · 去水印', kind: 'video', + endpoint: '/api/async/video_watermark_remover', note: '视频最长 120 秒、最高 2K。', + fields: [media('video', 'video', true), { key: 'prompt', kind: 'string' }, { key: 'seed', kind: 'number' }], + }, + { + id: 'video-enhance', label: 'DreamAPI · 视频增强', kind: 'video', + endpoint: '/api/async/video_super_resolution', note: '一条原视频。', + fields: [media('srcVideo', 'srcVideoUrl', true)], + }, + { + id: 'voice-clone', label: 'DreamAPI · 声音克隆', kind: 'voice', + endpoint: '/api/async/voice_clone', note: '返回 cloneId，供克隆音色配音使用。', + fields: [media('voiceUrl', 'voiceUrl', true)], + }, + { + id: 'tts-clone', label: 'DreamAPI · 克隆音色配音', kind: 'voice', + endpoint: '/api/async/do_tts_clone', note: '需要已保存的 cloneId。', + fields: [{ key: 'cloneId', kind: 'string', required: true }, { key: 'text', kind: 'string', required: true }, { key: 'lang', json: 'lan', kind: 'string', required: true }], + }, + { + id: 'tts-common', label: 'DreamAPI · 普通配音', kind: 'voice', + endpoint: '/api/async/do_tts_common', note: '音色 audioId 来自 DreamAPI 音色列表。', + fields: [{ key: 'audioId', kind: 'string', required: true }, { key: 'text', kind: 'string', required: true }, { key: 'lang', json: 'lan', kind: 'string', required: true }], + }, + { + id: 'tts-pro', label: 'DreamAPI · 高级配音', kind: 'voice', + endpoint: '/api/async/do_tts_pro', note: '音色 audioId 来自 Pro 列表。', + fields: [{ key: 'audioId', kind: 'string', required: true }, { key: 'text', kind: 'string', required: true }, { key: 'lang', json: 'lan', kind: 'string', required: true }], + }, +]; + +export const DREAMAPI_MODEL_IDS = DREAMAPI_MODELS.map((model) => model.id); + +export function dreamApiModel(id: string | undefined): DreamApiModel | undefined { + const trimmed = id?.trim(); + if (!trimmed) return undefined; + return DREAMAPI_MODELS.find((model) => model.id === trimmed); +} + +export function dreamApiModelsByKind(kind: DreamApiKind): DreamApiModel[] { + return DREAMAPI_MODELS.filter((model) => model.kind === kind); +} + +/** Copy catalog fields from a tool argument bag. Explicit `dreamapi` wins over top-level copies. */ +export function dreamApiInputFromArgs(modelId: string | undefined, args: Record): Record { + const model = dreamApiModel(modelId); + const nested = args.dreamapi; + const bag: Record = nested && typeof nested === 'object' ? { ...nested as Record } : {}; + if (!model) return bag; + for (const field of model.fields) { + if (bag[field.key] === undefined && args[field.key] !== undefined) bag[field.key] = args[field.key]; + } + return bag; +} + +const SEEDREAM_LIMITS: Record }> = { + 'seedream-5.0-pro': { min: 921600, max: 4624220, tiers: new Set(['1K', '1.5K', '2K']) }, + 'seedream-5.0-lite': { min: 3686400, max: 16777216, tiers: new Set(['2K', '3K', '4K']) }, + 'seedream-4.5': { min: 3686400, max: 16777216, tiers: new Set(['2K', '4K']) }, + 'seedream-4.0': { min: 921600, max: 16777216, tiers: new Set(['1K', '2K', '4K']) }, +}; + +function seedreamSizeError(model: string, size: string): string | undefined { + const limits = SEEDREAM_LIMITS[model]; + if (!limits) return `Unknown Seedream model: ${model}`; + const normalized = size.trim().toUpperCase().replace(/ /g, ''); + if (limits.tiers.has(normalized)) return undefined; + const parts = normalized.toLowerCase().replace('*', 'x').split('x'); + if (parts.length !== 2) return `Invalid size ${size} for ${model}`; + const width = Number(parts[0]); + const height = Number(parts[1]); + if (!Number.isInteger(width) || !Number.isInteger(height) || width <= 0 || height <= 0) { + return `Invalid size ${size}`; + } + const total = width * height; + if (total < limits.min || total > limits.max) return `size ${size} is outside ${model} pixel range`; + if (width * 16 < height || height * 16 < width) return `size ${size} aspect ratio is outside 1:16 to 16:1`; + return undefined; +} + +export interface DreamApiCall { + model: DreamApiModel; + body: Record; +} + +function readField(field: DreamApiField, value: unknown): unknown { + if (value === undefined || value === null || value === '') return undefined; + if (field.kind === 'string' || field.kind === 'media') { + if (typeof value !== 'string' || !value.trim()) throw new Error(`${field.key} must be a non-empty string`); + const text = value.trim(); + if (field.choices && !field.choices.includes(text)) throw new Error(`${field.key} must be one of ${field.choices.join(', ')}`); + if (field.maxLength !== undefined && text.length > field.maxLength) throw new Error(`${field.key} must be at most ${field.maxLength} characters`); + return text; + } + if (field.kind === 'number') { + if (typeof value !== 'number' || !Number.isFinite(value)) throw new Error(`${field.key} must be a number`); + if (field.min !== undefined && value < field.min) throw new Error(`${field.key} must be at least ${field.min}`); + if (field.max !== undefined && value > field.max) throw new Error(`${field.key} must be at most ${field.max}`); + if (field.multipleOf !== undefined && value % field.multipleOf !== 0) throw new Error(`${field.key} must be a multiple of ${field.multipleOf}`); + return value; + } + if (field.kind === 'boolean') { + if (typeof value !== 'boolean') throw new Error(`${field.key} must be a boolean`); + return value; + } + if (!Array.isArray(value) || value.some((item) => typeof item !== 'string' || !item.trim())) { + throw new Error(`${field.key} must be an array of strings`); + } + const items = value.map((item) => item.trim()); + if (field.required && items.length === 0) throw new Error(`${field.key} is required`); + if (field.maxItems !== undefined && items.length > field.maxItems) throw new Error(`${field.key} accepts at most ${field.maxItems} items`); + return items; +} + +function shapeBody(model: DreamApiModel, flat: Record): Record { + if (model.shape === 'outpaint') { + return { + url: flat.url, + outPaintSize: { left: flat.left ?? 0, right: flat.right ?? 0, top: flat.top ?? 0, bottom: flat.bottom ?? 0 }, + }; + } + if (model.shape === 'lipsync' || model.shape === 'lipsync2') { + const body: Record = { + srcVideoUrl: flat.srcVideoUrl, + audioUrl: flat.audioUrl, + videoParams: { + video_width: flat.videoWidth ?? 0, + video_height: flat.videoHeight ?? 0, + video_enhance: model.shape === 'lipsync' ? flat.videoEnhance === true : (flat.videoEnhance ?? 1), + }, + }; + if (flat.vocalAudioUrl) body.vocalAudioUrl = flat.vocalAudioUrl; + if (flat.fps) body.fps = flat.fps; + return body; + } + return flat; +} + +/** Validate one catalog id and build the DreamAPI JSON body. Does not touch the network. */ +export function buildDreamApiCall(modelId: string | undefined, input: Record): DreamApiCall { + const model = dreamApiModel(modelId); + if (!model) throw new Error('Choose a DreamAPI model in Settings or specify dreamapiModel'); + const allowed = new Set(model.fields.map((field) => field.key)); + for (const key of Object.keys(input)) { + if (input[key] !== undefined && !allowed.has(key)) throw new Error(`${key} is not supported by DreamAPI model ${model.id}`); + } + const flat: Record = { ...model.constants }; + for (const field of model.fields) { + const value = readField(field, input[field.key]); + if (value === undefined) { + if (field.required) throw new Error(`${field.key} is required for DreamAPI model ${model.id}`); + continue; + } + flat[field.json ?? field.key] = value; + } + if (model.id === 'try-on' && !flat.upperClothes && !flat.lowerClothes && !flat.overallClothes) { + throw new Error('try-on requires upper, lower, or overall'); + } + if (flat.endImageUrl && !flat.imageUrl) throw new Error('endImageUrl requires imageUrl'); + if (typeof flat.size === 'string') { + const seedream = String(flat.model ?? 'seedream-5.0-pro'); + const sizeError = seedreamSizeError(seedream, flat.size); + if (sizeError) throw new Error(sizeError); + } + if (model.id === 'video-translate-2' && flat.sourceLanguage === flat.targetLanguage) { + throw new Error('sourceLanguage and targetLanguage must differ'); + } + return { model, body: shapeBody(model, flat) }; +} diff --git a/src/agent/capabilities.ts b/src/agent/capabilities.ts index 4e9216232..5f458447f 100644 --- a/src/agent/capabilities.ts +++ b/src/agent/capabilities.ts @@ -1,4 +1,5 @@ import { FAL_MODELS } from '../../shared/fal-models'; +import { dreamApiModelsByKind } from '../../shared/dreamapi-models'; // Which key-gated capabilities are actually configured. The booleans are computed // SERVER-SIDE in config/vite.config.ts (from .env.local) and injected via `define` as // __CONFIGURED_CAPS__ — BOOLEANS ONLY, never any key value reaches the browser. @@ -66,6 +67,7 @@ const CAP_PROVIDERS: Partial> = { { label: 'WaveSpeed', arg: 'wavespeed', argKey: 'model', need: [['WAVESPEED_API_KEY']] }, { label: 'BytePlus', arg: 'byteplus', argKey: 'model', need: [['BYTEPLUS_API_KEY']] }, { label: 'xAI Grok', arg: 'grok-imagine', argKey: 'model', need: [['LLM_XAI_OAUTH_API_KEY'], ['LLM_XAI_API_KEY']] }, + { label: 'DreamAPI', arg: 'dreamapi', argKey: 'model', need: [['DREAMAPI_API_KEY']] }, ], voice: [ { label: 'ElevenLabs', arg: 'elevenlabs', argKey: 'provider', need: [['ELEVENLABS_API_KEY']] }, @@ -78,6 +80,7 @@ const CAP_PROVIDERS: Partial> = { { label: 'Gemini', arg: 'gemini', argKey: 'provider', need: [['GEMINI_API_KEY']] }, { label: 'Mistral Voxtral', arg: 'mistral', argKey: 'provider', need: [['LLM_MISTRAL_API_KEY']] }, { label: 'Cartesia', arg: 'cartesia', argKey: 'provider', need: [['CARTESIA_API_KEY']] }, + { label: 'DreamAPI', arg: 'dreamapi', argKey: 'provider', need: [['DREAMAPI_API_KEY']] }, ], video: [ { label: 'OFox', arg: 'ofox', argKey: 'model', need: [['LLM_OFOX_API_KEY']] }, @@ -87,6 +90,7 @@ const CAP_PROVIDERS: Partial> = { { label: 'Hailuo', arg: 'hailuo', argKey: 'model', need: [['MINIMAX_API_KEY']] }, { label: 'xAI Grok', arg: 'grok-imagine-video', argKey: 'model', need: [['LLM_XAI_OAUTH_API_KEY'], ['LLM_XAI_API_KEY']] }, { label: 'BytePlus', arg: 'byteplus', argKey: 'model', need: [['BYTEPLUS_API_KEY']] }, + { label: 'DreamAPI', arg: 'dreamapi', argKey: 'model', need: [['DREAMAPI_API_KEY']] }, ], music: [ { label: 'Mureka', arg: 'mureka', argKey: 'provider', need: [['MUREKA_API_KEY']] }, @@ -137,6 +141,19 @@ function falModelSuffix(cap: CapabilityKey): string { + ' Only the documented common inputs are supported; use the selected model constraints in the tool schema.'; } +function dreamApiSuffix(cap: CapabilityKey): string { + if (cap !== 'image' && cap !== 'video' && cap !== 'voice') return ''; + if (!liveKeys?.DREAMAPI_API_KEY?.configured) return ''; + const models = dreamApiModelsByKind(cap); + const preferredKey = cap === 'image' ? 'DREAMAPI_IMAGE_MODEL' : cap === 'video' ? 'DREAMAPI_VIDEO_MODEL' : 'DREAMAPI_VOICE_MODEL'; + const preferred = liveModels?.[preferredKey]; + const selected = models.find((model) => model.id === preferred); + const arg = cap === 'voice' ? 'provider=dreamapi and dreamapiModel' : 'model=dreamapi and dreamapiModel'; + return `\nDreamAPI ${cap} models (use ${arg}=): ${models.map((model) => `${model.label}=${model.id}`).join(', ')}. ` + + (selected ? `Saved DreamAPI default: ${selected.id}; honor it unless the user explicitly requests another model.` + : 'No DreamAPI model selected: ask which DreamAPI model to use before submitting.'); +} + function doubaoSpeakerSuffix(): string { if (!liveKeys?.DOUBAO_TTS_API_KEY?.configured) return ''; const speaker = (liveModels?.DOUBAO_TTS_VOICE_ID ?? '').trim(); @@ -199,7 +216,8 @@ export function capabilitiesPrompt( } return `\n\n# Available capabilities (based on configured API keys; local editing is always available without keys)\n` + `✅ Configured: ${on.length ? on.join(', ') : '(no key-gated capabilities)'}.\n` - + falModelSuffix('image') + falModelSuffix('video') + doubaoSpeakerSuffix() + '\n' + + falModelSuffix('image') + falModelSuffix('video') + + dreamApiSuffix('image') + dreamApiSuffix('video') + dreamApiSuffix('voice') + doubaoSpeakerSuffix() + '\n' + `⬜ Not configured — do not promise these in a plan or call them; they return "not configured" and waste a turn:\n` + (off.length ? off.map((s) => ` - ${s}`).join('\n') : ' (none)') + '\nWhen an unavailable capability is needed, follow its fallback above or tell the user that the capability is not configured' diff --git a/src/agent/capabilities.verify.ts b/src/agent/capabilities.verify.ts index fd078678c..bc9608fa2 100644 --- a/src/agent/capabilities.verify.ts +++ b/src/agent/capabilities.verify.ts @@ -63,6 +63,13 @@ applyLiveModels({ DOUBAO_TTS_VOICE_ID: 'S_5wulnA7i2' }); const doubaoDefault = capabilitiesPrompt({ ...ALL_OFF, voice: true }); assert.ok(doubaoDefault.includes('Doubao(provider=doubao)'), 'configured Doubao is selectable'); assert.ok(doubaoDefault.includes('Doubao default speaker ID: S_5wulnA7i2'), 'saved speaker reaches the agent'); + +// Merging provider additions must preserve both providers' saved routing hints. +applyLiveKeyStatus({ DOUBAO_TTS_API_KEY: { configured: true }, DREAMAPI_API_KEY: { configured: true } }); +applyLiveModels({ DOUBAO_TTS_VOICE_ID: 'S_5wulnA7i2', DREAMAPI_VOICE_MODEL: 'tts-pro' }); +const combinedVoiceProviders = capabilitiesPrompt({ ...ALL_OFF, voice: true }); +assert.ok(combinedVoiceProviders.includes('Doubao default speaker ID: S_5wulnA7i2'), 'DreamAPI does not hide the Doubao speaker'); +assert.ok(combinedVoiceProviders.includes('Saved DreamAPI default: tts-pro'), 'Doubao does not hide the DreamAPI default'); applyLiveKeyStatus({}); applyLiveModels({}); diff --git a/src/agent/tools/generate-schemas.ts b/src/agent/tools/generate-schemas.ts index 4f295c290..4679d521d 100644 --- a/src/agent/tools/generate-schemas.ts +++ b/src/agent/tools/generate-schemas.ts @@ -1,4 +1,5 @@ import { FAL_IMAGE_MODELS, FAL_VIDEO_MODELS, falModelSummary } from '../../../shared/fal-models'; +import { DREAMAPI_MODELS } from '../../../shared/dreamapi-models'; import type { AgentToolSchema } from '../tool-schema'; import { MINIMAX_LANGUAGE_BOOSTS } from '../../../shared/media-provider-params'; @@ -10,8 +11,10 @@ export const GENERATE_TOOL_SCHEMAS: AgentToolSchema[] = [ input_schema: { type: 'object', properties: { - model: { type: 'string', enum: ['gpt-image-2', 'nano-banana', 'image-01', 'wavespeed', 'byteplus', 'grok-imagine', 'fal'], description: 'gpt-image-2 is the default; nano-banana is best for reference-heavy work; image-01 is MiniMax (at most 9 outputs; one subject reference when R2 is configured); wavespeed is WaveSpeed AI (fast generic image models, no references); byteplus is BytePlus ModelArk Seedream (no references yet); grok-imagine is xAI Grok Imagine (text-to-image only, no references, at most 4 outputs, 1K/2K).' }, + model: { type: 'string', enum: ['gpt-image-2', 'nano-banana', 'image-01', 'wavespeed', 'byteplus', 'grok-imagine', 'fal', 'dreamapi'], description: 'gpt-image-2 is the default; nano-banana is best for reference-heavy work; image-01 is MiniMax (at most 9 outputs; one subject reference when R2 is configured); wavespeed is WaveSpeed AI (fast generic image models, no references); byteplus is BytePlus ModelArk Seedream (no references yet); grok-imagine is xAI Grok Imagine (text-to-image only, no references, at most 4 outputs, 1K/2K). dreamapi is DreamAPI; pair it with dreamapiModel.' }, falModel: { type: 'string', enum: FAL_IMAGE_MODELS.map((model) => model.id), description: `With model=fal, select a curated Fal image model; omitted uses the saved Fal image default. ${FAL_IMAGE_MODELS.map((model) => `${model.id}: ${falModelSummary(model)}`).join('; ')}. Model-specific limits are validated before submission.` }, + dreamapiModel: { type: 'string', enum: DREAMAPI_MODELS.filter((model) => model.kind === 'image').map((model) => model.id), description: `With model=dreamapi, select a DreamAPI image model. ${DREAMAPI_MODELS.filter((model) => model.kind === 'image').map((model) => `${model.id}: ${model.note}`).join('; ')}` }, + dreamapi: { type: 'object', description: 'DreamAPI model inputs such as image, images, url, face, width, height, seedreamModel, size, text. Keys are validated against the selected dreamapiModel before any request is sent.' }, prompt: { type: 'string', description: 'Detailed description of the image to generate.' }, name: { type: 'string', description: 'Short descriptive asset name shown in the media pool.' }, addToTimeline: { type: 'boolean', description: 'Defaults to true. Set false when the user asks to keep the result in the media pool/library only or says not to modify the timeline.' }, @@ -40,7 +43,9 @@ export const GENERATE_TOOL_SCHEMAS: AgentToolSchema[] = [ input_schema: { type: 'object', properties: { - provider: { type: 'string', enum: ['elevenlabs', 'doubao', 'minimax', 'inworld', 'fishaudio', 'speechify', 'openai', 'gemini', 'mistral', 'cartesia'], description: 'Required opt-in provider. Use only a provider listed as configured in capabilities. Voice IDs and catalogs are never interchangeable.' }, + provider: { type: 'string', enum: ['elevenlabs', 'doubao', 'minimax', 'inworld', 'fishaudio', 'speechify', 'openai', 'gemini', 'mistral', 'cartesia', 'dreamapi'], description: 'Required opt-in provider. Use only a provider listed as configured in capabilities. Voice IDs and catalogs are never interchangeable. dreamapi is DreamAPI TTS or voice clone.' }, + dreamapiModel: { type: 'string', enum: DREAMAPI_MODELS.filter((model) => model.kind === 'voice').map((model) => model.id), description: 'With provider=dreamapi, select tts-common, tts-pro, tts-clone, or voice-clone.' }, + dreamapi: { type: 'object', description: 'DreamAPI voice inputs: text, lang, audioId, cloneId, or voiceUrl.' }, text: { type: 'string', minLength: 1, description: 'Text to synthesize.' }, voiceId: { type: 'string', description: 'Concrete provider-specific voice ID confirmed by the user. Required except MiniMax timbreWeights mixing and Doubao when a default speaker ID is saved in Settings. For Doubao, pass the exact Volcengine speaker ID verbatim (for example S_5wulnA7i2 or zh_female_vv_uranus_bigtts); it is sent unchanged as references[].speaker. Do not invent a preset or reuse an ID from another provider.' }, modelId: { type: 'string', description: 'ElevenLabs, Inworld, Fish Audio, Speechify, OpenAI, Gemini, Mistral, or Cartesia only. Omit to use the configured model.' }, @@ -149,8 +154,10 @@ export const GENERATE_TOOL_SCHEMAS: AgentToolSchema[] = [ input_schema: { type: 'object', properties: { - model: { type: 'string', enum: ['seedance2', 'kling', 'hailuo', 'byteplus', 'grok-imagine-video', 'ofox', 'fal'], description: 'hailuo is MiniMax: 6 or 10s; firstFrame optional; lastFrame allowed with firstFrame; no multi-ref or multi-shot. 1080p is 6s only. byteplus is BytePlus ModelArk Seedance — same request shape/limits as seedance2. grok-imagine-video is xAI Grok Imagine: text-to-video only, 1–15s, audio track included, no references/frames. ofox is the OFox multi-model gateway (Seedance/Wan and more behind one key): 2–30s with per-model limits enforced by the API; supports firstFrame (and optional lastFrame), or up to 9 refImages (frames and refImages are mutually exclusive); no refVideos/refAudios yet.' }, // minimax: hailuo enum + model: { type: 'string', enum: ['seedance2', 'kling', 'hailuo', 'byteplus', 'grok-imagine-video', 'ofox', 'fal', 'dreamapi'], description: 'hailuo is MiniMax: 6 or 10s; firstFrame optional; lastFrame allowed with firstFrame; no multi-ref or multi-shot. 1080p is 6s only. byteplus is BytePlus ModelArk Seedance — same request shape/limits as seedance2. grok-imagine-video is xAI Grok Imagine: text-to-video only, 1–15s, audio track included, no references/frames. ofox is the OFox multi-model gateway (Seedance/Wan and more behind one key): 2–30s with per-model limits enforced by the API; supports firstFrame (and optional lastFrame), or up to 9 refImages (frames and refImages are mutually exclusive); no refVideos/refAudios yet. dreamapi is DreamAPI; pair it with dreamapiModel.' }, // minimax: hailuo enum falModel: { type: 'string', enum: FAL_VIDEO_MODELS.map((model) => model.id), description: `With model=fal, select a curated Fal video model; omitted uses the saved Fal video default. ${FAL_VIDEO_MODELS.map((model) => `${model.id}: ${falModelSummary(model)}`).join('; ')}. Omit duration/resolution to use model defaults.` }, + dreamapiModel: { type: 'string', enum: DREAMAPI_MODELS.filter((model) => model.kind === 'video').map((model) => model.id), description: `With model=dreamapi, select a DreamAPI video model. ${DREAMAPI_MODELS.filter((model) => model.kind === 'video').map((model) => `${model.id}: ${model.note}`).join('; ')}` }, + dreamapi: { type: 'object', description: 'DreamAPI video inputs such as prompt, image, images, srcVideo, audio, resolution, duration, sourceLanguage, and targetLanguage. Keys are validated against dreamapiModel before any request is sent.' }, prompt: { type: 'string', description: 'Required for normal generation and Kling intelligence; omit for Kling customize.' }, name: { type: 'string' }, durationSeconds: { anyOf: [{ type: 'number' }, { type: 'string' }], description: 'For Fal, see falModel limits and omit to use model default. Native providers: integer seconds, 2–15 for Seedance, 3–15 for Kling, exactly 6 or 10 for Hailuo (Hailuo 1080p → 6 only), 1–15 for grok-imagine-video, 2–30 for ofox (per-model limits enforced by the API).' }, // minimax: hailuo durations diff --git a/src/agent/tools/generate-tool-handlers.ts b/src/agent/tools/generate-tool-handlers.ts index f7f02e14e..6459f3aef 100644 --- a/src/agent/tools/generate-tool-handlers.ts +++ b/src/agent/tools/generate-tool-handlers.ts @@ -66,6 +66,9 @@ const submitImageHandler: Handler = async (args, ctx) => { const submitVoiceHandler: Handler = async (args, ctx) => { const input = buildSubmitVoiceArgs(args); const asset = await submitVoice(input, ctx.getState()); + if (!('kind' in asset)) { + return { ok: true, provider: input.provider, cloneId: asset.cloneId, addedTo: 'dreamapi-voice-clone' }; + } addAsset(ctx, asset); return { ok: true, provider: input.provider, voiceId: input.voiceId, assetId: asset.id, diff --git a/src/agent/tools/generate-tool-input.ts b/src/agent/tools/generate-tool-input.ts index c34c6b86e..960c29fc7 100644 --- a/src/agent/tools/generate-tool-input.ts +++ b/src/agent/tools/generate-tool-input.ts @@ -3,6 +3,7 @@ import type { SubmitMusicArgs } from '../../generate/music'; import type { SubmitSoundArgs } from '../../generate/sound'; import type { SubmitVideoArgs } from '../../generate/video'; import type { SubmitVoiceArgs, VoiceProvider } from '../../generate/voice'; +import { dreamApiInputFromArgs } from '../../../shared/dreamapi-models'; export type GenerateArgs = Record; export const shouldAddImageToTimeline = (args: GenerateArgs): boolean => args.addToTimeline !== false; @@ -29,6 +30,9 @@ export function buildSubmitImageArgs(args: GenerateArgs): SubmitImageArgs { model, falModel: str(args.falModel), prompt: String(args.prompt ?? ''), name: String(args.name ?? ''), aspectRatio, referenceAssetIds: references, count: num(args.count), }; + if (model === 'dreamapi') { + return { ...shared, dreamapiModel: str(args.dreamapiModel), dreamapi: dreamApiInputFromArgs(str(args.dreamapiModel), args) }; + } if (model === 'fal') { for (const key of ['width', 'height', 'quality', 'maskAssetId', 'background', 'moderation', 'inputFidelity', 'outputFormat', 'outputCompression', 'seed', 'promptOptimizer']) { if (args[key] !== undefined) throw new Error(`${key} is not supported by the Fal image integration`); @@ -130,6 +134,11 @@ const VOICE_STRATEGIES = { gemini: (args) => genericVoice(args, 'gemini'), mistral: (args) => genericVoice(args, 'mistral'), cartesia: (args) => genericVoice(args, 'cartesia'), + dreamapi: (args) => ({ + ...voiceBase(args, 'dreamapi'), dreamapiModel: str(args.dreamapiModel), + dreamapi: dreamApiInputFromArgs(str(args.dreamapiModel), args), + text: String(args.text ?? ''), + }), } satisfies Record SubmitVoiceArgs>; export function buildSubmitVoiceArgs(args: GenerateArgs): SubmitVoiceArgs { @@ -235,10 +244,14 @@ const falVideo = (args: GenerateArgs): SubmitVideoArgs => { }); }; -const VIDEO_STRATEGIES = { fal: falVideo, seedance2: seedanceVideo, kling: klingVideo, hailuo: hailuoVideo, byteplus: byteplusVideo, 'grok-imagine-video': grokVideo, ofox: ofoxVideo } as const; +const dreamapiVideo = (args: GenerateArgs): SubmitVideoArgs => ({ + ...videoBase(args, 'dreamapi'), dreamapiModel: str(args.dreamapiModel), + dreamapi: dreamApiInputFromArgs(str(args.dreamapiModel), args), +}); +const VIDEO_STRATEGIES = { fal: falVideo, dreamapi: dreamapiVideo, seedance2: seedanceVideo, kling: klingVideo, hailuo: hailuoVideo, byteplus: byteplusVideo, 'grok-imagine-video': grokVideo, ofox: ofoxVideo } as const; export function buildSubmitVideoArgs(args: GenerateArgs): SubmitVideoArgs { const model = args.model === undefined ? 'seedance2' : args.model; - if (model !== 'seedance2' && model !== 'kling' && model !== 'hailuo' && model !== 'byteplus' && model !== 'grok-imagine-video' && model !== 'ofox' && model !== 'fal') { + if (model !== 'seedance2' && model !== 'kling' && model !== 'hailuo' && model !== 'byteplus' && model !== 'grok-imagine-video' && model !== 'ofox' && model !== 'fal' && model !== 'dreamapi') { throw new Error('Unsupported video model; select an available provider before submitting or rerunning.'); } return VIDEO_STRATEGIES[model](args); diff --git a/src/components/settings/settingsMediaProviders.ts b/src/components/settings/settingsMediaProviders.ts index 1ec102711..671f1b457 100644 --- a/src/components/settings/settingsMediaProviders.ts +++ b/src/components/settings/settingsMediaProviders.ts @@ -1,5 +1,6 @@ +import { dreamApiModelsByKind } from '../../../shared/dreamapi-models'; import { - modelPicker, + modelPicker, modelText, routeSelect, secret, @@ -109,8 +110,9 @@ export const VOICE_SETTINGS_GROUP: SettingsGroup = { { value: 'openai', label: 'OpenAI' }, { value: 'gemini', label: 'Google Gemini' }, { value: 'mistral', label: 'Mistral Voxtral' }, - { value: 'cartesia', label: 'Cartesia' }, - ]), + { value: 'cartesia', label: 'Cartesia' }, + { value: 'dreamapi', label: 'DreamAPI' }, + ]), vendors: [ { key: 'voice/elevenlabs', vendor: 'elevenlabs', title: 'ElevenLabs', @@ -185,11 +187,25 @@ export const VOICE_SETTINGS_GROUP: SettingsGroup = { }, { key: 'voice/cartesia', vendor: 'cartesia', title: 'Cartesia', fields: [ - secret('CARTESIA_API_KEY', 'API Key'), - modelText('CARTESIA_TTS_MODEL', '配音模型', 'sonic-3'), - ], - }, - ], + secret('CARTESIA_API_KEY', 'API Key'), + modelText('CARTESIA_TTS_MODEL', '配音模型', 'sonic-3'), + ], + }, + { + key: 'voice/dreamapi', vendor: 'dreamapi', title: 'DreamAPI', + note: 'DreamAPI 配音与声音克隆。密钥只保存在本机服务里。', + fields: [ + secret('DREAMAPI_API_KEY', 'API Key'), + { + name: 'DREAMAPI_VOICE_MODEL', label: '默认模型', kind: 'select', + options: [ + { value: '', label: '每次询问（默认）' }, + ...dreamApiModelsByKind('voice').map(({ id, label }) => ({ value: id, label })), + ], + }, + ], + }, + ], }; export const TRANSCRIPTION_SETTINGS_GROUP: SettingsGroup = { @@ -241,6 +257,7 @@ export const TRANSCRIPTION_SETTINGS_GROUP: SettingsGroup = { export const ROUTE_NEEDS: Record = { fal: [['FAL_KEY']], + dreamapi: [['DREAMAPI_API_KEY']], 'gpt-image-2': [['IMAGE_API_KEY'], ['OPENAI_API_KEY']], 'nano-banana': [['GEMINI_API_KEY']], 'image-01': [['MINIMAX_API_KEY']], diff --git a/src/components/settings/settingsSchema.ts b/src/components/settings/settingsSchema.ts index f5be85823..c979c577d 100644 --- a/src/components/settings/settingsSchema.ts +++ b/src/components/settings/settingsSchema.ts @@ -1,4 +1,5 @@ import { FAL_MODELS } from '../../../shared/fal-models'; +import { dreamApiModelsByKind, type DreamApiKind } from '../../../shared/dreamapi-models'; // Set the information architecture of the panel (first-level classification → second-level capability group → third-level provider page → fields) and pure display logic. // Three columns: Left tree = Category → Capability; Middle column = Vendor list under this capability; Right column = Configuration page of the selected vendor. // Agent LLM saves independent API URLs, API Keys and models for each vendor; the capability to generate classes can be additionally provided @@ -70,6 +71,23 @@ const byteplusPage = (cap: string, modelField: SettingsField, title = 'BytePlus ], }); +const DREAMAPI_NOTE = 'DreamAPI。密钥只保存在本机服务里，请到 DreamAPI 控制台自行注册充值后填写。聊天里指定的模型优先于这里的默认值。'; +const dreamApiPage = (cap: DreamApiKind): SettingsVendorPage => ({ + key: `${cap}/dreamapi`, vendor: 'dreamapi', title: 'DreamAPI', + note: DREAMAPI_NOTE, + fields: [ + secret('DREAMAPI_API_KEY', 'API Key'), + { + name: cap === 'image' ? 'DREAMAPI_IMAGE_MODEL' : cap === 'video' ? 'DREAMAPI_VIDEO_MODEL' : 'DREAMAPI_VOICE_MODEL', + label: '默认模型', kind: 'select', + options: [ + { value: '', label: '每次询问（默认）' }, + ...dreamApiModelsByKind(cap).map(({ id, label }) => ({ value: id, label })), + ], + }, + ], +}); + const falPage = (cap: 'image' | 'video'): SettingsVendorPage => ({ key: `${cap}/fal`, vendor: 'fal', title: 'Fal.ai', note: '选择 Fal.ai 作为默认厂商，然后选择模型。聊天中指定的模型优先于此默认值。', @@ -113,6 +131,7 @@ export const SETTINGS_CATEGORIES: readonly SettingsCategory[] = [ { value: 'wavespeed', label: 'WaveSpeed' }, { value: 'byteplus', label: 'BytePlus · Seedream' }, { value: 'grok-imagine', label: 'xAI Grok Imagine' }, + { value: 'dreamapi', label: 'DreamAPI' }, ]), vendors: [ { key: 'image/openai', vendor: 'openai', title: 'OpenAI', fields: [ @@ -125,6 +144,7 @@ export const SETTINGS_CATEGORIES: readonly SettingsCategory[] = [ modelText('GEMINI_IMAGE_MODEL', '生图模型', 'gemini-3.1-flash-image'), ] }, falPage('image'), + dreamApiPage('image'), minimaxPage('image', modelPicker('MINIMAX_IMAGE_MODEL', '生图模型', 'image-01', ['image-01', 'image-01-live'])), { key: 'image/wavespeed', vendor: 'wavespeed', title: 'WaveSpeed', fields: [ secret('WAVESPEED_API_KEY', 'API Key'), @@ -150,9 +170,11 @@ export const SETTINGS_CATEGORIES: readonly SettingsCategory[] = [ { value: 'byteplus', label: 'BytePlus · Seedance' }, { value: 'grok-imagine-video', label: 'xAI Grok Imagine' }, { value: 'ofox', label: 'OFox · 多模型' }, + { value: 'dreamapi', label: 'DreamAPI' }, ]), vendors: [ falPage('video'), + dreamApiPage('video'), { key: 'video/seedance', vendor: 'seedance', title: 'Seedance · 火山', fields: [ secret('SEEDANCE_API_KEY', 'API Key'), text('SEEDANCE_BASE_URL', 'Base URL', '默认 https://ark.cn-beijing.volces.com/api/v3'), diff --git a/src/components/settings/vendorIcons.tsx b/src/components/settings/vendorIcons.tsx index a1dff873a..40c16fa87 100644 --- a/src/components/settings/vendorIcons.tsx +++ b/src/components/settings/vendorIcons.tsx @@ -44,7 +44,7 @@ export type VendorId = | 'seedance' | 'kling' | 'mureka' | 'sonilo' | 'pexels' | 'pixabay' | 'unsplash' | 'freesound' | 'assemblyai' | 'deepgram' | 'groq' | 'cartesia' | 'e2b' | 'firecrawl' | 'r2' | 'localdisk' | 'localasr' | 'stepfun' | 'byteplus' | 'inworld' | 'fishaudio' | 'speechify' | 'wavespeed' - | 'vision' | 'proxy' | 'atlas' | 'xai' | 'xai-oauth' | 'copilot' | 'ofox' | 'fal' | 'uploadpost'; + | 'vision' | 'proxy' | 'atlas' | 'xai' | 'xai-oauth' | 'copilot' | 'ofox' | 'fal' | 'dreamapi' | 'uploadpost'; interface SvgIcon { readonly svg: string; @@ -107,6 +107,7 @@ const MONOGRAMS: Partial; prompt: string; name: string; aspectRatio?: '1:1' | '16:9' | '9:16' | '4:3' | '3:4' | '3:2' | '2:3' | '4:5' | '5:4' | '21:9' | 'auto' | '4:1' | '1:4' | '8:1' | '1:8'; @@ -44,7 +48,9 @@ const newId = () => crypto.randomUUID?.() ?? `generated_${Date.now()}_${Math.ran export async function submitImage(args: SubmitImageArgs, state: TimelineState): Promise { const prompt = args.prompt.trim(); const name = args.name.trim(); - if (!prompt) throw new Error('prompt is required'); + const dreamModel = args.model === 'dreamapi' ? dreamApiModel(args.dreamapiModel) : undefined; + const promptRequired = !dreamModel || dreamModel.fields.some((field) => (field.key === 'prompt' || field.key === 'text') && field.required); + if (promptRequired && !prompt) throw new Error('prompt is required'); if (!name) throw new Error('name is required'); const referencePaths = (args.referenceAssetIds ?? []).map((id) => { const asset = (state.assets ?? []).find((candidate) => candidate.id === id); diff --git a/src/generate/video.ts b/src/generate/video.ts index 3a2bb9118..8ac18f96e 100644 --- a/src/generate/video.ts +++ b/src/generate/video.ts @@ -3,9 +3,11 @@ import { sourceWindowForTimelineRange } from '../editor/sourceLimit'; export interface SubmitVideoArgs { operationId?: string; - model: 'seedance2' | 'kling' | 'hailuo' | 'byteplus' | 'grok-imagine-video' | 'ofox' | 'fal'; + model: 'seedance2' | 'kling' | 'hailuo' | 'byteplus' | 'grok-imagine-video' | 'ofox' | 'fal' | 'dreamapi'; /** Explicit curated Fal model ID; omitted uses the saved Fal default. */ falModel?: string; + dreamapiModel?: string; + dreamapi?: Record; prompt?: string; name?: string; durationSeconds?: number | string; diff --git a/src/generate/voice.ts b/src/generate/voice.ts index 3a14fc5fd..a3f39346e 100644 --- a/src/generate/voice.ts +++ b/src/generate/voice.ts @@ -11,10 +11,13 @@ export type VoiceProvider = | 'openai' | 'gemini' | 'mistral' - | 'cartesia'; + | 'cartesia' + | 'dreamapi'; export interface SubmitVoiceArgs { provider: VoiceProvider; + dreamapiModel?: string; + dreamapi?: Record; text: string; voiceId: string; modelId?: string; @@ -64,9 +67,12 @@ interface VoiceResponse { path?: string; subtitlePath?: string; durationSeconds?: number; + cloneId?: string; error?: string; } +export type VoiceGenerationResult = MediaAsset | { cloneId: string }; + const newId = () => crypto.randomUUID?.() ?? `generated_${Date.now()}_${Math.random().toString(36).slice(2)}`; function probeAudio(src: string, fps: number): Promise { @@ -86,11 +92,11 @@ function probeAudio(src: string, fps: number): Promise { }); } -export async function submitVoice(args: SubmitVoiceArgs, state: TimelineState): Promise { +export async function submitVoice(args: SubmitVoiceArgs, state: TimelineState): Promise { const text = args.text.trim(); const voiceId = args.voiceId.trim(); - if (!text) throw new Error('text is required'); - if (!voiceId && !args.timbreWeights?.length && args.provider !== 'doubao') { + if (args.provider !== 'dreamapi' && !text) throw new Error('text is required'); + if (args.provider !== 'dreamapi' && !voiceId && !args.timbreWeights?.length && args.provider !== 'doubao') { throw new Error('voiceId is required unless MiniMax timbreWeights are provided'); } const response = await fetch('/generate/voice', { @@ -100,6 +106,7 @@ export async function submitVoice(args: SubmitVoiceArgs, state: TimelineState): }); const result = await response.json().catch(() => ({})) as VoiceResponse; if (!response.ok) throw new Error(result.error ?? `voice generation failed (${response.status})`); + if (!result.path && result.cloneId) return { cloneId: result.cloneId }; if (!result.path) throw new Error('voice generation returned no audio asset'); const durationInFrames = result.durationSeconds && Number.isFinite(result.durationSeconds) ? Math.max(1, Math.round(result.durationSeconds * state.fps)) diff --git a/src/i18n/dict/en/settings.ts b/src/i18n/dict/en/settings.ts index ecb00a99b..3bf715d7a 100644 --- a/src/i18n/dict/en/settings.ts +++ b/src/i18n/dict/en/settings.ts @@ -610,4 +610,33 @@ export default { '桌面端点击“选择目录”；也可手动输入绝对路径（可用 ~/ 开头）。清除后回到默认目录。': 'On desktop, click "Choose folder"; you can also type an absolute path (~/ accepted). Clear it to return to the default folder.', '已保存 · 重启应用后新的工程存储目录才会生效': 'Saved · the new project storage folder takes effect after a restart', '该模型不在内置目录，以上数值为估算（上下文 {context} / 输出 {output}）。若与实际不符，点「展开」手动修改。': 'This model is not in the built-in catalog, so these values are estimates (context {context} / output {output}). If they do not match the real model, expand and adjust them manually.', + 'DreamAPI。密钥只保存在本机服务里，请到 DreamAPI 控制台自行注册充值后填写。聊天里指定的模型优先于这里的默认值。': 'DreamAPI. The key stays in the local server. Register and top up in the DreamAPI dashboard, then paste it here. A model named in chat overrides this default.', + 'DreamAPI 配音与声音克隆。密钥只保存在本机服务里。': 'DreamAPI voice and voice clone. The key stays in the local server.', + 'DreamAPI · DreamVideo 3.0 文生视频': 'DreamAPI · DreamVideo 3.0 text to video', + 'DreamAPI · DreamVideo 3.0 图生视频': 'DreamAPI · DreamVideo 3.0 image to video', + 'DreamAPI · Wan2.1 文生视频': 'DreamAPI · Wan2.1 text to video', + 'DreamAPI · Wan2.1 图生视频': 'DreamAPI · Wan2.1 image to video', + 'DreamAPI · Wan2.1 首尾帧': 'DreamAPI · Wan2.1 first and last frame', + 'DreamAPI · Flux 文生图': 'DreamAPI · Flux text to image', + 'DreamAPI · Flux 图生图': 'DreamAPI · Flux image to image', + 'DreamAPI · GPT Image 2.5 Flare 文生图': 'DreamAPI · GPT Image 2.5 Flare text to image', + 'DreamAPI · GPT Image 2.5 Flare 图生图': 'DreamAPI · GPT Image 2.5 Flare image to image', + 'DreamAPI · GPT Image 2.5 Sunburst 文生图': 'DreamAPI · GPT Image 2.5 Sunburst text to image', + 'DreamAPI · GPT Image 2.5 Sunburst 图生图': 'DreamAPI · GPT Image 2.5 Sunburst image to image', + 'DreamAPI · 黑白上色': 'DreamAPI · Colorize', + 'DreamAPI · 图像增强': 'DreamAPI · Image enhance', + 'DreamAPI · 扩图': 'DreamAPI · Outpaint', + 'DreamAPI · 图片换脸': 'DreamAPI · Face swap', + 'DreamAPI · 去背景': 'DreamAPI · Remove background', + 'DreamAPI · 虚拟试衣': 'DreamAPI · Virtual try-on', + 'DreamAPI · 视频翻译 2.0': 'DreamAPI · Video translate 2.0', + 'DreamAPI · 视频换脸': 'DreamAPI · Video face swap', + 'DreamAPI · 视频抠像': 'DreamAPI · Video matting', + 'DreamAPI · 抠像合成': 'DreamAPI · Matting composite', + 'DreamAPI · 去水印': 'DreamAPI · Watermark removal', + 'DreamAPI · 视频增强': 'DreamAPI · Video enhance', + 'DreamAPI · 声音克隆': 'DreamAPI · Voice clone', + 'DreamAPI · 克隆音色配音': 'DreamAPI · Cloned-voice speech', + 'DreamAPI · 普通配音': 'DreamAPI · Standard speech', + 'DreamAPI · 高级配音': 'DreamAPI · Pro speech', } as Record; diff --git a/src/i18n/dict/it/settings.ts b/src/i18n/dict/it/settings.ts index 92da37c96..4bd66c5f9 100644 --- a/src/i18n/dict/it/settings.ts +++ b/src/i18n/dict/it/settings.ts @@ -447,6 +447,35 @@ const overrides: Record = { '默认工程位置': 'Posizione predefinita dei progetti', '新建工程、历史版本和应用生成的素材保存在这里。桌面端从外部拖入的文件和文件夹保留在原位置，工程只建立引用；浏览器运行时会上传托管副本。修改后重启应用生效。': 'Qui vengono salvati i nuovi progetti, la cronologia delle versioni e i media generati dall’app. Sul desktop i file e le cartelle trascinati dall’esterno restano nella posizione originale e il progetto crea solo un riferimento; nel browser viene caricata una copia gestita. La modifica si applica dopo il riavvio.', '未配置时素材只存本机。配置后：每次上传同步写入 R2（桶保持私有，读取经本地服务回源，src 路径不变）；本机缺文件时自动从云端取回。改动即时生效。R2 控制台建桶 → R2 API Token（Object Read & Write）即可拿到下面四个值。': 'Senza configurazione i media restano solo sul computer. Dopo la configurazione ogni caricamento viene scritto anche su R2; il bucket resta privato, la lettura passa dal servizio locale e il percorso src non cambia. Se manca un file locale, viene recuperato automaticamente dal cloud. Le modifiche hanno effetto immediato. Crea un bucket nella console R2 e un token API R2 con Object Read & Write per ottenere i quattro valori seguenti.', + 'DreamAPI。密钥只保存在本机服务里，请到 DreamAPI 控制台自行注册充值后填写。聊天里指定的模型优先于这里的默认值。': 'DreamAPI. La chiave resta nel server locale. Registrati e ricarica nella console DreamAPI, poi incollala qui. Un modello indicato in chat ha la precedenza su questo valore.', + 'DreamAPI 配音与声音克隆。密钥只保存在本机服务里。': 'Voce e clonazione vocale DreamAPI. La chiave resta nel server locale.', + 'DreamAPI · DreamVideo 3.0 文生视频': 'DreamAPI · DreamVideo 3.0 da testo a video', + 'DreamAPI · DreamVideo 3.0 图生视频': 'DreamAPI · DreamVideo 3.0 da immagine a video', + 'DreamAPI · Wan2.1 文生视频': 'DreamAPI · Wan2.1 da testo a video', + 'DreamAPI · Wan2.1 图生视频': 'DreamAPI · Wan2.1 da immagine a video', + 'DreamAPI · Wan2.1 首尾帧': 'DreamAPI · Wan2.1 primo e ultimo fotogramma', + 'DreamAPI · Flux 文生图': 'DreamAPI · Flux da testo a immagine', + 'DreamAPI · Flux 图生图': 'DreamAPI · Flux da immagine a immagine', + 'DreamAPI · GPT Image 2.5 Flare 文生图': 'DreamAPI · GPT Image 2.5 Flare da testo a immagine', + 'DreamAPI · GPT Image 2.5 Flare 图生图': 'DreamAPI · GPT Image 2.5 Flare da immagine a immagine', + 'DreamAPI · GPT Image 2.5 Sunburst 文生图': 'DreamAPI · GPT Image 2.5 Sunburst da testo a immagine', + 'DreamAPI · GPT Image 2.5 Sunburst 图生图': 'DreamAPI · GPT Image 2.5 Sunburst da immagine a immagine', + 'DreamAPI · 黑白上色': 'DreamAPI · Colorazione', + 'DreamAPI · 图像增强': 'DreamAPI · Migliora immagine', + 'DreamAPI · 扩图': 'DreamAPI · Estensione immagine', + 'DreamAPI · 图片换脸': 'DreamAPI · Scambio volto', + 'DreamAPI · 去背景': 'DreamAPI · Rimuovi sfondo', + 'DreamAPI · 虚拟试衣': 'DreamAPI · Prova abiti', + 'DreamAPI · 视频翻译 2.0': 'DreamAPI · Traduzione video 2.0', + 'DreamAPI · 视频换脸': 'DreamAPI · Scambio volto video', + 'DreamAPI · 视频抠像': 'DreamAPI · Scontorno video', + 'DreamAPI · 抠像合成': 'DreamAPI · Composito scontorno', + 'DreamAPI · 去水印': 'DreamAPI · Rimozione filigrana', + 'DreamAPI · 视频增强': 'DreamAPI · Migliora video', + 'DreamAPI · 声音克隆': 'DreamAPI · Clonazione voce', + 'DreamAPI · 克隆音色配音': 'DreamAPI · Sintesi con voce clonata', + 'DreamAPI · 普通配音': 'DreamAPI · Sintesi standard', + 'DreamAPI · 高级配音': 'DreamAPI · Sintesi Pro', }; export default overrides;