From 9450ee796844f7e5b0e966ddb28a1c225c3bb38b Mon Sep 17 00:00:00 2001 From: Rob Losch Date: Wed, 7 Oct 2026 16:40:06 -0700 Subject: [PATCH 01/56] feat(website): Openjutsu app, interface on a sample backend Character swap for video on MiniMax H3 and the Akatz Labs Character Swap LoRA. The page runs end to end against a stand-in transport in astro dev; no deployment or app-proxy entry exists yet, so it reports itself unavailable everywhere else. --- .../components/workshop/ModelsCatalogue.vue | 7 +- .../cinematic-studio/CinematicStudio.test.ts | 9 +- .../workshop/openjutsu/OpenjutsuMediaSlot.vue | 133 ++++++ .../workshop/openjutsu/OpenjutsuPage.vue | 66 +++ .../workshop/openjutsu/OpenjutsuSide.vue | 211 +++++++++ .../workshop/openjutsu/OpenjutsuStage.vue | 334 +++++++++++++ .../workshop/openjutsu/OpenjutsuStudio.vue | 119 +++++ .../workshop/openjutsu/OpenjutsuTrim.vue | 171 +++++++ apps/website/src/composables/useOpenjutsu.ts | 440 ++++++++++++++++++ apps/website/src/config/hub-app-names.json | 2 +- apps/website/src/config/hub-models.test.ts | 2 +- .../src/config/models-catalogue-data.ts | 2 +- apps/website/src/config/models-catalogue.ts | 2 +- apps/website/src/config/routes.ts | 8 +- .../src/config/workshop-app-content.test.ts | 6 +- apps/website/src/config/workshop-env.ts | 13 + .../workshop-workflow-catalog-schema.ts | 2 +- .../website/src/content/workshop-display.json | 1 + .../src/content/workshop-workflows.jsonl | 1 + .../src/data/workshop-model-availability.json | 4 + apps/website/src/env.d.ts | 4 + apps/website/src/lib/workshop/apps.ts | 7 + .../src/lib/workshop/catalogue-apps.ts | 6 +- .../src/lib/workshop/openjutsu/clip.test.ts | 80 ++++ .../src/lib/workshop/openjutsu/clip.ts | 126 +++++ .../openjutsu/sample-transport.test.ts | 54 +++ .../workshop/openjutsu/sample-transport.ts | 119 +++++ .../src/lib/workshop/openjutsu/swap.api.json | 204 ++++++++ .../workshop/openjutsu/transport-config.ts | 31 ++ .../lib/workshop/openjutsu/workflow.test.ts | 71 +++ .../src/lib/workshop/openjutsu/workflow.ts | 68 +++ apps/website/src/locales/en/main.json | 79 +++- apps/website/src/locales/zh-CN/main.json | 79 +++- apps/website/src/routes/models/app.astro | 17 +- apps/website/vercel.json | 10 + apps/website/workflows/openjutsu/README.md | 97 ++++ 36 files changed, 2562 insertions(+), 23 deletions(-) create mode 100644 apps/website/src/components/workshop/openjutsu/OpenjutsuMediaSlot.vue create mode 100644 apps/website/src/components/workshop/openjutsu/OpenjutsuPage.vue create mode 100644 apps/website/src/components/workshop/openjutsu/OpenjutsuSide.vue create mode 100644 apps/website/src/components/workshop/openjutsu/OpenjutsuStage.vue create mode 100644 apps/website/src/components/workshop/openjutsu/OpenjutsuStudio.vue create mode 100644 apps/website/src/components/workshop/openjutsu/OpenjutsuTrim.vue create mode 100644 apps/website/src/composables/useOpenjutsu.ts create mode 100644 apps/website/src/lib/workshop/openjutsu/clip.test.ts create mode 100644 apps/website/src/lib/workshop/openjutsu/clip.ts create mode 100644 apps/website/src/lib/workshop/openjutsu/sample-transport.test.ts create mode 100644 apps/website/src/lib/workshop/openjutsu/sample-transport.ts create mode 100644 apps/website/src/lib/workshop/openjutsu/swap.api.json create mode 100644 apps/website/src/lib/workshop/openjutsu/transport-config.ts create mode 100644 apps/website/src/lib/workshop/openjutsu/workflow.test.ts create mode 100644 apps/website/src/lib/workshop/openjutsu/workflow.ts create mode 100644 apps/website/workflows/openjutsu/README.md diff --git a/apps/website/src/components/workshop/ModelsCatalogue.vue b/apps/website/src/components/workshop/ModelsCatalogue.vue index f68b338c63b..8e3ebd34f8a 100644 --- a/apps/website/src/components/workshop/ModelsCatalogue.vue +++ b/apps/website/src/components/workshop/ModelsCatalogue.vue @@ -69,11 +69,16 @@ const apps = computed(() => (model): model is AppWorkshopModel => model.type === 'APP' ) ) +const APP_TASKS = { + studio: 'studioTask', + reshoot: 'reshootTask', + openjutsu: 'openjutsuTask' +} as const satisfies Record const appCards = computed(() => apps.value.map((app) => ({ key: app.slug, name: app.name, - task: ac(app.appId === 'studio' ? 'studioTask' : 'reshootTask', locale), + task: ac(APP_TASKS[app.appId], locale), href: app.href, thumbnail: app.thumbnail })) diff --git a/apps/website/src/components/workshop/cinematic-studio/CinematicStudio.test.ts b/apps/website/src/components/workshop/cinematic-studio/CinematicStudio.test.ts index e5fbed1927f..6158972c09f 100644 --- a/apps/website/src/components/workshop/cinematic-studio/CinematicStudio.test.ts +++ b/apps/website/src/components/workshop/cinematic-studio/CinematicStudio.test.ts @@ -1780,7 +1780,7 @@ describe('CinematicStudio', () => { expect(document.title).toBe('Re-shoot a video - Comfy') }) - it('lists only Cinematic Studio and Re-shoot a video in the Hub apps tab', async () => { + it('lists only the apps that open in the Hub apps tab', async () => { window.history.replaceState( null, '', @@ -1791,14 +1791,17 @@ describe('CinematicStudio', () => { const tab = await screen.findByRole('button', { name: 'Apps' }) expect(tab).toHaveAttribute('aria-pressed', 'true') const apps = screen.getAllByRole('listitem') - expect(apps, 'Apps that do not open yet stay off the Hub').toHaveLength(2) - const [firstApp, secondApp] = apps + expect(apps, 'Apps that do not open yet stay off the Hub').toHaveLength(3) + const [firstApp, secondApp, thirdApp] = apps expect( within(firstApp).getByRole('link', { name: 'Cinematic Studio' }) ).toHaveAttribute('href', '/hub/apps/cinematic-studio/') expect( within(secondApp).getByRole('link', { name: 'Re-shoot a video' }) ).toHaveAttribute('href', '/hub/apps/reshoot/') + expect( + within(thirdApp).getByRole('link', { name: 'Openjutsu' }) + ).toHaveAttribute('href', '/hub/apps/openjutsu/') }) it('runs a shot from the side panel on the model picked there', async () => { diff --git a/apps/website/src/components/workshop/openjutsu/OpenjutsuMediaSlot.vue b/apps/website/src/components/workshop/openjutsu/OpenjutsuMediaSlot.vue new file mode 100644 index 00000000000..26f6c1815f3 --- /dev/null +++ b/apps/website/src/components/workshop/openjutsu/OpenjutsuMediaSlot.vue @@ -0,0 +1,133 @@ + + + diff --git a/apps/website/src/components/workshop/openjutsu/OpenjutsuPage.vue b/apps/website/src/components/workshop/openjutsu/OpenjutsuPage.vue new file mode 100644 index 00000000000..e0c58e5708c --- /dev/null +++ b/apps/website/src/components/workshop/openjutsu/OpenjutsuPage.vue @@ -0,0 +1,66 @@ + + + diff --git a/apps/website/src/components/workshop/openjutsu/OpenjutsuSide.vue b/apps/website/src/components/workshop/openjutsu/OpenjutsuSide.vue new file mode 100644 index 00000000000..ecfc6f4a27c --- /dev/null +++ b/apps/website/src/components/workshop/openjutsu/OpenjutsuSide.vue @@ -0,0 +1,211 @@ + + + diff --git a/apps/website/src/components/workshop/openjutsu/OpenjutsuStage.vue b/apps/website/src/components/workshop/openjutsu/OpenjutsuStage.vue new file mode 100644 index 00000000000..e3fa793fa2a --- /dev/null +++ b/apps/website/src/components/workshop/openjutsu/OpenjutsuStage.vue @@ -0,0 +1,334 @@ + + + diff --git a/apps/website/src/components/workshop/openjutsu/OpenjutsuStudio.vue b/apps/website/src/components/workshop/openjutsu/OpenjutsuStudio.vue new file mode 100644 index 00000000000..9e12e9168ce --- /dev/null +++ b/apps/website/src/components/workshop/openjutsu/OpenjutsuStudio.vue @@ -0,0 +1,119 @@ + + + diff --git a/apps/website/src/components/workshop/openjutsu/OpenjutsuTrim.vue b/apps/website/src/components/workshop/openjutsu/OpenjutsuTrim.vue new file mode 100644 index 00000000000..4469b0e30c4 --- /dev/null +++ b/apps/website/src/components/workshop/openjutsu/OpenjutsuTrim.vue @@ -0,0 +1,171 @@ + + + diff --git a/apps/website/src/composables/useOpenjutsu.ts b/apps/website/src/composables/useOpenjutsu.ts new file mode 100644 index 00000000000..20af68d6bd5 --- /dev/null +++ b/apps/website/src/composables/useOpenjutsu.ts @@ -0,0 +1,440 @@ +import { useMounted, useObjectUrl } from '@vueuse/core' +import { computed, onScopeDispose, ref, shallowRef, watch } from 'vue' + +import { refreshWorkshopCredits } from '@/config/workshop-credits' +import { useWorkshopSession } from '@/config/workshop-session-state' +import { workshopIdempotencyKey } from '@/config/workshop-snippets' +import type { Locale } from '@/i18n/translations' +import { translationsFor } from '@/i18n/translations' +import type { StudioGate } from '@/lib/workshop/cinematic-studio/gate' +import { studioGate } from '@/lib/workshop/cinematic-studio/gate' +import { + failureNote, + quoteNote +} from '@/lib/workshop/cinematic-studio/reshoot-engine/notes' +import type { ReshootRunPhase } from '@/lib/workshop/cinematic-studio/reshoot-engine/run' +import { + downloadOutput, + runJob +} from '@/lib/workshop/cinematic-studio/reshoot-engine/run' +import type { ReshootQuote } from '@/lib/workshop/cinematic-studio/reshoot-engine/transport' +import { ReshootError } from '@/lib/workshop/cinematic-studio/reshoot-engine/transport' +import type { ClipFacts, SwapWindow } from '@/lib/workshop/openjutsu/clip' +import { + clipFactsOf, + clipFits, + defaultWindow, + fitWindow, + gridFrames, + pickCanvas +} from '@/lib/workshop/openjutsu/clip' +import { + OPENJUTSU_SAMPLE_MODE, + openjutsuTransport +} from '@/lib/workshop/openjutsu/transport-config' +import { + resolveSeed, + swapSeconds, + swapWorkflow +} from '@/lib/workshop/openjutsu/workflow' +import { captureWorkshopEvent, useWorkshopAuthFlag } from '@/scripts/posthog' +import type { WorkshopRunAnalytics } from '@/scripts/workshop-analytics' + +const OPENJUTSU_APP_SLUG = 'apps/openjutsu' + +/** Waits before asking for the price again after a failed quote. */ +const QUOTE_RETRY_MS = [5_000, 15_000, 30_000, 60_000] +/** Quote failures that waiting cannot fix: no such app, or signed out. */ +const QUOTE_FINAL = new Set(['not_found', 'unauthorized', 'app_unavailable']) +const UNAVAILABLE = new Set(['app_unavailable', 'not_found']) + +type SwapPhase = 'uploading' | ReshootRunPhase | 'fetching' + +export interface SwapTake { + readonly id: string + readonly n: number + /** Who was replaced, as typed. */ + readonly target: string + readonly window: SwapWindow + readonly frames: number + readonly seed: number + readonly status: 'rendering' | 'done' | 'cancelled' | 'failed' + readonly phase?: SwapPhase + readonly url?: string + readonly note?: string +} + +/** + * The Openjutsu app's state and its one metered run: a clip, a character + * image and who to replace go in, the swapped clip with the source's own + * sound comes out. + */ +export function useOpenjutsu({ locale = 'en' }: { locale?: Locale } = {}) { + const { t } = translationsFor(locale) + const { user, session, sessionFailure, settled, ensureFresh } = + useWorkshopSession() + const authEnabled = useWorkshopAuthFlag() + const mounted = useMounted() + const transport = openjutsuTransport(async () => { + const credential = await ensureFresh() + if (credential?.status !== 'ok') throw new ReshootError('unauthorized') + return credential.session.token + }) + + // --- the two inputs. A clip is checked before it is taken; see `takeVideo`. + const video = shallowRef() + const videoUrl = useObjectUrl(video) + const facts = shallowRef() + const character = shallowRef() + const characterUrl = useObjectUrl(character) + const target = ref('') + /** A fixed seed, or none: then every take draws its own. */ + const seed = ref() + const range = ref({ start: 0, seconds: 0 }) + + watch(videoUrl, async (url) => { + facts.value = undefined + if (!url) return + const read = await clipFactsOf(url) + if (url !== videoUrl.value) return + facts.value = read + if (read) range.value = defaultWindow(read.seconds) + }) + + const clipSeconds = computed(() => facts.value?.seconds) + const frames = computed(() => + facts.value ? gridFrames(range.value.seconds) : undefined + ) + const canvas = computed(() => + facts.value ? pickCanvas(facts.value.width, facts.value.height) : undefined + ) + function setWindow(next: SwapWindow) { + if (facts.value) range.value = fitWindow(next, facts.value.seconds) + } + + /** Why the last chosen file was turned away; the clip in use stays. */ + const videoRejected = ref() + async function takeVideo(file: File) { + const url = URL.createObjectURL(file) + const read = await clipFactsOf(url) + URL.revokeObjectURL(url) + if (!read) { + videoRejected.value = t('openjutsu.video.unreadable', { name: file.name }) + return + } + if (!clipFits(read.seconds)) { + videoRejected.value = t('openjutsu.video.length', { + name: file.name, + seconds: read.seconds.toFixed(1) + }) + return + } + videoRejected.value = undefined + video.value = file + selected.value = 'source' + } + function takeCharacter(file: File) { + character.value = file + } + + // --- takes + const takes = ref([]) + /** `source` shows the clip and its trim; anything else is a take's id. */ + const selected = ref('source') + const current = computed(() => + takes.value.find((take) => take.id === selected.value) + ) + const rendering = computed(() => + takes.value.some((take) => take.status === 'rendering') + ) + function updateTake(id: string, patch: Partial) { + takes.value = takes.value.map((take) => + take.id === id ? { ...take, ...patch } : take + ) + } + + // --- what the next run costs, and whether it may start + const quote = shallowRef() + const quoteSettled = ref(false) + const quoteFailed = ref(false) + const unavailable = ref(transport === undefined) + let quoteRequest = 0 + let quoteRetry: ReturnType | undefined + let quoteFailures = 0 + + const signedIn = () => OPENJUTSU_SAMPLE_MODE || !!session.value + async function refreshQuote() { + const request = ++quoteRequest + clearTimeout(quoteRetry) + if (!transport || !signedIn()) { + quote.value = undefined + quoteSettled.value = false + return + } + try { + const next = await transport.quote() + if (request !== quoteRequest) return + quote.value = next + quoteSettled.value = true + quoteFailed.value = false + quoteFailures = 0 + unavailable.value = false + } catch (error) { + if (request !== quoteRequest) return + const code = error instanceof ReshootError ? error.code : '' + quote.value = undefined + quoteSettled.value = false + if (UNAVAILABLE.has(code)) unavailable.value = true + quoteFailed.value = !QUOTE_FINAL.has(code) + if (!quoteFailed.value) return + const delay = + QUOTE_RETRY_MS[Math.min(quoteFailures, QUOTE_RETRY_MS.length - 1)] + quoteFailures += 1 + quoteRetry = setTimeout(() => void refreshQuote(), delay) + } + } + watch( + () => [mounted.value, session.value?.workspace.id] as const, + ([isMounted]) => { + if (isMounted) void refreshQuote() + }, + { immediate: true } + ) + + const quoteRefusesCredit = computed( + () => + !rendering.value && quote.value?.blocked_reason === 'insufficient_credits' + ) + const gate = computed(() => { + if (OPENJUTSU_SAMPLE_MODE && !unavailable.value) + return !mounted.value + ? 'pending' + : quoteRefusesCredit.value + ? 'noCredits' + : 'ready' + return studioGate({ + runEnabled: !unavailable.value, + modelRunnable: true, + mounted: mounted.value, + authAvailable: authEnabled.value && !sessionFailure.value, + sessionSettled: settled.value && !(user.value && !session.value), + role: session.value?.role, + credits: quoteRefusesCredit.value ? 0 : undefined + }) + }) + /** What is still missing before Generate, in the order the panel asks. */ + const missing = computed(() => { + if (!video.value || !facts.value) return 'video' + if (!character.value) return 'character' + if (!target.value.trim()) return 'target' + return undefined + }) + const canGenerate = computed( + () => + gate.value === 'ready' && + missing.value === undefined && + !rendering.value && + quoteSettled.value && + quote.value?.next_run !== 'blocked' + ) + const priceNote = computed(() => { + if (quote.value) + return quoteNote(quote.value, locale, { + size: '768p', + seconds: frames.value && swapSeconds(frames.value) + }) + return quoteFailed.value ? t('reshoot.quote.failed') : undefined + }) + + function noteFor(error: unknown): string { + const code = error instanceof ReshootError ? error.code : '' + if (code === 'insufficient_credits') + return t( + session.value?.role === 'member' + ? 'workshop.error.memberNoCredits' + : 'workshop.error.noCreditsCloud', + { workspace: session.value?.workspace.name ?? '' } + ) + if (UNAVAILABLE.has(code)) return t('openjutsu.unavailable') + if (code === 'unauthorized') return t('openjutsu.signIn') + if (code === 'job_failed') return t('openjutsu.error.swap') + return failureNote(error, locale, quote.value?.price_credits) + } + + // --- the run + const uploads = new WeakMap>() + const runs = new Map() + const objectUrls: string[] = [] + + /** Bytes already sent keep their name, so a retake uploads nothing twice. */ + function uploaded(file: File, signal: AbortSignal): Promise { + if (!transport) return Promise.reject(new ReshootError('app_unavailable')) + const known = uploads.get(file) + if (known) return known + const sending = transport.upload(file, signal) + uploads.set(file, sending) + sending.catch(() => uploads.delete(file)) + return sending + } + + async function generate() { + const clip = video.value + const image = character.value + const shape = canvas.value + const length = frames.value + if ( + !transport || + !canGenerate.value || + !clip || + !image || + !shape || + !length + ) + return + // Everything the run uses is read now: edits made while it renders belong + // to the next take, and a blank seed is drawn once, here. + const request = { + target: target.value.trim(), + window: { ...range.value }, + frames: length, + seed: resolveSeed(seed.value) + } + const id = crypto.randomUUID() + const n = takes.value.length + 1 + takes.value = [ + ...takes.value, + { id, n, ...request, status: 'rendering', phase: 'uploading' } + ] + selected.value = id + const controller = new AbortController() + runs.set(id, controller) + const { signal } = controller + const startedFor = session.value + const analytics: WorkshopRunAnalytics | undefined = startedFor && { + model_slug: OPENJUTSU_APP_SLUG, + page_type: 'app', + app_slug: OPENJUTSU_APP_SLUG, + user_id: startedFor.uid, + workspace_id: startedFor.workspace.id, + attempt_id: workshopIdempotencyKey() + } + const startedAt = Date.now() + const finish = ( + outcome: + | { status: 'succeeded'; output_count: number } + | { status: 'cancelled' } + | { status: 'failed'; reason: 'provider' | 'client' } + ) => { + if (analytics) + captureWorkshopEvent({ + name: 'run_finished', + properties: { + ...analytics, + duration_ms: Date.now() - startedAt, + ...outcome + } + }) + } + if (analytics) + captureWorkshopEvent({ name: 'run_started', properties: analytics }) + try { + const [videoName, characterName] = await Promise.all([ + uploaded(clip, signal), + uploaded(image, signal) + ]) + const job = await runJob( + transport, + swapWorkflow({ + video: videoName, + character: characterName, + target: request.target, + start: request.window.start, + frames: request.frames, + canvas: shape, + seed: request.seed + }), + (phase) => updateTake(id, { phase }), + signal + ) + updateTake(id, { phase: 'fetching' }) + const result = await downloadOutput(transport, job, 'result', signal) + if (signal.aborted) return finish({ status: 'cancelled' }) + const url = URL.createObjectURL(result) + objectUrls.push(url) + updateTake(id, { status: 'done', phase: undefined, url }) + finish({ status: 'succeeded', output_count: 1 }) + } catch (error) { + if (signal.aborted) return finish({ status: 'cancelled' }) + updateTake(id, { + status: 'failed', + phase: undefined, + note: noteFor(error) + }) + finish({ + status: 'failed', + reason: error instanceof ReshootError ? 'provider' : 'client' + }) + } finally { + runs.delete(id) + void refreshQuote() + if (startedFor) void refreshWorkshopCredits({ force: true }) + } + } + + function cancel() { + runs.forEach((controller) => controller.abort()) + takes.value = takes.value.map((take) => + take.status === 'rendering' + ? { ...take, status: 'cancelled', phase: undefined } + : take + ) + } + + /** Puts a take's own settings back, ready to run again or adjust. */ + function reuse(id: string) { + const take = takes.value.find((entry) => entry.id === id) + if (!take) return + target.value = take.target + setWindow(take.window) + seed.value = take.seed + selected.value = 'source' + } + + onScopeDispose(() => { + clearTimeout(quoteRetry) + quoteRequest += 1 + runs.forEach((controller) => controller.abort()) + objectUrls.forEach((url) => URL.revokeObjectURL(url)) + }) + + return { + sample: OPENJUTSU_SAMPLE_MODE, + video, + videoUrl, + videoRejected, + clipSeconds, + character, + characterUrl, + target, + seed, + range, + frames, + canvas, + takes, + selected, + current, + rendering, + gate, + missing, + canGenerate, + priceNote, + unavailable, + session, + takeVideo, + takeCharacter, + setWindow, + generate, + cancel, + reuse + } +} diff --git a/apps/website/src/config/hub-app-names.json b/apps/website/src/config/hub-app-names.json index 0af3f922ce3..ed5466acb9c 100644 --- a/apps/website/src/config/hub-app-names.json +++ b/apps/website/src/config/hub-app-names.json @@ -1 +1 @@ -["cinematic-studio", "reshoot"] +["cinematic-studio", "openjutsu", "reshoot"] diff --git a/apps/website/src/config/hub-models.test.ts b/apps/website/src/config/hub-models.test.ts index 4a75c52a123..935815e1329 100644 --- a/apps/website/src/config/hub-models.test.ts +++ b/apps/website/src/config/hub-models.test.ts @@ -19,7 +19,7 @@ describe('hub model addresses', () => { `${HUB_WORKFLOWS_PATH}/`, `${HUB_APPS_PATH}/` ]) - expect([routes.cinematicStudio, routes.reshoot]).toEqual( + expect([routes.cinematicStudio, routes.openjutsu, routes.reshoot]).toEqual( hubAppSlugs.map(hubAppHref) ) }) diff --git a/apps/website/src/config/models-catalogue-data.ts b/apps/website/src/config/models-catalogue-data.ts index 5b35e32b88e..b2d802d125c 100644 --- a/apps/website/src/config/models-catalogue-data.ts +++ b/apps/website/src/config/models-catalogue-data.ts @@ -59,7 +59,7 @@ export const workflowModelSchema = presentationSchema.extend({ const appModelSchema = presentationSchema.extend({ type: z.literal('APP'), - appId: z.enum(['studio', 'reshoot']), + appId: z.enum(['studio', 'reshoot', 'openjutsu']), routerId: z.never().optional(), workflowId: z.never().optional() }) diff --git a/apps/website/src/config/models-catalogue.ts b/apps/website/src/config/models-catalogue.ts index 4af1fcd7bc7..0dcc83c66a7 100644 --- a/apps/website/src/config/models-catalogue.ts +++ b/apps/website/src/config/models-catalogue.ts @@ -169,7 +169,7 @@ export type AppWorkshopModel = WorkshopPresentation & { readonly type: 'APP' readonly href: string /** Which app page runs it: see `WorkshopAppEntry.app`. */ - readonly appId: 'studio' | 'reshoot' + readonly appId: 'studio' | 'reshoot' | 'openjutsu' readonly routerId?: never readonly workflowId?: never } diff --git a/apps/website/src/config/routes.ts b/apps/website/src/config/routes.ts index 36937c8d9c8..69ca852b66c 100644 --- a/apps/website/src/config/routes.ts +++ b/apps/website/src/config/routes.ts @@ -60,7 +60,8 @@ const baseRoutes = { hubApps: '/hub/apps/', workshopSignIn: '/login/', cinematicStudio: '/hub/apps/cinematic-studio/', - reshoot: '/hub/apps/reshoot/' + reshoot: '/hub/apps/reshoot/', + openjutsu: '/hub/apps/openjutsu/' } as const type RouteKey = keyof typeof baseRoutes @@ -88,8 +89,8 @@ type Routes = Readonly> // form, so no localized variant exists. See the comment header in // src/pages/minimax/license/professional-request.astro. // -// workshop, hubWorkflows, hubApps, workshopSignIn, cinematicStudio, reshoot: -// English only. Every locale links the one catalogue. +// workshop, hubWorkflows, hubApps, workshopSignIn, cinematicStudio, reshoot, +// openjutsu: English only. Every locale links the one catalogue. // // customerVideoBlackMath / customerVideoSilversideAi: dedicated watch pages // built from a single English-language caption track — a "translated" watch @@ -108,6 +109,7 @@ const LOCALE_INVARIANT_ROUTE_KEYS = new Set([ 'workshopSignIn', 'cinematicStudio', 'reshoot', + 'openjutsu', 'customerVideoBlackMath', 'customerVideoSilversideAi' ]) diff --git a/apps/website/src/config/workshop-app-content.test.ts b/apps/website/src/config/workshop-app-content.test.ts index a08a97fb374..63ea8dac837 100644 --- a/apps/website/src/config/workshop-app-content.test.ts +++ b/apps/website/src/config/workshop-app-content.test.ts @@ -11,7 +11,8 @@ describe('Workshop apps', () => { it('lists every app declared in the catalog, each at /hub/apps//', () => { expect(appModels.map(({ appId, href }) => ({ appId, href }))).toEqual([ { appId: 'studio', href: '/hub/apps/cinematic-studio/' }, - { appId: 'reshoot', href: '/hub/apps/reshoot/' } + { appId: 'reshoot', href: '/hub/apps/reshoot/' }, + { appId: 'openjutsu', href: '/hub/apps/openjutsu/' } ]) }) @@ -20,7 +21,8 @@ describe('Workshop apps', () => { appPagePaths().map(({ params, props }) => [params.app, props.model.appId]) ).toEqual([ ['cinematic-studio', 'studio'], - ['reshoot', 'reshoot'] + ['reshoot', 'reshoot'], + ['openjutsu', 'openjutsu'] ]) }) diff --git a/apps/website/src/config/workshop-env.ts b/apps/website/src/config/workshop-env.ts index 60f37a755b8..cc236a5b430 100644 --- a/apps/website/src/config/workshop-env.ts +++ b/apps/website/src/config/workshop-env.ts @@ -124,3 +124,16 @@ const RESHOOT_PROXY_IDS: Record = { } export const WORKSHOP_RESHOOT_PROXY_ID = RESHOOT_PROXY_IDS[WORKSHOP_CLOUD_ENV] + +/** + * Openjutsu's app proxy id in each family's comfy-api catalog. None exists + * yet, so the page reports itself unavailable outside `astro dev`. + */ +const OPENJUTSU_PROXY_IDS: Record = { + prod: undefined, + staging: undefined, + test: undefined +} + +export const WORKSHOP_OPENJUTSU_PROXY_ID = + OPENJUTSU_PROXY_IDS[WORKSHOP_CLOUD_ENV] diff --git a/apps/website/src/config/workshop-workflow-catalog-schema.ts b/apps/website/src/config/workshop-workflow-catalog-schema.ts index 0f3b7a933fd..26c6ee4acc9 100644 --- a/apps/website/src/config/workshop-workflow-catalog-schema.ts +++ b/apps/website/src/config/workshop-workflow-catalog-schema.ts @@ -79,7 +79,7 @@ const appSchema = z .object({ id: z.string().regex(/^apps\/[a-z0-9]+(?:-[a-z0-9]+)*$/), type: z.literal('APP'), - app: z.enum(['studio', 'reshoot']) + app: z.enum(['studio', 'reshoot', 'openjutsu']) }) .strict() diff --git a/apps/website/src/content/workshop-display.json b/apps/website/src/content/workshop-display.json index 5ee8cae2704..d2714ba1690 100644 --- a/apps/website/src/content/workshop-display.json +++ b/apps/website/src/content/workshop-display.json @@ -1,6 +1,7 @@ [ {"id":"apps/cinematic-studio","slug":"apps/cinematic-studio","modelId":"apps/cinematic-studio","type":"APP","displayName":"Cinematic Studio","description":"Direct a shot with a real camera, light and grade, then run it on any image model.","media":{"thumbnail":{"url":"https://media.comfy.org/website/workshop/apps/cinematic-studio/thumbnail-480.mp4","kind":"video","poster":"https://media.comfy.org/website/workshop/apps/cinematic-studio/poster.jpg"}},"examples":[],"advancedFields":[],"pricing":null,"status":"active","license":null,"mediaConfidence":"exact","needsReview":false,"useCase":"generate-images","recommendedRank":0}, {"id":"apps/reshoot","slug":"apps/reshoot","modelId":"apps/reshoot","type":"APP","displayName":"Re-shoot a video","description":"Aim a new camera at your clip and generate the scene from that angle.","media":{"thumbnail":{"url":"https://media.comfy.org/website/workshop/apps/reshoot/thumbnail-480.mp4","kind":"video","poster":"https://media.comfy.org/website/workshop/apps/reshoot/poster.jpg"}},"examples":[],"advancedFields":[],"pricing":null,"status":"active","license":null,"mediaConfidence":"exact","needsReview":true,"useCase":"edit-videos","recommendedRank":1}, +{"id":"apps/openjutsu","slug":"apps/openjutsu","modelId":"apps/openjutsu","type":"APP","displayName":"Openjutsu","description":"Swap a character in your video for one from a single image, and keep the scene and its sound.","media":{},"examples":[],"advancedFields":[],"pricing":null,"status":"active","license":null,"mediaConfidence":"none","needsReview":true,"useCase":"edit-videos","recommendedRank":2}, {"displayName":"Beeble SwitchX Image Edit","media":{"thumbnail":{"url":"https://media.comfy.org/website/workshop/beeble/switchx-image-edit/studio-to-neon-alley.png","kind":"image"},"samples":[{"url":"https://media.comfy.org/website/workshop/beeble/switchx-image-edit/studio-to-neon-alley.png","kind":"image","prompt":"A rain-soaked neon alley at night: wet asphalt mirroring magenta and cyan signs, a fire escape and steam drifting from a grate behind the figure, a hard on-camera flash freezing raindrops in mid-air and throwing a crisp shadow onto the wall, cool blue fill from the left, high contrast and a touch of film grain. The figure, the electric-blue windbreaker, the trousers, the pose and the full-length framing stay exactly the same. No readable signs, no text."}]},"examples":[{"title":"Studio to neon alley","description":"The subject's pixels are preserved while the studio backdrop and lighting are switched to a flash-lit neon street, showing Beeble SwitchX's scene switching.","values":{"prompt":"A rain-soaked neon alley at night: wet asphalt mirroring magenta and cyan signs, a fire escape and steam drifting from a grate behind the figure, a hard on-camera flash freezing raindrops in mid-air and throwing a crisp shadow onto the wall, cool blue fill from the left, high contrast and a touch of film grain. The figure, the electric-blue windbreaker, the trousers, the pose and the full-length framing stay exactly the same. No readable signs, no text.","alpha_mode":"auto","max_resolution":"1080p","medias":[{"role":"image","value":"https://media.comfy.org/website/workshop/beeble/switchx-image-edit/switchx-image-edit-input-1.1.png"}]}}],"advancedFields":["alpha_mode","max_resolution"],"pricing":{"creditsPerRun":6},"status":"active","license":null,"mediaConfidence":"exact","needsReview":false,"id":"beeble--switchx-image-edit--edit-images","slug":"beeble--switchx-image-edit--edit-images","modelId":"beeble/switchx-image-edit","useCase":"edit-images"}, {"displayName":"Beeble SwitchX Video Edit","media":{"thumbnail":{"url":"https://media.comfy.org/website/workshop/beeble/switchx-video-edit/overcast-to-neon-blue-hour.mp4","kind":"video"},"samples":[{"url":"https://media.comfy.org/website/workshop/beeble/switchx-video-edit/overcast-to-neon-blue-hour.mp4","kind":"video","prompt":"Same plaza, same bare plane trees, same chain-link fence and concrete ledge, relit from flat overcast noon to blue hour just after rain: a deep cobalt sky, hard magenta and cyan neon from off-frame throwing coloured rim light along the rider and the board, the wet asphalt mirroring long streaks of pink and teal, cool blue fill in the shadows, a warm sodium glow on the far trees. No signs, no lettering, no new buildings. Keep the rider, the board, every movement, the camera track and the framing exactly as they are; only the lighting, sky and reflections change."}]},"examples":[{"title":"Overcast to neon blue hour","description":"Relights a 5-second overcast skate clip to blue hour with neon reflections while the skater, motion, camera and plaza stay untouched (alpha_mode fill).","values":{"prompt":"Same plaza, same bare plane trees, same chain-link fence and concrete ledge, relit from flat overcast noon to blue hour just after rain: a deep cobalt sky, hard magenta and cyan neon from off-frame throwing coloured rim light along the rider and the board, the wet asphalt mirroring long streaks of pink and teal, cool blue fill in the shadows, a warm sodium glow on the far trees. No signs, no lettering, no new buildings. Keep the rider, the board, every movement, the camera track and the framing exactly as they are; only the lighting, sky and reflections change.","alpha_mode":"fill","max_resolution":"1080p"}}],"advancedFields":["alpha_mode","max_resolution"],"pricing":{"creditsPerRun":9},"status":"active","license":null,"mediaConfidence":"exact","needsReview":false,"id":"beeble--switchx-video-edit--edit-videos","slug":"beeble--switchx-video-edit--edit-videos","modelId":"beeble/switchx-video-edit","useCase":"edit-videos"}, {"displayName":"FLUX 2 Max Text-to-Image","media":{"thumbnail":{"url":"https://comfy-hub-assets.comfy.org/templates/a644a267-23f9-4a55-b967-4a7c05bd9da8.png","kind":"image"},"samples":[{"url":"https://cdn.jsdelivr.net/gh/Comfy-Org/workflow_templates@main/output/api_bfl_flux2_max_sofa_swap.png","kind":"image","prompt":"Replace the current sofa with the one in Image 2, then place the character from Image 3 on the new sofa. Keep the scene lighting, environment, color scheme, and all other furniture completely unchanged.\n"}]},"examples":[{"title":"FLUX.2 [max]: Object Swap","description":"Replace objects in images with unmatched quality using FLUX.2 [max]. Perfect for product photography, furniture swaps, and maintaining scene consistency with highest editing precision.","values":{"prompt":"Replace the current sofa with the one in Image 2, then place the character from Image 3 on the new sofa. Keep the scene lighting, environment, color scheme, and all other furniture completely unchanged.\n","width":1024,"height":768,"medias":[{"role":"image","value":"https://cdn.jsdelivr.net/gh/Comfy-Org/workflow_templates@main/input/minimalist_wooden_sofa_living_room.png"},{"role":"image","value":"https://cdn.jsdelivr.net/gh/Comfy-Org/workflow_templates@main/input/fluffy_beige_sofa.png"},{"role":"image","value":"https://cdn.jsdelivr.net/gh/Comfy-Org/workflow_templates@main/input/black_coat_model.png"}]}}],"advancedFields":["seed","height","prompt_upsampling","width"],"pricing":{"creditsPerRun":26},"status":"active","license":null,"mediaConfidence":"exact","needsReview":false,"id":"bfl--flux-2-max--generate-images","slug":"bfl--flux-2-max--generate-images","modelId":"bfl/flux-2-max","useCase":"generate-images"}, diff --git a/apps/website/src/content/workshop-workflows.jsonl b/apps/website/src/content/workshop-workflows.jsonl index 319e8223046..6819189063c 100644 --- a/apps/website/src/content/workshop-workflows.jsonl +++ b/apps/website/src/content/workshop-workflows.jsonl @@ -30,3 +30,4 @@ {"id":"workflows/remove-object","type":"CLOUD","definitionVersion":"1","source":{"repository":"Comfy-Org/ComfyUI_frontend","commit":"43fd83e5f8dcfa2ac558b6f0e42a88231c940738","path":"apps/website/src/data/workflows/api_bria_eraser.json"},"inputSchema":{"type":"object","properties":{"image":{"type":"string","format":"uri","pattern":"^https://"},"mask":{"type":"string","format":"uri","pattern":"^https://"}},"required":["image","mask"],"additionalProperties":false},"cloud":{"workflow":{"2":{"inputs":{"image":"vintage_girl_yellow_apple.png"},"class_type":"LoadImage","_meta":{"title":"Load Image"}},"4":{"inputs":{"filename_prefix":"Bria_Eraser","format":"png","images":["15",0],"format.bit_depth":"8-bit","format.input_color_space":"sRGB"},"class_type":"SaveImageAdvanced","_meta":{"title":"Save Image (Advanced)"}},"15":{"inputs":{"mask_type":"manual","moderation":"false","image":["2",0],"mask":["workshop-mask",0]},"class_type":"BriaEraser","_meta":{"title":"Bria Eraser"}},"workshop-mask":{"class_type":"LoadImageMask","inputs":{"image":"","channel":"red"},"_meta":{"title":"Edit mask (white = edit)"}}},"inputBindings":{"image":{"encoding":"cloud-asset","targets":[{"nodeId":"2","inputName":"image"}],"mediaKind":"image"},"mask":{"encoding":"cloud-asset","targets":[{"nodeId":"workshop-mask","inputName":"image"}],"mediaKind":"image"}}},"outputs":[{"id":"image","nodeId":"4","key":"images","kind":"image"}]} {"id":"apps/cinematic-studio","type":"APP","app":"studio"} {"id":"apps/reshoot","type":"APP","app":"reshoot"} +{"id":"apps/openjutsu","type":"APP","app":"openjutsu"} diff --git a/apps/website/src/data/workshop-model-availability.json b/apps/website/src/data/workshop-model-availability.json index 79e49fdfb1f..01f2434eb24 100644 --- a/apps/website/src/data/workshop-model-availability.json +++ b/apps/website/src/data/workshop-model-availability.json @@ -1,4 +1,8 @@ { + "apps/openjutsu": { + "flag": "workshop-openjutsu-app-enabled", + "reason": "Openjutsu has no deployment or app-proxy entry yet; shown only where PostHog turns it on." + }, "apps/reshoot": { "flag": "workshop-reshoot-app-enabled", "reason": "Re-shoot runs on the app proxy (comfy-api BE-17741/BE-17804); shown only where PostHog turns it on while it rolls out." diff --git a/apps/website/src/env.d.ts b/apps/website/src/env.d.ts index 1e4759003e7..b48e022c3e0 100644 --- a/apps/website/src/env.d.ts +++ b/apps/website/src/env.d.ts @@ -33,6 +33,10 @@ declare global { readonly PUBLIC_WORKSHOP_SAVE_ASSETS?: string /** `astro dev` only: the local CrossView dev proxy, e.g. http://127.0.0.1:4329. */ readonly PUBLIC_CROSSVIEW_PROXY?: string + /** `astro dev` only: '1' answers Openjutsu from the page itself, with no backend. */ + readonly PUBLIC_OPENJUTSU_SAMPLE?: string + /** `astro dev` only: a local dev proxy in front of Openjutsu's deployment. */ + readonly PUBLIC_OPENJUTSU_PROXY?: string /** Optional Turnstile mode override: off, shadow, or enforce. */ readonly PUBLIC_WORKSHOP_TURNSTILE_MODE?: string readonly PUBLIC_POSTHOG_KEY?: string diff --git a/apps/website/src/lib/workshop/apps.ts b/apps/website/src/lib/workshop/apps.ts index fc69dd66280..082e9b22bc7 100644 --- a/apps/website/src/lib/workshop/apps.ts +++ b/apps/website/src/lib/workshop/apps.ts @@ -7,6 +7,7 @@ export type WorkshopAppId = AppWorkshopModel['appId'] export function workshopAppHref(app: WorkshopAppId, locale: Locale): string { const routes = getRoutes(locale) + if (app === 'openjutsu') return routes.openjutsu return app === 'reshoot' ? routes.reshoot : routes.cinematicStudio } @@ -37,6 +38,12 @@ const appCopy = { summary: 'cinematic.hub.reshootSummary', badge: 'cinematic.hub.prototype', meta: 'cinematic.hub.reshootMeta' + }, + openjutsu: { + name: 'cinematic.hub.openjutsu', + summary: 'cinematic.hub.openjutsuSummary', + badge: 'cinematic.hub.prototype', + meta: 'cinematic.hub.openjutsuMeta' } } as const satisfies Record< WorkshopAppId, diff --git a/apps/website/src/lib/workshop/catalogue-apps.ts b/apps/website/src/lib/workshop/catalogue-apps.ts index 8d854fc24f6..82603e49788 100644 --- a/apps/website/src/lib/workshop/catalogue-apps.ts +++ b/apps/website/src/lib/workshop/catalogue-apps.ts @@ -15,7 +15,11 @@ const copy = { 'zh-CN': '像在片场一样执导镜头' }, reshootName: { en: 'Re-shoot a video', 'zh-CN': '重拍视频' }, - reshootTask: { en: 'Re-shoot from any angle', 'zh-CN': '从任意角度重拍' } + reshootTask: { en: 'Re-shoot from any angle', 'zh-CN': '从任意角度重拍' }, + openjutsuTask: { + en: 'Swap a character in a video', + 'zh-CN': '替换视频中的角色' + } } as const satisfies Record export function ac(key: keyof typeof copy, locale: Locale = 'en'): string { diff --git a/apps/website/src/lib/workshop/openjutsu/clip.test.ts b/apps/website/src/lib/workshop/openjutsu/clip.test.ts new file mode 100644 index 00000000000..d07fdad3902 --- /dev/null +++ b/apps/website/src/lib/workshop/openjutsu/clip.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it } from 'vitest' + +import { + clipFits, + defaultWindow, + fitWindow, + gridFrames, + pickCanvas +} from './clip' + +describe('clipFits', () => { + it.for<[number, boolean]>([ + [4.5, false], + [4.97, true], + [5, true], + [15, true], + [15.04, true], + [15.5, false], + [30, false] + ])('says whether a %f second clip may be used: %s', ([seconds, fits]) => { + expect(clipFits(seconds)).toBe(fits) + }) +}) + +describe('gridFrames', () => { + it.for<[number, number]>([ + [3.1, 73], + [5, 107], + [5.2, 124], + [10, 226], + [14.4, 345], + [15, 345] + ])('fits %f seconds into %i frames of 17k + 5', ([seconds, frames]) => { + expect(gridFrames(seconds)).toBe(frames) + expect((frames - 5) % 17).toBe(0) + }) + + it('never asks for more frames than the seconds hold', () => { + for (let seconds = 4; seconds <= 15; seconds += 0.37) + expect(gridFrames(seconds) / 24).toBeLessThanOrEqual(seconds + 1e-6) + }) +}) + +describe('fitWindow', () => { + it('starts a new clip on its first five seconds or so', () => { + expect(defaultWindow(12)).toEqual({ start: 0, seconds: 124 / 24 }) + }) + + it('takes all it can of a clip barely five seconds long', () => { + expect(defaultWindow(5.1)).toEqual({ start: 0, seconds: 107 / 24 }) + }) + + it('snaps a dragged length to one the model can run', () => { + const fitted = fitWindow({ start: 1, seconds: 7.9 }, 12) + expect(fitted.seconds * 24).toBe(192) + expect(fitted.start).toBe(1) + }) + + it('pulls a window back inside the clip', () => { + const fitted = fitWindow({ start: 11, seconds: 5 }, 12) + expect(fitted.start + fitted.seconds).toBeCloseTo(12) + }) + + it('stops at the longest and shortest swap', () => { + expect(fitWindow({ start: 0, seconds: 40 }, 15).seconds).toBe(345 / 24) + expect(fitWindow({ start: 0, seconds: 0.2 }, 15).seconds).toBe(73 / 24) + }) +}) + +describe('pickCanvas', () => { + it.for<[number, number, number, number]>([ + [1920, 1080, 1344, 768], + [1080, 1920, 768, 1344], + [1080, 1080, 768, 768], + [1440, 1080, 1024, 768], + [2560, 1080, 1536, 672] + ])('gives a %ix%i source a %ix%i canvas', ([w, h, width, height]) => { + expect(pickCanvas(w, h)).toEqual({ width, height }) + }) +}) diff --git a/apps/website/src/lib/workshop/openjutsu/clip.ts b/apps/website/src/lib/workshop/openjutsu/clip.ts new file mode 100644 index 00000000000..b1ed789033b --- /dev/null +++ b/apps/website/src/lib/workshop/openjutsu/clip.ts @@ -0,0 +1,126 @@ +/** MiniMax H3 reads and writes 24 frames a second, in lengths of 17k + 5 frames. */ +export const FPS = 24 +const GRID_STEP = 17 +const GRID_OFFSET = 5 + +/** The clip lengths a visitor may upload, in seconds. */ +const MIN_CLIP_SECONDS = 5 +const MAX_CLIP_SECONDS = 15 +/** A container rounds its length; a "5 second" file often reads 4.97. */ +const LENGTH_TOLERANCE = 0.05 + +/** The shortest swap: 73 frames, the length the LoRA's own training clips had. */ +const MIN_FRAMES = 73 +/** The longest: 345 frames (14.4 s). The next grid length passes H3's 15 s. */ +const MAX_FRAMES = 345 + +export const MIN_WINDOW_SECONDS = MIN_FRAMES / FPS +/** Short, continuous shots are where the LoRA holds up best. */ +const DEFAULT_WINDOW_SECONDS = 5 + +export function clipFits(seconds: number): boolean { + return ( + seconds >= MIN_CLIP_SECONDS - LENGTH_TOLERANCE && + seconds <= MAX_CLIP_SECONDS + LENGTH_TOLERANCE + ) +} + +/** The most frames H3's grid fits into `seconds` of source at 24 fps. */ +export function gridFrames(seconds: number): number { + const available = Math.floor(seconds * FPS + 1e-6) + const fitted = + available - + ((((available - GRID_OFFSET) % GRID_STEP) + GRID_STEP) % GRID_STEP) + return Math.min(MAX_FRAMES, Math.max(MIN_FRAMES, fitted)) +} + +/** The part of the clip a swap runs on. */ +export interface SwapWindow { + /** Seconds into the clip. */ + readonly start: number + /** Seconds asked for; the run uses `gridFrames` of it. */ + readonly seconds: number +} + +/** The longest window a clip of this length allows. */ +export const maxWindowSeconds = (clipSeconds: number) => + Math.min(clipSeconds, MAX_FRAMES / FPS) + +/** One grid step, in seconds: the least a swap can grow or shrink by. */ +export const WINDOW_STEP_SECONDS = GRID_STEP / FPS + +/** + * A window moved and resized until it lies inside the clip, its length the + * nearest one H3's grid allows, so what is shown is what will run. + */ +export function fitWindow(window: SwapWindow, clipSeconds: number): SwapWindow { + const steps = Math.round((window.seconds * FPS - GRID_OFFSET) / GRID_STEP) + const frames = Math.min( + gridFrames(maxWindowSeconds(clipSeconds)), + Math.max(MIN_FRAMES, steps * GRID_STEP + GRID_OFFSET) + ) + const seconds = frames / FPS + const start = Math.min( + Math.max(0, clipSeconds - seconds), + Math.max(0, window.start) + ) + return { start, seconds } +} + +export const defaultWindow = (clipSeconds: number): SwapWindow => + fitWindow({ start: 0, seconds: DEFAULT_WINDOW_SECONDS }, clipSeconds) + +export interface SwapCanvas { + readonly width: number + readonly height: number +} + +/** + * One canvas per aspect family at the LoRA's own 768 short edge (its training + * bucket is 1344x768). Each side is a multiple of 32, as H3 needs. + */ +const CANVASES: readonly SwapCanvas[] = [ + { width: 1344, height: 768 }, + { width: 768, height: 1344 }, + { width: 768, height: 768 }, + { width: 1024, height: 768 }, + { width: 768, height: 1024 }, + { width: 1536, height: 672 } +] + +/** The canvas closest in shape to the source, so the shot is not recomposed. */ +export function pickCanvas(width: number, height: number): SwapCanvas { + const ratio = Math.log(width / height) + return CANVASES.reduce((best, canvas) => + Math.abs(Math.log(canvas.width / canvas.height) - ratio) < + Math.abs(Math.log(best.width / best.height) - ratio) + ? canvas + : best + ) +} + +export interface ClipFacts { + readonly seconds: number + readonly width: number + readonly height: number +} + +/** A video's length and frame size from its metadata; undefined if unreadable. */ +export function clipFactsOf(url: string): Promise { + return new Promise((resolve) => { + const video = document.createElement('video') + video.preload = 'metadata' + video.onloadedmetadata = () => + resolve( + Number.isFinite(video.duration) && video.videoWidth > 0 + ? { + seconds: video.duration, + width: video.videoWidth, + height: video.videoHeight + } + : undefined + ) + video.onerror = () => resolve(undefined) + video.src = url + }) +} diff --git a/apps/website/src/lib/workshop/openjutsu/sample-transport.test.ts b/apps/website/src/lib/workshop/openjutsu/sample-transport.test.ts new file mode 100644 index 00000000000..0ad866f46f1 --- /dev/null +++ b/apps/website/src/lib/workshop/openjutsu/sample-transport.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from 'vitest' + +import { ReshootError } from '@/lib/workshop/cinematic-studio/reshoot-engine/transport' +import { sampleScenario, sampleTransport } from './sample-transport' + +const clip = new File(['clip'], 'clip.mp4', { type: 'video/mp4' }) + +describe('sampleScenario', () => { + it.for<[string, string]>([ + ['', 'free'], + ['?sample=paid', 'paid'], + ['?sample=nonsense', 'free'] + ])('reads %s as %s', ([search, scenario]) => { + expect(sampleScenario(search)).toBe(scenario) + }) +}) + +describe('sampleTransport', () => { + it('queues, runs, then hands the uploaded clip back as the result', async () => { + const transport = sampleTransport(() => 'free') + await transport.upload(clip) + const job = await transport.submit({}, 'key') + const statuses = [] + for (let poll = 0; poll < 4; poll++) + statuses.push((await transport.job(job.id)).status) + expect(statuses).toEqual(['queued', 'running', 'running', 'succeeded']) + const done = await transport.job(job.id) + expect(done.outputs?.at(0)?.filename).toContain('result') + await expect(transport.output(done, {})).resolves.toBe(clip) + }) + + it('ends a run in failure when asked to', async () => { + const transport = sampleTransport(() => 'fails') + const job = await transport.submit({}, 'key') + for (let poll = 0; poll < 3; poll++) await transport.job(job.id) + expect((await transport.job(job.id)).status).toBe('failed') + }) + + it.for<[ReturnType, string, string | undefined]>([ + ['paid', 'paid', undefined], + ['exhausted', 'blocked', 'free_runs_exhausted'], + ['no-credits', 'blocked', 'insufficient_credits'] + ])('quotes %s as %s', async ([scenario, next, reason]) => { + const quote = await sampleTransport(() => scenario).quote() + expect(quote?.next_run).toBe(next) + expect(quote?.blocked_reason).toBe(reason) + }) + + it('refuses the quote when the app is unavailable', async () => { + await expect(sampleTransport(() => 'unavailable').quote()).rejects.toThrow( + ReshootError + ) + }) +}) diff --git a/apps/website/src/lib/workshop/openjutsu/sample-transport.ts b/apps/website/src/lib/workshop/openjutsu/sample-transport.ts new file mode 100644 index 00000000000..3f5dc26d5f1 --- /dev/null +++ b/apps/website/src/lib/workshop/openjutsu/sample-transport.ts @@ -0,0 +1,119 @@ +import type { + ReshootJob, + ReshootQuote, + ReshootTransport +} from '@/lib/workshop/cinematic-studio/reshoot-engine/transport' +import { ReshootError } from '@/lib/workshop/cinematic-studio/reshoot-engine/transport' + +/** + * A stand-in backend that lives in the page, for building the interface before + * a deployment exists. No model runs: a "result" is the uploaded clip handed + * back unchanged. `?sample=` picks which answer the app proxy would give. + */ +const SAMPLE_SCENARIOS = [ + 'free', + 'paid', + 'exhausted', + 'no-credits', + 'unavailable', + 'fails' +] as const +type SampleScenario = (typeof SAMPLE_SCENARIOS)[number] + +const ALLOWANCE = { runs: 5, period: 'P1D', period_seconds: 86_400 } +const PRICE = 60 + +const QUOTES: Readonly> = { + free: { + free_runs_allowance: ALLOWANCE, + free_runs_remaining: 3, + resets_at: null, + price_credits: PRICE, + next_run: 'free' + }, + fails: { + free_runs_allowance: ALLOWANCE, + free_runs_remaining: 3, + resets_at: null, + price_credits: PRICE, + next_run: 'free' + }, + paid: { + free_runs_allowance: ALLOWANCE, + free_runs_remaining: 0, + resets_at: null, + price_credits: PRICE, + next_run: 'paid' + }, + exhausted: { + free_runs_allowance: ALLOWANCE, + free_runs_remaining: 0, + resets_at: new Date(Date.now() + 5 * 3_600_000).toISOString(), + price_credits: 0, + next_run: 'blocked', + blocked_reason: 'free_runs_exhausted' + }, + 'no-credits': { + free_runs_allowance: ALLOWANCE, + free_runs_remaining: 0, + resets_at: null, + price_credits: PRICE, + next_run: 'blocked', + blocked_reason: 'insufficient_credits' + }, + unavailable: undefined +} + +/** Polls a job answers `queued`, then `running`, before it settles. */ +const QUEUED_POLLS = 1 +const RUNNING_POLLS = 2 + +export function sampleScenario(search: string): SampleScenario { + const asked = new URLSearchParams(search).get('sample') + return SAMPLE_SCENARIOS.find((scenario) => scenario === asked) ?? 'free' +} + +/** `pick` is asked on each call: the page's address is only there in a browser. */ +export function sampleTransport(pick: () => SampleScenario): ReshootTransport { + const uploads = new Map() + const polls = new Map() + let lastVideo: File | undefined + + return { + async quote() { + const scenario = pick() + if (scenario === 'unavailable') throw new ReshootError('app_unavailable') + return QUOTES[scenario] + }, + async upload(file) { + const name = `sample-${uploads.size + 1}-${file.name}` + uploads.set(name, file) + if (file.type.startsWith('video/')) lastVideo = file + return name + }, + async submit(): Promise { + const id = `sample-job-${polls.size + 1}` + polls.set(id, 0) + return { id, status: 'queued' } + }, + async job(id): Promise { + const seen = (polls.get(id) ?? 0) + 1 + polls.set(id, seen) + if (seen <= QUEUED_POLLS) return { id, status: 'queued' } + if (seen <= QUEUED_POLLS + RUNNING_POLLS) return { id, status: 'running' } + if (pick() === 'fails') return { id, status: 'failed' } + return { + id, + status: 'succeeded', + outputs: [{ id: 'result', filename: 'openjutsu/result_00001_.mp4' }] + } + }, + async output() { + if (!lastVideo) throw new ReshootError('missing_output') + return lastVideo + }, + async cancel() { + // nothing is running anywhere, so there is nothing to stop + } + } +} diff --git a/apps/website/src/lib/workshop/openjutsu/swap.api.json b/apps/website/src/lib/workshop/openjutsu/swap.api.json new file mode 100644 index 00000000000..1fd5f9563a3 --- /dev/null +++ b/apps/website/src/lib/workshop/openjutsu/swap.api.json @@ -0,0 +1,204 @@ +{ + "119": { + "class_type": "VAELoader", + "inputs": { + "vae_name": "minimax_h3_video_vae_fp16.safetensors" + } + }, + "120": { + "class_type": "VAELoader", + "inputs": { + "vae_name": "minimax_h3_audio_vae_fp32.safetensors" + } + }, + "123": { + "class_type": "KSamplerSelect", + "inputs": { + "sampler_name": "res_multistep" + } + }, + "124": { + "class_type": "BasicScheduler", + "inputs": { + "scheduler": "simple", + "denoise": 1, + "model": ["145", 0], + "steps": 20 + } + }, + "125": { + "class_type": "SamplerCustomAdvanced", + "inputs": { + "noise": ["129", 0], + "guider": ["126", 0], + "sampler": ["123", 0], + "sigmas": ["124", 0], + "latent_image": ["136", 1] + } + }, + "126": { + "class_type": "BasicGuider", + "inputs": { + "model": ["145", 0], + "conditioning": ["136", 0] + } + }, + "127": { + "class_type": "UNETLoader", + "inputs": { + "unet_name": "minimax_h3_ref2va_pruned_int8_convrot.safetensors", + "weight_dtype": "default" + } + }, + "128": { + "class_type": "CLIPLoader", + "inputs": { + "clip_name": "qwen3vl_32b_minimax_h3_nvfp4_awq.safetensors", + "type": "minimax", + "device": "default" + } + }, + "129": { + "class_type": "RandomNoise", + "inputs": { + "noise_seed": 0 + } + }, + "130": { + "class_type": "CreateVideo", + "inputs": { + "fps": 24, + "bit_depth": 8, + "color_space": "sRGB", + "codec": "none", + "images": ["decode", 0], + "audio": ["components", 1] + } + }, + "136": { + "class_type": "MiniMaxH3ReferenceToVideo", + "inputs": { + "clip": ["128", 0], + "vae": ["119", 0], + "audio_vae": ["120", 0], + "prompt": "Swap the person in