From 5ee1ea7d76b9990da92dd666866482fb2ba767e4 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 02:13:18 +0000 Subject: [PATCH 1/3] Keep karaoke captions still while the highlight moves libass has no CSS transform: scaling the active word changed its advance width, so the centred line re-flowed and every other word slid sideways on each word change, and the word settled at 109% while the preview settles at 100%. Each word event is now drawn as two layers from the same text: a base layer with the active word transparent, and a pop layer showing only the active word, which scales about its own centre. Adds scripts/bench-caption-stability.ts, which renders every style through libass and measures word drift against a no-pop render: before 17-26px settled drift and 109.1% active size, after 0px and 100.0% on all 12 styles. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_019QYatd8aXSLCVxPx1SggNZ --- CHANGELOG.md | 1 + README.md | 2 +- scripts/bench-caption-stability.ts | 217 +++++++++++++++++++++++++++++ src/main/pipeline/captions.ts | 56 ++++++-- tests/captions.test.ts | 26 ++++ 5 files changed, 289 insertions(+), 13 deletions(-) create mode 100644 scripts/bench-caption-stability.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index cefab17..1034dc3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ still pre-1.0, minor bumps carry new features and patch bumps carry fixes. ### Improved +- Exported karaoke captions now hold still while the highlight moves from word to word. Previously the highlighted word was scaled up inside the line, which pushed the other words 17-26 pixels sideways on every word change and left it about 9% larger than the preview showed. The word now pops in place, exactly as in the preview, and every other word stays put (measured with `scripts/bench-caption-stability.ts`). - The setup wizard no longer shows the unavailable "Claude subscription" card. Claude models can be used through OpenRouter instead. ## [0.13.0] - 2026-09-25 diff --git a/README.md b/README.md index 1d56799..5c842b3 100644 --- a/README.md +++ b/README.md @@ -192,7 +192,7 @@ npm run typecheck npm run lint ``` -Integration test scripts live in `scripts/` (`test-pipeline`, `test-e2e`, `test-quality`, `test-wholevideo`, `test-encoders`, `test-resilience`, `test-broll`, `test-youtube`, `test-asd`, `smoke-test.sh`). See each file's header for what it covers. The e2e ones need `OPENAI_API_KEY`. +Integration test scripts live in `scripts/` (`test-pipeline`, `test-e2e`, `test-quality`, `test-wholevideo`, `test-encoders`, `test-resilience`, `test-broll`, `test-youtube`, `test-asd`, `smoke-test.sh`). `bench-caption-stability` renders karaoke captions through libass and checks that no word moves when the highlight changes and that the highlighted word settles at preview size (`--check` fails otherwise). See each file's header for what it covers. The e2e ones need `OPENAI_API_KEY`. To measure clip quality on your own projects, `scripts/eval-clips.ts` reads every saved project and reports how many clips open mid-sentence, cut a sentence off, or trail into dead air, plus the length spread. Add `--rerun` (needs `OPENAI_API_KEY`, a few cents per project, no transcription cost) to re-run clip detection on the saved transcripts with the current prompts and compare, which is how to check a prompt change actually helps: diff --git a/scripts/bench-caption-stability.ts b/scripts/bench-caption-stability.ts new file mode 100644 index 0000000..80444a4 --- /dev/null +++ b/scripts/bench-caption-stability.ts @@ -0,0 +1,217 @@ +/** + * Measures how steady burned-in karaoke captions are while the highlight moves + * from word to word, and whether the highlighted word ends up the size the + * live preview shows. + * + * The preview pops the active word with a CSS transform, which never moves the + * other words. libass has no transform: scaling one word changes its advance + * width, so a centred line re-flows and every other word slides sideways. + * This renders real frames through libass and reports, per caption style: + * + * settled largest ink-edge difference from the same captions rendered + * with no pop at all (the preview's settled layout), measured + * once each word's pop has finished (px) + * during pop the same, sampled every 20ms through the first 160ms of each + * word while the pop animates (px) + * active size settled width of the highlighted word / its width unpopped + +Both drift figures track the words that are not highlighted (their own colour) +against the no-pop render, so group changes and pill outlines cancel out: +0px means no other word ever moves. Styles whose highlight shares the base +colour fall back to all visible ink. + * + * Makes no API calls. Run with: + * npx tsx --tsconfig tsconfig.node.json scripts/bench-caption-stability.ts [--check] + * + * `--check` fails when a settled style moves more than 1px or settles off + * preview size. + */ +import { mkdir, rm, writeFile, readFile } from 'node:fs/promises' +import { join } from 'node:path' +import { spawn } from 'node:child_process' +import { buildAss } from '../src/main/pipeline/captions' +import { parseFontMetrics } from '../src/main/fonts' +import { CAPTION_STYLES } from '../src/shared/captionStyles' +import { FFMPEG_PATH } from '../src/main/pipeline/ffmpeg' +import type { Transcript } from '../src/shared/types' + +const WORK = join(process.cwd(), '.tmp', 'caption-stability') +const W = 1080 +const H = 1920 +const FPS = 100 +/** Long enough for any pop animation to finish before the measured frame. */ +const WORD_SEC = 0.6 +const SETTLE_SEC = 0.4 +/** Mixed widths, so a re-flow cannot cancel out across words. */ +const WORDS = ['I', 'ABSOLUTELY', 'LOVE', 'IT', 'WHEN', 'EVERYTHING', 'WORKS', 'OUT'] + +const FONT_FILES: Record = { + Anton: 'resources/fonts/Anton-Regular.ttf', + Poppins: 'resources/fonts/Poppins-Bold.ttf', + 'Poppins Medium': 'resources/fonts/Poppins-Medium.ttf' +} + +function transcript(): Transcript { + const words = WORDS.map((text, i) => ({ text, start: i * WORD_SEC, end: (i + 1) * WORD_SEC - 0.05 })) + const end = WORDS.length * WORD_SEC + return { + language: 'en', + durationSec: end, + segments: [{ id: 0, start: 0, end, text: WORDS.join(' '), words }] + } +} + +function esc(p: string): string { + return p.replace(/\\/g, '/').replace(/:/g, '\\:').replace(/'/g, "\\'") +} + +function run(args: string[]): Promise { + return new Promise((resolve, reject) => { + const child = spawn(FFMPEG_PATH, ['-hide_banner', '-loglevel', 'error', '-y', ...args], { windowsHide: true }) + const out: Buffer[] = [] + child.stdout.on('data', (c: Buffer) => out.push(c)) + child.on('error', reject) + child.on('close', (code) => (code === 0 ? resolve(Buffer.concat(out)) : reject(new Error(`ffmpeg exited ${code}`)))) + }) +} + +/** Renders the given times (seconds) of an ASS file onto black, as RGB frames. */ +async function renderFrames(assPath: string, times: number[]): Promise { + const frames = times.map((t) => Math.round(t * FPS)) + const select = [...new Set(frames)].map((n) => `eq(n\\,${n})`).join('+') + const duration = (Math.max(...frames) + 2) / FPS + const raw = await run([ + '-f', 'lavfi', '-i', `color=c=black:s=${W}x${H}:r=${FPS}:d=${duration}`, + '-vf', `ass=filename='${esc(assPath)}':fontsdir='${esc('resources/fonts')}',select='${select}'`, + '-fps_mode', 'passthrough', '-pix_fmt', 'rgb24', '-f', 'rawvideo', '-' + ]) + // select emits frames in stream order, whatever order they were asked for. + const order = [...new Set(frames)].sort((a, b) => a - b) + const size = W * H * 3 + if (raw.length !== size * order.length) throw new Error(`expected ${order.length} frames, got ${raw.length / size}`) + return frames.map((n) => { + const i = order.indexOf(n) + return raw.subarray(i * size, (i + 1) * size) + }) +} + +interface Box { left: number; right: number } + +/** Horizontal ink extent of pixels matching a colour (or any visible ink). */ +function inkBox(frame: Buffer, rgb?: number[]): Box | null { + let left = W + let right = -1 + for (let y = 0; y < H; y++) { + for (let x = 0; x < W; x++) { + const p = (y * W + x) * 3 + const hit = rgb + ? Math.abs(frame[p] - rgb[0]) < 40 && Math.abs(frame[p + 1] - rgb[1]) < 40 && Math.abs(frame[p + 2] - rgb[2]) < 40 + : frame[p] + frame[p + 1] + frame[p + 2] > 60 + if (hit) { + if (x < left) left = x + if (x > right) right = x + } + } + } + return right < 0 ? null : { left, right } +} + +function hexRgb(hex: string): number[] { + const h = hex.replace('#', '') + return [0, 2, 4].map((i) => parseInt(h.slice(i, i + 2), 16)) +} + +function distinct(a: number[], b: number[]): boolean { + return a.some((v, i) => Math.abs(v - b[i]) > 80) +} + +/** The same captions with every scale override removed: the preview's settled look. */ +function withoutPop(ass: string): string { + return ass.replace(/\\t\([^)]*\)/g, '').replace(/\\fsc[xy]\d+/g, '') +} + +interface Row { style: string; settled: number; transient: number; activeSize: number | null } + +/** Sample offsets into each word: settled first, then through the pop. */ +const OFFSETS = [SETTLE_SEC, 0.01, 0.03, 0.05, 0.07, 0.09, 0.11, 0.13, 0.15] + +async function measure(styleId: string, fontFamily: string, highlight: string, text: string): Promise { + const file = FONT_FILES[fontFamily] + if (!file) return null + const metrics = parseFontMetrics(await readFile(file)) + const clipEnd = WORDS.length * WORD_SEC + const ass = buildAss(transcript(), { styleId, width: W, height: H, clipStart: 0, clipEnd, fontMetrics: metrics }) + const popPath = join(WORK, `${styleId}.ass`) + const staticPath = join(WORK, `${styleId}-static.ass`) + await writeFile(popPath, ass, 'utf8') + await writeFile(staticPath, withoutPop(ass), 'utf8') + + const times = WORDS.flatMap((_, i) => OFFSETS.map((o) => i * WORD_SEC + o)) + const [popFrames, staticFrames] = await Promise.all([renderFrames(popPath, times), renderFrames(staticPath, times)]) + + let settled = 0 + let transient = 0 + const ratios: number[] = [] + const hl = hexRgb(highlight) + const base = hexRgb(text) + const visible = (c: number[]): boolean => c.some((v) => v > 96) + const sizeMeasurable = distinct(hl, base) && visible(hl) + // Track the words that are not highlighted (base colour) when that colour + // stands apart; otherwise fall back to all visible ink. + const othersColour = distinct(hl, base) && visible(base) ? base : undefined + for (let i = 0; i < times.length; i++) { + const box = inkBox(popFrames[i], othersColour) + const ref = inkBox(staticFrames[i], othersColour) + if (!box || !ref) continue + const drift = Math.max(Math.abs(box.left - ref.left), Math.abs(box.right - ref.right)) + const isSettled = i % OFFSETS.length === 0 + if (isSettled) settled = Math.max(settled, drift) + else transient = Math.max(transient, drift) + if (isSettled && sizeMeasurable) { + const a = inkBox(popFrames[i], hl) + const b = inkBox(staticFrames[i], hl) + if (a && b && b.right > b.left) ratios.push((a.right - a.left + 1) / (b.right - b.left + 1)) + } + } + const activeSize = ratios.length ? ratios.reduce((s, r) => s + r, 0) / ratios.length : null + return { style: styleId, settled, transient, activeSize } +} + +async function main(): Promise { + const check = process.argv.includes('--check') + await rm(WORK, { recursive: true, force: true }) + await mkdir(WORK, { recursive: true }) + + const rows: Row[] = [] + for (const s of CAPTION_STYLES) { + const row = await measure(s.id, s.fontFamily, s.highlightColor, s.textColor) + if (row) rows.push(row) + } + + console.log('style settled during pop active size') + for (const r of rows) { + console.log( + `${r.style.padEnd(10)} ${`${r.settled}px`.padStart(10)} ${`${r.transient}px`.padStart(11)} ` + + (r.activeSize === null ? ' n/a' : `${(r.activeSize * 100).toFixed(1)}%`.padStart(6)) + ) + } + const worst = Math.max(...rows.map((r) => r.settled)) + const worstPop = Math.max(...rows.map((r) => r.transient)) + const sizes = rows.flatMap((r) => (r.activeSize === null ? [] : [r.activeSize])) + const meanSize = sizes.reduce((s, v) => s + v, 0) / sizes.length + console.log(`\n${rows.length} styles · worst settled drift ${worst}px · worst drift during pop ${worstPop}px · mean settled active-word size ${(meanSize * 100).toFixed(1)}%`) + + if (check) { + const bad = rows.filter((r) => r.settled > 1 || (r.activeSize !== null && Math.abs(r.activeSize - 1) > 0.02)) + if (bad.length) { + console.error(`Unstable or off-size captions: ${bad.map((r) => r.style).join(', ')}`) + process.exit(1) + } + console.log('All styles hold their layout and settle at preview size.') + } +} + +main().catch((err) => { + console.error(err) + process.exit(1) +}) diff --git a/src/main/pipeline/captions.ts b/src/main/pipeline/captions.ts index 4982d72..37fea5d 100644 --- a/src/main/pipeline/captions.ts +++ b/src/main/pipeline/captions.ts @@ -62,7 +62,38 @@ function escapeAss(text: string): string { return text.replace(/\\/g, '').replace(/[{}]/g, '').replace(/\n/g, ' ') } -function renderGroupText(group: WordGroup, activeIndex: number, style: CaptionStyle): string { +/** Fully transparent text, outline and shadow: keeps a word's space in the layout. */ +const HIDDEN = '\\alpha&HFF&' + +/** + * Pop for the word being spoken, matching the preview's `captionPop` + * (scale 0.94 -> 1 over 90ms, ease-out). Accel 0.5 decelerates like ease-out. + */ +const POP_IN = '\\fscx94\\fscy94\\t(0,90,0.5,\\fscx100\\fscy100)' + +/** + * One layer of a karaoke caption. + * + * libass has no CSS transform: scaling a word changes its advance width, so a + * centred line re-flows and every other word slides sideways each time the + * highlight moves on (17-26px on a 1080px frame, see + * scripts/bench-caption-stability.ts). The preview scales the word with a + * transform that moves nothing else. To match it, each word event is drawn + * twice from the same text, so both layers share one layout: + * + * - `base` draws every word except the active one, which stays in the layout + * but is transparent. + * - `pop` draws only the active word. Scaling one word re-centres the line + * by half its growth, which leaves that word's own centre where it was, so + * the word pops in place. Every other word in this layer is transparent. + */ +function renderGroupText( + group: WordGroup, + activeIndex: number, + style: CaptionStyle, + layer: 'base' | 'pop', + animate: boolean +): string { const highlight = assColor(style.highlightColor) const base = assColor(style.textColor) const lines: string[] = [] @@ -72,17 +103,14 @@ function renderGroupText(group: WordGroup, activeIndex: number, style: CaptionSt for (const word of line) { const raw = escapeAss(word.text) const text = style.uppercase ? raw.toUpperCase() : raw - if (index === activeIndex) { - const pop = '\\t(0,70,\\fscx109\\fscy109)' - if (style.highlightBoxColor) { - // Fat outline in the pill colour approximates a rounded label. - const pill = assColor(style.highlightBoxColor) - parts.push(`{\\c${highlight}\\bord10\\3c${pill}\\fscx100\\fscy100${pop}}${text}{\\r}`) - } else { - parts.push(`{\\c${highlight}\\fscx100\\fscy100${pop}}${text}{\\r}`) - } + const active = index === activeIndex + if (active) { + // Fat outline in the pill colour approximates a rounded label. + const pill = style.highlightBoxColor ? `\\bord10\\3c${assColor(style.highlightBoxColor)}` : '' + const look = layer === 'pop' ? (animate ? POP_IN : '') : HIDDEN + parts.push(`{\\c${highlight}${pill}${look}}${text}{\\r}`) } else { - parts.push(`{\\c${base}}${text}{\\r}`) + parts.push(layer === 'pop' ? `{${HIDDEN}}${text}{\\r}` : `{\\c${base}}${text}{\\r}`) } index++ } @@ -231,7 +259,11 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text const from = cuts[part], until = cuts[part + 1] const position = captionPositionAt(opts.positionRanges, opts.clipStart + (from + until) / 2, style.positionY) const anchor = `{\\an5\\q2\\pos(${captionX},${Math.round(position * opts.height)})}` - lines.push(`Dialogue: 0,${assTime(from)},${assTime(until)},Caption,,0,0,0,,${anchor}${renderGroupText(group, i, style)}`) + const span = `${assTime(from)},${assTime(until)},Caption,,0,0,0,,${anchor}` + lines.push(`Dialogue: 0,${span}${renderGroupText(group, i, style, 'base', false)}`) + // Layer 2 sits above the base (the title uses layer 1). The pop only + // plays when the word starts, not again at a layout band boundary. + lines.push(`Dialogue: 2,${span}${renderGroupText(group, i, style, 'pop', part === 0)}`) } } }) diff --git a/tests/captions.test.ts b/tests/captions.test.ts index 218689a..1658ea4 100644 --- a/tests/captions.test.ts +++ b/tests/captions.test.ts @@ -19,6 +19,32 @@ describe('buildAss', () => { expect(events.length).toBe(4) }) + it('pops the active word on its own layer so the other words never move', () => { + const ass = buildAss(transcript, base) + const baseLayer = ass.split('\n').filter((l) => l.startsWith('Dialogue: 0,')) + const popLayer = ass.split('\n').filter((l) => l.startsWith('Dialogue: 2,')) + expect(popLayer).toHaveLength(baseLayer.length) + // Scaling a word re-flows a centred libass line, so the layer that draws + // the other words never scales anything. + for (const line of baseLayer) expect(line).not.toMatch(/\\fsc|\\t\(/) + // Both layers carry the same words and anchor, so they share one layout. + const strip = (l: string): string => l.slice(l.indexOf(',', 12)).replace(/\{[^}]*\}/g, '') + baseLayer.forEach((line, i) => expect(strip(popLayer[i])).toBe(strip(line))) + // Second word active: hidden in the base layer, the only visible word on top. + expect(baseLayer[1]).toMatch(/\{\\c&H00D4FF&\\alpha&HFF&\}BRAVE/) + expect(popLayer[1]).toContain('{\\alpha&HFF&}HELLO') + expect(popLayer[1]).toMatch(/\\fscx94\\fscy94\\t\(0,90,0\.5,\\fscx100\\fscy100\)\}BRAVE/) + }) + + it('plays the pop once per word, not again at a layout boundary', () => { + const ass = buildAss(transcript, { ...base, positionRanges: [{ start: 0.25, end: 0.4, positionY: 0.38 }] }) + const pops = ass.split('\n').filter((l) => l.startsWith('Dialogue: 2,')) + // The first word is split in three by the band; only its first part pops. + expect(pops[0]).toContain('\\t(') + expect(pops[1]).not.toContain('\\t(') + expect(pops[2]).not.toContain('\\t(') + }) + it('splits a caption event at a layout boundary to keep preview and export positions aligned', () => { const ass = buildAss(transcript, { ...base, positionRanges: [{ start: .25, end: .4, positionY: .38 }] }) const events = ass.split('\n').filter(l => l.startsWith('Dialogue: 0,')) From 23a0e86e4a153ba197363e808dbd945ba835664d Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 02:21:45 +0000 Subject: [PATCH 2/3] Hit the loudness target on mono and peak-limited clips Exports are stereo, but loudness was measured on the source layout. A mono source is spread to both channels 3 dB down, so measuring the mono peak sent ordinary speech down the limiter path and limited it about 3 dB harder than needed. Measure and normalise the stereo mix instead. The limiter path applied TARGET_I - inputI and then lost loudness to the limiter, so spiky speech landed short. When a clip needs limiting, re-measure the limited result (audio only, at most two passes) and raise the gain into the same limiter. Adds scripts/bench-loudness.ts: mean error 1.30 LU mono / 0.52 LU stereo before, 0.26 LU both after; true peak <= -2.0 dBTP throughout. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_019QYatd8aXSLCVxPx1SggNZ --- CHANGELOG.md | 1 + README.md | 2 +- scripts/bench-loudness.ts | 115 ++++++++++++++++++++++++++++++++++ src/main/pipeline/loudness.ts | 73 +++++++++++++++++++-- src/main/pipeline/render.ts | 9 +-- tests/loudness.test.ts | 36 ++++++++++- tests/render.test.ts | 5 ++ 7 files changed, 229 insertions(+), 12 deletions(-) create mode 100644 scripts/bench-loudness.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 1034dc3..7316e20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ still pre-1.0, minor bumps carry new features and patch bumps carry fixes. ### Improved - Exported karaoke captions now hold still while the highlight moves from word to word. Previously the highlighted word was scaled up inside the line, which pushed the other words 17-26 pixels sideways on every word change and left it about 9% larger than the preview showed. The word now pops in place, exactly as in the preview, and every other word stays put (measured with `scripts/bench-caption-stability.ts`). +- Exports land closer to the -14 LUFS loudness target. Mono recordings (common for podcasts and calls) were measured before being spread to stereo, so they were limited about 3 dB harder than needed and could come out up to 2.4 LU quiet. Clips that need peak limiting are now re-measured after the limiter and topped up, so laughs, claps and plosives no longer pull the whole clip under target. Across the benchmark signals, mean error fell from 1.3 LU (mono) and 0.5 LU (stereo) to 0.3 LU, with true peaks still at or below -2 dBTP (`scripts/bench-loudness.ts`). - The setup wizard no longer shows the unavailable "Claude subscription" card. Claude models can be used through OpenRouter instead. ## [0.13.0] - 2026-09-25 diff --git a/README.md b/README.md index 5c842b3..fa1bc25 100644 --- a/README.md +++ b/README.md @@ -192,7 +192,7 @@ npm run typecheck npm run lint ``` -Integration test scripts live in `scripts/` (`test-pipeline`, `test-e2e`, `test-quality`, `test-wholevideo`, `test-encoders`, `test-resilience`, `test-broll`, `test-youtube`, `test-asd`, `smoke-test.sh`). `bench-caption-stability` renders karaoke captions through libass and checks that no word moves when the highlight changes and that the highlighted word settles at preview size (`--check` fails otherwise). See each file's header for what it covers. The e2e ones need `OPENAI_API_KEY`. +Integration test scripts live in `scripts/` (`test-pipeline`, `test-e2e`, `test-quality`, `test-wholevideo`, `test-encoders`, `test-resilience`, `test-broll`, `test-youtube`, `test-asd`, `smoke-test.sh`). `bench-caption-stability` renders karaoke captions through libass and checks that no word moves when the highlight changes and that the highlighted word settles at preview size (`--check` fails otherwise). `bench-loudness` runs mono and stereo test signals through the export's audio master and reports loudness error and true peak. See each file's header for what it covers. The e2e ones need `OPENAI_API_KEY`. To measure clip quality on your own projects, `scripts/eval-clips.ts` reads every saved project and reports how many clips open mid-sentence, cut a sentence off, or trail into dead air, plus the length spread. Add `--rerun` (needs `OPENAI_API_KEY`, a few cents per project, no transcription cost) to re-run clip detection on the saved transcripts with the current prompts and compare, which is how to check a prompt change actually helps: diff --git a/scripts/bench-loudness.ts b/scripts/bench-loudness.ts new file mode 100644 index 0000000..ce3f95d --- /dev/null +++ b/scripts/bench-loudness.ts @@ -0,0 +1,115 @@ +/** + * Checks that exports land on the loudness target without overshooting the + * true-peak ceiling, for mono and stereo sources. + * + * Generates deterministic speech-like test signals (a voiced, syllable-paced + * buzz with pauses, plus variants: quiet, with sharp transients, over a bright + * music bed, already hot), runs each through the export's real master chain + * (measure, then linear gain or gain + limiter, then AAC at the export bitrate) + * and measures the result with ffmpeg's EBU R128 scanner. + * + * Makes no API calls. Run with: + * npx tsx --tsconfig tsconfig.node.json scripts/bench-loudness.ts [--check] + * + * `--check` fails when any export misses the target by more than 1.5 LU or + * has a true peak above -1 dBTP. + */ +import { mkdir, rm } from 'node:fs/promises' +import { join } from 'node:path' +import { spawn } from 'node:child_process' +import { FFMPEG_PATH } from '../src/main/pipeline/ffmpeg' +import { STEREO, TARGET_I, loudnormFilter, measureLoudness, normalisationMode } from '../src/main/pipeline/loudness' + +const WORK = join(process.cwd(), '.tmp', 'loudness-bench') +const SECONDS = 20 + +/** + * Speech-like source: a 110-190 Hz buzz with harmonics, gated into ~4 Hz + * syllables with a pause every few seconds, plus a little breath noise. + */ +const VOICE = + "(0.55*sin(2*PI*(150+40*sin(2*PI*0.3*t))*t)+0.3*sin(4*PI*(150+40*sin(2*PI*0.3*t))*t)+0.15*sin(6*PI*(150+40*sin(2*PI*0.3*t))*t))" + + '*pow(max(0,sin(2*PI*4.1*t)),2)*(0.6+0.4*sin(2*PI*0.23*t))*gt(mod(t,3.7),0.45)' + +interface Signal { name: string; expr: string; gainDb: number } + +const SIGNALS: Signal[] = [ + { name: 'speech', expr: VOICE, gainDb: -6 }, + { name: 'quiet', expr: VOICE, gainDb: -26 }, + { name: 'transients', expr: `${VOICE}*0.5+if(lt(mod(t,1.7),0.004),0.95*sin(2*PI*3000*t),0)`, gainDb: -3 }, + { name: 'music bed', expr: `${VOICE}*0.7+0.12*sgn(sin(2*PI*220*t))`, gainDb: -4 }, + { name: 'hot', expr: `min(0.98,max(-0.98,4*${VOICE}))`, gainDb: 0 } +] + +function ffmpeg(args: string[]): Promise { + return new Promise((resolve, reject) => { + const child = spawn(FFMPEG_PATH, ['-hide_banner', '-nostats', '-y', ...args], { windowsHide: true }) + let err = '' + child.stderr.on('data', (c: Buffer) => (err += c.toString())) + child.on('error', reject) + child.on('close', (code) => (code === 0 ? resolve(err) : reject(new Error(`ffmpeg exited ${code}: ${err.slice(-400)}`)))) + }) +} + +async function measureOutput(path: string): Promise<{ i: number; tp: number }> { + const log = await ffmpeg(['-i', path, '-af', 'ebur128=peak=true', '-f', 'null', '-']) + const i = Number(/Integrated loudness:\s+I:\s+(-?[\d.]+)/.exec(log)?.[1]) + const tp = Number(/True peak:\s+Peak:\s+(-?[\d.]+)/.exec(log)?.[1]) + return { i, tp } +} + +interface Row { name: string; layout: string; mode: string; i: number; tp: number } + +async function run(signal: Signal, channels: 1 | 2): Promise { + const slug = `${signal.name.replace(/\s+/g, '-')}-${channels}ch` + const source = join(WORK, `${slug}.wav`) + await ffmpeg([ + '-f', 'lavfi', '-i', `aevalsrc='${signal.expr}':s=48000:d=${SECONDS}`, + '-af', `volume=${signal.gainDb}dB`, '-ac', String(channels), source + ]) + // The export's master chain, as render.ts builds it (minus the end fade). + const stats = await measureLoudness(source, 0, SECONDS) + const chain = `${STEREO},${loudnormFilter(stats)},aresample=48000,${STEREO}` + const out = join(WORK, `${slug}.m4a`) + await ffmpeg(['-i', source, '-af', chain, '-c:a', 'aac', '-b:a', '160k', '-ar', '48000', out]) + const { i, tp } = await measureOutput(out) + return { name: signal.name, layout: channels === 1 ? 'mono' : 'stereo', mode: normalisationMode(stats), i, tp } +} + +async function main(): Promise { + const check = process.argv.includes('--check') + await rm(WORK, { recursive: true, force: true }) + await mkdir(WORK, { recursive: true }) + + const rows: Row[] = [] + for (const s of SIGNALS) for (const ch of [1, 2] as const) rows.push(await run(s, ch)) + + console.log(`signal layout path loudness error true peak (target ${TARGET_I} LUFS)`) + for (const r of rows) { + console.log( + `${r.name.padEnd(12)} ${r.layout.padEnd(7)} ${r.mode.padEnd(8)} ${r.i.toFixed(1).padStart(6)} LUFS ` + + `${(r.i - TARGET_I >= 0 ? '+' : '') + (r.i - TARGET_I).toFixed(1)} LU ${r.tp.toFixed(1).padStart(6)} dBTP` + ) + } + for (const layout of ['mono', 'stereo']) { + const set = rows.filter((r) => r.layout === layout) + const worst = Math.max(...set.map((r) => Math.abs(r.i - TARGET_I))) + const mean = set.reduce((s, r) => s + Math.abs(r.i - TARGET_I), 0) / set.length + const peak = Math.max(...set.map((r) => r.tp)) + console.log(`${layout.padEnd(6)}: mean error ${mean.toFixed(2)} LU, worst ${worst.toFixed(1)} LU, highest true peak ${peak.toFixed(1)} dBTP`) + } + + if (check) { + const bad = rows.filter((r) => Math.abs(r.i - TARGET_I) > 1.5 || r.tp > -1) + if (bad.length) { + console.error(`Off target: ${bad.map((r) => `${r.name} (${r.layout})`).join(', ')}`) + process.exit(1) + } + console.log('Every export is within 1.5 LU of target and under -1 dBTP.') + } +} + +main().catch((err) => { + console.error(err) + process.exit(1) +}) diff --git a/src/main/pipeline/loudness.ts b/src/main/pipeline/loudness.ts index df26c49..f294f54 100644 --- a/src/main/pipeline/loudness.ts +++ b/src/main/pipeline/loudness.ts @@ -25,12 +25,29 @@ export interface LoudnessStats { inputLra: number inputThresh: number targetOffset: number + /** + * Gain into the limiter, when the limiter path was measured through. The + * limiter takes loudness off the peaks it catches, so the plain + * `TARGET_I - inputI` gain lands short on spiky speech; this is that gain + * raised until the limited result reaches the target. + */ + limiterGainDb?: number } const TARGETS = `I=${TARGET_I}:TP=${TARGET_TP}:LRA=${TARGET_LRA}` +/** + * Exports are stereo, so loudness is measured and normalised on the stereo + * mix rather than the source layout. A mono source is spread to both channels + * 3 dB down each; measuring the mono peak instead sent ordinary mono speech + * down the limiter path and limited it about 3 dB harder than needed, so it + * landed up to 2.4 LU under target (scripts/bench-loudness.ts). Surround + * sources are judged on the same downmix the export carries. + */ +export const STEREO = 'aformat=channel_layouts=stereo' + /** The measurement filter, for a first pass whose output is discarded. */ -export const MEASURE_FILTER = `loudnorm=${TARGETS}:print_format=json` +export const MEASURE_FILTER = `${STEREO},loudnorm=${TARGETS}:print_format=json` /** * Most gain the limiter path will apply. A source this quiet is broken @@ -73,9 +90,39 @@ export function loudnormFilter(stats: LoudnessStats | null): string { `:offset=${f(stats.targetOffset)}:linear=true` ) } - const gain = Math.min(MAX_GAIN_DB, gainDb) + return limiterChain(Math.min(MAX_GAIN_DB, stats.limiterGainDb ?? gainDb)) +} + +function limiterChain(gainDb: number): string { const ceiling = Math.pow(10, LIMITER_CEILING_DB / 20) - return `volume=${f(gain)}dB,alimiter=limit=${ceiling.toFixed(4)}:attack=5:release=50:level=false` + return `volume=${gainDb.toFixed(2)}dB,alimiter=limit=${ceiling.toFixed(4)}:attack=5:release=50:level=false` +} + +/** Passes allowed to correct the limiter-path gain; each is audio-only. */ +const LIMITER_PASSES = 2 +/** Close enough to the target to stop refining. */ +const LIMITER_TOLERANCE_LU = 0.3 + +/** + * Raise the limiter-path gain until the limited result reaches the target: + * measure what the limiter actually leaves, add the shortfall, repeat. Each + * extra dB drives the limiter harder, so one pass under-corrects slightly; + * two get within a few tenths on spiky material. Gains only go up from the + * plain estimate and stay under MAX_GAIN_DB. Pure: `measure` runs the pass. + */ +export async function refineLimiterGain( + stats: LoudnessStats, + measure: (chain: string) => Promise +): Promise { + let gain = Math.min(MAX_GAIN_DB, TARGET_I - stats.inputI) + for (let pass = 0; pass < LIMITER_PASSES; pass++) { + const limited = await measure(limiterChain(gain)) + if (limited === null) break + const shortfall = TARGET_I - limited + if (shortfall < LIMITER_TOLERANCE_LU) break + gain = Math.min(MAX_GAIN_DB, gain + shortfall) + } + return { ...stats, limiterGainDb: gain } } /** Which path loudnormFilter takes for a measurement; exported for tests and logs. */ @@ -150,7 +197,25 @@ export async function measureLoudness( ], { signal } ) - return parseLoudnormStats(stderr) + const stats = parseLoudnormStats(stderr) + if (!stats || normalisationMode(stats) !== 'limited') return stats + return await refineLimiterGain(stats, async (chain) => { + const pass = await runBinaryFull( + FFMPEG_PATH, + [ + '-hide_banner', + '-nostats', + '-ss', startSec.toFixed(3), + '-t', durationSec.toFixed(3), + '-i', sourcePath, + '-vn', + '-af', `${STEREO},${chain},loudnorm=${TARGETS}:print_format=json`, + '-f', 'null', '-' + ], + { signal } + ) + return parseLoudnormStats(pass.stderr)?.inputI ?? null + }) } catch (err) { if (signal?.aborted) throw err console.error('Loudness measurement failed; using single-pass normalisation:', err) diff --git a/src/main/pipeline/render.ts b/src/main/pipeline/render.ts index ea46e55..ef58769 100644 --- a/src/main/pipeline/render.ts +++ b/src/main/pipeline/render.ts @@ -23,7 +23,7 @@ import { computeZoomEvents, fitZoomEvents, remapZoomEvents, type ZoomEvent } fro import { planUploadEncode, type UploadEncodePlan } from '@shared/uploadBudget' import { FFMPEG_PATH, probeVideo, runFfmpegWith } from './ffmpeg' import { timed } from './timing' -import { loudnormFilter, measureLoudness, normalisationMode, type LoudnessStats } from './loudness' +import { STEREO, loudnormFilter, measureLoudness, normalisationMode, type LoudnessStats } from './loudness' import { buildAss, fontsDir } from './captions' import { fontMetricsForFamily } from '../fonts' import { mediaJobs } from './mediaJobs' @@ -72,9 +72,10 @@ export function speechSafeFade(transcript: Transcript | null, start: number, end } function audioChain(clipDuration: number, loudness: LoudnessStats | null, tailSec = 0): string { - // FFmpeg 6 on macOS cannot always infer a layout after loudnorm's resampler. - // Exports use stereo AAC; constrain the filter link too, before the fade. - const master = `${loudnormFilter(loudness)},aresample=48000,aformat=channel_layouts=stereo` + // Normalise the stereo mix the export carries, as it was measured. FFmpeg 6 + // on macOS cannot always infer a layout after loudnorm's resampler, so the + // link is constrained again after it, before the fade. + const master = `${STEREO},${loudnormFilter(loudness)},aresample=48000,${STEREO}` const fade = Math.min(END_FADE_SEC, Math.max(0, tailSec)) if (fade < 0.01 || clipDuration <= fade * 3) return master const st = (clipDuration - fade).toFixed(3) diff --git a/tests/loudness.test.ts b/tests/loudness.test.ts index de7dbcb..eb80681 100644 --- a/tests/loudness.test.ts +++ b/tests/loudness.test.ts @@ -4,7 +4,8 @@ import { MAX_GAIN_DB, MEASURE_FILTER, normalisationMode, - parseLoudnormStats + parseLoudnormStats, + refineLimiterGain } from '../src/main/pipeline/loudness' const STDERR = `Input #0, mov,mp4,m4a,3gp,3g2,mj2, from 'source.mp4': @@ -82,7 +83,36 @@ describe('loudnormFilter', () => { expect(loudnormFilter(stats)).toContain(`volume=${MAX_GAIN_DB.toFixed(2)}dB`) }) - it('measures against the same targets it normalises to', () => { - expect(MEASURE_FILTER).toBe('loudnorm=I=-14:TP=-1.5:LRA=11:print_format=json') + it('measures the stereo mix against the same targets it normalises to', () => { + expect(MEASURE_FILTER).toBe('aformat=channel_layouts=stereo,loudnorm=I=-14:TP=-1.5:LRA=11:print_format=json') + }) +}) + +describe('refineLimiterGain', () => { + const limited = { inputI: -24, inputTp: -6, inputLra: 9, inputThresh: -34, targetOffset: 0 } + /** A limiter that eats a fixed share of every dB pushed into it. */ + const limiter = (lossPerDb: number) => async (chain: string): Promise => { + const gain = Number(/volume=(-?[\d.]+)dB/.exec(chain)![1]) + return limited.inputI + gain - lossPerDb * gain + } + + it('raises the gain until the limited clip reaches the target', async () => { + const refined = await refineLimiterGain(limited, limiter(0.15)) + // Plain estimate is +10 dB, which the limiter leaves 1.5 LU short. + expect(refined.limiterGainDb).toBeGreaterThan(10) + const result = await limiter(0.15)(loudnormFilter(refined)) + expect(Math.abs(result - -14)).toBeLessThan(0.3) + expect(loudnormFilter(refined)).toContain('alimiter=') + }) + + it('keeps the plain gain when the limiter already lands on target', async () => { + const refined = await refineLimiterGain(limited, limiter(0)) + expect(refined.limiterGainDb).toBe(10) + }) + + it('never exceeds the gain cap and survives a failed measurement', async () => { + const silent = { ...limited, inputI: -60 } + expect((await refineLimiterGain(silent, limiter(0.5))).limiterGainDb).toBe(MAX_GAIN_DB) + expect((await refineLimiterGain(limited, async () => null)).limiterGainDb).toBe(10) }) }) diff --git a/tests/render.test.ts b/tests/render.test.ts index 8d03864..6d50579 100644 --- a/tests/render.test.ts +++ b/tests/render.test.ts @@ -113,6 +113,11 @@ describe('buildFilterGraph', () => { expect(graph.filterComplex).toContain('linear=true,aresample=48000') }) + it('normalises the stereo mix it measured, so mono sources are not over-limited', () => { + const graph = buildFilterGraph(makeClip(), source, null, 30, null) + expect(graph.filterComplex).toContain('aformat=channel_layouts=stereo,loudnorm=I=-14') + }) + it('gains into a limiter when a linear gain would clip the true peak', () => { const graph = buildFilterGraph(makeClip(), source, null, 30, null, { loudness: { inputI: -23.4, inputTp: -6.1, inputLra: 9.2, inputThresh: -33.7, targetOffset: 0.3 } From 8cc5bab1b305500dc5a443716301682720866581 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 02:27:40 +0000 Subject: [PATCH 3/3] Remove clicks at tighten-cut audio joins Concatenating trimmed audio at arbitrary samples left a waveform step at every join, audible as a click under music, game sound or room tone. Fade 5ms out and 5ms in at each internal join; durations are unchanged, so picture and captions stay in sync. Adds scripts/bench-tighten-joins.ts, which renders the real tighten graph over continuous beds: worst join step was 12.7x / 118x / 14.2x the material's 99.9th percentile step (music, bass, room tone), now 0.2x / 0.6x / 0.2x with zero drift. A duration-preserving crossfade was also measured; it dips less but leaves larger steps and cancels low notes that meet out of phase. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_019QYatd8aXSLCVxPx1SggNZ --- CHANGELOG.md | 1 + README.md | 2 +- scripts/bench-tighten-joins.ts | 133 +++++++++++++++++++++++++++++++++ src/main/pipeline/render.ts | 27 ++++++- tests/render.test.ts | 12 ++- 5 files changed, 169 insertions(+), 6 deletions(-) create mode 100644 scripts/bench-tighten-joins.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 7316e20..3e75fa3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ still pre-1.0, minor bumps carry new features and patch bumps carry fixes. - Exported karaoke captions now hold still while the highlight moves from word to word. Previously the highlighted word was scaled up inside the line, which pushed the other words 17-26 pixels sideways on every word change and left it about 9% larger than the preview showed. The word now pops in place, exactly as in the preview, and every other word stays put (measured with `scripts/bench-caption-stability.ts`). - Exports land closer to the -14 LUFS loudness target. Mono recordings (common for podcasts and calls) were measured before being spread to stereo, so they were limited about 3 dB harder than needed and could come out up to 2.4 LU quiet. Clips that need peak limiting are now re-measured after the limiter and topped up, so laughs, claps and plosives no longer pull the whole clip under target. Across the benchmark signals, mean error fell from 1.3 LU (mono) and 0.5 LU (stereo) to 0.3 LU, with true peaks still at or below -2 dBTP (`scripts/bench-loudness.ts`). +- **Tighten cuts** no longer click where pauses are removed. Splicing audio at an arbitrary point left a step in any music, game sound or room tone running underneath, measured at 13-118 times the largest step in the material itself. Each join now fades out and back in over 5 milliseconds, which removes the click without changing any timing (`scripts/bench-tighten-joins.ts`). - The setup wizard no longer shows the unavailable "Claude subscription" card. Claude models can be used through OpenRouter instead. ## [0.13.0] - 2026-09-25 diff --git a/README.md b/README.md index fa1bc25..3bed38c 100644 --- a/README.md +++ b/README.md @@ -192,7 +192,7 @@ npm run typecheck npm run lint ``` -Integration test scripts live in `scripts/` (`test-pipeline`, `test-e2e`, `test-quality`, `test-wholevideo`, `test-encoders`, `test-resilience`, `test-broll`, `test-youtube`, `test-asd`, `smoke-test.sh`). `bench-caption-stability` renders karaoke captions through libass and checks that no word moves when the highlight changes and that the highlighted word settles at preview size (`--check` fails otherwise). `bench-loudness` runs mono and stereo test signals through the export's audio master and reports loudness error and true peak. See each file's header for what it covers. The e2e ones need `OPENAI_API_KEY`. +Integration test scripts live in `scripts/` (`test-pipeline`, `test-e2e`, `test-quality`, `test-wholevideo`, `test-encoders`, `test-resilience`, `test-broll`, `test-youtube`, `test-asd`, `smoke-test.sh`). `bench-caption-stability` renders karaoke captions through libass and checks that no word moves when the highlight changes and that the highlighted word settles at preview size (`--check` fails otherwise). `bench-loudness` runs mono and stereo test signals through the export's audio master and reports loudness error and true peak. `bench-tighten-joins` checks tighten-cut audio joins for clicks and timing drift. See each file's header for what it covers. The e2e ones need `OPENAI_API_KEY`. To measure clip quality on your own projects, `scripts/eval-clips.ts` reads every saved project and reports how many clips open mid-sentence, cut a sentence off, or trail into dead air, plus the length spread. Add `--rerun` (needs `OPENAI_API_KEY`, a few cents per project, no transcription cost) to re-run clip detection on the saved transcripts with the current prompts and compare, which is how to check a prompt change actually helps: diff --git a/scripts/bench-tighten-joins.ts b/scripts/bench-tighten-joins.ts new file mode 100644 index 0000000..18f1cc3 --- /dev/null +++ b/scripts/bench-tighten-joins.ts @@ -0,0 +1,133 @@ +/** + * Measures clicks at the audio joins tighten cuts create. + * + * Removing a pause splices two pieces of audio together. Speech is quiet at + * the cut, but music, game sound or room tone under it is not, so a splice at + * an arbitrary sample leaves a step in the waveform that is heard as a click. + * This renders the real tighten graph over a continuous music bed and reports: + * + * click ratio largest sample-to-sample step within 1ms of a join, divided + * by the 99.9th percentile step everywhere else (>1 = a step + * bigger than anything the music itself does) + * dip lowest 5ms RMS within 10ms of a join, relative to the lowest + * 5ms RMS away from joins (the cost of fading: how far the + * audio briefly drops below anything it does on its own) + * duration output length minus the kept length (sync must not drift) + * + * Makes no API calls. Run with: + * npx tsx --tsconfig tsconfig.node.json scripts/bench-tighten-joins.ts [--check] + * + * `--check` fails on a click ratio above 1 or any duration drift over 1ms. + */ +import { mkdir, rm } from 'node:fs/promises' +import { join } from 'node:path' +import { spawn } from 'node:child_process' +import { FFMPEG_PATH } from '../src/main/pipeline/ffmpeg' +import { tightenGraph } from '../src/main/pipeline/render' + +const WORK = join(process.cwd(), '.tmp', 'tighten-joins') +const RATE = 48000 +/** Kept ranges with odd boundaries, so joins land at arbitrary phases. */ +const SEGMENTS = [ + { start: 0, end: 1.137 }, + { start: 1.71, end: 2.903 }, + { start: 3.4, end: 4.555 }, + { start: 5.21, end: 6.0 }, + { start: 6.9, end: 8.333 }, + { start: 8.71, end: 9.5 } +] + +const BEDS: Record = { + 'music bed': '0.25*sin(2*PI*220*t)+0.15*sin(2*PI*330.5*t)+0.05*sin(2*PI*1250*t)', + 'bass note': '0.5*sin(2*PI*55*t)', + 'room tone': '0.02*sin(2*PI*120*t)+0.01*sin(2*PI*240*t)+0.004*sin(2*PI*3100*t)' +} + +function ffmpeg(args: string[]): Promise { + return new Promise((resolve, reject) => { + const child = spawn(FFMPEG_PATH, ['-hide_banner', '-loglevel', 'error', '-y', ...args], { windowsHide: true }) + const out: Buffer[] = [] + child.stdout.on('data', (c: Buffer) => out.push(c)) + child.on('error', reject) + child.on('close', (code) => (code === 0 ? resolve(Buffer.concat(out)) : reject(new Error(`ffmpeg exited ${code}`)))) + }) +} + +async function render(source: string): Promise { + const graph = tightenGraph(SEGMENTS, 0, true) + const raw = await ffmpeg([ + '-i', source, '-filter_complex', graph, + '-map', '[acat]', '-ac', '1', '-f', 'f32le', '-', + '-map', '[vcat]', '-f', 'null', '-' + ]) + return new Float32Array(raw.buffer.slice(raw.byteOffset, raw.byteOffset + raw.byteLength)) +} + +function rms(x: Float32Array, from: number, to: number): number { + let sum = 0 + for (let i = from; i < to; i++) sum += x[i] * x[i] + return Math.sqrt(sum / Math.max(1, to - from)) +} + +async function main(): Promise { + const check = process.argv.includes('--check') + await rm(WORK, { recursive: true, force: true }) + await mkdir(WORK, { recursive: true }) + + const joins: number[] = [] + let t = 0 + for (const seg of SEGMENTS.slice(0, -1)) { + t += seg.end - seg.start + joins.push(Math.round(t * RATE)) + } + const kept = SEGMENTS.reduce((s, seg) => s + seg.end - seg.start, 0) + + console.log('bed click ratio dip duration') + let failed = false + for (const [name, expr] of Object.entries(BEDS)) { + const source = join(WORK, `${name.replace(/\s+/g, '-')}.mkv`) + await ffmpeg([ + '-f', 'lavfi', '-i', 'color=c=gray:s=160x90:r=25:d=10', + '-f', 'lavfi', '-i', `aevalsrc='${expr}':s=${RATE}:d=10`, + '-ac', '2', '-c:v', 'libx264', '-preset', 'ultrafast', '-c:a', 'pcm_s16le', '-shortest', source + ]) + const x = await render(source) + const steps = new Float32Array(x.length - 1) + for (let i = 1; i < x.length; i++) steps[i - 1] = Math.abs(x[i] - x[i - 1]) + const near = new Set() + for (const j of joins) for (let k = j - 48; k <= j + 48; k++) near.add(k) + const elsewhere = Array.from(steps).filter((_, i) => !near.has(i)).sort((a, b) => a - b) + const p999 = elsewhere[Math.floor(elsewhere.length * 0.999)] + const atJoins = Math.max(...joins.map((j) => Math.max(...steps.subarray(j - 48, j + 48)))) + const lowest = (from: number, to: number): number => { + let low = Infinity + for (let k = from; k + 240 <= to; k += 24) low = Math.min(low, rms(x, k, k + 240)) + return low + } + const dip = Math.min(...joins.map((j) => lowest(j - 480, j + 480))) + let natural = Infinity + for (let k = 0; k < joins.length; k++) { + const from = (k === 0 ? 0 : joins[k - 1]) + 960 + natural = Math.min(natural, lowest(from, joins[k] - 960)) + } + const dipDb = 20 * Math.log10(Math.max(1e-9, dip) / natural) + const drift = x.length / RATE - kept + const ratio = atJoins / p999 + console.log( + `${name.padEnd(12)} ${ratio.toFixed(1).padStart(10)}x ${dipDb.toFixed(1).padStart(6)} dB ${(drift * 1000).toFixed(1).padStart(7)} ms` + ) + if (ratio > 1 || Math.abs(drift) > 0.001) failed = true + } + if (check) { + if (failed) { + console.error('Clicks at tighten joins, or the audio length drifted.') + process.exit(1) + } + console.log('No join steps above the material itself, and no drift.') + } +} + +main().catch((err) => { + console.error(err) + process.exit(1) +}) diff --git a/src/main/pipeline/render.ts b/src/main/pipeline/render.ts index ef58769..fd83ebd 100644 --- a/src/main/pipeline/render.ts +++ b/src/main/pipeline/render.ts @@ -312,12 +312,25 @@ function reframeGraph( ) } +/** + * Fade on each side of a tighten join. Cutting audio at an arbitrary sample + * leaves a step in the waveform, an audible click wherever music, game sound + * or room tone runs under the pause: 12.7x the largest natural step on a + * music bed (scripts/bench-tighten-joins.ts). 5ms out and 5ms in removes it + * and changes no durations, so picture and captions stay in sync. A + * duration-preserving crossfade was measured too: it dips less (-2 to -8 dB + * against -12 to -20 dB, for about 10ms) but leaves steps up to 1.5x the + * material's own, and longer ones cancel low notes that meet out of phase. + */ +export const JOIN_FADE_SEC = 0.005 + /** * Trim+concat prefix for tightened clips: cuts the kept segments out of the - * (already -ss seeked, so clip-relative) input and concatenates them. - * Produces [vcat] and, when audio is present, [acat]. + * (already -ss seeked, so clip-relative) input and concatenates them, with a + * de-click fade at every join (not at the clip's own start or end). + * Produces [vcat] and, when audio is present, [acat]. Exported for tests. */ -function tightenGraph(segments: KeptSegment[], clipStart: number, hasAudio: boolean): string { +export function tightenGraph(segments: KeptSegment[], clipStart: number, hasAudio: boolean): string { const parts: string[] = [] const vLabels: string[] = [] const aLabels: string[] = [] @@ -327,7 +340,13 @@ function tightenGraph(segments: KeptSegment[], clipStart: number, hasAudio: bool parts.push(`[0:v]trim=start=${s}:end=${e},setpts=PTS-STARTPTS[vs${i}]`) vLabels.push(`[vs${i}]`) if (hasAudio) { - parts.push(`[0:a]atrim=start=${s}:end=${e},asetpts=PTS-STARTPTS[as${i}]`) + const fade = Math.min(JOIN_FADE_SEC, (Number(e) - Number(s)) / 4) + const fades: string[] = [] + if (fade > 0 && i > 0) fades.push(`,afade=t=in:st=0:d=${fade.toFixed(4)}`) + if (fade > 0 && i < segments.length - 1) { + fades.push(`,afade=t=out:st=${(Number(e) - Number(s) - fade).toFixed(4)}:d=${fade.toFixed(4)}`) + } + parts.push(`[0:a]atrim=start=${s}:end=${e},asetpts=PTS-STARTPTS${fades.join('')}[as${i}]`) aLabels.push(`[as${i}]`) } }) diff --git a/tests/render.test.ts b/tests/render.test.ts index 6d50579..10399bf 100644 --- a/tests/render.test.ts +++ b/tests/render.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest' -import { buildFilterGraph, speechSafeFade } from '../src/main/pipeline/render' +import { buildFilterGraph, speechSafeFade, tightenGraph } from '../src/main/pipeline/render' import { makeTranscript } from './helpers' import { DEFAULT_BRAND_COLORS, DEFAULT_CAPTION_STYLE_ID } from '@shared/captionStyles' import type { BrandingSettings, Clip, VideoInfo } from '@shared/types' @@ -244,4 +244,14 @@ describe('buildFilterGraph', () => { }) expect(graph.filterComplex).toContain("fontsdir='/custom/fonts'") }) + + it('fades audio briefly at every tighten join, never at the clip edges', () => { + const graph = tightenGraph([{ start: 10, end: 12 }, { start: 13, end: 15.5 }, { start: 16, end: 17 }], 10, true) + expect(graph).toContain('[0:a]atrim=start=0.000:end=2.000,asetpts=PTS-STARTPTS,afade=t=out:st=1.9950:d=0.0050[as0]') + expect(graph).toContain('atrim=start=3.000:end=5.500,asetpts=PTS-STARTPTS,afade=t=in:st=0:d=0.0050,afade=t=out:st=2.4950:d=0.0050[as1]') + expect(graph).toContain('atrim=start=6.000:end=7.000,asetpts=PTS-STARTPTS,afade=t=in:st=0:d=0.0050[as2]') + // Video joins stay hard cuts and nothing changes length. + expect(graph).not.toMatch(/\[0:v\][^;]*fade/) + expect(graph).toContain('concat=n=3:v=0:a=1[acat]') + }) })