Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions plugins/infermatrix-copilot/skills/explainer-video/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ The video is a deterministic web page. `renderAt(t)` draws the exact frame for t
```mermaid
flowchart LR
A[narration.json] -->|tts.py| B[audio/*.mp3 + timeline.json]
B --> C[stage/timeline.js]
B -->|render.py count| C[stage/timeline.js]
C --> D[stage: renderAt t]
D -->|render.py stills| E[QA frames]
D -->|render.py seg K| F[segs/seg_KK.mp4]
Expand Down Expand Up @@ -77,9 +77,11 @@ Requires `ffmpeg` and `ffprobe` on PATH.

```bash
python tts.py .
python render.py . count
```

- Writes `build/audio/<scene>_<i>.mp3` and `build/timeline.json`, and prints each scene's start and length.
- After each TTS run, `render.py . count` exports the current timeline and configured FPS to `stage/timeline.js` for browser preview, and prints the segment count. Run both commands again after narration or timing changes.
- Existing MP3 files are reused only when their adjacent JSON metadata matches `say`, `voice` and `rate`. Changing any of these, or using an older cache without metadata, regenerates that line. Caption-only edits reuse the audio.
- For a strict total, shorten narration first, then use modest `atempo` adjustment only if speech stays clear. Measure the fitted audio with `ffprobe`, retain lead/tail space, and require every line to end within its scene. The supplied `tts.py` uses narration-driven lengths; fixed-budget timing needs a project-local adaptation, not just smaller `min_len` values. Update audio cues, captions and JS animation timings together.
- When adding, reordering or lengthening shots, update the full timeline, transitions, captions and narration cues, then remix continuous music and ducking against the new cues. Deliver the complete revised video when requested.
Expand All @@ -89,7 +91,7 @@ python tts.py .

## 4. Preview and QA before the full render

1. Open `stage/index.html` in a browser for live playback, or `index.html?t=12.5` to freeze one moment. A short animated preview can establish composition and motion for a substantial structural change; apply only the approval gate established in §1.
1. After both commands in §3, open `stage/index.html` in a browser for live playback, or `index.html?t=12.5` to freeze one moment. A short animated preview can establish composition and motion for a substantial structural change; apply only the approval gate established in §1.
2. Capture stills at the busiest moments of each scene:
`python render.py . stills 3.2 12.5 24 ...` → `build/still_*.png`.
3. Read every still. Check, in this order: caption overlapping content, text cut off or wrapped badly, elements crossing each other (lines through numbers, labels on nodes), and wrong facts.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,7 @@ def mix(pj: Project, silent: Path) -> None:
args += ["-i", str(pj.music)]
m = len(lines) + 1
parts += [
f"{vins}amix=inputs={len(lines)}:normalize=0,asplit=2[voice][sc]",
f"{vins}amix=inputs={len(lines)}:normalize=0,apad=whole_dur={total},asplit=2[voice][sc]",
f"[{m}:a]atrim=0:{total},asetpts=PTS-STARTPTS,volume={gain:.2f}dB,"
f"afade=t=in:d=2,afade=t=out:st={total - 3:.3f}:d=3[bed]",
# voice above threshold pushes the bed down ~10 dB; 400 ms release avoids pumping between lines
Expand Down
69 changes: 69 additions & 0 deletions test/test_explainer_video_segments.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
import array
import asyncio
import importlib.util
import json
import math
import shutil
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
Expand Down Expand Up @@ -91,6 +94,72 @@ def test_fingerprint_tracks_inputs_but_ignores_music_only_edits(self):
finally:
setattr(self.renderer, name, previous)

@unittest.skipUnless(importlib.util.find_spec("playwright"),
"optional preview check requires Playwright/Chromium")
def test_count_prepares_fresh_and_updated_browser_preview(self):
from playwright.sync_api import sync_playwright

exported = self.root / "stage" / "timeline.js"
self.assertFalse(exported.exists())
command = [sys.executable, str(self.root / "render.py"), str(self.root), "count"]
with sync_playwright() as engine:
if not Path(engine.chromium.executable_path).is_file():
self.skipTest("Chromium is absent; run python -m playwright install chromium")
browser = engine.chromium.launch()
try:
page = browser.new_page()
errors = []
page.on("pageerror", lambda error: errors.append(str(error)))
for caption in ["Title", "Updated preview"]:
with self.subTest(caption=caption):
self.timeline["scenes"][0]["lines"][0]["cap"] = caption
self.save_timeline()
result = subprocess.run(command, check=True, capture_output=True, text=True)
self.assertEqual(result.stdout.strip(), "1")
page.goto((self.root / "stage" / "index.html").as_uri() + "?t=0.05")
page.wait_for_function("window.READY === true")
self.assertEqual(page.evaluate("window.TL.fps"), self.renderer.FPS)
self.assertEqual(page.locator("#cap").inner_text(), caption)
self.assertFalse(errors, errors)
finally:
browser.close()

@unittest.skipUnless(shutil.which("ffmpeg"), "optional audio check requires FFmpeg")
def test_music_survives_narration_end_and_fades_during_scene_tail(self):
silent = self.root / "build" / "silent.mp4"
voice = self.root / "voice.wav"
music = self.root / "music.wav"
subprocess.run(["ffmpeg", "-y", "-v", "error", "-f", "lavfi", "-i",
"color=c=black:s=16x16:r=30:d=6", "-c:v", "libx264", "-threads", "1",
"-pix_fmt", "yuv420p", str(silent)], check=True)
for path, frequency, seconds in [(voice, 880, 1), (music, 220, 6)]:
subprocess.run(["ffmpeg", "-y", "-v", "error", "-f", "lavfi", "-i",
f"sine=frequency={frequency}:sample_rate=48000:duration={seconds}",
str(path)], check=True)
(self.root / "narration.json").write_text(json.dumps({
"output": "out.mp4", "music": music.name,
}), encoding="utf-8")
self.timeline.update(total=6)
self.timeline["scenes"][0].update(len=3, lines=[{
"file": str(voice), "start": .5, "dur": 1, "cap": "Short narration",
}])
self.timeline["scenes"][1].update(start=3, len=3, lines=[])
self.save_timeline()
project = self.renderer.Project(self.root)
self.renderer.mix(project, silent)
decoded = subprocess.run(["ffmpeg", "-v", "error", "-xerror", "-i", str(project.out),
"-vn", "-ac", "1", "-ar", "48000", "-f", "f32le", "-"],
check=True, capture_output=True)
samples = array.array("f", decoded.stdout)
levels = []
for start in [2, 3.5, 4.5, 5.5]:
window = samples[round(start * 48000):round((start + .1) * 48000)]
self.assertEqual(len(window), 4800)
rms = math.sqrt(sum(value * value for value in window) / len(window))
self.assertGreater(rms, .0001, f"music is silent at {start}s")
levels.append(rms)
self.assertLess(levels[-1], levels[-2] * .7)

@unittest.skipUnless(importlib.util.find_spec("playwright") and shutil.which("ffmpeg"),
"optional media check requires Playwright/Chromium and FFmpeg")
def test_real_render_resume_invalidation_and_final_guard(self):
Expand Down
Loading