From 7d2334fb69d8f908c76e752e1c7e76db3a55b44b Mon Sep 17 00:00:00 2001 From: Aarav Subberwal Date: Wed, 8 Jul 2026 12:32:04 +0530 Subject: [PATCH 01/11] Fix Clip Lab waveform playhead, seek, and peaks flicker. Stabilize audio revision keys and skip redundant peak refetches so the editor stays in sync during playback and region edits. Co-authored-by: Cursor --- frontend/src/WaveformPane.test.tsx | 86 ++++++++++++++++++- frontend/src/WaveformPane.tsx | 114 +++++++++++++++++++------- frontend/src/workspace/EditorPane.tsx | 65 ++++++--------- 3 files changed, 193 insertions(+), 72 deletions(-) diff --git a/frontend/src/WaveformPane.test.tsx b/frontend/src/WaveformPane.test.tsx index e8fc5ad..5847b77 100644 --- a/frontend/src/WaveformPane.test.tsx +++ b/frontend/src/WaveformPane.test.tsx @@ -7,17 +7,22 @@ type DeferredLoad = { }; const deferredLoads: DeferredLoad[] = []; +const readyOnceHandlers: Array<() => void> = []; const waveSurferLoad = vi.fn(() => new Promise((resolve, reject) => { deferredLoads.push({ - resolve: () => resolve(), + resolve: () => { + for (const handler of readyOnceHandlers.splice(0, readyOnceHandlers.length)) { + handler(); + } + resolve(); + }, reject, }); }), ); const waveSurferDestroy = vi.fn(); const waveSurferOn = vi.fn(); -const waveSurferOnce = vi.fn(); const waveSurferSeekTo = vi.fn(); const waveSurferGetDuration = vi.fn(() => 2); const waveSurferGetCurrentTime = vi.fn(() => 0); @@ -32,7 +37,11 @@ vi.mock("wavesurfer.js", () => ({ load: waveSurferLoad, destroy: waveSurferDestroy, on: waveSurferOn, - once: waveSurferOnce, + once: vi.fn((event: string, handler: () => void) => { + if (event === "ready") { + readyOnceHandlers.push(handler); + } + }), seekTo: waveSurferSeekTo, getDuration: waveSurferGetDuration, getCurrentTime: waveSurferGetCurrentTime, @@ -71,8 +80,9 @@ async function resolveAllPendingLoads() { describe("WaveformPane", () => { beforeEach(() => { deferredLoads.length = 0; + readyOnceHandlers.length = 0; waveSurferLoad.mockClear(); - waveSurferOnce.mockClear(); + waveSurferSeekTo.mockClear(); enableDragSelection.mockClear(); }); @@ -346,6 +356,74 @@ describe("WaveformPane", () => { expect(waveSurferLoad).toHaveBeenCalledTimes(1); }); + it("does not reload or re-seek when peaks flicker after the revision is loaded", async () => { + const revisionKey = "rev-a"; + const peaks = [0.5, 0.4]; + + const { rerender } = render( + {}} + onCursorChange={() => {}} + />, + ); + + await waitFor(() => { + expect(waveSurferLoad).toHaveBeenCalledTimes(1); + }); + await resolveAllPendingLoads(); + expect(waveSurferSeekTo).toHaveBeenCalledTimes(1); + + rerender( + {}} + onCursorChange={() => {}} + />, + ); + + await act(async () => { + await Promise.resolve(); + }); + + rerender( + {}} + onCursorChange={() => {}} + />, + ); + + await act(async () => { + await Promise.resolve(); + }); + + expect(waveSurferLoad).toHaveBeenCalledTimes(1); + expect(waveSurferSeekTo).toHaveBeenCalledTimes(1); + }); + it("ignores stale load completion when an older revision resolves late", async () => { const peaksA = [0.1, 0.2]; const peaksB = [0.3, 0.4]; diff --git a/frontend/src/WaveformPane.tsx b/frontend/src/WaveformPane.tsx index 0bf72a2..48b5ca5 100644 --- a/frontend/src/WaveformPane.tsx +++ b/frontend/src/WaveformPane.tsx @@ -55,6 +55,8 @@ export default function WaveformPane({ const loadGenerationRef = useRef(0); const peaksRef = useRef(peaks); const desiredCursorRef = useRef(desiredCursorSeconds); + const userSeekSecondsRef = useRef(null); + const syncingRegionFromPropsRef = useRef(false); const [audioState, setAudioState] = useState<"loading" | "ready" | "error">("loading"); const [audioError, setAudioError] = useState(null); @@ -90,9 +92,24 @@ export default function WaveformPane({ peaksRef.current = peaks; }, [peaks]); - useEffect(() => { - lastLoadedRevisionRef.current = null; - }, [loadRevisionKey, audioUrl]); + function rememberUserSeek(time: number) { + userSeekSecondsRef.current = roundTime(time); + } + + function seekWaveSurferToSeconds(time: number) { + const waveSurfer = waveSurferRef.current; + if (!waveSurfer) { + return; + } + const duration = waveSurfer.getDuration(); + if (duration <= 0) { + return; + } + const clamped = Math.max(0, Math.min(time, duration)); + waveSurfer.seekTo(clamped / duration); + rememberUserSeek(clamped); + cursorChangeRef.current(clamped); + } useEffect(() => { if (!containerRef.current) { @@ -178,12 +195,14 @@ export default function WaveformPane({ if (draggedThisGestureRef.current) { return; } + rememberUserSeek(waveSurfer.getCurrentTime()); cursorChangeRef.current(roundTime(waveSurfer.getCurrentTime())); }; const handleSeeking = (time: number) => { if (draggedThisGestureRef.current) { return; } + rememberUserSeek(time); cursorChangeRef.current(roundTime(time)); }; const handleClick = () => { @@ -191,6 +210,7 @@ export default function WaveformPane({ return; } const time = roundTime(waveSurfer.getCurrentTime()); + rememberUserSeek(time); cursorChangeRef.current(time); selectionChangeRef.current(time, time); }; @@ -223,6 +243,9 @@ export default function WaveformPane({ }); regions.on("region-created", (region: any) => { + if (syncingRegionFromPropsRef.current) { + return; + } for (const candidate of regions.getRegions()) { if (candidate.id !== region.id) { candidate.remove(); @@ -235,6 +258,9 @@ export default function WaveformPane({ }); regions.on("region-updated", (region: any) => { + if (syncingRegionFromPropsRef.current) { + return; + } selectionChangeRef.current( roundTime(region.start), roundTime(region.end), @@ -283,6 +309,12 @@ export default function WaveformPane({ } draggedThisGestureRef.current = false; + + // DAW-style click: move playhead to click point and keep it there. + if (endTime !== null) { + seekWaveSurferToSeconds(endTime); + selectionChangeRef.current(endTime, endTime); + } }; const handleWheel = (event: WheelEvent) => { @@ -333,6 +365,12 @@ export default function WaveformPane({ } const revisionKey = loadRevisionKey ?? audioUrl; + + // Already decoded this revision — ignore peaks-state churn and user playhead moves. + if (lastLoadedRevisionRef.current === revisionKey) { + return; + } + const waitForPeaks = requirePeaksBeforeLoad && peaksLoadState !== "ready" @@ -344,21 +382,30 @@ export default function WaveformPane({ return; } - if (lastLoadedRevisionRef.current === revisionKey) { - return; - } - const isClipChange = lastAudioUrlRef.current !== audioUrl; lastAudioUrlRef.current = audioUrl; const targetTime = isClipChange ? 0 - : Math.max(0, Math.min(desiredCursorRef.current, durationSeconds)); + : Math.max( + 0, + Math.min( + userSeekSecondsRef.current ?? desiredCursorRef.current, + durationSeconds, + ), + ); + const seekGeneration = ++loadGenerationRef.current; const seekOnReady = () => { + if (seekGeneration !== loadGenerationRef.current) { + return; + } const duration = waveSurfer.getDuration(); if (duration > 0) { waveSurfer.seekTo(Math.max(0, Math.min(targetTime / duration, 1))); } - cursorChangeRef.current(roundTime(targetTime)); + if (isClipChange) { + userSeekSecondsRef.current = null; + cursorChangeRef.current(roundTime(targetTime)); + } }; waveSurfer.once("ready", seekOnReady); setAudioState("loading"); @@ -370,7 +417,7 @@ export default function WaveformPane({ ? [currentPeaks] : undefined; - const generation = ++loadGenerationRef.current; + const generation = seekGeneration; const revisionKeyAtStart = revisionKey; void waveSurfer @@ -408,29 +455,36 @@ export default function WaveformPane({ const end = Math.max(selectionStart, selectionEnd); const currentRegion = regions.getRegions()[0]; - if (end <= start + 0.01) { - if (currentRegion) { - currentRegion.remove(); + syncingRegionFromPropsRef.current = true; + try { + if (end <= start + 0.01) { + if (currentRegion) { + currentRegion.remove(); + } + return; } - return; - } - if (!currentRegion) { - regions.addRegion({ - start, - end, - color: "rgba(247, 203, 104, 0.2)", - drag: true, - resize: true, - }); - return; - } + if (!currentRegion) { + regions.addRegion({ + start, + end, + color: "rgba(247, 203, 104, 0.2)", + drag: true, + resize: true, + }); + return; + } - if ( - Math.abs(currentRegion.start - start) > 0.02 || - Math.abs(currentRegion.end - end) > 0.02 - ) { - currentRegion.setOptions({ start, end }); + if ( + Math.abs(currentRegion.start - start) > 0.02 || + Math.abs(currentRegion.end - end) > 0.02 + ) { + currentRegion.setOptions({ start, end }); + } + } finally { + queueMicrotask(() => { + syncingRegionFromPropsRef.current = false; + }); } }, [selectionStart, selectionEnd]); diff --git a/frontend/src/workspace/EditorPane.tsx b/frontend/src/workspace/EditorPane.tsx index f777cda..24078d1 100644 --- a/frontend/src/workspace/EditorPane.tsx +++ b/frontend/src/workspace/EditorPane.tsx @@ -132,22 +132,24 @@ export default function EditorPane({ const activeRevisionKeyRef = useRef(null); const activeDuration = activeClip ? getSliceDuration(activeClip) : 0; - const activeAudioRevisionKey = activeClip - ? JSON.stringify({ - audio_url: activeClip.audio_url, - active_variant_id: activeClip.active_variant?.id ?? null, - active_commit_id: activeClip.active_commit?.id ?? null, - edl_operations: activeClip.active_commit?.edl_operations ?? [], - waveform_peaks_url: - activeClip.item_metadata && "waveform_peaks_url" in activeClip.item_metadata - ? activeClip.item_metadata.waveform_peaks_url - : null, - effective_audio_revision_key: - activeClip.item_metadata && "effective_audio_revision_key" in activeClip.item_metadata - ? activeClip.item_metadata.effective_audio_revision_key - : null, - }) - : null; + const activeAudioRevisionKey = useMemo(() => { + if (!activeClip) { + return null; + } + const metadata = activeClip.item_metadata; + return JSON.stringify({ + audio_url: activeClip.audio_url, + active_variant_id: activeClip.active_variant?.id ?? null, + active_commit_id: activeClip.active_commit?.id ?? null, + edl_operations: activeClip.active_commit?.edl_operations ?? [], + waveform_peaks_url: + metadata && "waveform_peaks_url" in metadata ? metadata.waveform_peaks_url : null, + effective_audio_revision_key: + metadata && "effective_audio_revision_key" in metadata + ? metadata.effective_audio_revision_key + : null, + }); + }, [activeClip]); const datasetRenderStatus = activeClip?.item_metadata && "render_status" in activeClip.item_metadata ? (activeClip.item_metadata.render_status as string | null) @@ -187,28 +189,8 @@ export default function EditorPane({ return Array.from({ length: Math.floor(activeDuration) + 1 }, (_, second) => second); }, [activeClip?.id, activeDuration]); - const pendingPlayheadSecondsRef = useRef(null); - const playheadRafRef = useRef(null); - const handleWaveformCursorChange = useCallback((time: number) => { - pendingPlayheadSecondsRef.current = time; - if (playheadRafRef.current !== null) { - return; - } - playheadRafRef.current = window.requestAnimationFrame(() => { - playheadRafRef.current = null; - if (pendingPlayheadSecondsRef.current !== null) { - setPlayheadSeconds(pendingPlayheadSecondsRef.current); - } - }); - }, []); - - useEffect(() => { - return () => { - if (playheadRafRef.current !== null) { - window.cancelAnimationFrame(playheadRafRef.current); - } - }; + setPlayheadSeconds(time); }, []); useEffect(() => { @@ -248,6 +230,13 @@ export default function EditorPane({ return; } + if ( + revisionPeaks.revisionKey === activeAudioRevisionKey + && revisionPeaks.status === "ready" + ) { + return; + } + const revisionKeyAtStart = activeAudioRevisionKey; setRevisionPeaks(initialRevisionPeaksForFetch(revisionKeyAtStart)); setWaveformError(null); @@ -289,7 +278,7 @@ export default function EditorPane({ return () => { cancelled = true; }; - }, [activeClip?.id, activeAudioRevisionKey, waveformPeaksUrl, datasetRenderStatus]); + }, [activeClip?.id, activeAudioRevisionKey, waveformPeaksUrl, datasetRenderStatus, revisionPeaks.revisionKey, revisionPeaks.status]); useEffect(() => { const editor = transcriptEditorRef.current; From b9c7badbea26653a217827f4e395d1bcfecff544 Mon Sep 17 00:00:00 2001 From: Aarav Subberwal Date: Sun, 12 Jul 2026 23:19:47 +0530 Subject: [PATCH 02/11] Add Clip Lab tagging and canonical export flow with worker runtime fixes --- backend/app/canonical_export.py | 409 +++++++++++++++++ backend/app/clip_lab_audio_ops.py | 10 + backend/app/main.py | 72 +-- backend/app/models.py | 51 +-- backend/app/repository.py | 88 ---- backend/tests/test_api_integration.py | 8 +- backend/tests/test_clip_lab_routes.py | 418 +++++++++++++++++- backend/tests/test_repository_media.py | 4 - frontend/src/App.tsx | 16 +- frontend/src/api.ts | 42 +- frontend/src/pages/ExportPage.tsx | 98 ---- .../src/pages/LabelPage.integration.test.tsx | 89 +++- frontend/src/pages/LabelPage.tsx | 115 ++++- frontend/src/pages/ProcessingPage.test.tsx | 2 +- frontend/src/pages/QcPage.test.tsx | 1 - frontend/src/pages/ReferencePage.test.tsx | 2 +- frontend/src/pages/SlicerPage.test.tsx | 2 +- frontend/src/pages/SpeakersPage.test.tsx | 2 +- frontend/src/pages/labelPageDatasetHelpers.ts | 2 +- frontend/src/pipeline/PipelineContext.tsx | 2 +- frontend/src/styles.css | 25 +- frontend/src/types.ts | 46 +- frontend/src/workspace/EditorPane.tsx | 7 + frontend/src/workspace/InspectorPane.tsx | 101 +++-- .../workspace/dataset-clip-lab-patch.test.ts | 15 +- workers/dataset/README.md | 4 +- workers/dataset/scripts/preflight.py | 12 +- .../analyze_ctc_transcript_qc.py | 31 +- workers/dataset/speechcraft_dataset/mfa.py | 97 +++- .../tests/test_analyze_ctc_transcript_qc.py | 98 ++++ .../dataset/tests/test_mfa_runtime_config.py | 97 ++++ 31 files changed, 1581 insertions(+), 385 deletions(-) create mode 100644 backend/app/canonical_export.py delete mode 100644 frontend/src/pages/ExportPage.tsx create mode 100644 workers/dataset/tests/test_mfa_runtime_config.py diff --git a/backend/app/canonical_export.py b/backend/app/canonical_export.py new file mode 100644 index 0000000..cf2f5d9 --- /dev/null +++ b/backend/app/canonical_export.py @@ -0,0 +1,409 @@ +from __future__ import annotations + +import json +import os +import shutil +import wave +from datetime import datetime, timezone +from pathlib import Path +from typing import Any +from uuid import uuid4 + +from sqlmodel import Session + +from .clip_lab_audio import sha256_file +from .clip_lab_audio_ops import render_cache_path +from .clip_lab_state import ( + ClipLabValidationError, + build_clip_lab_view, + clip_lab_run_lock, + index_manifest_by_clip_id, + load_candidate_manifest, + load_clip_lab_state, +) +from .dataset_runs import _run_root +from .models import ( + CanonicalExportBlockedReasonView, + CanonicalExportPreviewView, + CanonicalExportSummaryView, + ProcessingRun, +) + +SCHEMA_VERSION = 1 +EXPORTS_RELATIVE_DIR = Path("artifacts/canonical_exports") +MANIFEST_FILENAME = "speechcraft_dataset.jsonl" +METADATA_FILENAME = "speechcraft_export.json" +REPORT_FILENAME = "export_report.json" + + +class CanonicalExportConflictError(ValueError): + """Current Clip Lab state cannot produce a canonical export.""" + + +def _utc_now() -> datetime: + return datetime.now(timezone.utc).replace(microsecond=0) + + +def _utc_now_iso() -> str: + return _utc_now().isoformat().replace("+00:00", "Z") + + +def _canonical_exports_root(run_root: Path) -> Path: + return run_root / EXPORTS_RELATIVE_DIR + + +def _format_export_id(now: datetime) -> str: + return f"canonical_export_{now.strftime('%Y-%m-%d_%H%M%S')}_{uuid4().hex[:8]}" + + +def _next_export_id(exports_root: Path) -> str: + while True: + candidate = _format_export_id(_utc_now()) + if not (exports_root / candidate).exists(): + return candidate + + +def _read_wave_metadata(path: Path) -> dict[str, int | float]: + with wave.open(str(path), "rb") as handle: + frame_count = int(handle.getnframes()) + sample_rate_hz = int(handle.getframerate()) + channels = int(handle.getnchannels()) + if sample_rate_hz <= 0: + raise ClipLabValidationError(f"audio file has invalid sample rate: {path}") + return { + "sample_rate_hz": sample_rate_hz, + "channels": channels, + "duration_sec": round(frame_count / sample_rate_hz, 6), + } + + +def _resolve_manifest_audio_path(run_root: Path, manifest_row: dict[str, Any], *, clip_id: str) -> Path: + audio_path = manifest_row.get("audio_path") + if not isinstance(audio_path, str) or not audio_path.strip(): + raise ClipLabValidationError(f"{clip_id} candidate manifest row is missing audio_path") + return run_root / audio_path + + +def _relative_path(from_dir: Path, to_path: Path) -> str: + return os.path.relpath(to_path, start=from_dir) + + +def _serialize_json(path: Path, payload: dict[str, Any]) -> None: + path.write_text(json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8") + + +def _quality_payload(clip_view: dict[str, Any]) -> dict[str, float] | None: + quality: dict[str, float] = {} + transcript_match = clip_view.get("transcript_match") + speaker_check = clip_view.get("speaker_check") + if isinstance(transcript_match, (int, float)): + quality["transcript_match"] = round(float(transcript_match), 2) + if isinstance(speaker_check, (int, float)): + quality["speaker_check"] = round(float(speaker_check), 2) + return quality or None + + +def _build_clip_export_row( + *, + export_dir: Path, + run_root: Path, + manifest_row: dict[str, Any], + clip_view: dict[str, Any], + clip_entry: dict[str, Any] | None = None, +) -> tuple[dict[str, Any] | None, list[str], str | None, float]: + clip_id = str(clip_view["clip_id"]) + transcript = clip_view["transcript"] + if not isinstance(transcript, str): + return None, ["transcript_missing"], None, 0.0 + + source_audio_sha256 = clip_view.get("source_audio_sha256") + if not isinstance(source_audio_sha256, str) or not source_audio_sha256: + return None, ["source_audio_sha256_missing"], None, 0.0 + + audio_path: Path | None = None + audio_sha256: str | None = None + audio_revision_hash: str | None = None + reasons: list[str] = [] + audio_kind: str + audio_edit = clip_entry.get("audio_edit") if isinstance(clip_entry, dict) else None + has_active_audio_edit = isinstance(audio_edit, dict) and bool(audio_edit.get("ops") or []) + + if has_active_audio_edit: + audio_kind = "rendered_revision" + render_status = audio_edit.get("render_status") + audio_revision_hash = audio_edit.get("audio_revision_hash") + rendered_audio_sha256 = audio_edit.get("rendered_audio_sha256") + if render_status != "ready": + reasons.append("rendered_audio_not_ready") + if not isinstance(audio_revision_hash, str) or not audio_revision_hash: + reasons.append("audio_revision_hash_missing") + if not isinstance(rendered_audio_sha256, str) or not rendered_audio_sha256: + reasons.append("rendered_audio_sha256_missing") + if not reasons: + audio_path = render_cache_path(run_root, clip_id, audio_revision_hash) + if not audio_path.is_file(): + reasons.append("rendered_audio_missing") + elif sha256_file(audio_path) != rendered_audio_sha256: + reasons.append("rendered_audio_hash_mismatch") + else: + audio_sha256 = rendered_audio_sha256 + else: + audio_kind = "candidate_original" + audio_path = _resolve_manifest_audio_path(run_root, manifest_row, clip_id=clip_id) + if not audio_path.is_file(): + reasons.append("candidate_audio_missing") + elif sha256_file(audio_path) != source_audio_sha256: + reasons.append("candidate_audio_hash_mismatch") + else: + audio_sha256 = source_audio_sha256 + + if reasons or audio_path is None or audio_sha256 is None: + return None, reasons, audio_kind, 0.0 + + audio_meta = _read_wave_metadata(audio_path) + audio_payload: dict[str, Any] = { + "path": _relative_path(export_dir, audio_path), + "kind": audio_kind, + "sha256": audio_sha256, + "source_audio_sha256": source_audio_sha256, + "sample_rate_hz": audio_meta["sample_rate_hz"], + "channels": audio_meta["channels"], + "duration_sec": audio_meta["duration_sec"], + } + if audio_kind == "rendered_revision" and audio_revision_hash: + audio_payload["audio_revision_hash"] = audio_revision_hash + + row: dict[str, Any] = { + "schema_version": SCHEMA_VERSION, + "clip_id": clip_id, + "transcript": transcript, + "lineage": { + "source_clip_id": clip_id, + "parent_clip_ids": [], + }, + "audio": audio_payload, + "review": { + "status": "accepted", + "reviewer_tags": list(clip_view.get("reviewer_tags") or []), + }, + } + quality = _quality_payload(clip_view) + if quality: + row["quality"] = quality + return row, [], audio_kind, float(audio_meta["duration_sec"]) + + +def _collect_export_material( + repository: Any, + run_id: str, +) -> tuple[ProcessingRun, Path, list[tuple[dict[str, Any], dict[str, Any], dict[str, Any] | None]], CanonicalExportPreviewView]: + with Session(repository.engine) as session: + run = session.get(ProcessingRun, run_id) + if run is None: + raise KeyError("Dataset run not found") + run_root = _run_root(repository, run) + + with clip_lab_run_lock(run_root): + view = build_clip_lab_view(run_root, run_id=run_id) + if view.get("stale_state"): + raise CanonicalExportConflictError( + f"Clip Lab state is stale: {view.get('stale_reason') or 'candidate manifest changed'}" + ) + if view.get("invalid_state"): + raise CanonicalExportConflictError( + f"Clip Lab state is invalid: {view.get('invalid_state_reason') or 'validation failed'}" + ) + + manifest = load_candidate_manifest(run_root) + manifest_by_id = index_manifest_by_clip_id(manifest) + saved_state = load_clip_lab_state(run_root) + stored_clips = saved_state.get("clips") if isinstance(saved_state, dict) else None + accepted_clip_views = [dict(clip) for clip in view.get("clips") or [] if clip.get("review_status") == "accepted"] + accepted_manifest_rows: dict[str, dict[str, Any]] = {} + accepted_clip_entries: dict[str, dict[str, Any] | None] = {} + for clip_view in accepted_clip_views: + clip_id = str(clip_view["clip_id"]) + manifest_row = manifest_by_id.get(clip_id) + if manifest_row is not None: + accepted_manifest_rows[clip_id] = dict(manifest_row) + if isinstance(stored_clips, dict) and isinstance(stored_clips.get(clip_id), dict): + accepted_clip_entries[clip_id] = dict(stored_clips[clip_id]) + else: + accepted_clip_entries[clip_id] = None + + export_inputs: list[tuple[dict[str, Any], dict[str, Any], dict[str, Any] | None]] = [] + blocked_reasons: list[CanonicalExportBlockedReasonView] = [] + total_duration_sec = 0.0 + original_audio_count = 0 + edited_audio_count = 0 + preview_export_dir = _canonical_exports_root(run_root) / "preview" + for clip_view in accepted_clip_views: + clip_id = str(clip_view["clip_id"]) + manifest_row = accepted_manifest_rows.get(clip_id) + if manifest_row is None: + blocked_reasons.append( + CanonicalExportBlockedReasonView(clip_id=clip_id, reasons=["manifest_row_missing"]) + ) + continue + row, reasons, audio_kind, duration_sec = _build_clip_export_row( + export_dir=preview_export_dir, + run_root=run_root, + manifest_row=manifest_row, + clip_view=clip_view, + clip_entry=accepted_clip_entries.get(clip_id), + ) + if reasons: + blocked_reasons.append(CanonicalExportBlockedReasonView(clip_id=clip_id, reasons=reasons)) + continue + assert row is not None + export_inputs.append((manifest_row, clip_view, accepted_clip_entries.get(clip_id))) + total_duration_sec += duration_sec + if audio_kind == "rendered_revision": + edited_audio_count += 1 + else: + original_audio_count += 1 + + preview = CanonicalExportPreviewView( + run_id=run_id, + accepted_clip_count=len(accepted_clip_views), + total_duration_sec=round(total_duration_sec, 6), + original_audio_count=original_audio_count, + edited_audio_count=edited_audio_count, + blocked_clip_count=len(blocked_reasons), + blocked_reasons=blocked_reasons, + ) + return run, run_root, export_inputs, preview + + +def preview_canonical_export(repository: Any, run_id: str) -> CanonicalExportPreviewView: + _run, _run_root_path, _rows, preview = _collect_export_material(repository, run_id) + return preview + + +def create_canonical_export(repository: Any, run_id: str) -> CanonicalExportSummaryView: + run, run_root, export_inputs, preview = _collect_export_material(repository, run_id) + if preview.accepted_clip_count == 0: + raise CanonicalExportConflictError("no accepted clips available for canonical export") + if preview.blocked_clip_count > 0: + raise CanonicalExportConflictError("accepted clips are blocked; resolve Clip Lab export blockers first") + + exports_root = _canonical_exports_root(run_root) + exports_root.mkdir(parents=True, exist_ok=True) + export_id = _next_export_id(exports_root) + tmp_dir = exports_root / f".tmp_{export_id}_{uuid4().hex}" + final_dir = exports_root / export_id + if final_dir.exists(): + raise CanonicalExportConflictError("canonical export snapshot already exists; retry") + tmp_dir.mkdir(parents=True, exist_ok=False) + + try: + export_rows: list[dict[str, Any]] = [] + for manifest_row, clip_view, clip_entry in export_inputs: + row, reasons, _audio_kind, _duration_sec = _build_clip_export_row( + export_dir=final_dir, + run_root=run_root, + manifest_row=manifest_row, + clip_view=clip_view, + clip_entry=clip_entry, + ) + if reasons or row is None: + raise CanonicalExportConflictError("canonical export inputs changed while exporting; retry") + export_rows.append(row) + + manifest_path = tmp_dir / MANIFEST_FILENAME + with manifest_path.open("w", encoding="utf-8") as handle: + for row in export_rows: + handle.write(json.dumps(row, ensure_ascii=False, sort_keys=True)) + handle.write("\n") + + metadata = { + "schema_version": SCHEMA_VERSION, + "export_id": export_id, + "run_id": run.id, + "project_id": run.project_id, + "created_at": _utc_now_iso(), + "path_mode": "snapshot_relative", + "audio_storage_mode": "project_artifact_reference", + "portable": False, + "dataset_manifest": MANIFEST_FILENAME, + "accepted_clip_count": preview.accepted_clip_count, + "total_duration_sec": preview.total_duration_sec, + "original_audio_count": preview.original_audio_count, + "edited_audio_count": preview.edited_audio_count, + "blocked_clip_count": preview.blocked_clip_count, + } + _serialize_json(tmp_dir / METADATA_FILENAME, metadata) + _serialize_json( + tmp_dir / REPORT_FILENAME, + { + "run_id": run.id, + "export_id": export_id, + "accepted_clip_count": preview.accepted_clip_count, + "blocked_clip_count": preview.blocked_clip_count, + "blocked_reasons": [item.model_dump() for item in preview.blocked_reasons], + }, + ) + + tmp_dir.rename(final_dir) + except Exception: + shutil.rmtree(tmp_dir, ignore_errors=True) + raise + return CanonicalExportSummaryView( + export_id=export_id, + run_id=run.id, + project_id=run.project_id, + created_at=str(metadata["created_at"]), + accepted_clip_count=preview.accepted_clip_count, + total_duration_sec=preview.total_duration_sec, + snapshot_dir=str(final_dir), + manifest_path=str(final_dir / MANIFEST_FILENAME), + ) + + +def _summary_from_snapshot(run_id: str, snapshot_dir: Path) -> CanonicalExportSummaryView | None: + metadata_path = snapshot_dir / METADATA_FILENAME + try: + payload = json.loads(metadata_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return None + if not isinstance(payload, dict): + return None + export_id = payload.get("export_id") + project_id = payload.get("project_id") + created_at = payload.get("created_at") + accepted_clip_count = payload.get("accepted_clip_count") + total_duration_sec = payload.get("total_duration_sec") + if not isinstance(export_id, str) or not isinstance(project_id, str) or not isinstance(created_at, str): + return None + if not isinstance(accepted_clip_count, int) or not isinstance(total_duration_sec, (int, float)): + return None + return CanonicalExportSummaryView( + export_id=export_id, + run_id=run_id, + project_id=project_id, + created_at=created_at, + accepted_clip_count=accepted_clip_count, + total_duration_sec=round(float(total_duration_sec), 6), + snapshot_dir=str(snapshot_dir), + manifest_path=str(snapshot_dir / MANIFEST_FILENAME), + ) + + +def list_canonical_exports(repository: Any, run_id: str) -> list[CanonicalExportSummaryView]: + with Session(repository.engine) as session: + run = session.get(ProcessingRun, run_id) + if run is None: + raise KeyError("Dataset run not found") + + run_root = _run_root(repository, run) + exports_root = _canonical_exports_root(run_root) + if not exports_root.exists(): + return [] + summaries: list[CanonicalExportSummaryView] = [] + for child in exports_root.iterdir(): + if not child.is_dir() or child.name.startswith(".tmp_"): + continue + summary = _summary_from_snapshot(run_id, child) + if summary is not None: + summaries.append(summary) + return sorted(summaries, key=lambda item: item.created_at, reverse=True) diff --git a/backend/app/clip_lab_audio_ops.py b/backend/app/clip_lab_audio_ops.py index 80b38d4..9862102 100644 --- a/backend/app/clip_lab_audio_ops.py +++ b/backend/app/clip_lab_audio_ops.py @@ -76,6 +76,13 @@ def _manifest_wav_path(run_root: Path, manifest_row: dict[str, Any]) -> Path: return run_root / str(manifest_row["audio_path"]) +NATIVE_EXPORT_CLIPS_DIR = "artifacts/native_export_clips" + + +def _native_export_wav_path(run_root: Path, clip_id: str) -> Path: + return run_root / NATIVE_EXPORT_CLIPS_DIR / f"{clip_id}.wav" + + def _source_identity(manifest_row: dict[str, Any], *, clip_id: str) -> str: source_sha = resolve_manifest_source_audio_hash(manifest_row, clip_id=clip_id) if source_sha is None: @@ -705,6 +712,9 @@ def resolve_revision_media_bytes( raise ClipLabRevisionNotFoundError(f"revision key {revision_key!r} is not current for clip {clip_id}") source_sha = _source_identity(manifest_row, clip_id=clip_id) if revision_key == source_sha: + native_wav = _native_export_wav_path(run_root, clip_id) + if native_wav.is_file(): + return native_wav.read_bytes() return _manifest_wav_path(run_root, manifest_row).read_bytes() cache_path = render_cache_path(run_root, clip_id, revision_key) if not cache_path.is_file(): diff --git a/backend/app/main.py b/backend/app/main.py index 25c1bd4..d26f1e9 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -8,6 +8,12 @@ from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import FileResponse, Response +from .canonical_export import ( + CanonicalExportConflictError, + create_canonical_export, + list_canonical_exports, + preview_canonical_export, +) from .reference_clip_candidates import mark_dataset_clip_as_reference_candidate from .dataset_worker_client import run_dataset_worker_preflight from .defaults import resolve_asr_device_and_compute_type, resolve_whisper_model @@ -54,6 +60,8 @@ ) from .native_cliplab import NativeClipLabStore from .models import ( + CanonicalExportPreviewView, + CanonicalExportSummaryView, DatasetClipLabClipView, DatasetClipLabAudioOperationRequest, DatasetClipLabAudioStackRequest, @@ -74,8 +82,6 @@ DatasetRunView, DatasetSlicerResultsView, DatasetSlicerRerunRequest, - ExportPreview, - ExportRun, ImportBatchCreate, MarkReferenceClipCandidateRequest, ProcessingJobView, @@ -351,6 +357,42 @@ def read_dataset_clip_lab(run_id: str) -> DatasetClipLabView: raise HTTPException(status_code=503, detail=str(exc)) from exc +@app.get("/api/dataset-runs/{run_id}/canonical-export-preview", response_model=CanonicalExportPreviewView) +def read_canonical_export_preview(run_id: str) -> CanonicalExportPreviewView: + try: + return preview_canonical_export(repository, run_id) + except KeyError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except CanonicalExportConflictError as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + except ClipLabValidationError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + except ClipLabStateError as exc: + raise HTTPException(status_code=503, detail=str(exc)) from exc + + +@app.post("/api/dataset-runs/{run_id}/canonical-exports", response_model=CanonicalExportSummaryView) +def post_canonical_export(run_id: str) -> CanonicalExportSummaryView: + try: + return create_canonical_export(repository, run_id) + except KeyError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except CanonicalExportConflictError as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + except ClipLabValidationError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + except ClipLabStateError as exc: + raise HTTPException(status_code=503, detail=str(exc)) from exc + + +@app.get("/api/dataset-runs/{run_id}/canonical-exports", response_model=list[CanonicalExportSummaryView]) +def read_canonical_exports(run_id: str) -> list[CanonicalExportSummaryView]: + try: + return list_canonical_exports(repository, run_id) + except KeyError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + @app.post( "/api/dataset-runs/{run_id}/clips/{clip_id}/audio/operations", response_model=DatasetClipLabClipView, @@ -694,22 +736,6 @@ def get_source_recording_artifact(recording_id: str) -> SourceRecordingArtifactV raise HTTPException(status_code=404, detail="Source recording not found") from exc -@app.get("/api/projects/{project_id}/export-preview", response_model=ExportPreview) -def get_export_preview(project_id: str) -> ExportPreview: - try: - return repository.get_export_preview(project_id) - except KeyError as exc: - raise HTTPException(status_code=404, detail="Project not found") from exc - - -@app.get("/api/projects/{project_id}/exports", response_model=list[ExportRun]) -def list_export_runs(project_id: str) -> list[ExportRun]: - try: - return repository.list_export_runs(project_id) - except KeyError as exc: - raise HTTPException(status_code=404, detail="Project not found") from exc - - @app.post("/api/projects/{project_id}/media-cleanup") def cleanup_project_media(project_id: str) -> dict[str, object]: try: @@ -819,16 +845,6 @@ def redo_slice(clip_id: str) -> dict[str, object]: raise HTTPException(status_code=400, detail=str(exc)) from exc -@app.post("/api/projects/{project_id}/export", response_model=ExportRun) -def export_project(project_id: str) -> ExportRun: - try: - return repository.export_project(project_id) - except KeyError as exc: - raise HTTPException(status_code=404, detail="Project not found") from exc - except ValueError as exc: - raise HTTPException(status_code=400, detail=str(exc)) from exc - - @app.post("/api/recordings/{recording_id}/jobs/transcription", response_model=ProcessingJobView) def enqueue_source_transcription( recording_id: str, diff --git a/backend/app/models.py b/backend/app/models.py index a063680..03fc221 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -293,7 +293,6 @@ class ImportBatch(SQLModel, table=True): active_preparation_job_id: str | None = None recordings: list["SourceRecording"] = Relationship(back_populates="batch", cascade_delete=True) - exports: list["ExportRun"] = Relationship(back_populates="batch", cascade_delete=True) class SourceRecording(SQLModel, table=True): @@ -913,6 +912,32 @@ class DatasetClipLabView(SQLModel): clips: list[DatasetClipLabClipView] = Field(default_factory=list) +class CanonicalExportBlockedReasonView(SQLModel): + clip_id: str + reasons: list[str] = Field(default_factory=list) + + +class CanonicalExportPreviewView(SQLModel): + run_id: str + accepted_clip_count: int = 0 + total_duration_sec: float = 0.0 + original_audio_count: int = 0 + edited_audio_count: int = 0 + blocked_clip_count: int = 0 + blocked_reasons: list[CanonicalExportBlockedReasonView] = Field(default_factory=list) + + +class CanonicalExportSummaryView(SQLModel): + export_id: str + run_id: str + project_id: str + created_at: str + accepted_clip_count: int + total_duration_sec: float + snapshot_dir: str + manifest_path: str + + class DatasetClipLabPatchRequest(SQLModel): expected_manifest_sha256: str expected_clip_version: int @@ -1011,29 +1036,6 @@ def duration_s(self) -> float: return self.num_samples / self.sample_rate if self.sample_rate else 0.0 -class ExportRun(SQLModel, table=True): - """Project-level export history for the current UI.""" - - id: str = Field(primary_key=True) - batch_id: str = Field(foreign_key="importbatch.id") - status: JobStatus = Field(default=JobStatus.PENDING, sa_column=Column(sql_enum(JobStatus))) - output_root: str - manifest_path: str - accepted_clip_count: int = 0 - failed_clip_count: int = 0 - created_at: datetime = Field(default_factory=utc_now) - completed_at: datetime | None = None - - batch: ImportBatch | None = Relationship(back_populates="exports") - - -class ExportPreview(SQLModel): - project_id: str - manifest_path: str - accepted_slice_count: int - lines: list[str] - - class ImportBatchCreate(SQLModel): id: str name: str @@ -1044,7 +1046,6 @@ class ProjectSummary(SQLModel): name: str created_at: datetime updated_at: datetime - export_status: JobStatus | None = None active_prepared_output_group_id: str | None = None active_preparation_job_id: str | None = None diff --git a/backend/app/repository.py b/backend/app/repository.py index c669bec..ee0c77a 100644 --- a/backend/app/repository.py +++ b/backend/app/repository.py @@ -45,8 +45,6 @@ load_cutpoint_windows, ) from .models import ( - ExportRun, - ExportPreview, ImportBatch, ImportBatchCreate, JobKind, @@ -827,45 +825,6 @@ def list_source_recording_jobs(self, recording_id: str) -> list[ProcessingJobVie ).all() return [self._processing_job_view(job) for job in jobs] - def list_export_runs(self, project_id: str) -> list[ExportRun]: - with self._session() as session: - self._get_batch(session, project_id) - runs = session.exec( - select(ExportRun).where(ExportRun.batch_id == project_id).order_by(ExportRun.created_at) - ).all() - return [self._normalize_export_run(run) for run in runs] - - def get_export_preview(self, project_id: str) -> ExportPreview: - with self._session() as session: - batch = self._get_batch(session, project_id) - return ExportPreview( - project_id=batch.id, - manifest_path=f"exports/{batch.id}/dataset.list", - accepted_slice_count=0, - lines=[], - ) - - - def export_project(self, project_id: str) -> ExportRun: - with self._session() as session: - self._get_batch(session, project_id) - export_id = self._new_id("export") - output_root = self.exports_root / project_id / export_id - manifest_path = output_root / "dataset.list" - export_run = ExportRun( - id=export_id, - batch_id=project_id, - status=JobStatus.COMPLETED, - output_root=str(output_root), - manifest_path=str(manifest_path), - accepted_clip_count=0, - completed_at=utc_now(), - ) - session.add(export_run) - session.commit() - return self._normalize_export_run(export_run) - - def get_source_recording_window_media_path( self, recording_id: str, @@ -927,7 +886,6 @@ def delete_project(self, project_id: str) -> dict[str, int | str]: session.exec(delete(SourceRecordingArtifact).where(SourceRecordingArtifact.source_recording_id.in_(recording_ids))) session.exec(delete(ProcessingJob).where(ProcessingJob.source_recording_id.in_(recording_ids))) session.exec(delete(SourceRecording).where(SourceRecording.id.in_(recording_ids))) - session.exec(delete(ExportRun).where(ExportRun.batch_id == project_id)) session.exec(delete(ImportBatch).where(ImportBatch.id == project_id)) session.commit() @@ -1936,7 +1894,6 @@ def _seed_from_legacy_json(self, session: Session) -> None: source_recordings: dict[tuple[str, str], SourceRecording] = {} projects = legacy.get("projects", {}) clips_by_project = legacy.get("clips_by_project", {}) - exports_by_project = legacy.get("exports_by_project", {}) source_audio_overrides = self._legacy_seed_source_audio_overrides(clips_by_project) for project_id, batch_payload in projects.items(): @@ -1995,24 +1952,6 @@ def _seed_from_legacy_json(self, session: Session) -> None: source_recordings[source_key] = source_recording session.flush() - for export_payload in exports_by_project.get(batch.id, []): - session.add( - ExportRun( - id=export_payload["id"], - batch_id=batch.id, - status=self._job_status_from_legacy(export_payload["status"]), - output_root=export_payload["output_root"], - manifest_path=export_payload["manifest_path"], - accepted_clip_count=export_payload["accepted_clip_count"], - failed_clip_count=export_payload["failed_clip_count"], - created_at=datetime.fromisoformat(export_payload["created_at"].replace("Z", "+00:00")), - completed_at=( - datetime.fromisoformat(export_payload["completed_at"].replace("Z", "+00:00")) - if export_payload.get("completed_at") - else None - ), - ) - ) def _seed_demo(self, session: Session) -> None: batch = ImportBatch(id="phase1-demo", name="Phase 1 Demo Project") @@ -2080,34 +2019,16 @@ def _coerce_datetime(self, value: datetime | str | None) -> datetime: def _project_summary(self, session: Session, batch: ImportBatch) -> ProjectSummary: created_at = self._as_utc(batch.created_at) updated_at = created_at - latest_export = session.exec( - select(ExportRun).where(ExportRun.batch_id == batch.id).order_by(ExportRun.created_at.desc()) - ).first() - export_status = None - if latest_export is not None: - export_status = latest_export.status - updated_at = max( - updated_at, - self._as_utc(latest_export.completed_at or latest_export.created_at), - ) return ProjectSummary( id=batch.id, name=batch.name, created_at=created_at, updated_at=updated_at, - export_status=export_status, active_prepared_output_group_id=batch.active_prepared_output_group_id, active_preparation_job_id=batch.active_preparation_job_id, ) - def _normalize_export_run(self, run: ExportRun) -> ExportRun: - run.created_at = self._as_utc(run.created_at) - if run.completed_at is not None: - run.completed_at = self._as_utc(run.completed_at) - return run - - def _source_recording_view( self, @@ -4505,15 +4426,6 @@ def _job_status_to_api(self, status: JobStatus) -> str: } return mapping[status] - def _export_status_from_job(self, status: JobStatus) -> str: - mapping = { - JobStatus.PENDING: "export_in_progress", - JobStatus.RUNNING: "export_in_progress", - JobStatus.COMPLETED: "export_succeeded", - JobStatus.FAILED: "export_failed", - } - return mapping[status] - _repository_instance: SQLiteRepository | None = None diff --git a/backend/tests/test_api_integration.py b/backend/tests/test_api_integration.py index 46aa785..f01b5f2 100644 --- a/backend/tests/test_api_integration.py +++ b/backend/tests/test_api_integration.py @@ -37,14 +37,8 @@ def test_project_routes_return_seeded_demo_project(self) -> None: self.assertGreaterEqual(len(recordings), 1) self.assertEqual(recordings[0]["slice_count"], 0) - preview_response = self.client.get("/api/projects/phase1-demo/export-preview") - self.assertEqual(preview_response.status_code, 200) - preview = preview_response.json() - self.assertEqual(preview["project_id"], "phase1-demo") - self.assertEqual(preview["accepted_slice_count"], 0) - def test_missing_project_route_returns_404(self) -> None: - response = self.client.get("/api/projects/missing-project/export-preview") + response = self.client.get("/api/projects/missing-project") self.assertEqual(response.status_code, 404) self.assertEqual(response.json()["detail"], "Project not found") diff --git a/backend/tests/test_clip_lab_routes.py b/backend/tests/test_clip_lab_routes.py index 605daa4..2853ab9 100644 --- a/backend/tests/test_clip_lab_routes.py +++ b/backend/tests/test_clip_lab_routes.py @@ -12,14 +12,29 @@ import numpy as np +from fastapi import HTTPException from fastapi.testclient import TestClient from sqlmodel import Session from app.clip_lab_audio import MAX_INSERT_SILENCE_SEC -from app.clip_lab_state import ClipLabStateError, compute_content_hash, compute_manifest_sha256 +from app.clip_lab_state import ( + ClipLabStateError, + compute_content_hash, + compute_manifest_sha256, + save_clip_lab_state, +) from app.dataset_runs import create_dataset_run -from app.main import app, patch_dataset_clip_lab_route, read_dataset_clip_lab +from app.main import ( + app, + patch_dataset_clip_lab_route, + post_dataset_clip_audio_operation_route, + post_canonical_export, + read_canonical_exports, + read_canonical_export_preview, + read_dataset_clip_lab, +) from app.models import ( + DatasetClipLabAudioOperationRequest, DatasetClipLabClipView, DatasetClipLabPatchRequest, DatasetClipLabPipelineFindingView, @@ -178,6 +193,405 @@ def test_get_clip_lab_returns_manifest_clips(self) -> None: self.assertTrue(payload["qc_available"]) self.assertEqual(len(payload["clips"]), 4) + def test_canonical_export_preview_allows_zero_accepted(self) -> None: + with self.repo(): + response = read_canonical_export_preview(self.run.id) + + self.assertEqual(response.accepted_clip_count, 0) + self.assertEqual(response.blocked_clip_count, 0) + + def test_create_canonical_export_rejects_zero_accepted(self) -> None: + with self.repo(): + with self.assertRaises(HTTPException) as exc_info: + post_canonical_export(self.run.id) + self.assertEqual(exc_info.exception.status_code, 409) + + def test_create_canonical_export_writes_manifest_snapshot(self) -> None: + with self.repo(): + accept_response = patch_dataset_clip_lab_route( + self.run.id, + "candidate_review_clip_000001", + DatasetClipLabPatchRequest( + expected_manifest_sha256=self.manifest_sha, + expected_clip_version=0, + review_status="accepted", + transcript_override="Final transcript.", + reviewer_tags=["good energy"], + ), + ) + self.assertEqual(accept_response.review_status, "accepted") + + export_response = post_canonical_export(self.run.id) + + export_dir = Path(export_response.snapshot_dir) + self.assertTrue((export_dir / "speechcraft_dataset.jsonl").is_file()) + self.assertTrue((export_dir / "speechcraft_export.json").is_file()) + self.assertTrue((export_dir / "export_report.json").is_file()) + + rows = [ + json.loads(line) + for line in (export_dir / "speechcraft_dataset.jsonl").read_text(encoding="utf-8").splitlines() + if line.strip() + ] + self.assertEqual(len(rows), 1) + row = rows[0] + self.assertEqual(row["schema_version"], 1) + self.assertEqual(row["clip_id"], "candidate_review_clip_000001") + self.assertEqual(row["transcript"], "Final transcript.") + self.assertEqual(row["lineage"]["source_clip_id"], "candidate_review_clip_000001") + self.assertEqual(row["lineage"]["parent_clip_ids"], []) + self.assertEqual(row["review"]["status"], "accepted") + self.assertEqual(row["review"]["reviewer_tags"], ["good energy"]) + self.assertEqual(row["audio"]["kind"], "candidate_original") + self.assertEqual(row["audio"]["source_audio_sha256"], row["audio"]["sha256"]) + self.assertIn("candidate_review_clips/candidate_review_clip_000001.wav", row["audio"]["path"]) + metadata = json.loads((export_dir / "speechcraft_export.json").read_text(encoding="utf-8")) + self.assertEqual(metadata["path_mode"], "snapshot_relative") + self.assertEqual(metadata["audio_storage_mode"], "project_artifact_reference") + self.assertFalse(metadata["portable"]) + self.assertEqual(export_dir.parent, self.run_root / "artifacts" / "canonical_exports") + + def test_create_canonical_export_uses_rendered_revision_for_accepted_audio_edit(self) -> None: + with self.repo(): + edited = post_dataset_clip_audio_operation_route( + self.run.id, + "candidate_review_clip_000001", + DatasetClipLabAudioOperationRequest( + expected_manifest_sha256=self.manifest_sha, + expected_clip_version=0, + operation={ + "kind": "insert_silence", + "at_sample": 10, + "duration_samples": 16, + }, + ), + ) + self.assertEqual(edited.effective_audio_kind, "rendered_revision") + accepted = patch_dataset_clip_lab_route( + self.run.id, + "candidate_review_clip_000001", + DatasetClipLabPatchRequest( + expected_manifest_sha256=self.manifest_sha, + expected_clip_version=edited.clip_version, + review_status="accepted", + ), + ) + self.assertEqual(accepted.review_status, "accepted") + + export_response = post_canonical_export(self.run.id) + + rows = [ + json.loads(line) + for line in (Path(export_response.snapshot_dir) / "speechcraft_dataset.jsonl") + .read_text(encoding="utf-8") + .splitlines() + if line.strip() + ] + row = rows[0] + self.assertEqual(row["audio"]["kind"], "rendered_revision") + self.assertIn("clip_lab_renders", row["audio"]["path"]) + self.assertIsInstance(row["audio"]["audio_revision_hash"], str) + + def test_canonical_export_preview_blocks_pending_rendered_audio(self) -> None: + manifest = json.loads((self.run_root / "artifacts" / "candidate_review_manifest.json").read_text(encoding="utf-8")) + clip_row = next(row for row in manifest if row["id"] == "candidate_review_clip_000001") + source_sha = str(clip_row["audio_sha256"]) + audio_revision_hash = "ab" * 32 + accepted_hash = compute_content_hash( + manifest_transcript=str(clip_row["training_text"]), + transcript_override=None, + audio_revision_hash=audio_revision_hash, + base_audio_hash=source_sha, + ) + save_clip_lab_state( + self.run_root, + { + "schema_version": 1, + "stage": "clip_lab_state", + "candidate_manifest_sha256": self.manifest_sha, + "updated_at": "2026-07-08T00:00:00Z", + "clips": { + "candidate_review_clip_000001": { + "clip_version": 1, + "review_status": "accepted", + "accepted_content_hash": accepted_hash, + "accepted_at": "2026-07-08T00:00:00Z", + "reviewer_tags": [], + "updated_at": "2026-07-08T00:00:00Z", + "audio_edit": { + "schema_version": 1, + "source_audio_sha256": source_sha, + "source_sample_rate_hz": SAMPLE_RATE, + "ops": [{"kind": "insert_silence", "at_sample": 10, "duration_samples": 16}], + "redo_ops": [], + "audio_revision_hash": audio_revision_hash, + "rendered_audio_sha256": None, + "render_status": "pending", + }, + } + }, + }, + ) + + with self.repo(): + preview = read_canonical_export_preview(self.run.id) + + self.assertEqual(preview.accepted_clip_count, 1) + self.assertEqual(preview.blocked_clip_count, 1) + self.assertIn("rendered_audio_not_ready", preview.blocked_reasons[0].reasons) + + def test_canonical_export_preview_blocks_failed_rendered_audio(self) -> None: + manifest = json.loads((self.run_root / "artifacts" / "candidate_review_manifest.json").read_text(encoding="utf-8")) + clip_row = next(row for row in manifest if row["id"] == "candidate_review_clip_000001") + source_sha = str(clip_row["audio_sha256"]) + audio_revision_hash = "cd" * 32 + accepted_hash = compute_content_hash( + manifest_transcript=str(clip_row["training_text"]), + transcript_override=None, + audio_revision_hash=audio_revision_hash, + base_audio_hash=source_sha, + ) + save_clip_lab_state( + self.run_root, + { + "schema_version": 1, + "stage": "clip_lab_state", + "candidate_manifest_sha256": self.manifest_sha, + "updated_at": "2026-07-08T00:00:00Z", + "clips": { + "candidate_review_clip_000001": { + "clip_version": 1, + "review_status": "accepted", + "accepted_content_hash": accepted_hash, + "accepted_at": "2026-07-08T00:00:00Z", + "reviewer_tags": [], + "updated_at": "2026-07-08T00:00:00Z", + "audio_edit": { + "schema_version": 1, + "source_audio_sha256": source_sha, + "source_sample_rate_hz": SAMPLE_RATE, + "ops": [{"kind": "insert_silence", "at_sample": 10, "duration_samples": 16}], + "redo_ops": [], + "audio_revision_hash": audio_revision_hash, + "rendered_audio_sha256": None, + "render_status": "failed", + }, + } + }, + }, + ) + + with self.repo(): + preview = read_canonical_export_preview(self.run.id) + + self.assertEqual(preview.accepted_clip_count, 1) + self.assertEqual(preview.blocked_clip_count, 1) + self.assertIn("rendered_audio_not_ready", preview.blocked_reasons[0].reasons) + + def test_canonical_export_preview_blocks_missing_rendered_wav(self) -> None: + manifest = json.loads((self.run_root / "artifacts" / "candidate_review_manifest.json").read_text(encoding="utf-8")) + clip_row = next(row for row in manifest if row["id"] == "candidate_review_clip_000001") + source_sha = str(clip_row["audio_sha256"]) + audio_revision_hash = "ef" * 32 + rendered_sha = "11" * 32 + accepted_hash = compute_content_hash( + manifest_transcript=str(clip_row["training_text"]), + transcript_override=None, + audio_revision_hash=audio_revision_hash, + base_audio_hash=source_sha, + ) + save_clip_lab_state( + self.run_root, + { + "schema_version": 1, + "stage": "clip_lab_state", + "candidate_manifest_sha256": self.manifest_sha, + "updated_at": "2026-07-08T00:00:00Z", + "clips": { + "candidate_review_clip_000001": { + "clip_version": 1, + "review_status": "accepted", + "accepted_content_hash": accepted_hash, + "accepted_at": "2026-07-08T00:00:00Z", + "reviewer_tags": [], + "updated_at": "2026-07-08T00:00:00Z", + "audio_edit": { + "schema_version": 1, + "source_audio_sha256": source_sha, + "source_sample_rate_hz": SAMPLE_RATE, + "ops": [{"kind": "insert_silence", "at_sample": 10, "duration_samples": 16}], + "redo_ops": [], + "audio_revision_hash": audio_revision_hash, + "rendered_audio_sha256": rendered_sha, + "render_status": "ready", + }, + } + }, + }, + ) + + with self.repo(): + preview = read_canonical_export_preview(self.run.id) + + self.assertEqual(preview.blocked_clip_count, 1) + self.assertIn("rendered_audio_missing", preview.blocked_reasons[0].reasons) + + def test_canonical_export_preview_blocks_rendered_hash_mismatch(self) -> None: + manifest = json.loads((self.run_root / "artifacts" / "candidate_review_manifest.json").read_text(encoding="utf-8")) + clip_row = next(row for row in manifest if row["id"] == "candidate_review_clip_000001") + source_sha = str(clip_row["audio_sha256"]) + audio_revision_hash = "12" * 32 + accepted_hash = compute_content_hash( + manifest_transcript=str(clip_row["training_text"]), + transcript_override=None, + audio_revision_hash=audio_revision_hash, + base_audio_hash=source_sha, + ) + render_path = self.run_root / "artifacts" / "clip_lab_renders" / "candidate_review_clip_000001" / f"{audio_revision_hash}.wav" + render_path.parent.mkdir(parents=True, exist_ok=True) + render_path.write_bytes(b"not-the-expected-hash") + save_clip_lab_state( + self.run_root, + { + "schema_version": 1, + "stage": "clip_lab_state", + "candidate_manifest_sha256": self.manifest_sha, + "updated_at": "2026-07-08T00:00:00Z", + "clips": { + "candidate_review_clip_000001": { + "clip_version": 1, + "review_status": "accepted", + "accepted_content_hash": accepted_hash, + "accepted_at": "2026-07-08T00:00:00Z", + "reviewer_tags": [], + "updated_at": "2026-07-08T00:00:00Z", + "audio_edit": { + "schema_version": 1, + "source_audio_sha256": source_sha, + "source_sample_rate_hz": SAMPLE_RATE, + "ops": [{"kind": "insert_silence", "at_sample": 10, "duration_samples": 16}], + "redo_ops": [], + "audio_revision_hash": audio_revision_hash, + "rendered_audio_sha256": "22" * 32, + "render_status": "ready", + }, + } + }, + }, + ) + + with self.repo(): + preview = read_canonical_export_preview(self.run.id) + + self.assertEqual(preview.blocked_clip_count, 1) + self.assertIn("rendered_audio_hash_mismatch", preview.blocked_reasons[0].reasons) + + def test_canonical_export_preview_blocks_candidate_hash_mismatch(self) -> None: + with self.repo(): + accepted = patch_dataset_clip_lab_route( + self.run.id, + "candidate_review_clip_000001", + DatasetClipLabPatchRequest( + expected_manifest_sha256=self.manifest_sha, + expected_clip_version=0, + review_status="accepted", + ), + ) + self.assertEqual(accepted.review_status, "accepted") + + candidate_wav = self.run_root / "artifacts" / "candidate_review_clips" / "candidate_review_clip_000001.wav" + candidate_wav.write_bytes(b"mutated-audio") + + with self.repo(): + preview = read_canonical_export_preview(self.run.id) + + self.assertEqual(preview.accepted_clip_count, 1) + self.assertEqual(preview.blocked_clip_count, 1) + self.assertIn("candidate_audio_hash_mismatch", preview.blocked_reasons[0].reasons) + + def test_canonical_export_preview_rejects_stale_state(self) -> None: + save_clip_lab_state( + self.run_root, + { + "schema_version": 1, + "stage": "clip_lab_state", + "candidate_manifest_sha256": "stale-manifest-sha", + "updated_at": "2026-07-08T00:00:00Z", + "clips": {}, + }, + ) + + with self.repo(): + with self.assertRaises(HTTPException) as exc_info: + read_canonical_export_preview(self.run.id) + self.assertEqual(exc_info.exception.status_code, 409) + + def test_canonical_export_create_rejects_invalid_state(self) -> None: + save_clip_lab_state( + self.run_root, + { + "schema_version": 1, + "stage": "clip_lab_state", + "candidate_manifest_sha256": self.manifest_sha, + "updated_at": "2026-07-08T00:00:00Z", + "clips": { + "candidate_review_clip_000001": { + "clip_version": 1, + "review_status": "accepted", + "reviewer_tags": ["Accepted"], + } + }, + }, + ) + + with self.repo(): + with self.assertRaises(HTTPException) as exc_info: + post_canonical_export(self.run.id) + self.assertEqual(exc_info.exception.status_code, 409) + + def test_list_canonical_exports_returns_newest_first_and_skips_corrupt_dirs(self) -> None: + exports_root = self.run_root / "artifacts" / "canonical_exports" + older = exports_root / "canonical_export_2026-07-08_010101_aaaaaaaa" + newer = exports_root / "canonical_export_2026-07-08_020202_bbbbbbbb" + corrupt = exports_root / "canonical_export_corrupt" + older.mkdir(parents=True, exist_ok=True) + newer.mkdir(parents=True, exist_ok=True) + corrupt.mkdir(parents=True, exist_ok=True) + (older / "speechcraft_export.json").write_text( + json.dumps( + { + "schema_version": 1, + "export_id": older.name, + "run_id": self.run.id, + "project_id": "project-1", + "created_at": "2026-07-08T01:01:01Z", + "accepted_clip_count": 2, + "total_duration_sec": 12.5, + } + ), + encoding="utf-8", + ) + (newer / "speechcraft_export.json").write_text( + json.dumps( + { + "schema_version": 1, + "export_id": newer.name, + "run_id": self.run.id, + "project_id": "project-1", + "created_at": "2026-07-08T02:02:02Z", + "accepted_clip_count": 3, + "total_duration_sec": 18.0, + } + ), + encoding="utf-8", + ) + (corrupt / "speechcraft_export.json").write_text("{bad json", encoding="utf-8") + + with self.repo(): + exports = read_canonical_exports(self.run.id) + + self.assertEqual([item.export_id for item in exports], [newer.name, older.name]) + def test_patch_transcript_override_then_get(self) -> None: with self.repo(): patch_response = self.client.patch( diff --git a/backend/tests/test_repository_media.py b/backend/tests/test_repository_media.py index 8214994..6651d2c 100644 --- a/backend/tests/test_repository_media.py +++ b/backend/tests/test_repository_media.py @@ -278,7 +278,6 @@ def test_legacy_seed_imports_all_projects_without_source_collisions(self) -> Non "id": "project-a", "name": "Project A", "status": "ready", - "export_status": "idle", "created_at": "2026-03-18T15:16:30Z", "updated_at": "2026-03-18T15:16:30Z", }, @@ -286,7 +285,6 @@ def test_legacy_seed_imports_all_projects_without_source_collisions(self) -> Non "id": "project-b", "name": "Project B", "status": "ready", - "export_status": "idle", "created_at": "2026-03-18T15:16:30Z", "updated_at": "2026-03-18T15:16:30Z", }, @@ -336,7 +334,6 @@ def test_seed_from_legacy_json_uses_real_audio_for_one_to_one_source_recordings( "id": "project-a", "name": "Project A", "status": "ready", - "export_status": "idle", "created_at": "2026-03-18T15:16:30Z", "updated_at": "2026-03-18T15:16:30Z", } @@ -387,7 +384,6 @@ def test_migrate_legacy_seed_source_recordings_backfills_real_audio(self) -> Non "id": "project-a", "name": "Project A", "status": "ready", - "export_status": "idle", "created_at": "2026-03-18T15:16:30Z", "updated_at": "2026-03-18T15:16:30Z", } diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 27ba32f..96afaef 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -8,7 +8,6 @@ import { type PipelineSelectionState, type PipelineStage, } from "./pipeline/PipelineContext"; -import ExportPage from "./pages/ExportPage"; import IngestPage from "./pages/IngestPage"; import LabelPage from "./pages/LabelPage"; import OverviewPage from "./pages/OverviewPage"; @@ -55,7 +54,6 @@ const stepDefinitions: StepDefinition[] = [ { id: "slicer", label: "Slicer", shortLabel: "Sl", glyph: "C", tone: "Generate candidate review clips" }, { id: "qc", label: "QC", shortLabel: "QC", glyph: "Q", tone: "Machine triage for one run" }, { id: "lab", label: "Lab", shortLabel: "La", glyph: "L", tone: "Human review and override" }, - { id: "export", label: "Export", shortLabel: "Ex", glyph: "E", tone: "Emit training-ready data" }, ]; function getErrorMessage(error: unknown, fallback: string): string { @@ -102,8 +100,6 @@ function readLocationState(): AppLocationState { enhance: "overview", segment: "slicer", label: "lab", - train: "export", - deploy: "export", }; const maybeStep = path.length > 0 ? legacyStepMap[path] ?? path : "ingest"; const step = isAppStep(maybeStep) ? maybeStep : "ingest"; @@ -242,9 +238,10 @@ function getPageHeaderContent(step: AppStep, activeProject: Project | null): Pag } return { - eyebrow: "Step 08", - title: "Export", - description: "Emit the selected reviewed or machine-triaged dataset for downstream training.", + eyebrow: "Reference", + title: activeProject?.name ?? "Reference Workstation", + description: + "Reference remains available as its existing workstation, outside the current sprint path.", }; } @@ -479,7 +476,7 @@ export default function App() { } else if (route.step === "reference") { pageContent = ; } else { - pageContent = ; + pageContent = ; } return ( @@ -592,9 +589,6 @@ export default function App() {
{activeProject.name} updated {new Date(activeProject.updated_at).toLocaleDateString()} - - export {activeProject.export_status ? activeProject.export_status.replace(/_/g, " ") : "n/a"} - {stepDefinitions[visibleStepIndex]?.shortLabel ?? "In"} active
) : null} diff --git a/frontend/src/api.ts b/frontend/src/api.ts index 4f4e13a..4fea069 100644 --- a/frontend/src/api.ts +++ b/frontend/src/api.ts @@ -1,4 +1,6 @@ import type { + CanonicalExportPreview, + CanonicalExportSummary, ClipLabItem, DatasetClipLabClipRow, DatasetClipLabAudioOperationRequest, @@ -17,8 +19,6 @@ import type { DatasetRunLog, DatasetSpeakerSelection, DatasetSlicerResults, - ExportPreview, - ExportRun, ImportBatch, MediaCleanupResult, ProjectAlignmentSettings, @@ -307,6 +307,27 @@ export async function fetchDatasetClipLab(runId: string): Promise(`${API_BASE}/api/dataset-runs/${runId}/clip-lab`); } +export async function fetchCanonicalExportPreview(runId: string): Promise { + return await requestJson( + `${API_BASE}/api/dataset-runs/${runId}/canonical-export-preview`, + ); +} + +export async function createCanonicalExport(runId: string): Promise { + return await requestJson( + `${API_BASE}/api/dataset-runs/${runId}/canonical-exports`, + { + method: "POST", + }, + ); +} + +export async function fetchCanonicalExports(runId: string): Promise { + return await requestJson( + `${API_BASE}/api/dataset-runs/${runId}/canonical-exports`, + ); +} + export async function patchDatasetClipLabClip( runId: string, clipId: string, @@ -556,23 +577,6 @@ export async function fetchClipLabItem(sliceId: string): Promise { return await parseJson(response); } -export async function fetchProjectExports(projectId: string): Promise { - const response = await fetch(`${API_BASE}/api/projects/${projectId}/exports`); - return await parseJson(response); -} - -export async function fetchExportPreview(projectId: string): Promise { - const response = await fetch(`${API_BASE}/api/projects/${projectId}/export-preview`); - return await parseJson(response); -} - -export async function runProjectExport(projectId: string): Promise { - const response = await fetch(`${API_BASE}/api/projects/${projectId}/export`, { - method: "POST", - }); - return await parseJson(response); -} - export async function cleanupProjectMedia(projectId: string): Promise { const response = await fetch(`${API_BASE}/api/projects/${projectId}/media-cleanup`, { method: "POST", diff --git a/frontend/src/pages/ExportPage.tsx b/frontend/src/pages/ExportPage.tsx deleted file mode 100644 index fef65f6..0000000 --- a/frontend/src/pages/ExportPage.tsx +++ /dev/null @@ -1,98 +0,0 @@ -import JobActivityPanel from "../components/JobActivityPanel"; -import type { Project } from "../types"; -import WorkspaceStatePanel from "../workspace/WorkspaceStatePanel"; - -type ExportPageProps = { - activeProject: Project | null; - projectLoadStatus: "loading" | "ready" | "error"; - projectLoadError: string | null; - onRetryProjects: () => void; -}; - -export default function ExportPage({ - activeProject, - projectLoadStatus, - projectLoadError, - onRetryProjects, -}: ExportPageProps) { - if (projectLoadStatus === "error") { - return ( - - ); - } - - if (projectLoadStatus === "loading") { - return ; - } - - if (!activeProject) { - return ( - - ); - } - - return ( -
-
- - -
-
-

Export shell

-

{activeProject.name}

-

- Export is included in the workflow navigation so Phase 1 has a complete stage path. - The export backend and selection policy remain outside this phase. -

-
- - -
- - -
-
- ); -} diff --git a/frontend/src/pages/LabelPage.integration.test.tsx b/frontend/src/pages/LabelPage.integration.test.tsx index 4ec156f..542b619 100644 --- a/frontend/src/pages/LabelPage.integration.test.tsx +++ b/frontend/src/pages/LabelPage.integration.test.tsx @@ -2,7 +2,13 @@ import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/re import { useState } from "react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import type { ClipLabItemRef, DatasetClipLabClipRow, DatasetClipLabView, Project } from "../types"; +import type { + CanonicalExportPreview, + ClipLabItemRef, + DatasetClipLabClipRow, + DatasetClipLabView, + Project, +} from "../types"; import LabelPage from "./LabelPage"; const pipelineState = { @@ -19,6 +25,10 @@ const clipLabFetchControls = vi.hoisted(() => ({ staleRun1Fetch: null as Promise | null, })); +const canonicalExportStore = vi.hoisted(() => ({ + preview: null as CanonicalExportPreview | null, +})); + vi.mock("../pipeline/PipelineContext", () => ({ usePipelineContext: () => ({ selectedLabDatasetRunId: pipelineState.selectedLabDatasetRunId, @@ -53,7 +63,18 @@ vi.mock("../api", () => ({ API_BASE: "http://127.0.0.1:8010", appendClipEdlOperation: vi.fn(), appendDatasetAudioOperation: vi.fn(), + createCanonicalExport: vi.fn(async () => ({ + export_id: "canonical_export_2026-07-08_120000", + run_id: "dataset-run-1", + project_id: "project-1", + created_at: "2026-07-08T12:00:00Z", + accepted_clip_count: canonicalExportStore.preview?.accepted_clip_count ?? 0, + total_duration_sec: canonicalExportStore.preview?.total_duration_sec ?? 0, + snapshot_dir: "/tmp/canonical", + manifest_path: "/tmp/canonical/speechcraft_dataset.jsonl", + })), fetchClipLabItem: vi.fn(), + fetchCanonicalExportPreview: vi.fn(async () => canonicalExportStore.preview), fetchDatasetClipLab: vi.fn(async () => clipLabStore.view), fetchDatasetQc: vi.fn(async () => ({ run_id: "dataset-run-1", @@ -66,7 +87,6 @@ vi.mock("../api", () => ({ })), fetchDatasetSlicerResults: vi.fn(), fetchProjectDatasetRuns: vi.fn(), - fetchProjectExports: vi.fn(async () => []), fetchProjectRecordings: vi.fn(), fetchProjectReferenceAssets: vi.fn(async () => []), markDatasetClipAsReferenceCandidate: vi.fn(), @@ -92,6 +112,8 @@ vi.mock("../api", () => ({ })); import { + createCanonicalExport, + fetchCanonicalExportPreview, fetchDatasetClipLab, fetchDatasetSlicerResults, fetchProjectDatasetRuns, @@ -104,7 +126,6 @@ const activeProject: Project = { name: "Test Project", created_at: "2026-01-01T00:00:00.000Z", updated_at: "2026-01-01T00:00:00.000Z", - export_status: null, }; const datasetRun = { @@ -157,6 +178,8 @@ const recording = { num_samples: 160000, processing_recipe: null, duration_seconds: 10, + slice_count: 0, + processing_state: "idle", }; function makeClipLabRow(overrides: Partial = {}): DatasetClipLabClipRow { @@ -210,6 +233,27 @@ function makeClipLabView(overrides: Partial = {}): DatasetCl }; } +function makeCanonicalExportPreview( + overrides: Partial = {}, +): CanonicalExportPreview { + return { + run_id: "dataset-run-1", + accepted_clip_count: 1, + total_duration_sec: 1.5, + original_audio_count: 1, + edited_audio_count: 0, + blocked_clip_count: 0, + blocked_reasons: [], + ...overrides, + }; +} + +function canonicalPreviewFromClipLabView(view: DatasetClipLabView | null): CanonicalExportPreview { + const acceptedClipCount = + view?.clips.filter((clip) => clip.review_status === "accepted").length ?? 0; + return makeCanonicalExportPreview({ accepted_clip_count: acceptedClipCount }); +} + function Harness({ datasetRunId = "dataset-run-1", remountKey, @@ -247,6 +291,7 @@ afterEach(() => { beforeEach(() => { clipLabFetchControls.staleRun1Fetch = null; clipLabStore.view = makeClipLabView(); + canonicalExportStore.preview = canonicalPreviewFromClipLabView(clipLabStore.view); vi.mocked(fetchProjectDatasetRuns).mockResolvedValue([datasetRun, datasetRun2]); vi.mocked(fetchProjectRecordings).mockResolvedValue([recording]); @@ -282,6 +327,9 @@ beforeEach(() => { } return clipLabStore.view ?? makeClipLabView(); }); + vi.mocked(fetchCanonicalExportPreview).mockImplementation( + async () => canonicalExportStore.preview ?? canonicalPreviewFromClipLabView(clipLabStore.view), + ); vi.mocked(patchDatasetClipLabClip).mockImplementation(async (_runId, clipId, payload) => { const current = clipLabStore.view?.clips.find((clip) => clip.clip_id === clipId); if (!current) { @@ -305,6 +353,7 @@ beforeEach(() => { ...clipLabStore.view!, clips: clipLabStore.view!.clips.map((clip) => (clip.clip_id === clipId ? updated : clip)), }; + canonicalExportStore.preview = canonicalPreviewFromClipLabView(clipLabStore.view); return updated; }); }); @@ -326,7 +375,8 @@ describe("LabelPage dataset Clip Lab integration", () => { expect(patchDatasetClipLabClip).toHaveBeenCalled(); }); - const lastCall = vi.mocked(patchDatasetClipLabClip).mock.calls.at(-1); + const patchCalls = vi.mocked(patchDatasetClipLabClip).mock.calls; + const lastCall = patchCalls[patchCalls.length - 1]; expect(lastCall?.[2]).toMatchObject({ expected_clip_version: 0, reviewer_tags: ["good energy", "mouth noise"], @@ -428,6 +478,37 @@ describe("LabelPage dataset Clip Lab integration", () => { } }); + it("shows the canonical export button and runs export from the inspector header", async () => { + render(); + + const button = await screen.findByRole("button", { name: "Export accepted clips (1)" }); + expect((button as HTMLButtonElement).disabled).toBe(false); + + fireEvent.click(button); + + await waitFor(() => { + expect(createCanonicalExport).toHaveBeenCalledWith("dataset-run-1"); + }); + expect(await screen.findByText("Exported 1 clips.")).toBeTruthy(); + }); + + it("refreshes the canonical export count after accepting a clip", async () => { + clipLabStore.view = makeClipLabView({ + clips: [makeClipLabRow({ review_status: "unresolved", accepted_content_hash: null, accepted_at: null })], + }); + canonicalExportStore.preview = canonicalPreviewFromClipLabView(clipLabStore.view); + + render(); + + expect(await screen.findByRole("button", { name: "Export accepted clips (0)" })).toBeTruthy(); + + fireEvent.click(screen.getByRole("button", { name: "Accepted" })); + + await waitFor(() => { + expect(screen.getByRole("button", { name: "Export accepted clips (1)" })).toBeTruthy(); + }); + }); + it("treats stale clip lab state as read-only across editor and inspector", async () => { clipLabStore.view = makeClipLabView({ stale_state: true, diff --git a/frontend/src/pages/LabelPage.tsx b/frontend/src/pages/LabelPage.tsx index 99158b9..3173724 100644 --- a/frontend/src/pages/LabelPage.tsx +++ b/frontend/src/pages/LabelPage.tsx @@ -3,12 +3,13 @@ import { ApiError, appendClipEdlOperation, appendDatasetAudioOperation, + createCanonicalExport, fetchClipLabItem, + fetchCanonicalExportPreview, fetchDatasetClipLab, fetchDatasetQc, fetchDatasetSlicerResults, fetchProjectDatasetRuns, - fetchProjectExports, fetchProjectRecordings, fetchProjectReferenceAssets, markDatasetClipAsReferenceCandidate, @@ -54,6 +55,7 @@ import { type DatasetQcScores, } from "../workspace/workspace-helpers"; import type { + CanonicalExportPreview, ClipLabItem, ClipLabItemRef, ClipLabCapabilities, @@ -67,7 +69,6 @@ import type { DatasetQcPayload, DatasetRun, DatasetSlicerResults, - ExportRun, Project, ReferenceAssetSummary, ReviewStatus, @@ -418,7 +419,6 @@ export default function LabelPage({ const [recordings, setRecordings] = useState([]); const [activeClip, setActiveClip] = useState(null); const [visibleQueueClipIds, setVisibleQueueClipIds] = useState([]); - const [exportRuns, setExportRuns] = useState([]); const [referenceAssets, setReferenceAssets] = useState([]); const [datasetSlicerResults, setDatasetSlicerResults] = useState(null); const [datasetRuns, setDatasetRuns] = useState([]); @@ -435,6 +435,14 @@ export default function LabelPage({ const [datasetClipLab, setDatasetClipLabState] = useState(null); const [datasetClipLabLoadState, setDatasetClipLabLoadStateInternal] = useState("idle"); + const [canonicalExportPreview, setCanonicalExportPreview] = useState(null); + const [canonicalExportPreviewStatus, setCanonicalExportPreviewStatus] = + useState<"idle" | "loading" | "ready" | "error">("idle"); + const [canonicalExportFeedback, setCanonicalExportFeedback] = useState<{ + tone: "error" | "success"; + message: string; + } | null>(null); + const [isCreatingCanonicalExport, setIsCreatingCanonicalExport] = useState(false); const datasetClipLabRef = useRef(null); const datasetClipLabLoadStateRef = useRef("idle"); const activeClipItemRef = useRef(activeClipItem); @@ -457,6 +465,30 @@ export default function LabelPage({ setDatasetClipLabLoadStateInternal(next); } + async function refreshCanonicalExportPreview( + runId: string, + options?: { suppress409Notice?: boolean }, + ): Promise { + setCanonicalExportPreviewStatus("loading"); + try { + const preview = await fetchCanonicalExportPreview(runId); + if (datasetClipLabRef.current?.run_id !== runId) { + return; + } + setCanonicalExportPreview(preview); + setCanonicalExportPreviewStatus("ready"); + } catch (error) { + if (datasetClipLabRef.current?.run_id !== runId) { + return; + } + setCanonicalExportPreview(null); + setCanonicalExportPreviewStatus("error"); + if (!(error instanceof ApiError && error.status === 409 && options?.suppress409Notice)) { + setWorkspaceNotice(getErrorMessage(error, "Canonical export preview failed to load.")); + } + } + } + const patchCoordinator = useMemo( () => createClipLabPatchCoordinator({ @@ -510,6 +542,7 @@ export default function LabelPage({ if (activeClipItemRef.current?.id === updated.clip_id) { refreshActiveDatasetClip(updated.clip_id, updated); } + void refreshCanonicalExportPreview(runId, { suppress409Notice: true }); }; conflictRef.current = async (runId, clipId) => { @@ -537,6 +570,7 @@ export default function LabelPage({ refreshActiveDatasetClip(clipId, refreshedRow); } } + await refreshCanonicalExportPreview(runId, { suppress409Notice: true }); setWorkspaceNotice("Clip Lab state was out of date and has been reloaded."); }; @@ -626,10 +660,11 @@ export default function LabelPage({ if (!projectId) { setSlices([]); setRecordings([]); - setExportRuns([]); setReferenceAssets([]); setDatasetClipLab(null); setDatasetClipLabLoadState("idle"); + setCanonicalExportPreview(null); + setCanonicalExportPreviewStatus("idle"); onActiveClipItemChange(null); setVisibleQueueClipIds([]); setWorkspaceStatus("ready"); @@ -645,9 +680,10 @@ export default function LabelPage({ setDatasetSlicerResults(null); setDatasetClipLab(null); setDatasetClipLabLoadState("idle"); + setCanonicalExportPreview(null); + setCanonicalExportPreviewStatus("idle"); setSlices([]); setRecordings([]); - setExportRuns([]); setReferenceAssets([]); onActiveClipItemChange(null); setVisibleQueueClipIds([]); @@ -656,10 +692,9 @@ export default function LabelPage({ return; } - const [results, nextRecordings, nextExports] = await Promise.all([ + const [results, nextRecordings] = await Promise.all([ fetchDatasetSlicerResults(datasetRunId), fetchProjectRecordings(projectId), - fetchProjectExports(projectId), ]); let nextReferenceAssets: ReferenceAssetSummary[] = []; let datasetQcPayload: DatasetQcPayload | null = null; @@ -699,6 +734,10 @@ export default function LabelPage({ setDatasetClipLab(nextClipLab); setDatasetClipLabLoadState(nextClipLabLoadState); + await refreshCanonicalExportPreview(datasetRunId, { suppress409Notice: true }); + if (latestWorkspaceRequestRef.current !== requestId) { + return; + } if (nextClipLabLoadState === "unavailable") { setWorkspaceNotice( getErrorMessage( @@ -727,7 +766,6 @@ export default function LabelPage({ if (!options?.silent) { setActiveClip(null); } - setExportRuns(nextExports); setReferenceAssets(nextReferenceAssets); const sortedSlices = sortClipsForQueue(nextSlices, queueSortMode); const nextActiveClip = @@ -759,7 +797,6 @@ export default function LabelPage({ setSlices([]); setRecordings([]); setActiveClip(null); - setExportRuns([]); setReferenceAssets([]); onActiveClipItemChange(null); setVisibleQueueClipIds([]); @@ -806,6 +843,9 @@ export default function LabelPage({ selectLabDatasetRun(runId); setDatasetClipLab(null); setDatasetClipLabLoadState("idle"); + setCanonicalExportPreview(null); + setCanonicalExportPreviewStatus("idle"); + setCanonicalExportFeedback(null); onActiveClipItemChange(null); setActiveClip(null); } @@ -1349,6 +1389,14 @@ export default function LabelPage({ : null; const canUndo = Boolean(activeClip?.can_undo); const canRedo = Boolean(activeClip?.can_redo); + const canTriggerCanonicalExport = + Boolean(datasetRunId) + && canonicalExportPreviewStatus === "ready" + && (canonicalExportPreview?.accepted_clip_count ?? 0) > 0 + && (canonicalExportPreview?.blocked_clip_count ?? 0) === 0 + && !datasetClipLab?.stale_state + && !datasetClipLab?.invalid_state + && !isCreatingCanonicalExport; const activeCommitId = activeClip?.active_commit?.id ?? null; const existingReferenceForCurrentState = useMemo(() => { if (!activeCommitId || !activeClip) { @@ -1370,6 +1418,29 @@ export default function LabelPage({ }; }, [onHeaderActionsChange]); + async function handleCreateCanonicalExport() { + if (!datasetRunId || !canTriggerCanonicalExport) { + return; + } + setCanonicalExportFeedback(null); + setIsCreatingCanonicalExport(true); + try { + const result = await createCanonicalExport(datasetRunId); + await refreshCanonicalExportPreview(datasetRunId, { suppress409Notice: true }); + setCanonicalExportFeedback({ + tone: "success", + message: `Exported ${result.accepted_clip_count} clips.`, + }); + } catch (error) { + setCanonicalExportFeedback({ + tone: "error", + message: getErrorMessage(error, "Canonical export failed."), + }); + } finally { + setIsCreatingCanonicalExport(false); + } + } + const datasetClipLabEditable = isDatasetClipLabEditable(datasetMode, datasetClipLabLoadState, datasetClipLab); const activeClipLabRow = datasetClipLabEditable && activeClipItem ? clipLabRowById.get(activeClipItem.id) : undefined; @@ -1379,18 +1450,21 @@ export default function LabelPage({ machineFindings: activeClipLabRow.pipeline_findings, reviewerTags: activeClipLabRow.reviewer_tags, acceptanceStale: activeClipLabRow.acceptance_stale, - onReviewStatusChange: (status: ReviewStatus) => - patchDatasetClipLab(activeClipLabRow.clip_id, () => ({ review_status: status })), - onAddReviewerTag: (tag: string) => - patchDatasetClipLab(activeClipLabRow.clip_id, (row) => ({ + onReviewStatusChange: async (status: ReviewStatus) => { + await patchDatasetClipLab(activeClipLabRow.clip_id, () => ({ review_status: status })); + }, + onAddReviewerTag: async (tag: string) => { + await patchDatasetClipLab(activeClipLabRow.clip_id, (row) => ({ reviewer_tags: [...row.reviewer_tags, tag], - })), - onRemoveReviewerTag: (tag: string) => - patchDatasetClipLab(activeClipLabRow.clip_id, (row) => ({ + })); + }, + onRemoveReviewerTag: async (tag: string) => { + await patchDatasetClipLab(activeClipLabRow.clip_id, (row) => ({ reviewer_tags: row.reviewer_tags.filter( (entry) => entry.toLowerCase() !== tag.toLowerCase(), ), - })), + })); + }, } : undefined; const datasetTagReadOnly = buildDatasetTagReadOnlyConfig( @@ -1496,7 +1570,12 @@ export default function LabelPage({ acceptedRejectedRatio={acceptedRejectedRatio} predictedOutputSeconds={predictedOutputSeconds} progressPercent={progressPercent} - exportRuns={exportRuns} + canonicalExportPreview={canonicalExportPreview} + canonicalExportPreviewStatus={canonicalExportPreviewStatus} + canonicalExportFeedback={canonicalExportFeedback} + canTriggerCanonicalExport={canTriggerCanonicalExport} + isCreatingCanonicalExport={isCreatingCanonicalExport} + onCreateCanonicalExport={() => void handleCreateCanonicalExport()} onRetryLoad={() => void loadWorkspace(activeProject?.id)} onStatusChange={(status) => { if (!activeClip) { diff --git a/frontend/src/pages/ProcessingPage.test.tsx b/frontend/src/pages/ProcessingPage.test.tsx index d934cb1..11877c0 100644 --- a/frontend/src/pages/ProcessingPage.test.tsx +++ b/frontend/src/pages/ProcessingPage.test.tsx @@ -93,7 +93,7 @@ function runWithStatus(status: "pending" | "running" | "completed" | "failed", s function renderPage(onOpenSlicerWithRun = vi.fn(), onOpenSpeakers = vi.fn()) { return render( {}} diff --git a/frontend/src/pages/QcPage.test.tsx b/frontend/src/pages/QcPage.test.tsx index f0a3a19..b651280 100644 --- a/frontend/src/pages/QcPage.test.tsx +++ b/frontend/src/pages/QcPage.test.tsx @@ -34,7 +34,6 @@ const project = { name: "Project", created_at: "", updated_at: "", - export_status: null, }; const run = { diff --git a/frontend/src/pages/ReferencePage.test.tsx b/frontend/src/pages/ReferencePage.test.tsx index 1269646..64e1c83 100644 --- a/frontend/src/pages/ReferencePage.test.tsx +++ b/frontend/src/pages/ReferencePage.test.tsx @@ -141,7 +141,7 @@ function makeCandidate(runId: string, candidateId: string) { function renderReferencePage() { return render( {}} diff --git a/frontend/src/pages/SlicerPage.test.tsx b/frontend/src/pages/SlicerPage.test.tsx index 8b5903c..85159d8 100644 --- a/frontend/src/pages/SlicerPage.test.tsx +++ b/frontend/src/pages/SlicerPage.test.tsx @@ -68,7 +68,7 @@ const slicedRun = { function renderPage() { return render( {}} diff --git a/frontend/src/pages/SpeakersPage.test.tsx b/frontend/src/pages/SpeakersPage.test.tsx index 6322aeb..5b63a25 100644 --- a/frontend/src/pages/SpeakersPage.test.tsx +++ b/frontend/src/pages/SpeakersPage.test.tsx @@ -73,7 +73,7 @@ const diarizationRun = { function renderPage(onOpenProcessing = vi.fn(), onOpenProcessingWithRun = vi.fn()) { return render( {}} diff --git a/frontend/src/pages/labelPageDatasetHelpers.ts b/frontend/src/pages/labelPageDatasetHelpers.ts index 2df76cd..bdc42bc 100644 --- a/frontend/src/pages/labelPageDatasetHelpers.ts +++ b/frontend/src/pages/labelPageDatasetHelpers.ts @@ -46,7 +46,7 @@ export function buildDatasetTagReadOnlyConfig( } return { - reviewStatus: activeClip.status, + reviewStatus: activeClip.status ?? "unresolved", tags: activeClip.tags.filter(isReviewerDisplayTag), message: buildDatasetTagReadOnlyMessage(view, loadState), }; diff --git a/frontend/src/pipeline/PipelineContext.tsx b/frontend/src/pipeline/PipelineContext.tsx index 6d45de0..520ec10 100644 --- a/frontend/src/pipeline/PipelineContext.tsx +++ b/frontend/src/pipeline/PipelineContext.tsx @@ -1,6 +1,6 @@ import { createContext, useContext, type ReactNode } from "react"; -export type PipelineStage = "ingest" | "overview" | "speakers" | "processing" | "slicer" | "qc" | "lab" | "export"; +export type PipelineStage = "ingest" | "overview" | "speakers" | "processing" | "slicer" | "qc" | "lab"; export type PipelineSelectionState = { selectedSpeakersRunId: string | null; diff --git a/frontend/src/styles.css b/frontend/src/styles.css index d8a4ab7..fbce14e 100644 --- a/frontend/src/styles.css +++ b/frontend/src/styles.css @@ -761,6 +761,26 @@ button:disabled.is-busy { justify-content: flex-end; } +.inspector-header-actions { + display: grid; + justify-items: end; + gap: 0.35rem; +} + +.inspector-inline-feedback { + margin: 0; + font-size: 0.82rem; + text-align: right; +} + +.inspector-inline-feedback.success { + color: #226355; +} + +.inspector-inline-feedback.error { + color: #8d3c38; +} + .editor-notice { margin: 0 0 0.9rem; padding: 0.7rem 0.85rem; @@ -1724,11 +1744,6 @@ button:disabled.is-busy { color: #66757c; } -.export-preview { - display: grid; - gap: 0.55rem; -} - .manifest-path { margin: 0; font-size: 0.82rem; diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 93059a6..9d8c0be 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -300,6 +300,32 @@ export type DatasetClipLabView = { clips: DatasetClipLabClipRow[]; }; +export type CanonicalExportBlockedReason = { + clip_id: string; + reasons: string[]; +}; + +export type CanonicalExportPreview = { + run_id: string; + accepted_clip_count: number; + total_duration_sec: number; + original_audio_count: number; + edited_audio_count: number; + blocked_clip_count: number; + blocked_reasons: CanonicalExportBlockedReason[]; +}; + +export type CanonicalExportSummary = { + export_id: string; + run_id: string; + project_id: string; + created_at: string; + accepted_clip_count: number; + total_duration_sec: number; + snapshot_dir: string; + manifest_path: string; +}; + export type DatasetClipLabPatchRequest = { expected_manifest_sha256: string; expected_clip_version: number; @@ -527,32 +553,12 @@ export type ImportBatch = { name: string; created_at: string; updated_at: string; - export_status?: "pending" | "running" | "completed" | "failed" | null; active_prepared_output_group_id?: string | null; active_preparation_job_id?: string | null; }; export type Project = ImportBatch; -export type ExportRun = { - id: string; - batch_id: string; - status: "pending" | "running" | "completed" | "failed"; - output_root: string; - manifest_path: string; - accepted_clip_count: number; - failed_clip_count: number; - created_at: string; - completed_at?: string | null; -}; - -export type ExportPreview = { - project_id: string; - manifest_path: string; - accepted_slice_count: number; - lines: string[]; -}; - export type MediaCleanupResult = { project_id: string; deleted_slice_count: number; diff --git a/frontend/src/workspace/EditorPane.tsx b/frontend/src/workspace/EditorPane.tsx index 24078d1..f1e38ff 100644 --- a/frontend/src/workspace/EditorPane.tsx +++ b/frontend/src/workspace/EditorPane.tsx @@ -1,5 +1,12 @@ import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import type WaveSurfer from "wavesurfer.js"; +import type { + ClipLabItem, + ClipLabItemRef, + DatasetClipLabPipelineFinding, + ReviewStatus, + Tag, +} from "../types"; import WaveformPane from "../WaveformPane"; import { fetchClipLabWaveformPeaks, fetchDatasetClipLabWaveformPeaks } from "../api"; import { diff --git a/frontend/src/workspace/InspectorPane.tsx b/frontend/src/workspace/InspectorPane.tsx index 3d2e0ae..6e4d830 100644 --- a/frontend/src/workspace/InspectorPane.tsx +++ b/frontend/src/workspace/InspectorPane.tsx @@ -1,5 +1,11 @@ import { useEffect, useState, type FormEvent } from "react"; -import type { ClipLabItem, DatasetRun, ExportRun, ReferenceAssetSummary, ReviewStatus } from "../types"; +import type { + CanonicalExportPreview, + ClipLabItem, + DatasetRun, + ReferenceAssetSummary, + ReviewStatus, +} from "../types"; import WorkspaceStatePanel from "./WorkspaceStatePanel"; import { formatDurationCompact, @@ -35,7 +41,12 @@ type InspectorPaneProps = { acceptedRejectedRatio: number | null; predictedOutputSeconds: number | null; progressPercent: number | null; - exportRuns: ExportRun[]; + canonicalExportPreview: CanonicalExportPreview | null; + canonicalExportPreviewStatus: "idle" | "loading" | "ready" | "error"; + canonicalExportFeedback: { tone: "error" | "success"; message: string } | null; + canTriggerCanonicalExport: boolean; + isCreatingCanonicalExport: boolean; + onCreateCanonicalExport: () => void; onRetryLoad: () => void; onStatusChange: (status: ReviewStatus) => void; onVariantSelect: (variantId: string) => void; @@ -62,7 +73,12 @@ export default function InspectorPane({ acceptedRejectedRatio, predictedOutputSeconds, progressPercent, - exportRuns, + canonicalExportPreview, + canonicalExportPreviewStatus, + canonicalExportFeedback, + canTriggerCanonicalExport, + isCreatingCanonicalExport, + onCreateCanonicalExport, onRetryLoad, onStatusChange, onVariantSelect, @@ -135,6 +151,8 @@ export default function InspectorPane({ ); } + const exportButtonLabel = `Export accepted clips (${canonicalExportPreview?.accepted_clip_count ?? 0})`; + return (