Repository navigation
Generate previews for ProRes outputs #17674
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,5 +1,5 @@ | ||
| """Previews of uploaded EXRs: decoded, stored and linked on upload. Best-effort, since an | ||
| asset without a preview is a normal state.""" | ||
| """Previews of uploads browsers can't show, EXRs and ProRes: made, stored and linked on upload. | ||
| Best-effort, since an asset without a preview is a normal state.""" | ||
|
|
||
| from __future__ import annotations | ||
|
|
||
|
|
@@ -20,6 +20,8 @@ | |
| from app.assets.database.queries.records import create_content, create_record | ||
| from app.assets.event_log import emit, error_type | ||
| from app.assets.services.image_dimensions import read_exr_windows | ||
| from app.assets.services.ingest import link_preview | ||
| from app.assets.services.video_proxy import needs_video_proxy, write_video_proxy | ||
| from app.database.db import create_write_session | ||
| from comfy_execution.preview_tonemap import linear_to_preview | ||
|
|
||
|
|
@@ -105,22 +107,30 @@ def _emit_error(reason: str, exc: BaseException) -> None: | |
| emit("previews.generation_failed", reason=reason, error_type=error_type(exc)) | ||
|
|
||
|
|
||
| async def generate_upload_preview(asset_id: str, path: str | None, preview_id: str | None) -> str | None: | ||
| """The preview_id an upload ends with: the one it has, else one made now for an EXR.""" | ||
| if preview_id is not None or not path or not is_exr(path): | ||
| async def generate_upload_preview( | ||
| asset_id: str, path: str | None, preview_id: str | None, system_metadata: dict | None | ||
| ) -> str | None: | ||
| """The preview_id an upload ends with: the one it has, else one made now for an EXR or ProRes.""" | ||
| if preview_id is not None or not path: | ||
| return preview_id | ||
| if is_exr(path): | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. At some point I think we should turn this into a way to register preview providers rather than a elif chain |
||
| make, link = _make_preview, lambda made: _store_and_link(asset_id, *made) | ||
| elif needs_video_proxy(system_metadata): | ||
| make, link = write_video_proxy, lambda made: link_preview(asset_id, made) | ||
| else: | ||
| return None | ||
| started = time.monotonic() | ||
| try: | ||
| async with _DECODE_SLOTS: | ||
| webp, width, height = await asyncio.to_thread(_make_preview, path) | ||
| made = await asyncio.to_thread(make, path) | ||
| except PreviewSkipped as skipped: | ||
| _emit_failed(skipped.args[0]) | ||
| return None | ||
| except Exception as exc: | ||
| _emit_error("decode_failed", exc) | ||
| return None | ||
| try: | ||
| linked = await asyncio.to_thread(_store_and_link, asset_id, webp, width, height) | ||
| linked = await asyncio.to_thread(link, made) | ||
| except Exception as exc: | ||
| _emit_error("write_failed", exc) | ||
| return None | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -52,6 +52,7 @@ | |
| UserMetadata, | ||
| ) | ||
| from app.assets.services.snapshot_hash import snapshot_hash | ||
| from app.assets.services.video_proxy import needs_video_proxy, write_video_proxy | ||
| from app.database.db import create_session, create_write_session | ||
|
|
||
|
|
||
|
|
@@ -108,11 +109,12 @@ def _live_sibling_preview_id(session: Session, content_id: str) -> str | None: | |
| return next((preview_id for preview_id, path in rows if os.path.isfile(path)), None) | ||
|
|
||
|
|
||
| _PREVIEW_FILENAME = re.compile(r"[0-9a-f]{64}\.(jpg|webp)") | ||
| _PREVIEW_FILENAME = re.compile(r"[0-9a-f]{64}\.(jpg|webp|mp4)") | ||
| _PREVIEW_MEDIA = {"jpg": ("image/jpeg", "image"), "webp": ("image/webp", "image"), "mp4": ("video/mp4", "video")} | ||
|
|
||
|
|
||
| def _probe_output_preview(preview_ref: dict | None) -> tuple[str, str, int, int, os.stat_result] | None: | ||
| """The file a save node named as its output's preview: trusted by name, so stat only.""" | ||
| def _probe_preview(preview_ref: dict | None) -> tuple[str, str, int, int, os.stat_result] | None: | ||
| """The file named as an asset's preview: trusted by name, so stat only.""" | ||
| if preview_ref is None: | ||
| return None | ||
| try: | ||
|
|
@@ -122,12 +124,12 @@ def _probe_output_preview(preview_ref: dict | None) -> tuple[str, str, int, int, | |
| path = os.path.join(folder_paths.get_previews_directory(), filename) | ||
| return path, filename, int(preview_ref["width"]), int(preview_ref["height"]), os.stat(path) | ||
| except Exception: | ||
| logging.warning("Ignoring the preview named for an output: %r", preview_ref, exc_info=True) | ||
| logging.warning("Ignoring the preview named for an asset: %r", preview_ref, exc_info=True) | ||
| return None | ||
|
|
||
|
|
||
| def _link_output_preview(session: Session, record: Asset, preview: tuple[str, str, int, int, os.stat_result]) -> None: | ||
| """Link the output to its preview's record, creating it on first use; never fails the output.""" | ||
| def _link_preview(session: Session, record: Asset, preview: tuple[str, str, int, int, os.stat_result]) -> None: | ||
| """Link the asset to its preview's record, creating it on first use; never fails the asset.""" | ||
| path, filename, width, height, stat_result = preview | ||
| try: | ||
| with session.begin_nested(): | ||
|
|
@@ -143,20 +145,47 @@ def _link_output_preview(session: Session, record: Asset, preview: tuple[str, st | |
| .limit(1) | ||
| ) | ||
| if preview_id is None: | ||
| mime_type, kind = _PREVIEW_MEDIA[filename.rpartition(".")[2]] | ||
| preview_id = create_record( | ||
| session, | ||
| content.id, | ||
| filename, | ||
| mime_type="image/webp" if filename.endswith(".webp") else "image/jpeg", | ||
| mime_type=mime_type, | ||
| tags=["preview"], | ||
| system_metadata={"kind": "image", "width": width, "height": height}, | ||
| system_metadata={"kind": kind, "width": width, "height": height}, | ||
| ).id | ||
| record.preview_id = preview_id | ||
| session.flush() | ||
| except Exception: | ||
| logging.warning("Could not link the preview %s", filename, exc_info=True) | ||
|
|
||
|
|
||
| def _write_preview_proxy(abs_path: str, system_metadata: dict | None) -> dict | None: | ||
| """The preview ref of a browser-playable proxy, for a video that needs one.""" | ||
| if not needs_video_proxy(system_metadata): | ||
| return None | ||
| try: | ||
| return write_video_proxy(abs_path) | ||
| except Exception: | ||
| logging.warning("Could not write a preview proxy for %s", abs_path, exc_info=True) | ||
| return None | ||
|
|
||
|
|
||
| def link_preview(asset_id: str, preview_ref: dict) -> str | None: | ||
| """Link a generated preview to an asset that has none; None if it is gone or got one meanwhile.""" | ||
| preview = _probe_preview(preview_ref) | ||
| if preview is None: | ||
| return None | ||
| with create_write_session() as session: | ||
| record = session.get(Asset, asset_id) | ||
| if record is None or record.preview_id is not None: | ||
| return None | ||
| _link_preview(session, record, preview) | ||
| preview_id = record.preview_id | ||
| session.commit() | ||
| return preview_id | ||
|
|
||
|
|
||
| def _discard_unreferenced_content(session: Session, content_id: str) -> None: | ||
| """Remove a content row left orphaned by a failed registration. | ||
|
|
||
|
|
@@ -715,7 +744,9 @@ def create_from_hash( | |
| return None | ||
| content_id = content.id | ||
| # Read the file before the claim below opens the write transaction. | ||
| system_metadata = _extract_system_metadata_sync(content.path, mime_type) | ||
| system_metadata = _extract_system_metadata_sync( | ||
| content.path, mime_type or mimetypes.guess_type(content.path, strict=False)[0] | ||
| ) | ||
| if not claim_qualified_content(session, content_id, stored_hash): | ||
| session.rollback() | ||
| return None | ||
|
|
@@ -815,7 +846,6 @@ def register_executed_output( | |
| abs_path: str, job_id: str | None = None, preview_ref: dict | None = None | ||
| ) -> RegisteredAsset | None: | ||
| locator = os.path.abspath(abs_path) | ||
| preview = _probe_output_preview(preview_ref) | ||
| try: | ||
| stat_result = os.stat(locator, follow_symlinks=True) | ||
| size_bytes = stat_result.st_size | ||
|
|
@@ -825,6 +855,7 @@ def register_executed_output( | |
| system_metadata = _extract_system_metadata_sync( | ||
| locator, mime_type, stat_result | ||
| ) | ||
| preview = _probe_preview(preview_ref or _write_preview_proxy(locator, system_metadata)) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
| with create_write_session() as session: | ||
| created_content_id: str | None = None | ||
| try: | ||
|
|
@@ -852,7 +883,7 @@ def register_executed_output( | |
| system_metadata=system_metadata, | ||
| ) | ||
| if preview is not None: | ||
| _link_output_preview(session, record, preview) | ||
| _link_preview(session, record, preview) | ||
| # Read before commit: expiry would reload them in a second write transaction. | ||
| record_id = record.id | ||
| record_content_id = record.content_id | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,87 @@ | ||
| """Browser-playable H.264 proxies of videos browsers can't play, written to previews/ under their blake3.""" | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. As discussed, |
||
|
|
||
| import os | ||
| import uuid | ||
|
|
||
| import av | ||
| from av.video.reformatter import ColorPrimaries, Colorspace, ColorTrc | ||
|
|
||
| import folder_paths | ||
| from app.assets.services.hashing import compute_blake3_hash | ||
|
|
||
| PROXY_LONG_SIDE = 1920 | ||
| PROXY_SHORT_SIDE = 1080 | ||
|
|
||
|
|
||
| def needs_video_proxy(system_metadata: dict | None) -> bool: | ||
| return (system_metadata or {}).get("codec") == "prores" | ||
|
|
||
|
|
||
| def _proxy_size(width: int, height: int) -> tuple[int, int]: | ||
| scale = min(1.0, PROXY_LONG_SIDE / max(width, height), PROXY_SHORT_SIDE / min(width, height)) | ||
| return max(2, round(width * scale / 2) * 2), max(2, round(height * scale / 2) * 2) | ||
|
|
||
|
|
||
| def _transcode(source: str, dest: str) -> tuple[int, int]: | ||
| with av.open(source) as src, av.open(dest, "w", format="mp4", options={"movflags": "+faststart"}) as out: | ||
| video = src.streams.video[0] | ||
| video.thread_type = "AUTO" | ||
| audio = src.streams.audio[0] if src.streams.audio else None | ||
| width, height = _proxy_size(video.codec_context.width, video.codec_context.height) | ||
|
|
||
| out_video = out.add_stream("libx264", rate=video.average_rate or video.guessed_rate) | ||
| out_video.width, out_video.height, out_video.pix_fmt = width, height, "yuv420p" | ||
| out_video.options = {"preset": "veryfast", "crf": "18"} | ||
| out_video.codec_context.color_primaries = ColorPrimaries.BT709 | ||
| out_video.codec_context.color_trc = ColorTrc.IEC61966_2_1 | ||
| out_video.codec_context.colorspace = Colorspace.ITU709 | ||
| out_audio = out.add_stream("aac", rate=audio.rate, layout=audio.layout) if audio is not None else None | ||
|
|
||
| graph = av.filter.Graph() | ||
| graph.link_nodes( | ||
| graph.add_buffer(template=video), | ||
| graph.add( | ||
| "scale", | ||
| f"w={width}:h={height}:out_color_matrix=bt709:out_range=tv:out_primaries=bt709:out_transfer=iec61966-2-1:intent=perceptual", | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Thoughts on these "magic" strings having comments justifying why each flag is used? |
||
| ), | ||
| graph.add("format", "yuv420p"), | ||
| graph.add("buffersink"), | ||
| ).configure() | ||
|
|
||
| def encode_filtered(): | ||
| while True: | ||
| try: | ||
| frame = graph.pull() | ||
| except (av.BlockingIOError, av.EOFError): | ||
| return | ||
| out.mux(out_video.encode(frame)) | ||
|
|
||
| for packet in src.demux(*(s for s in (video, audio) if s is not None)): | ||
| for frame in packet.decode(): | ||
| if packet.stream is video: | ||
| graph.push(frame) | ||
| encode_filtered() | ||
| else: | ||
| out.mux(out_audio.encode(frame)) | ||
| graph.push(None) | ||
| encode_filtered() | ||
| out.mux(out_video.encode(None)) | ||
| if out_audio is not None: | ||
| out.mux(out_audio.encode(None)) | ||
| return width, height | ||
|
|
||
|
|
||
| def write_video_proxy(source: str) -> dict: | ||
| """Transcode ``source`` to previews/<blake3>.mp4 and return its preview ref.""" | ||
| directory = folder_paths.get_previews_directory() | ||
| os.makedirs(directory, exist_ok=True) | ||
| temp_path = os.path.join(directory, f"{uuid.uuid4().hex}.mp4.tmp") | ||
| try: | ||
| width, height = _transcode(source, temp_path) | ||
| digest, _ = compute_blake3_hash(temp_path) | ||
| filename = f"{digest}.mp4" | ||
| os.replace(temp_path, os.path.join(directory, filename)) | ||
| finally: | ||
| if os.path.exists(temp_path): | ||
| os.remove(temp_path) | ||
| return {"filename": filename, "width": width, "height": height} | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -53,3 +53,33 @@ def duplicate_data_window(data: bytes) -> bytes: | |
| at = data.find(key) | ||
| end = at + len(key) + 4 + 16 | ||
| return data[:end] + data[at:end] + data[end:] | ||
|
|
||
|
|
||
| def write_prores(path: Path, width: int, height: int, frames: int = 2, value=0.5, transfer=None, alpha=False, audio=False) -> Path: | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. types? 🥺 |
||
| """A ProRes MOV the way SaveVideo writes one; BT.2020 with ``transfer`` given, PCM audio with ``audio``.""" | ||
| from av.video.reformatter import ColorPrimaries | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Move the Line 60 imports ♻️ Proposed fix
As per coding guidelines: "Keep imports at module scope. Avoid inline imports unless they are already part of an established optional-backend probe or are needed to avoid an import cycle."
|
||
|
|
||
| with av.open(str(path), "w", format="mov") as out: | ||
| stream = out.add_stream("prores_ks", rate=24) | ||
| stream.width, stream.height = width, height | ||
| stream.pix_fmt = "yuva444p10le" if alpha else "yuv422p10le" | ||
| if alpha: | ||
| stream.options = {"profile": "4444"} | ||
| if transfer is not None: | ||
| stream.codec_context.color_trc, stream.codec_context.color_primaries, stream.codec_context.colorspace = transfer, ColorPrimaries.BT2020, 9 | ||
| sound = out.add_stream("pcm_s16le", rate=48000, layout="stereo") if audio else None | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Consider breaking out sample rate to a value rather than repeating |
||
| pixels = np.full((height, width, 4 if alpha else 3), value, np.float32) | ||
| for i in range(frames): | ||
| frame = av.VideoFrame.from_ndarray(pixels, format="gbrapf32le" if alpha else "gbrpf32le") | ||
| frame = frame.reformat(format=stream.pix_fmt, dst_colorspace=9 if transfer is not None else 1) | ||
| if transfer is not None: | ||
| frame.color_trc, frame.color_primaries = transfer, ColorPrimaries.BT2020 | ||
| frame.pts = i | ||
| out.mux(stream.encode(frame)) | ||
| out.mux(stream.encode(None)) | ||
| if sound is not None: | ||
| samples = av.AudioFrame.from_ndarray(np.zeros((1, 2 * 2000 * frames), np.int16), format="s16", layout="stereo") | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. why 2000? |
||
| samples.sample_rate, samples.pts = 48000, 0 | ||
| out.mux(sound.encode(samples)) | ||
| out.mux(sound.encode(None)) | ||
| return path | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
We should treat
generate_upload_previewas a public API, that other consumers may be using. Considersystem_metadata=None