From 12e8ae9b803b0c1194daa6e53756c921719fff18 Mon Sep 17 00:00:00 2001 From: flier268 Date: Thu, 17 Sep 2026 14:28:09 +0800 Subject: [PATCH 1/2] feat(linux): VAAPI decode to DRM PRIME dma-buf Add a Linux-only decoder that maps VAAPI frames to DRM PRIME so callers can import dma-buf without a CPU RGB download. RAM decode is unchanged. --- build.rs | 6 + cpp/ffmpeg_ram/ffmpeg_ram_ffi.h | 23 ++++ cpp/ffmpeg_ram/ffmpeg_vaapi_prime.cpp | 185 ++++++++++++++++++++++++++ src/ffmpeg_ram/mod.rs | 2 + src/ffmpeg_ram/prime.rs | 108 +++++++++++++++ 5 files changed, 324 insertions(+) create mode 100644 cpp/ffmpeg_ram/ffmpeg_vaapi_prime.cpp create mode 100644 src/ffmpeg_ram/prime.rs diff --git a/build.rs b/build.rs index e8e6b97..87c730b 100644 --- a/build.rs +++ b/build.rs @@ -240,6 +240,12 @@ mod ffmpeg { builder.files( ["ffmpeg_ram_encode.cpp", "ffmpeg_ram_decode.cpp"].map(|f| ffmpeg_ram_dir.join(f)), ); + let target_os = std::env::var("CARGO_CFG_TARGET_OS").unwrap(); + if target_os == "linux" { + builder.file(ffmpeg_ram_dir.join("ffmpeg_vaapi_prime.cpp")); + println!("cargo:rustc-link-lib=va"); + println!("cargo:rustc-link-lib=va-drm"); + } } #[cfg(feature = "vram")] diff --git a/cpp/ffmpeg_ram/ffmpeg_ram_ffi.h b/cpp/ffmpeg_ram/ffmpeg_ram_ffi.h index b1057b2..505984d 100644 --- a/cpp/ffmpeg_ram/ffmpeg_ram_ffi.h +++ b/cpp/ffmpeg_ram/ffmpeg_ram_ffi.h @@ -31,4 +31,27 @@ int ffmpeg_ram_get_linesize_offset_length(int pix_fmt, int width, int height, int *length); int ffmpeg_ram_set_bitrate(void *encoder, int kbs); +#if defined(__linux__) +#define FFMPEG_GPU_FRAME_PRIME 2 + +typedef struct FFmpegPrimeFrame { + int kind; + int n_fds; + int fds[4]; + uint32_t fourcc; + uint64_t modifier; + int width; + int height; + int n_planes; + int pitches[4]; + int offsets[4]; + int obj_indices[4]; +} FFmpegPrimeFrame; + +void *ffmpeg_vaapi_prime_new(int hevc); +void ffmpeg_vaapi_prime_free(void *decoder); +int ffmpeg_vaapi_prime_decode(void *decoder, const uint8_t *data, int length, + FFmpegPrimeFrame *out); +#endif + #endif // FFMPEG_RAM_FFI_H \ No newline at end of file diff --git a/cpp/ffmpeg_ram/ffmpeg_vaapi_prime.cpp b/cpp/ffmpeg_ram/ffmpeg_vaapi_prime.cpp new file mode 100644 index 0000000..c5d2d13 --- /dev/null +++ b/cpp/ffmpeg_ram/ffmpeg_vaapi_prime.cpp @@ -0,0 +1,185 @@ +// VAAPI decode → DRM PRIME (dma-buf). Linux only. No RGB download. + +extern "C" { +#include +#include +#include +} + +#include +#include +#include + +#include "ffmpeg_ram_ffi.h" + +namespace { + +struct VaapiPrimeDec { + AVCodecContext *c = nullptr; + AVBufferRef *hw_device_ctx = nullptr; + AVFrame *frame = nullptr; + AVFrame *mapped = nullptr; + AVPacket *pkt = nullptr; +}; + +enum AVPixelFormat get_hw_format(AVCodecContext *ctx, + const enum AVPixelFormat *pix_fmts) { + (void)ctx; + for (const enum AVPixelFormat *p = pix_fmts; *p != AV_PIX_FMT_NONE; p++) { + if (*p == AV_PIX_FMT_VAAPI) { + return *p; + } + } + return AV_PIX_FMT_NONE; +} + +void close_mapped(VaapiPrimeDec *d) { + if (d->mapped) { + av_frame_unref(d->mapped); + } +} + +int fill_prime(AVFrame *mapped, FFmpegPrimeFrame *out) { + auto *desc = (AVDRMFrameDescriptor *)mapped->data[0]; + if (!desc || desc->nb_layers < 1 || desc->nb_objects < 1) { + return 0; + } + memset(out, 0, sizeof(*out)); + out->kind = FFMPEG_GPU_FRAME_PRIME; + out->n_fds = desc->nb_objects; + if (out->n_fds > 4) { + out->n_fds = 4; + } + for (int i = 0; i < out->n_fds; i++) { + int fd = dup(desc->objects[i].fd); + if (fd < 0) { + for (int j = 0; j < i; j++) { + close(out->fds[j]); + } + return 0; + } + out->fds[i] = fd; + if (i == 0) { + out->modifier = desc->objects[i].format_modifier; + } + } + out->width = mapped->width; + out->height = mapped->height; + int n = 0; + for (int L = 0; L < desc->nb_layers && n < 4; L++) { + AVDRMLayerDescriptor *layer = &desc->layers[L]; + for (int p = 0; p < layer->nb_planes && n < 4; p++) { + out->pitches[n] = (int)layer->planes[p].pitch; + out->offsets[n] = (int)layer->planes[p].offset; + out->obj_indices[n] = layer->planes[p].object_index; + n++; + } + } + out->n_planes = n; + if (desc->nb_layers == 1) { + out->fourcc = desc->layers[0].format; + } else { + out->fourcc = 0x3231564e; // DRM_FORMAT_NV12 + } + return n >= 1; +} + +} // namespace + +extern "C" void *ffmpeg_vaapi_prime_new(int hevc) { + auto *d = new (std::nothrow) VaapiPrimeDec(); + if (!d) { + return nullptr; + } + const AVCodec *codec = avcodec_find_decoder_by_name(hevc ? "hevc" : "h264"); + if (!codec) { + delete d; + return nullptr; + } + d->c = avcodec_alloc_context3(codec); + if (!d->c) { + delete d; + return nullptr; + } + d->c->flags |= AV_CODEC_FLAG_LOW_DELAY; + d->c->thread_count = 1; + d->c->get_format = get_hw_format; + d->c->extra_hw_frames = 8; + int ret = av_hwdevice_ctx_create(&d->hw_device_ctx, AV_HWDEVICE_TYPE_VAAPI, + nullptr, nullptr, 0); + if (ret < 0) { + avcodec_free_context(&d->c); + delete d; + return nullptr; + } + d->c->hw_device_ctx = av_buffer_ref(d->hw_device_ctx); + d->pkt = av_packet_alloc(); + d->frame = av_frame_alloc(); + d->mapped = av_frame_alloc(); + if (!d->pkt || !d->frame || !d->mapped) { + ffmpeg_vaapi_prime_free(d); + return nullptr; + } + if (avcodec_open2(d->c, codec, nullptr) != 0) { + ffmpeg_vaapi_prime_free(d); + return nullptr; + } + return d; +} + +extern "C" void ffmpeg_vaapi_prime_free(void *decoder) { + auto *d = (VaapiPrimeDec *)decoder; + if (!d) { + return; + } + close_mapped(d); + if (d->mapped) + av_frame_free(&d->mapped); + if (d->frame) + av_frame_free(&d->frame); + if (d->pkt) + av_packet_free(&d->pkt); + if (d->c) + avcodec_free_context(&d->c); + if (d->hw_device_ctx) + av_buffer_unref(&d->hw_device_ctx); + delete d; +} + +extern "C" int ffmpeg_vaapi_prime_decode(void *decoder, const uint8_t *data, + int length, FFmpegPrimeFrame *out) { + auto *d = (VaapiPrimeDec *)decoder; + if (!d || !data || length <= 0 || !out) { + return 0; + } + d->pkt->data = (uint8_t *)data; + d->pkt->size = length; + int ret = avcodec_send_packet(d->c, d->pkt); + av_packet_unref(d->pkt); + if (ret < 0) { + return 0; + } + int got = 0; + while (ret >= 0) { + ret = avcodec_receive_frame(d->c, d->frame); + if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) { + break; + } + if (ret < 0) { + break; + } + close_mapped(d); + d->mapped->format = AV_PIX_FMT_DRM_PRIME; + if (av_hwframe_map(d->mapped, d->frame, AV_HWFRAME_MAP_READ) < 0) { + av_frame_unref(d->frame); + continue; + } + if (fill_prime(d->mapped, out)) { + got = 1; + } else { + close_mapped(d); + } + av_frame_unref(d->frame); + } + return got; +} diff --git a/src/ffmpeg_ram/mod.rs b/src/ffmpeg_ram/mod.rs index 6fb98bf..df3c1dd 100644 --- a/src/ffmpeg_ram/mod.rs +++ b/src/ffmpeg_ram/mod.rs @@ -14,6 +14,8 @@ include!(concat!(env!("OUT_DIR"), "/ffmpeg_ram_ffi.rs")); pub mod decode; pub mod encode; +#[cfg(target_os = "linux")] +pub mod prime; pub enum Priority { Best = 0, diff --git a/src/ffmpeg_ram/prime.rs b/src/ffmpeg_ram/prime.rs new file mode 100644 index 0000000..c20948e --- /dev/null +++ b/src/ffmpeg_ram/prime.rs @@ -0,0 +1,108 @@ +use super::{ffmpeg_vaapi_prime_decode, ffmpeg_vaapi_prime_free, ffmpeg_vaapi_prime_new}; +use std::os::raw::{c_int, c_void}; + +pub const GPU_FRAME_PRIME: i32 = 2; + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct PrimeFrame { + pub kind: i32, + pub n_fds: i32, + pub fds: [i32; 4], + pub fourcc: u32, + pub modifier: u64, + pub width: i32, + pub height: i32, + pub n_planes: i32, + pub pitches: [i32; 4], + pub offsets: [i32; 4], + pub obj_indices: [i32; 4], +} + +impl Default for PrimeFrame { + fn default() -> Self { + Self { + kind: GPU_FRAME_PRIME, + n_fds: 0, + fds: [-1; 4], + fourcc: 0, + modifier: 0, + width: 0, + height: 0, + n_planes: 0, + pitches: [0; 4], + offsets: [0; 4], + obj_indices: [0; 4], + } + } +} + +impl PrimeFrame { + pub fn close_fds(&mut self) { + for i in 0..self.n_fds.max(0) as usize { + let fd = self.fds[i]; + if fd >= 0 { + unsafe { + libc::close(fd); + } + self.fds[i] = -1; + } + } + self.n_fds = 0; + } +} + +mod libc { + extern "C" { + pub fn close(fd: i32) -> i32; + } +} + +pub struct VaapiPrimeDecoder { + ctx: *mut c_void, + pub last: PrimeFrame, +} + +unsafe impl Send for VaapiPrimeDecoder {} + +impl VaapiPrimeDecoder { + pub fn new(hevc: bool) -> Result { + let ctx = unsafe { ffmpeg_vaapi_prime_new(if hevc { 1 } else { 0 }) }; + if ctx.is_null() { + return Err(()); + } + Ok(Self { + ctx, + last: PrimeFrame::default(), + }) + } + + pub fn decode(&mut self, data: &[u8]) -> Result { + self.last.close_fds(); + let mut out = PrimeFrame::default(); + let ok = unsafe { + ffmpeg_vaapi_prime_decode( + self.ctx, + data.as_ptr(), + data.len() as c_int, + &mut out as *mut PrimeFrame as *mut _, + ) + }; + if ok == 1 { + self.last = out; + Ok(true) + } else { + Ok(false) + } + } +} + +impl Drop for VaapiPrimeDecoder { + fn drop(&mut self) { + self.last.close_fds(); + if !self.ctx.is_null() { + unsafe { ffmpeg_vaapi_prime_free(self.ctx) }; + self.ctx = std::ptr::null_mut(); + } + } +} From f7ac38256dc1917381bc98f6df5d66cd020f0f6c Mon Sep 17 00:00:00 2001 From: flier268 Date: Thu, 17 Sep 2026 14:43:16 +0800 Subject: [PATCH 2/2] fix(linux): always free VAAPI device on prime decoder open failure av_hwdevice_ctx_create success followed by a later alloc failure must go through ffmpeg_vaapi_prime_free. Reject DRM descriptors with more than four objects instead of truncating fds. --- cpp/ffmpeg_ram/ffmpeg_vaapi_prime.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/cpp/ffmpeg_ram/ffmpeg_vaapi_prime.cpp b/cpp/ffmpeg_ram/ffmpeg_vaapi_prime.cpp index c5d2d13..2f5ea77 100644 --- a/cpp/ffmpeg_ram/ffmpeg_vaapi_prime.cpp +++ b/cpp/ffmpeg_ram/ffmpeg_vaapi_prime.cpp @@ -46,10 +46,10 @@ int fill_prime(AVFrame *mapped, FFmpegPrimeFrame *out) { } memset(out, 0, sizeof(*out)); out->kind = FFMPEG_GPU_FRAME_PRIME; - out->n_fds = desc->nb_objects; - if (out->n_fds > 4) { - out->n_fds = 4; + if (desc->nb_objects > 4) { + return 0; } + out->n_fds = desc->nb_objects; for (int i = 0; i < out->n_fds; i++) { int fd = dup(desc->objects[i].fd); if (fd < 0) { @@ -108,8 +108,7 @@ extern "C" void *ffmpeg_vaapi_prime_new(int hevc) { int ret = av_hwdevice_ctx_create(&d->hw_device_ctx, AV_HWDEVICE_TYPE_VAAPI, nullptr, nullptr, 0); if (ret < 0) { - avcodec_free_context(&d->c); - delete d; + ffmpeg_vaapi_prime_free(d); return nullptr; } d->c->hw_device_ctx = av_buffer_ref(d->hw_device_ctx);