From 686d09b8efae39746f50c81c7ad976602737f8bb Mon Sep 17 00:00:00 2001 From: mmdmcy Date: Fri, 11 Sep 2026 09:56:56 +0200 Subject: [PATCH 1/5] fix(vaapi): support constant-QP rate control Signed-off-by: mmdmcy --- cpp/common/util.cpp | 13 ++++++++++++- cpp/ffmpeg_ram/ffmpeg_ram_encode.cpp | 7 +++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/cpp/common/util.cpp b/cpp/common/util.cpp index 45810b4..558a77e 100644 --- a/cpp/common/util.cpp +++ b/cpp/common/util.cpp @@ -214,6 +214,7 @@ bool set_rate_control(AVCodecContext *c, const std::string &name, int rc, {"mediacodec", "bitrate_mode", {{RC_CBR, "cbr"}, {RC_VBR, "vbr"}, {RC_CQ, "cq"}}}, + {"vaapi", "rc_mode", {{RC_CQ, "CQP"}}}, // {"videotoolbox", "constant_bit_rate", {{RC_CBR, "1"}}}, }; @@ -235,6 +236,16 @@ bool set_rate_control(AVCodecContext *c, const std::string &name, int rc, } } } + if (name.find("vaapi") != std::string::npos && rc == RC_CQ) { + constexpr int default_qp = 23; + const int qp = q >= 0 && q <= 51 ? q : default_qp; + ret = av_opt_set_int(c->priv_data, "qp", qp, 0); + if (ret < 0) { + LOG_ERROR(std::string("vaapi set opt qp failed, ret = ") + + av_err2str(ret)); + return false; + } + } } break; } @@ -330,4 +341,4 @@ bool has_flag_could_not_find_ref_with_poc() { extern "C" void hwcodec_set_flag_could_not_find_ref_with_poc() { util_decode::g_flag_could_not_find_ref_with_poc = true; -} \ No newline at end of file +} diff --git a/cpp/ffmpeg_ram/ffmpeg_ram_encode.cpp b/cpp/ffmpeg_ram/ffmpeg_ram_encode.cpp index 0a8d1b1..bf90de5 100644 --- a/cpp/ffmpeg_ram/ffmpeg_ram_encode.cpp +++ b/cpp/ffmpeg_ram/ffmpeg_ram_encode.cpp @@ -236,7 +236,10 @@ class FFmpegRamEncoder { return false; } // util_encode::set_quality(c_->priv_data, name_, quality_); - util_encode::set_rate_control(c_, name_, rc_, q_); + if (!util_encode::set_rate_control(c_, name_, rc_, q_)) { + LOG_ERROR(std::string("set_rate_control failed, name: ") + name_); + return false; + } util_encode::set_gpu(c_->priv_data, name_, gpu_); util_encode::force_hw(c_->priv_data, name_); util_encode::set_others(c_->priv_data, name_); @@ -463,4 +466,4 @@ extern "C" int ffmpeg_ram_set_bitrate(FFmpegRamEncoder *encoder, int kbs) { LOG_ERROR(std::string("ffmpeg_ram_set_bitrate failed, ") + std::string(e.what())); } return -1; -} \ No newline at end of file +} From 24e46304028359a0e65f1a92b5c068d4e7944ce9 Mon Sep 17 00:00:00 2001 From: 21pages Date: Fri, 11 Sep 2026 17:03:59 +0800 Subject: [PATCH 2/5] fix(vaapi): probe CQP and normalize default QP Use CQP when probing VAAPI encoders and treat zero QP as unspecified. Preserve the existing best-effort rate-control option setup and cover CQP options with native tests. --- cpp/common/util.cpp | 3 +- cpp/ffmpeg_ram/ffmpeg_ram_encode.cpp | 5 +- src/ffmpeg_ram/encode.rs | 7 +++ tests/vaapi_cqp.cpp | 76 ++++++++++++++++++++++++++++ 4 files changed, 86 insertions(+), 5 deletions(-) create mode 100644 tests/vaapi_cqp.cpp diff --git a/cpp/common/util.cpp b/cpp/common/util.cpp index 558a77e..b08376c 100644 --- a/cpp/common/util.cpp +++ b/cpp/common/util.cpp @@ -238,7 +238,8 @@ bool set_rate_control(AVCodecContext *c, const std::string &name, int rc, } if (name.find("vaapi") != std::string::npos && rc == RC_CQ) { constexpr int default_qp = 23; - const int qp = q >= 0 && q <= 51 ? q : default_qp; + // FFmpeg treats qp = 0 as unspecified, not as an explicit QP. + const int qp = q > 0 && q <= 51 ? q : default_qp; ret = av_opt_set_int(c->priv_data, "qp", qp, 0); if (ret < 0) { LOG_ERROR(std::string("vaapi set opt qp failed, ret = ") + diff --git a/cpp/ffmpeg_ram/ffmpeg_ram_encode.cpp b/cpp/ffmpeg_ram/ffmpeg_ram_encode.cpp index bf90de5..18d7bc0 100644 --- a/cpp/ffmpeg_ram/ffmpeg_ram_encode.cpp +++ b/cpp/ffmpeg_ram/ffmpeg_ram_encode.cpp @@ -236,10 +236,7 @@ class FFmpegRamEncoder { return false; } // util_encode::set_quality(c_->priv_data, name_, quality_); - if (!util_encode::set_rate_control(c_, name_, rc_, q_)) { - LOG_ERROR(std::string("set_rate_control failed, name: ") + name_); - return false; - } + util_encode::set_rate_control(c_, name_, rc_, q_); util_encode::set_gpu(c_->priv_data, name_, gpu_); util_encode::force_hw(c_->priv_data, name_); util_encode::set_others(c_->priv_data, name_); diff --git a/src/ffmpeg_ram/encode.rs b/src/ffmpeg_ram/encode.rs index 73417b4..4d4da4a 100644 --- a/src/ffmpeg_ram/encode.rs +++ b/src/ffmpeg_ram/encode.rs @@ -312,6 +312,13 @@ impl Encoder { let c = EncodeContext { name: codec.name.clone(), mc_name: codec.mc_name.clone(), + // RustDesk uses CQP for VAAPI; probing with a bitrate-based + // mode would reject drivers that only support CQP. + rc: if codec.name.contains("vaapi") { + RateControl::RC_CQ + } else { + ctx.rc + }, ..ctx }; diff --git a/tests/vaapi_cqp.cpp b/tests/vaapi_cqp.cpp new file mode 100644 index 0000000..6b8c316 --- /dev/null +++ b/tests/vaapi_cqp.cpp @@ -0,0 +1,76 @@ +#include "common.h" +#include "util.h" + +extern "C" { +#include +} + +#include +#include +#include + +namespace gol { +void error(const std::string &message) { std::cerr << message << '\n'; } +} + +// Exercise the real AVOption writes without requiring a VAAPI device. +struct Options { + const AVClass *av_class; + int rc_mode; + int qp; +}; + +static const AVOption options[] = { + {"rc_mode", nullptr, offsetof(Options, rc_mode), AV_OPT_TYPE_INT, + {.i64 = 0}, 0, 6, 0, "rc_mode"}, + {"CQP", nullptr, 0, AV_OPT_TYPE_CONST, {.i64 = 1}, 0, 0, 0, "rc_mode"}, + {"qp", nullptr, offsetof(Options, qp), AV_OPT_TYPE_INT, {.i64 = 0}, 0, 51}, + {nullptr}, +}; + +int main() { + AVClass av_class = {}; + av_class.class_name = "VAAPI options test"; + av_class.item_name = av_default_item_name; + av_class.option = options; + av_class.version = LIBAVUTIL_VERSION_INT; + + Options priv = {&av_class, 0, 0}; + AVCodecContext ctx = {}; + ctx.priv_data = &priv; + ctx.bit_rate = 1000000; + + for (const auto &name : {"h264_vaapi", "hevc_vaapi"}) { + for (int qp : {1, 16, 23, 26, 51}) { + av_opt_set_defaults(&priv); + assert(util_encode::set_rate_control(&ctx, name, RC_CQ, qp)); + assert(priv.rc_mode == 1); + assert(priv.qp == qp); + assert(ctx.bit_rate == 1000000); + } + for (int qp : {-100, -1, 0, 52, 100}) { + av_opt_set_defaults(&priv); + assert(util_encode::set_rate_control(&ctx, name, RC_CQ, qp)); + assert(priv.rc_mode == 1); + assert(priv.qp == 23); + } + } + + // Explicit CQP configuration must not affect callers requesting other modes. + for (int rc : {RC_CBR, RC_VBR}) { + av_opt_set_defaults(&priv); + assert(util_encode::set_rate_control(&ctx, "h264_vaapi", rc, 16)); + assert(priv.rc_mode == 0); + assert(priv.qp == 0); + } + + // Missing options must be reported as failures, not as successful CQP setup. + AVOption mode_only[] = {options[0], options[1], {nullptr}}; + av_class.option = mode_only; + assert(!util_encode::set_rate_control(&ctx, "h264_vaapi", RC_CQ, 23)); + AVOption no_options[] = {{nullptr}}; + av_class.option = no_options; + assert(!util_encode::set_rate_control(&ctx, "h264_vaapi", RC_CQ, 23)); + + std::cout << "VAAPI CQP option tests passed\n"; +} From a3217ddba070c4bdd44a906a3b0bf558daf3966f Mon Sep 17 00:00:00 2001 From: mmdmcy Date: Fri, 11 Sep 2026 12:31:09 +0200 Subject: [PATCH 3/5] fix(vaapi): preserve non-VAAPI rate-control behavior Keep the existing best-effort initialization behavior for other hardware encoders while treating VAAPI CQP setup failures as fatal. Signed-off-by: mmdmcy --- cpp/ffmpeg_ram/ffmpeg_ram_encode.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cpp/ffmpeg_ram/ffmpeg_ram_encode.cpp b/cpp/ffmpeg_ram/ffmpeg_ram_encode.cpp index 18d7bc0..66781e8 100644 --- a/cpp/ffmpeg_ram/ffmpeg_ram_encode.cpp +++ b/cpp/ffmpeg_ram/ffmpeg_ram_encode.cpp @@ -236,7 +236,12 @@ class FFmpegRamEncoder { return false; } // util_encode::set_quality(c_->priv_data, name_, quality_); - util_encode::set_rate_control(c_, name_, rc_, q_); + const bool rate_control_set = + util_encode::set_rate_control(c_, name_, rc_, q_); + if (!rate_control_set && name_.find("vaapi") != std::string::npos) { + LOG_ERROR(std::string("set_rate_control failed, name: ") + name_); + return false; + } util_encode::set_gpu(c_->priv_data, name_, gpu_); util_encode::force_hw(c_->priv_data, name_); util_encode::set_others(c_->priv_data, name_); From 220c55a79d5857374e183d32c860818a05040d61 Mon Sep 17 00:00:00 2001 From: mmdmcy Date: Fri, 11 Sep 2026 12:42:30 +0200 Subject: [PATCH 4/5] fix(vaapi): reject invalid positive QP values Keep non-positive QP values as unspecified defaults, but let each FFmpeg VAAPI option validate positive values instead of silently replacing out-of-range inputs. Signed-off-by: mmdmcy --- cpp/common/util.cpp | 4 +++- tests/vaapi_cqp.cpp | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/cpp/common/util.cpp b/cpp/common/util.cpp index b08376c..7a73b31 100644 --- a/cpp/common/util.cpp +++ b/cpp/common/util.cpp @@ -239,7 +239,9 @@ bool set_rate_control(AVCodecContext *c, const std::string &name, int rc, if (name.find("vaapi") != std::string::npos && rc == RC_CQ) { constexpr int default_qp = 23; // FFmpeg treats qp = 0 as unspecified, not as an explicit QP. - const int qp = q > 0 && q <= 51 ? q : default_qp; + // Negative values are also unspecified in EncodeContext. Positive + // values are validated by the codec-specific FFmpeg option. + const int qp = q > 0 ? q : default_qp; ret = av_opt_set_int(c->priv_data, "qp", qp, 0); if (ret < 0) { LOG_ERROR(std::string("vaapi set opt qp failed, ret = ") + diff --git a/tests/vaapi_cqp.cpp b/tests/vaapi_cqp.cpp index 6b8c316..132285d 100644 --- a/tests/vaapi_cqp.cpp +++ b/tests/vaapi_cqp.cpp @@ -48,12 +48,17 @@ int main() { assert(priv.qp == qp); assert(ctx.bit_rate == 1000000); } - for (int qp : {-100, -1, 0, 52, 100}) { + for (int qp : {-100, -1, 0}) { av_opt_set_defaults(&priv); assert(util_encode::set_rate_control(&ctx, name, RC_CQ, qp)); assert(priv.rc_mode == 1); assert(priv.qp == 23); } + for (int qp : {52, 100}) { + av_opt_set_defaults(&priv); + assert(!util_encode::set_rate_control(&ctx, name, RC_CQ, qp)); + assert(priv.rc_mode == 1); + } } // Explicit CQP configuration must not affect callers requesting other modes. From 8425b2a9a272e754d6bcfdaf96f595598a883d8c Mon Sep 17 00:00:00 2001 From: 21pages Date: Fri, 11 Sep 2026 20:34:25 +0800 Subject: [PATCH 5/5] fix(vaapi): preserve RC setup behavior and correct QP tests Keep rate-control option setup best-effort as before this PR. Match the H.264 and HEVC VAAPI AVOption range in the native test: 52 is accepted by the option, while 53 is out of range. --- cpp/ffmpeg_ram/ffmpeg_ram_encode.cpp | 7 +------ tests/vaapi_cqp.cpp | 7 ++++--- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/cpp/ffmpeg_ram/ffmpeg_ram_encode.cpp b/cpp/ffmpeg_ram/ffmpeg_ram_encode.cpp index 66781e8..18d7bc0 100644 --- a/cpp/ffmpeg_ram/ffmpeg_ram_encode.cpp +++ b/cpp/ffmpeg_ram/ffmpeg_ram_encode.cpp @@ -236,12 +236,7 @@ class FFmpegRamEncoder { return false; } // util_encode::set_quality(c_->priv_data, name_, quality_); - const bool rate_control_set = - util_encode::set_rate_control(c_, name_, rc_, q_); - if (!rate_control_set && name_.find("vaapi") != std::string::npos) { - LOG_ERROR(std::string("set_rate_control failed, name: ") + name_); - return false; - } + util_encode::set_rate_control(c_, name_, rc_, q_); util_encode::set_gpu(c_->priv_data, name_, gpu_); util_encode::force_hw(c_->priv_data, name_); util_encode::set_others(c_->priv_data, name_); diff --git a/tests/vaapi_cqp.cpp b/tests/vaapi_cqp.cpp index 132285d..e6c8fed 100644 --- a/tests/vaapi_cqp.cpp +++ b/tests/vaapi_cqp.cpp @@ -24,7 +24,8 @@ static const AVOption options[] = { {"rc_mode", nullptr, offsetof(Options, rc_mode), AV_OPT_TYPE_INT, {.i64 = 0}, 0, 6, 0, "rc_mode"}, {"CQP", nullptr, 0, AV_OPT_TYPE_CONST, {.i64 = 1}, 0, 0, 0, "rc_mode"}, - {"qp", nullptr, offsetof(Options, qp), AV_OPT_TYPE_INT, {.i64 = 0}, 0, 51}, + // H.264/HEVC VAAPI accept 0..52 in AVOptions, then clamp CQP to 1..51. + {"qp", nullptr, offsetof(Options, qp), AV_OPT_TYPE_INT, {.i64 = 0}, 0, 52}, {nullptr}, }; @@ -41,7 +42,7 @@ int main() { ctx.bit_rate = 1000000; for (const auto &name : {"h264_vaapi", "hevc_vaapi"}) { - for (int qp : {1, 16, 23, 26, 51}) { + for (int qp : {1, 16, 23, 26, 51, 52}) { av_opt_set_defaults(&priv); assert(util_encode::set_rate_control(&ctx, name, RC_CQ, qp)); assert(priv.rc_mode == 1); @@ -54,7 +55,7 @@ int main() { assert(priv.rc_mode == 1); assert(priv.qp == 23); } - for (int qp : {52, 100}) { + for (int qp : {53, 100}) { av_opt_set_defaults(&priv); assert(!util_encode::set_rate_control(&ctx, name, RC_CQ, qp)); assert(priv.rc_mode == 1);