Support CQP rate control for FFmpeg VAAPI encoders

cpp/common/util.cpp

@@ -214,6 +214,7 @@ bool set_rate_control(AVCodecContext *c, const std::string &name, int rc,
       {"mediacodec",
        "bitrate_mode",
        {{RC_CBR, "cbr"}, {RC_VBR, "vbr"}, {RC_CQ, "cq"}}},
+      {"vaapi", "rc_mode", {{RC_CQ, "CQP"}}},
       // {"videotoolbox", "constant_bit_rate", {{RC_CBR, "1"}}},
     };
 
@@ -235,6 +236,19 @@ bool set_rate_control(AVCodecContext *c, const std::string &name, int rc,
             }
           }
         }
+        if (name.find("vaapi") != std::string::npos && rc == RC_CQ) {
+          constexpr int default_qp = 23;
+          // FFmpeg treats qp = 0 as unspecified, not as an explicit QP.
+          // Negative values are also unspecified in EncodeContext. Positive
+          // values are validated by the codec-specific FFmpeg option.
+          const int qp = q > 0 ? q : default_qp;
+          ret = av_opt_set_int(c->priv_data, "qp", qp, 0);
+          if (ret < 0) {
+            LOG_ERROR(std::string("vaapi set opt qp failed, ret = ") +
+                      av_err2str(ret));
+            return false;
+          }
+        }
       }
       break;
     }
@@ -330,4 +344,4 @@ bool has_flag_could_not_find_ref_with_poc() {
 
 extern "C" void hwcodec_set_flag_could_not_find_ref_with_poc() {
   util_decode::g_flag_could_not_find_ref_with_poc = true;
-}
+}

cpp/ffmpeg_ram/ffmpeg_ram_encode.cpp

@@ -463,4 +463,4 @@ extern "C" int ffmpeg_ram_set_bitrate(FFmpegRamEncoder *encoder, int kbs) {
     LOG_ERROR(std::string("ffmpeg_ram_set_bitrate failed, ") + std::string(e.what()));
   }
   return -1;
-}
+}

src/ffmpeg_ram/encode.rs

@@ -312,6 +312,13 @@ impl Encoder {
                 let c = EncodeContext {
                     name: codec.name.clone(),
                     mc_name: codec.mc_name.clone(),
+                    // RustDesk uses CQP for VAAPI; probing with a bitrate-based
+                    // mode would reject drivers that only support CQP.
+                    rc: if codec.name.contains("vaapi") {
+                        RateControl::RC_CQ
+                    } else {
+                        ctx.rc
+                    },
                     ..ctx
                 };
 

tests/vaapi_cqp.cpp

@@ -0,0 +1,82 @@
+#include "common.h"
+#include "util.h"
+
+extern "C" {
+#include <libavutil/opt.h>
+}
+
+#include <cassert>
+#include <cstddef>
+#include <iostream>
+
+namespace gol {
+void error(const std::string &message) { std::cerr << message << '\n'; }
+}
+
+// Exercise the real AVOption writes without requiring a VAAPI device.
+struct Options {
+  const AVClass *av_class;
+  int rc_mode;
+  int qp;
+};
+
+static const AVOption options[] = {
+    {"rc_mode", nullptr, offsetof(Options, rc_mode), AV_OPT_TYPE_INT,
+     {.i64 = 0}, 0, 6, 0, "rc_mode"},
+    {"CQP", nullptr, 0, AV_OPT_TYPE_CONST, {.i64 = 1}, 0, 0, 0, "rc_mode"},
+    // H.264/HEVC VAAPI accept 0..52 in AVOptions, then clamp CQP to 1..51.
+    {"qp", nullptr, offsetof(Options, qp), AV_OPT_TYPE_INT, {.i64 = 0}, 0, 52},
+    {nullptr},
+};
+
+int main() {
+  AVClass av_class = {};
+  av_class.class_name = "VAAPI options test";
+  av_class.item_name = av_default_item_name;
+  av_class.option = options;
+  av_class.version = LIBAVUTIL_VERSION_INT;
+
+  Options priv = {&av_class, 0, 0};
+  AVCodecContext ctx = {};
+  ctx.priv_data = &priv;
+  ctx.bit_rate = 1000000;
+
+  for (const auto &name : {"h264_vaapi", "hevc_vaapi"}) {
+    for (int qp : {1, 16, 23, 26, 51, 52}) {
+      av_opt_set_defaults(&priv);
+      assert(util_encode::set_rate_control(&ctx, name, RC_CQ, qp));
+      assert(priv.rc_mode == 1);
+      assert(priv.qp == qp);
+      assert(ctx.bit_rate == 1000000);
+    }
+    for (int qp : {-100, -1, 0}) {
+      av_opt_set_defaults(&priv);
+      assert(util_encode::set_rate_control(&ctx, name, RC_CQ, qp));
+      assert(priv.rc_mode == 1);
+      assert(priv.qp == 23);
+    }
+    for (int qp : {53, 100}) {
+      av_opt_set_defaults(&priv);
+      assert(!util_encode::set_rate_control(&ctx, name, RC_CQ, qp));
+      assert(priv.rc_mode == 1);
+    }
+  }
+
+  // Explicit CQP configuration must not affect callers requesting other modes.
+  for (int rc : {RC_CBR, RC_VBR}) {
+    av_opt_set_defaults(&priv);
+    assert(util_encode::set_rate_control(&ctx, "h264_vaapi", rc, 16));
+    assert(priv.rc_mode == 0);
+    assert(priv.qp == 0);
+  }
+
+  // Missing options must be reported as failures, not as successful CQP setup.
+  AVOption mode_only[] = {options[0], options[1], {nullptr}};
+  av_class.option = mode_only;
+  assert(!util_encode::set_rate_control(&ctx, "h264_vaapi", RC_CQ, 23));
+  AVOption no_options[] = {{nullptr}};
+  av_class.option = no_options;
+  assert(!util_encode::set_rate_control(&ctx, "h264_vaapi", RC_CQ, 23));
+
+  std::cout << "VAAPI CQP option tests passed\n";
+}