From 1cf5b3dbde29de149911958d5e9ba23ae926b36f Mon Sep 17 00:00:00 2001 From: merovex Date: Fri, 9 Oct 2026 15:15:36 -0400 Subject: [PATCH 1/4] Decode camera JPEG frames to 8-bit 4:2:0 before encoding Webcams deliver 4K30 as MJPEG. Qt decoded each frame to RGB and then encoded it as 10-bit 4:4:4 H.264, which most players cannot open and which is far slower to encode. Decode with libjpeg-turbo straight to video-range YUV 4:2:0 on the thread pool, keeping frames in capture order. Co-Authored-By: Claude Opus 5.5 --- README.md | 4 +-- pkgbuild/PKGBUILD | 2 +- src/common.pri | 2 +- src/mediautils.cpp | 67 +++++++++++++++++++++++++++++++++++++++++ src/mediautils.h | 5 +++ src/writer.cpp | 36 +++++++++++++++++----- src/writer.h | 4 ++- tests/backend_tests.cpp | 33 ++++++++++++++++++++ 8 files changed, 141 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 1168e5c..f710a7b 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ A simple webcam recorder for Omarchy. Choose your camera and microphone once, th ## Build and run -Install a C++17 compiler, `make`, `qt6-base`, `qt6-declarative`, `qt6-multimedia` (Qt 6.8 or newer), `libpulse`, and `ffmpeg`. Microphone capture requires a PulseAudio-compatible server, such as PipeWire-Pulse on Omarchy. Then: +Install a C++17 compiler, `make`, `qt6-base`, `qt6-declarative`, `qt6-multimedia` (Qt 6.8 or newer), `libpulse`, `libjpeg-turbo`, and `ffmpeg`. Microphone capture requires a PulseAudio-compatible server, such as PipeWire-Pulse on Omarchy. Then: ```sh ./bin/build @@ -75,7 +75,7 @@ A disconnected source or encoder error stops the take and preserves its files. A ./bin/test ``` -Tests cover format ranking, pause timing, PCM levels, atomic saving, the edit model, undo, reopening an interrupted take, take deletion and locking, filmstrip generation, edited exports (duration and content on both sides of a cut), the first frame showing on entering the editor, theme file changes and symlink swaps, actual H.264/AAC encoding and decoding, and native QML keyboard/layout behavior with simulated sources. They run offscreen and do not activate a camera or microphone. The capture-independent backend uses an injected file picker for save and recovery tests. The QML tests write inspection screenshots to `/tmp/monologue-ui-*.png`. +Tests cover format ranking, camera JPEG decoding, pause timing, PCM levels, atomic saving, the edit model, undo, reopening an interrupted take, take deletion and locking, filmstrip generation, edited exports (duration and content on both sides of a cut), the first frame showing on entering the editor, theme file changes and symlink swaps, actual H.264/AAC encoding and decoding, and native QML keyboard/layout behavior with simulated sources. They run offscreen and do not activate a camera or microphone. The capture-independent backend uses an injected file picker for save and recovery tests. The QML tests write inspection screenshots to `/tmp/monologue-ui-*.png`. The recording engine forwards native camera frames and one shared microphone PCM stream to timestamped Qt inputs. Microphone timestamps account for the audio server's measured capture latency, including buffered samples. The writer removes paused intervals by capture time and accepts delayed samples from before Pause or Finish. Finish briefly waits for those samples before closing the recording. Qt's audio encoder counts samples, so the writer pads genuine capture gaps and trims overlaps to maintain the common timeline. Preview and metering remain live while paused. Finish remuxes the MP4 without re-encoding to normalize packet durations and put playback metadata at the front of the file. diff --git a/pkgbuild/PKGBUILD b/pkgbuild/PKGBUILD index a971ac4..bb01621 100644 --- a/pkgbuild/PKGBUILD +++ b/pkgbuild/PKGBUILD @@ -5,7 +5,7 @@ pkgrel=1 pkgdesc='A simple, theme-synced webcam recorder for Omarchy' arch=('x86_64' 'aarch64') license=('MIT') -depends=('ffmpeg' 'libpulse' 'qt6-base' 'qt6-declarative' 'qt6-multimedia>=6.8' 'xdg-desktop-portal') +depends=('ffmpeg' 'libjpeg-turbo' 'libpulse' 'qt6-base' 'qt6-declarative' 'qt6-multimedia>=6.8' 'xdg-desktop-portal') makedepends=('gcc' 'make') source=() sha256sums=() diff --git a/src/common.pri b/src/common.pri index f7936b5..4e21ebd 100644 --- a/src/common.pri +++ b/src/common.pri @@ -1,5 +1,5 @@ CONFIG += link_pkgconfig -PKGCONFIG += libpulse +PKGCONFIG += libpulse libturbojpeg HEADERS += $$PWD/theme.h $$PWD/mediautils.h $$PWD/writer.h $$PWD/backend.h \ $$PWD/filepicker.h $$PWD/portalfilepicker.h $$PWD/audiocapture.h \ $$PWD/edit.h $$PWD/thumbnails.h diff --git a/src/mediautils.cpp b/src/mediautils.cpp index 190212f..dc68799 100644 --- a/src/mediautils.cpp +++ b/src/mediautils.cpp @@ -9,6 +9,9 @@ #include #include #include +#include +#include +#include double media::targetFps(double minimum, double maximum) { return std::clamp(30.0, minimum, maximum); } int media::bestFormat(const QList &formats) { @@ -35,6 +38,70 @@ QCameraFormat media::bestCameraFormat(const QCameraDevice &device) { const int index = bestFormat(choices); return index < 0 ? QCameraFormat() : formats[index]; } +QVideoFrame media::decodeJpeg(QByteArrayView jpeg) { + std::unique_ptr tj(tj3Init(TJINIT_DECOMPRESS), tj3Destroy); + const auto *data = reinterpret_cast(jpeg.data()); + if (!tj || tj3DecompressHeader(tj.get(), data, jpeg.size()) != 0) return {}; + const int width = tj3Get(tj.get(), TJPARAM_JPEGWIDTH), height = tj3Get(tj.get(), TJPARAM_JPEGHEIGHT); + const int sampling = tj3Get(tj.get(), TJPARAM_SUBSAMP), colorspace = tj3Get(tj.get(), TJPARAM_COLORSPACE); + if (width <= 0 || height <= 0 || width % 2 || height % 2 || sampling == TJSAMP_UNKNOWN || + (colorspace != TJCS_YCbCr && colorspace != TJCS_GRAY)) return {}; + // Decode to the JPEG's own planes, skipping libjpeg's RGB conversion. + const int planes = sampling == TJSAMP_GRAY ? 1 : 3; + std::vector source[3]; + unsigned char *destinations[3] = {}; + int strides[3] = {}, heights[3] = {}; + for (int i = 0; i < planes; ++i) { + strides[i] = tj3YUVPlaneWidth(i, width, sampling); + heights[i] = tj3YUVPlaneHeight(i, height, sampling); + source[i].resize(size_t(strides[i]) * heights[i]); + destinations[i] = source[i].data(); + } + if (tj3DecompressToYUVPlanes8(tj.get(), data, jpeg.size(), destinations, strides) != 0) return {}; + + QVideoFrameFormat format(QSize(width, height), QVideoFrameFormat::Format_YUV420P); + format.setColorSpace(QVideoFrameFormat::ColorSpace_BT601); + format.setColorRange(QVideoFrameFormat::ColorRange_Video); + QVideoFrame frame(format); + if (!frame.map(QVideoFrame::WriteOnly)) return {}; + // JPEG is full range; H.264 players expect video range. + unsigned char luma[256], chroma[256]; + for (int v = 0; v < 256; ++v) { luma[v] = 16 + (v * 219 + 127) / 255; chroma[v] = 16 + (v * 224 + 127) / 255; } + for (int y = 0; y < height; ++y) { + const unsigned char *in = source[0].data() + size_t(y) * strides[0]; + unsigned char *out = frame.bits(0) + qsizetype(y) * frame.bytesPerLine(0); + for (int x = 0; x < width; ++x) out[x] = luma[in[x]]; + } + // Box-filter each chroma plane, whatever its sampling, onto the 4:2:0 grid. + const int outWidth = width / 2, outHeight = height / 2; + std::vector columns(outWidth + 1); + for (int i = 1; i < 3; ++i) { + unsigned char *plane = frame.bits(i); + const int bytesPerLine = frame.bytesPerLine(i); + if (planes == 1) { + for (int y = 0; y < outHeight; ++y) std::memset(plane + qsizetype(y) * bytesPerLine, 128, outWidth); + continue; + } + for (int x = 0; x <= outWidth; ++x) columns[x] = qint64(x) * strides[i] / outWidth; + for (int y = 0; y < outHeight; ++y) { + const int top = qint64(y) * heights[i] / outHeight; + const int bottom = std::max(top + 1, int(qint64(y + 1) * heights[i] / outHeight)); + unsigned char *out = plane + qsizetype(y) * bytesPerLine; + for (int x = 0; x < outWidth; ++x) { + const int left = columns[x], right = std::max(left + 1, columns[x + 1]); + int sum = 0; + for (int row = top; row < bottom; ++row) { + const unsigned char *in = source[i].data() + size_t(row) * strides[i]; + for (int column = left; column < right; ++column) sum += in[column]; + } + const int count = (bottom - top) * (right - left); + out[x] = chroma[(sum + count / 2) / count]; + } + } + } + frame.unmap(); + return frame; +} double media::peak(const QByteArray &data, const QAudioFormat &format) { double result = 0; const int bytes = format.bytesPerSample(); diff --git a/src/mediautils.h b/src/mediautils.h index 17645d3..5045c93 100644 --- a/src/mediautils.h +++ b/src/mediautils.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include "edit.h" @@ -12,6 +13,10 @@ struct Format { QSize size; double minFps; double maxFps; int pixelFormat; }; int bestFormat(const QList &formats); QCameraFormat bestCameraFormat(const QCameraDevice &device); double targetFps(double minimum, double maximum); +// Decodes a camera JPEG to 8-bit 4:2:0 limited-range video, as delivered by +// webcams' MJPEG modes. Encoding Qt's RGB decode instead yields 10-bit 4:4:4 +// H.264, too slow to sustain 4K and unplayable in most players. +QVideoFrame decodeJpeg(QByteArrayView jpeg); double peak(const QByteArray &data, const QAudioFormat &format); QString copyAtomically(const QString &source, const QString &destination); QString normalizeMp4(const QString &path, double fps); diff --git a/src/writer.cpp b/src/writer.cpp index 3a29664..e6459e9 100644 --- a/src/writer.cpp +++ b/src/writer.cpp @@ -1,7 +1,9 @@ #include "writer.h" +#include "mediautils.h" #include #include #include +#include #include namespace { @@ -138,15 +140,28 @@ void Writer::videoAt(QVideoFrame frame, qint64 position) { const qint64 timestamp = qRound64(frameIndex * 1000000.0 / m_fps); if (timestamp <= m_lastVideo) return; m_tailFrame = frame; - frame = QVideoFrame(std::make_unique(frame)); - frame.setStartTime(timestamp); - frame.setEndTime(qRound64((frameIndex + 1) * 1000000.0 / m_fps)); - frame.setStreamFrameRate(m_fps); m_lastVideo = timestamp; // Bounded native-frame references; never build an unbounded 4K frame queue. if (m_frames.size() >= 6) { fail("Video encoding cannot keep up at this camera's maximum resolution, so the take was stopped."); return; } - m_frames.enqueue(frame); - drain(); + const auto stamp = [timestamp, end = qRound64((frameIndex + 1) * 1000000.0 / m_fps), fps = m_fps](QVideoFrame frame) { + frame.setStartTime(timestamp); + frame.setEndTime(end); + frame.setStreamFrameRate(fps); + return frame; + }; + if (frame.pixelFormat() != QVideoFrameFormat::Format_Jpeg) { + m_frames.enqueue(QtFuture::makeReadyValueFuture(stamp(QVideoFrame(std::make_unique(frame))))); + drain(); + return; + } + auto decoded = QtConcurrent::run([frame, stamp]() mutable { + if (!frame.map(QVideoFrame::ReadOnly)) return QVideoFrame(); + auto result = media::decodeJpeg(QByteArrayView(frame.bits(0), frame.mappedBytes(0))); + frame.unmap(); + return result.isValid() ? stamp(result) : result; + }); + m_frames.enqueue(decoded); + decoded.then(this, [this](const QVideoFrame &) { drain(); }); } void Writer::audio(QByteArray data, qint64 capturedAt) { if (m_stopping || m_failed || !m_audio || data.isEmpty()) return; @@ -194,11 +209,18 @@ void Writer::appendAudio(const QByteArray &data) { void Writer::drain() { if (m_draining || m_failed) return; m_draining = true; - while (!m_frames.isEmpty() && m_video->sendVideoFrame(m_frames.head())) m_frames.dequeue(); + bool undecodable = false; + while (!m_frames.isEmpty() && m_frames.head().isFinished()) { + const auto frame = m_frames.head().result(); + if (!frame.isValid()) { undecodable = true; break; } + if (!m_video->sendVideoFrame(frame)) break; + m_frames.dequeue(); + } while (!m_buffers.isEmpty() && m_audio->sendAudioBuffer(m_buffers.head())) { m_queuedAudioUs -= m_buffers.head().duration(); m_buffers.dequeue(); } m_draining = false; + if (undecodable) { fail("A camera frame could not be decoded, so the take was stopped."); return; } if (m_stopping && m_frames.isEmpty() && m_buffers.isEmpty()) m_recorder.stop(); } void Writer::finish() { diff --git a/src/writer.h b/src/writer.h index 9838276..1573b23 100644 --- a/src/writer.h +++ b/src/writer.h @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -58,7 +59,8 @@ class Writer : public QObject { QVideoFrameInput *m_video = nullptr; QAudioBufferInput *m_audio = nullptr; QAudioFormat m_audioFormat; - QQueue m_frames; + // In capture order; JPEG frames resolve once decoded off the GUI thread. + QQueue> m_frames; QVideoFrame m_tailFrame; QQueue m_buffers; QTimer m_flushTimeout; diff --git a/tests/backend_tests.cpp b/tests/backend_tests.cpp index 19745fe..d456b69 100644 --- a/tests/backend_tests.cpp +++ b/tests/backend_tests.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include "mediautils.h" #include "theme.h" #include "writer.h" @@ -117,6 +118,38 @@ private slots: QVERIFY(write(response,"{\"int\":8}")); QTRY_COMPARE(theme.radius(),8); QVERIFY(write(response,"unavailable")); QTest::qWait(1200); QCOMPARE(theme.radius(),8); } + void jpegDecodesToVideoRange_data() { + QTest::addColumn("sampling"); + QTest::newRow("4:2:2")<("sound"); QTest::newRow("silent")< Date: Fri, 9 Oct 2026 15:31:54 -0400 Subject: [PATCH 2/4] Encode with FFmpeg directly so 4K30 recordings are sustained Qt's recorder always runs libx264 at its medium preset, which manages about 17 fps at 4K, and never uses VAAPI for frames in system memory, so 4K takes stopped within seconds. Encode on a worker thread with FFmpeg: the GPU's h264_vaapi encoder when available, otherwise libx264 at veryfast. Takes keep the same timing model and MP4 output. Fixes #8 Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- src/common.pri | 6 +- src/encoder.cpp | 334 ++++++++++++++++++++++++++++++++++++++++ src/encoder.h | 74 +++++++++ src/main.cpp | 2 +- src/writer.cpp | 111 ++++++------- src/writer.h | 19 +-- tests/backend_tests.cpp | 7 +- 8 files changed, 472 insertions(+), 83 deletions(-) create mode 100644 src/encoder.cpp create mode 100644 src/encoder.h diff --git a/README.md b/README.md index f710a7b..49bfeb0 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,7 @@ A disconnected source or encoder error stops the take and preserves its files. A Tests cover format ranking, camera JPEG decoding, pause timing, PCM levels, atomic saving, the edit model, undo, reopening an interrupted take, take deletion and locking, filmstrip generation, edited exports (duration and content on both sides of a cut), the first frame showing on entering the editor, theme file changes and symlink swaps, actual H.264/AAC encoding and decoding, and native QML keyboard/layout behavior with simulated sources. They run offscreen and do not activate a camera or microphone. The capture-independent backend uses an injected file picker for save and recovery tests. The QML tests write inspection screenshots to `/tmp/monologue-ui-*.png`. -The recording engine forwards native camera frames and one shared microphone PCM stream to timestamped Qt inputs. Microphone timestamps account for the audio server's measured capture latency, including buffered samples. The writer removes paused intervals by capture time and accepts delayed samples from before Pause or Finish. Finish briefly waits for those samples before closing the recording. Qt's audio encoder counts samples, so the writer pads genuine capture gaps and trims overlaps to maintain the common timeline. Preview and metering remain live while paused. Finish remuxes the MP4 without re-encoding to normalize packet durations and put playback metadata at the front of the file. +The recording engine takes native camera frames and one shared microphone PCM stream and encodes them with FFmpeg on a worker thread, using the GPU's VAAPI H.264 encoder when available (libx264 at its `veryfast` preset otherwise), so 4K at 30 fps is sustained. Webcams deliver 4K30 as JPEG; each frame is decoded with libjpeg-turbo straight to 8-bit 4:2:0, so recordings are standard H.264 High profile that any player opens. Microphone timestamps account for the audio server's measured capture latency, including buffered samples. The writer removes paused intervals by capture time and accepts delayed samples from before Pause or Finish. Finish briefly waits for those samples before closing the recording. The audio encoder counts samples, so the writer pads genuine capture gaps and trims overlaps to maintain the common timeline. Preview and metering remain live while paused. Finish remuxes the MP4 without re-encoding to normalize packet durations and put playback metadata at the front of the file. A synchronization regression test encodes and decodes matching flashes and audio clicks with delayed delivery, including events just before Pause and Finish. It checks event alignment within 10 ms, rather than only comparing stream durations. diff --git a/src/common.pri b/src/common.pri index 4e21ebd..8140c55 100644 --- a/src/common.pri +++ b/src/common.pri @@ -1,8 +1,8 @@ CONFIG += link_pkgconfig -PKGCONFIG += libpulse libturbojpeg -HEADERS += $$PWD/theme.h $$PWD/mediautils.h $$PWD/writer.h $$PWD/backend.h \ +PKGCONFIG += libpulse libturbojpeg libavcodec libavformat libavutil libswscale libswresample +HEADERS += $$PWD/theme.h $$PWD/mediautils.h $$PWD/encoder.h $$PWD/writer.h $$PWD/backend.h \ $$PWD/filepicker.h $$PWD/portalfilepicker.h $$PWD/audiocapture.h \ $$PWD/edit.h $$PWD/thumbnails.h -SOURCES += $$PWD/theme.cpp $$PWD/mediautils.cpp $$PWD/writer.cpp $$PWD/backend.cpp \ +SOURCES += $$PWD/theme.cpp $$PWD/mediautils.cpp $$PWD/encoder.cpp $$PWD/writer.cpp $$PWD/backend.cpp \ $$PWD/portalfilepicker.cpp $$PWD/audiocapture.cpp \ $$PWD/edit.cpp $$PWD/thumbnails.cpp diff --git a/src/encoder.cpp b/src/encoder.cpp new file mode 100644 index 0000000..cdf24dd --- /dev/null +++ b/src/encoder.cpp @@ -0,0 +1,334 @@ +#include "encoder.h" +#include +extern "C" { +#include +#include +#include +#include +#include +#include +#include +} + +namespace { +QString failure(const char *what, int code) { + char reason[AV_ERROR_MAX_STRING_SIZE] = {}; + av_strerror(code, reason, sizeof reason); + return QString("%1: %2").arg(what, reason); +} +AVPixelFormat pixelFormat(QVideoFrameFormat::PixelFormat format) { + switch (format) { + case QVideoFrameFormat::Format_YUV420P: return AV_PIX_FMT_YUV420P; + case QVideoFrameFormat::Format_YUV422P: return AV_PIX_FMT_YUV422P; + case QVideoFrameFormat::Format_NV12: return AV_PIX_FMT_NV12; + case QVideoFrameFormat::Format_YUYV: return AV_PIX_FMT_YUYV422; + case QVideoFrameFormat::Format_UYVY: return AV_PIX_FMT_UYVY422; + case QVideoFrameFormat::Format_BGRA8888: return AV_PIX_FMT_BGRA; + case QVideoFrameFormat::Format_BGRX8888: return AV_PIX_FMT_BGR0; + case QVideoFrameFormat::Format_RGBA8888: return AV_PIX_FMT_RGBA; + case QVideoFrameFormat::Format_RGBX8888: return AV_PIX_FMT_RGB0; + case QVideoFrameFormat::Format_ARGB8888: return AV_PIX_FMT_ARGB; + case QVideoFrameFormat::Format_XRGB8888: return AV_PIX_FMT_0RGB; + case QVideoFrameFormat::Format_ABGR8888: return AV_PIX_FMT_ABGR; + case QVideoFrameFormat::Format_XBGR8888: return AV_PIX_FMT_0BGR; + default: return AV_PIX_FMT_NONE; + } +} +AVSampleFormat sampleFormat(QAudioFormat::SampleFormat format) { + switch (format) { + case QAudioFormat::UInt8: return AV_SAMPLE_FMT_U8; + case QAudioFormat::Int16: return AV_SAMPLE_FMT_S16; + case QAudioFormat::Int32: return AV_SAMPLE_FMT_S32; + case QAudioFormat::Float: return AV_SAMPLE_FMT_FLT; + default: return AV_SAMPLE_FMT_NONE; + } +} +} + +Encoder::~Encoder() { + { + std::lock_guard lock(m_mutex); + m_finishing = m_discard = true; + m_done = nullptr; m_failed = nullptr; + } + m_wake.notify_one(); + if (m_thread.joinable()) m_thread.join(); + release(); +} +bool Encoder::supported(bool audio) { + return av_guess_format("mp4", nullptr, nullptr) && + (avcodec_find_encoder_by_name("h264_vaapi") || avcodec_find_encoder_by_name("libx264")) && + (!audio || avcodec_find_encoder(AV_CODEC_ID_AAC)); +} +QString Encoder::open(const Settings &settings, std::function failed) { + m_failed = std::move(failed); + m_audioFormat = settings.audio; + const auto path = settings.path.toUtf8(); + int result = avformat_alloc_output_context2(&m_format, nullptr, "mp4", path.constData()); + if (result < 0) return failure("Could not create the MP4", result); + auto error = openVideo(settings, true); + if (!error.isEmpty()) error = openVideo(settings, false); + if (error.isEmpty() && settings.audio.isValid()) error = openAudio(settings.audio); + if (error.isEmpty() && (result = avio_open(&m_format->pb, path.constData(), AVIO_FLAG_WRITE)) < 0) + error = failure("Could not open the recording file", result); + if (error.isEmpty() && (result = avformat_write_header(m_format, nullptr)) < 0) + error = failure("Could not start the MP4", result); + if (!error.isEmpty()) { release(); return error; } + m_headerWritten = true; + m_thread = std::thread(&Encoder::run, this); + return {}; +} +QString Encoder::openVideo(const Settings &settings, bool hardware) { + const AVCodec *codec = avcodec_find_encoder_by_name(hardware ? "h264_vaapi" : "libx264"); + if (!codec) return "No H.264 encoder is available."; + int result = 0; + if (hardware && (result = av_hwdevice_ctx_create(&m_device, AV_HWDEVICE_TYPE_VAAPI, nullptr, nullptr, 0)) < 0) + return failure("No VAAPI device", result); + const AVRational rate = av_d2q(settings.fps, 1001000); + m_video = avcodec_alloc_context3(codec); + m_video->width = settings.size.width(); + m_video->height = settings.size.height(); + m_video->framerate = rate; + m_video->time_base = av_inv_q(rate); + // Keyframes every two seconds keep the editor's seeking quick. + m_video->gop_size = std::max(1, int(settings.fps * 2)); + m_video->color_range = AVCOL_RANGE_MPEG; + m_video->colorspace = AVCOL_SPC_BT470BG; + if (m_format->oformat->flags & AVFMT_GLOBALHEADER) m_video->flags |= AV_CODEC_FLAG_GLOBAL_HEADER; + AVDictionary *options = nullptr; + const AVPixelFormat software = hardware ? AV_PIX_FMT_NV12 : AV_PIX_FMT_YUV420P; + if (hardware) { + AVBufferRef *frames = av_hwframe_ctx_alloc(m_device); + auto *context = reinterpret_cast(frames->data); + context->format = AV_PIX_FMT_VAAPI; + context->sw_format = software; + context->width = m_video->width; + context->height = m_video->height; + context->initial_pool_size = 20; + if ((result = av_hwframe_ctx_init(frames)) < 0) { + av_buffer_unref(&frames); + avcodec_free_context(&m_video); av_buffer_unref(&m_device); + return failure("Could not allocate VAAPI frames", result); + } + m_video->hw_frames_ctx = frames; + m_video->pix_fmt = AV_PIX_FMT_VAAPI; + m_video->profile = AV_PROFILE_H264_HIGH; + // Constant QP 20 is close to the CRF 19 libx264 quality Qt used. + m_video->global_quality = 20; + av_dict_set(&options, "rc_mode", "CQP", 0); + } else { + m_video->pix_fmt = software; + av_dict_set(&options, "preset", "veryfast", 0); + av_dict_set(&options, "crf", "19", 0); + } + result = avcodec_open2(m_video, codec, &options); + av_dict_free(&options); + if (result < 0) { + avcodec_free_context(&m_video); av_buffer_unref(&m_device); + return failure("Could not open the H.264 encoder", result); + } + m_software = av_frame_alloc(); + m_software->format = software; + m_software->width = m_video->width; + m_software->height = m_video->height; + if ((result = av_frame_get_buffer(m_software, 0)) < 0) return failure("Could not allocate a video frame", result); + m_videoStream = avformat_new_stream(m_format, nullptr); + m_videoStream->time_base = m_video->time_base; + m_videoStream->avg_frame_rate = rate; + avcodec_parameters_from_context(m_videoStream->codecpar, m_video); + m_videoEncoderName = codec->name; + return {}; +} +QString Encoder::openAudio(const QAudioFormat &format) { + const AVCodec *codec = avcodec_find_encoder(AV_CODEC_ID_AAC); + const AVSampleFormat input = sampleFormat(format.sampleFormat()); + if (!codec || input == AV_SAMPLE_FMT_NONE) return "No AAC encoder is available for this microphone format."; + m_audio = avcodec_alloc_context3(codec); + m_audio->sample_fmt = AV_SAMPLE_FMT_FLTP; + m_audio->sample_rate = format.sampleRate(); + av_channel_layout_default(&m_audio->ch_layout, format.channelCount()); + m_audio->bit_rate = 192000; + m_audio->time_base = {1, format.sampleRate()}; + if (m_format->oformat->flags & AVFMT_GLOBALHEADER) m_audio->flags |= AV_CODEC_FLAG_GLOBAL_HEADER; + int result = avcodec_open2(m_audio, codec, nullptr); + if (result < 0) return failure("Could not open the AAC encoder", result); + if ((result = swr_alloc_set_opts2(&m_resample, &m_audio->ch_layout, AV_SAMPLE_FMT_FLTP, m_audio->sample_rate, + &m_audio->ch_layout, input, m_audio->sample_rate, 0, nullptr)) < 0 || + (result = swr_init(m_resample)) < 0) + return failure("Could not convert the microphone audio", result); + m_fifo = av_audio_fifo_alloc(AV_SAMPLE_FMT_FLTP, format.channelCount(), std::max(1, m_audio->frame_size)); + m_audioStream = avformat_new_stream(m_format, nullptr); + m_audioStream->time_base = m_audio->time_base; + avcodec_parameters_from_context(m_audioStream->codecpar, m_audio); + return {}; +} +void Encoder::video(const QVideoFrame &frame) { + { + std::lock_guard lock(m_mutex); + if (m_finishing) return; + m_jobs.push_back({frame, {}}); + ++m_queuedFrames; + } + m_wake.notify_one(); +} +void Encoder::audio(const QByteArray &pcm) { + { + std::lock_guard lock(m_mutex); + if (m_finishing || !m_audio) return; + m_jobs.push_back({{}, pcm}); + m_queuedAudioUs += m_audioFormat.durationForBytes(pcm.size()); + } + m_wake.notify_one(); +} +void Encoder::finish(bool discard, std::function done) { + { + std::lock_guard lock(m_mutex); + m_finishing = true; + m_discard = m_discard || discard; + m_done = std::move(done); + } + m_wake.notify_one(); +} +void Encoder::run() { + QString error; + for (;;) { + Job job; + { + std::unique_lock lock(m_mutex); + m_wake.wait(lock, [this] { return !m_jobs.empty() || m_finishing; }); + if (m_discard) { m_jobs.clear(); m_queuedFrames = 0; m_queuedAudioUs = 0; } + if (m_jobs.empty()) break; + job = std::move(m_jobs.front()); + m_jobs.pop_front(); + } + if (error.isEmpty()) { + error = job.frame.isValid() ? encodeVideo(job.frame) : encodeAudio(job.pcm, false); + if (!error.isEmpty()) { + std::lock_guard lock(m_mutex); + if (m_failed) m_failed(error); + } + } + if (job.frame.isValid()) --m_queuedFrames; + else m_queuedAudioUs -= m_audioFormat.durationForBytes(job.pcm.size()); + } + if (error.isEmpty() && m_audio) error = encodeAudio({}, true); + const auto closing = close(); + if (error.isEmpty()) error = closing; + // Hold the lock so the owner cannot be destroyed while it is notified. + std::lock_guard lock(m_mutex); + if (m_done) m_done(error); +} +QString Encoder::encodeVideo(const QVideoFrame &source) { + QVideoFrame frame = source; + AVPixelFormat format = pixelFormat(frame.pixelFormat()); + const uint8_t *planes[4] = {}; + int strides[4] = {}; + QImage image; + if (format == AV_PIX_FMT_NONE) { + image = frame.toImage().convertToFormat(QImage::Format_RGB32); + if (image.isNull()) return "A camera frame could not be converted for encoding."; + format = AV_PIX_FMT_BGR0; + planes[0] = image.constBits(); strides[0] = int(image.bytesPerLine()); + } else { + if (!frame.map(QVideoFrame::ReadOnly)) return "A camera frame could not be read for encoding."; + for (int i = 0; i < frame.planeCount(); ++i) { planes[i] = frame.bits(i); strides[i] = frame.bytesPerLine(i); } + } + const QSize size = image.isNull() ? frame.size() : image.size(); + m_scale = sws_getCachedContext(m_scale, size.width(), size.height(), format, + m_video->width, m_video->height, AVPixelFormat(m_software->format), + SWS_BILINEAR, nullptr, nullptr, nullptr); + int result = m_scale ? av_frame_make_writable(m_software) : AVERROR(EINVAL); + if (result >= 0) sws_scale(m_scale, planes, strides, 0, size.height(), m_software->data, m_software->linesize); + if (image.isNull()) frame.unmap(); + if (result < 0) return failure("Could not convert a camera frame", result); + // Frame times are already quantized to the output rate. + m_software->pts = av_rescale_q_rnd(frame.startTime(), {1, 1000000}, m_video->time_base, + AVRounding(AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX)); + if (!m_device) return send(m_video, m_videoStream, m_software); + AVFrame *upload = av_frame_alloc(); + if ((result = av_hwframe_get_buffer(m_video->hw_frames_ctx, upload, 0)) < 0 || + (result = av_hwframe_transfer_data(upload, m_software, 0)) < 0) { + av_frame_free(&upload); + return failure("Could not upload a frame to the GPU", result); + } + upload->pts = m_software->pts; + const auto error = send(m_video, m_videoStream, upload); + av_frame_free(&upload); + return error; +} +QString Encoder::encodeAudio(const QByteArray &pcm, bool flush) { + const int channels = m_audio->ch_layout.nb_channels; + if (!pcm.isEmpty()) { + const int samples = m_audioFormat.framesForBytes(pcm.size()); + uint8_t **converted = nullptr; + int result = av_samples_alloc_array_and_samples(&converted, nullptr, channels, samples, AV_SAMPLE_FMT_FLTP, 0); + if (result < 0) return failure("Could not convert microphone audio", result); + const uint8_t *input[1] = {reinterpret_cast(pcm.constData())}; + result = swr_convert(m_resample, converted, samples, input, samples); + if (result > 0) result = av_audio_fifo_write(m_fifo, reinterpret_cast(converted), result); + av_freep(&converted[0]); av_freep(&converted); + if (result < 0) return failure("Could not convert microphone audio", result); + } + const int frameSize = m_audio->frame_size > 0 ? m_audio->frame_size : 1024; + while (av_audio_fifo_size(m_fifo) >= frameSize || (flush && av_audio_fifo_size(m_fifo) > 0)) { + const int available = std::min(frameSize, av_audio_fifo_size(m_fifo)); + const bool padded = available < frameSize && !(m_audio->codec->capabilities & AV_CODEC_CAP_SMALL_LAST_FRAME); + AVFrame *frame = av_frame_alloc(); + frame->nb_samples = padded ? frameSize : available; + frame->format = AV_SAMPLE_FMT_FLTP; + frame->sample_rate = m_audio->sample_rate; + av_channel_layout_copy(&frame->ch_layout, &m_audio->ch_layout); + int result = av_frame_get_buffer(frame, 0); + if (result >= 0) { + if (padded) av_samples_set_silence(frame->data, 0, frameSize, channels, AV_SAMPLE_FMT_FLTP); + result = av_audio_fifo_read(m_fifo, reinterpret_cast(frame->data), available); + } + if (result < 0) { av_frame_free(&frame); return failure("Could not buffer microphone audio", result); } + frame->pts = m_audioSamples; + m_audioSamples += frame->nb_samples; + const auto error = send(m_audio, m_audioStream, frame); + av_frame_free(&frame); + if (!error.isEmpty()) return error; + } + return {}; +} +QString Encoder::send(AVCodecContext *codec, AVStream *stream, AVFrame *frame) { + int result = avcodec_send_frame(codec, frame); + if (result < 0) return failure("Encoding failed", result); + AVPacket *packet = av_packet_alloc(); + while ((result = avcodec_receive_packet(codec, packet)) >= 0) { + av_packet_rescale_ts(packet, codec->time_base, stream->time_base); + packet->stream_index = stream->index; + if ((result = av_interleaved_write_frame(m_format, packet)) < 0) break; + } + av_packet_free(&packet); + if (result == AVERROR(EAGAIN) || result == AVERROR_EOF) return {}; + return failure("Could not write the recording", result); +} +QString Encoder::close() { + QString error; + if (m_video) error = send(m_video, m_videoStream, nullptr); + if (m_audio) { const auto audio = send(m_audio, m_audioStream, nullptr); if (error.isEmpty()) error = audio; } + if (m_headerWritten) { + const int result = av_write_trailer(m_format); + if (result < 0 && error.isEmpty()) error = failure("Could not finish the MP4", result); + m_headerWritten = false; + } + release(); + return error; +} +void Encoder::release() { + avcodec_free_context(&m_video); + avcodec_free_context(&m_audio); + av_buffer_unref(&m_device); + av_frame_free(&m_software); + sws_freeContext(m_scale); m_scale = nullptr; + swr_free(&m_resample); + if (m_fifo) { av_audio_fifo_free(m_fifo); m_fifo = nullptr; } + if (m_format) { + if (m_format->pb) avio_closep(&m_format->pb); + avformat_free_context(m_format); + m_format = nullptr; + } + m_videoStream = m_audioStream = nullptr; +} diff --git a/src/encoder.h b/src/encoder.h new file mode 100644 index 0000000..209edba --- /dev/null +++ b/src/encoder.h @@ -0,0 +1,74 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct AVCodecContext; +struct AVFormatContext; +struct AVStream; +struct AVBufferRef; +struct AVFrame; +struct AVAudioFifo; +struct SwsContext; +struct SwrContext; + +// Encodes H.264/AAC into an MP4 on its own thread. Uses the GPU's VAAPI +// encoder when available, which sustains 4K30 where Qt's libx264 (always at +// its "medium" preset) cannot; otherwise libx264 at a realtime preset. +class Encoder { +public: + struct Settings { QString path; QSize size; double fps = 30; QAudioFormat audio; }; + ~Encoder(); + // Opens the file and codecs and starts the worker. Returns an error, or empty. + // failed runs on the worker thread at the first encoding error. + QString open(const Settings &settings, std::function failed); + // Queues a frame; its start time, quantized to the frame rate, sets its position. + void video(const QVideoFrame &frame); + // Queues PCM in the input format, contiguous with all earlier audio. + void audio(const QByteArray &pcm); + // Flushes and closes the file, dropping queued input first if discard is set. + // done runs on the worker thread with an error, or empty on success. + void finish(bool discard, std::function done); + int queuedFrames() const { return m_queuedFrames; } + qint64 queuedAudioUs() const { return m_queuedAudioUs; } + QString videoEncoder() const { return m_videoEncoderName; } + static bool supported(bool audio); +private: + struct Job { QVideoFrame frame; QByteArray pcm; }; + QString openVideo(const Settings &settings, bool hardware); + QString openAudio(const QAudioFormat &format); + void run(); + QString encodeVideo(const QVideoFrame &source); + QString encodeAudio(const QByteArray &pcm, bool flush); + QString send(AVCodecContext *codec, AVStream *stream, AVFrame *frame); + QString close(); + void release(); + AVFormatContext *m_format = nullptr; + AVCodecContext *m_video = nullptr, *m_audio = nullptr; + AVStream *m_videoStream = nullptr, *m_audioStream = nullptr; + AVBufferRef *m_device = nullptr; + AVFrame *m_software = nullptr; + SwsContext *m_scale = nullptr; + SwrContext *m_resample = nullptr; + AVAudioFifo *m_fifo = nullptr; + QAudioFormat m_audioFormat; + QString m_videoEncoderName; + qint64 m_audioSamples = 0; + bool m_headerWritten = false; + std::thread m_thread; + std::mutex m_mutex; + std::condition_variable m_wake; + std::deque m_jobs; + bool m_finishing = false, m_discard = false; + std::function m_done, m_failed; + std::atomic m_queuedFrames{0}; + std::atomic m_queuedAudioUs{0}; +}; diff --git a/src/main.cpp b/src/main.cpp index 262c9e0..c338992 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -9,7 +9,7 @@ #include "thumbprovider.h" int main(int argc,char *argv[]) { - // Custom timestamped frame/buffer inputs require Qt's FFmpeg backend. + // Qt's FFmpeg backend delivers timestamped native camera frames. qputenv("QT_MEDIA_BACKEND","ffmpeg"); QGuiApplication app(argc,argv); app.setApplicationName("monologue"); diff --git a/src/writer.cpp b/src/writer.cpp index e6459e9..5b4ecc7 100644 --- a/src/writer.cpp +++ b/src/writer.cpp @@ -1,7 +1,5 @@ #include "writer.h" #include "mediautils.h" -#include -#include #include #include #include @@ -77,48 +75,26 @@ QList TakeClock::joins() const { } Writer::Writer(QObject *parent) : QObject(parent) { - m_session.setRecorder(&m_recorder); m_flushTimeout.setSingleShot(true); m_flushTimeout.setInterval(10000); - connect(&m_flushTimeout, &QTimer::timeout, this, [this] { fail("The encoder stopped responding, so the take was stopped."); }); - connect(&m_recorder, &QMediaRecorder::errorOccurred, this, [this](auto, const QString &message) { fail(message); }); - connect(&m_recorder, &QMediaRecorder::recorderStateChanged, this, [this](auto state) { - if (state == QMediaRecorder::StoppedState && m_stopping) { - m_flushTimeout.stop(); - emit finished(); - } + connect(&m_flushTimeout, &QTimer::timeout, this, [this] { + if (!m_failed) { m_failed = true; emit failed("The encoder stopped responding, so the take was stopped."); } + emitFinished(); }); } -static QMediaFormat recordingFormat(bool audio) { - QMediaFormat f(QMediaFormat::MPEG4); - f.setVideoCodec(QMediaFormat::VideoCodec::H264); - if (audio) f.setAudioCodec(QMediaFormat::AudioCodec::AAC); - return f; -} -bool Writer::supported(bool audio) { return recordingFormat(audio).isSupported(QMediaFormat::Encode); } +Writer::~Writer() = default; +bool Writer::supported(bool audio) { return Encoder::supported(audio); } bool Writer::start(const QString &path, const QVideoFrameFormat &format, double fps, const QAudioFormat &audioFormat, qint64 now) { m_clock.start(now); m_frameDuration = qRound64(1000000.0 / fps); m_fps = fps; m_audioFormat = audioFormat; - // Let the first frame initialize the encoder. Supplying a format hint can - // leave Qt 6.11's canPushFrame cache false before its worker starts. - m_video = new QVideoFrameInput(this); - m_session.setVideoFrameInput(m_video); - connect(m_video, &QVideoFrameInput::readyToSendVideoFrame, this, &Writer::drain); - if (audioFormat.isValid()) { - m_audio = new QAudioBufferInput(this); - m_session.setAudioBufferInput(m_audio); - connect(m_audio, &QAudioBufferInput::readyToSendAudioBuffer, this, &Writer::drain); - } - m_recorder.setMediaFormat(recordingFormat(audioFormat.isValid())); - m_recorder.setVideoResolution(format.frameSize()); - m_recorder.setVideoFrameRate(fps); - m_recorder.setQuality(QMediaRecorder::VeryHighQuality); - m_recorder.setAudioBitRate(192000); - m_recorder.setOutputLocation(QUrl::fromLocalFile(path)); - m_recorder.record(); + m_encoder = std::make_unique(); + const auto error = m_encoder->open({path, format.frameSize(), fps, audioFormat}, [this](const QString &message) { + QMetaObject::invokeMethod(this, [this, message] { fail(message); }, Qt::QueuedConnection); + }); + if (!error.isEmpty()) { m_encoder.reset(); fail(error); } return !m_failed; } void Writer::pause(qint64 now) { @@ -142,7 +118,9 @@ void Writer::videoAt(QVideoFrame frame, qint64 position) { m_tailFrame = frame; m_lastVideo = timestamp; // Bounded native-frame references; never build an unbounded 4K frame queue. - if (m_frames.size() >= 6) { fail("Video encoding cannot keep up at this camera's maximum resolution, so the take was stopped."); return; } + if (m_frames.size() + m_encoder->queuedFrames() >= 8) { + fail("Video encoding cannot keep up at this camera's maximum resolution, so the take was stopped."); return; + } const auto stamp = [timestamp, end = qRound64((frameIndex + 1) * 1000000.0 / m_fps), fps = m_fps](QVideoFrame frame) { frame.setStartTime(timestamp); frame.setEndTime(end); @@ -164,7 +142,7 @@ void Writer::videoAt(QVideoFrame frame, qint64 position) { decoded.then(this, [this](const QVideoFrame &) { drain(); }); } void Writer::audio(QByteArray data, qint64 capturedAt) { - if (m_stopping || m_failed || !m_audio || data.isEmpty()) return; + if (m_stopping || m_failed || !m_audioFormat.isValid() || data.isEmpty()) return; const qint64 end = capturedAt + m_audioFormat.durationForBytes(data.size()); const int frameBytes = m_audioFormat.bytesPerFrame(); // Select by capture time, including closed intervals. A late buffer may @@ -177,8 +155,8 @@ void Writer::audio(QByteArray data, qint64 capturedAt) { const qint64 last = sampleAtOrAfter(span.end - capturedAt); QByteArray part = data.mid(first * frameBytes, (last - first) * frameBytes); const auto position = span.position + m_audioFormat.durationForFrames(first) - (span.begin - capturedAt); - // Qt counts audio samples rather than honoring PTS. Materialize capture - // gaps as silence and trim overlaps, keeping the original event times. + // The encoder counts audio samples rather than honoring PTS. Materialize + // capture gaps as silence and trim overlaps, keeping the original event times. const qint64 desiredFrame = qRound64(position * m_audioFormat.sampleRate() / 1000000.0); if (desiredFrame < m_audioFramesWritten) { const auto overlap = (m_audioFramesWritten - desiredFrame) * frameBytes; @@ -188,7 +166,7 @@ void Writer::audio(QByteArray data, qint64 capturedAt) { } } void Writer::padAudioTo(qint64 time) { - if (!m_audio || m_failed) return; + if (!m_audioFormat.isValid() || m_failed) return; const qint64 missing = qRound64(time * m_audioFormat.sampleRate() / 1000000.0) - m_audioFramesWritten; if (missing <= 0) return; if (m_audioFormat.durationForFrames(missing) > 500000) { @@ -199,29 +177,18 @@ void Writer::padAudioTo(qint64 time) { } void Writer::appendAudio(const QByteArray &data) { if (m_failed) return; - QAudioBuffer buffer(data, m_audioFormat, m_audioFormat.durationForFrames(m_audioFramesWritten)); - m_audioFramesWritten += buffer.frameCount(); - m_queuedAudioUs += buffer.duration(); - if (m_queuedAudioUs > 500000) { fail("Audio encoding cannot keep up, so the take was stopped."); return; } - m_buffers.enqueue(buffer); - drain(); + if (m_encoder->queuedAudioUs() > 500000) { fail("Audio encoding cannot keep up, so the take was stopped."); return; } + m_audioFramesWritten += m_audioFormat.framesForBytes(data.size()); + m_encoder->audio(data); } void Writer::drain() { - if (m_draining || m_failed) return; - m_draining = true; - bool undecodable = false; + if (m_failed || m_closing) return; while (!m_frames.isEmpty() && m_frames.head().isFinished()) { - const auto frame = m_frames.head().result(); - if (!frame.isValid()) { undecodable = true; break; } - if (!m_video->sendVideoFrame(frame)) break; - m_frames.dequeue(); + const auto frame = m_frames.dequeue().result(); + if (!frame.isValid()) { fail("A camera frame could not be decoded, so the take was stopped."); return; } + m_encoder->video(frame); } - while (!m_buffers.isEmpty() && m_audio->sendAudioBuffer(m_buffers.head())) { - m_queuedAudioUs -= m_buffers.head().duration(); m_buffers.dequeue(); - } - m_draining = false; - if (undecodable) { fail("A camera frame could not be decoded, so the take was stopped."); return; } - if (m_stopping && m_frames.isEmpty() && m_buffers.isEmpty()) m_recorder.stop(); + if (m_stopping && m_frames.isEmpty()) close(false); } void Writer::finish() { if (m_stopping) return; @@ -236,18 +203,30 @@ void Writer::finish() { m_stopping = true; m_flushTimeout.start(); drain(); - if (m_recorder.recorderState() == QMediaRecorder::StoppedState) { - m_flushTimeout.stop(); - QTimer::singleShot(0, this, &Writer::finished); - } +} +void Writer::close(bool discard) { + if (m_closing) return; + m_closing = true; + if (!m_encoder) { QTimer::singleShot(0, this, &Writer::emitFinished); return; } + m_encoder->finish(discard, [this](const QString &error) { + QMetaObject::invokeMethod(this, [this, error] { + if (!error.isEmpty() && !m_failed) { m_failed = true; emit failed(error); } + emitFinished(); + }, Qt::QueuedConnection); + }); +} +void Writer::emitFinished() { + m_flushTimeout.stop(); + if (m_finished) return; + m_finished = true; + emit finished(); } void Writer::fail(const QString &message) { if (m_failed) return; m_failed = true; m_stopping = true; emit failed(message); - m_frames.clear(); m_buffers.clear(); - m_recorder.stop(); - if (m_recorder.recorderState() == QMediaRecorder::StoppedState) - QTimer::singleShot(0, this, &Writer::finished); + m_frames.clear(); + // Keep what was already encoded: the file is still closed as a valid MP4. + close(true); } diff --git a/src/writer.h b/src/writer.h index 1573b23..1b7acec 100644 --- a/src/writer.h +++ b/src/writer.h @@ -1,14 +1,13 @@ #pragma once #include -#include -#include -#include -#include +#include #include #include #include #include +#include #include +#include "encoder.h" // Maps a monotonic capture clock onto a take with the paused intervals removed. class TakeClock { @@ -35,6 +34,7 @@ class Writer : public QObject { Q_OBJECT public: explicit Writer(QObject *parent = nullptr); + ~Writer() override; bool start(const QString &path, const QVideoFrameFormat &videoFormat, double fps, const QAudioFormat &audioFormat, qint64 now); void video(QVideoFrame frame, qint64 capturedAt); @@ -50,25 +50,22 @@ class Writer : public QObject { void failed(const QString &message); private: void drain(); + void close(bool discard); + void emitFinished(); void fail(const QString &message); void appendAudio(const QByteArray &data); void padAudioTo(qint64 time); void videoAt(QVideoFrame frame, qint64 position); - QMediaCaptureSession m_session; - QMediaRecorder m_recorder; - QVideoFrameInput *m_video = nullptr; - QAudioBufferInput *m_audio = nullptr; + std::unique_ptr m_encoder; QAudioFormat m_audioFormat; // In capture order; JPEG frames resolve once decoded off the GUI thread. QQueue> m_frames; QVideoFrame m_tailFrame; - QQueue m_buffers; QTimer m_flushTimeout; TakeClock m_clock; qint64 m_lastVideo = -1; qint64 m_frameDuration = 33333; double m_fps = 30; - qint64 m_queuedAudioUs = 0; qint64 m_audioFramesWritten = 0; - bool m_stopping = false, m_failed = false, m_draining = false; + bool m_stopping = false, m_failed = false, m_closing = false, m_finished = false; }; diff --git a/tests/backend_tests.cpp b/tests/backend_tests.cpp index d456b69..2bfe8c9 100644 --- a/tests/backend_tests.cpp +++ b/tests/backend_tests.cpp @@ -199,7 +199,12 @@ private slots: auto stream=entry.toObject(); const double duration=stream["duration"].toString().toDouble(); QVERIFY2(std::abs(duration-2)<.08,qPrintable(QString::number(duration))); - if(stream["codec_type"]=="video") QCOMPARE(stream["nb_frames"].toString().toInt(),60); + if(stream["codec_type"]=="video") { + QCOMPARE(stream["nb_frames"].toString().toInt(),60); + // Standard 8-bit 4:2:0 High profile, which every player decodes. + QCOMPARE(stream["pix_fmt"].toString(),QString("yuv420p")); + QCOMPARE(stream["profile"].toString(),QString("High")); + } } if(sound) { QProcess pcm; pcm.start("ffmpeg",{"-v","error","-i",directory.filePath("take.mp4"),"-vn","-ac","1","-ar","48000","-f","s16le","-"}); From d1b30f45993db400badb437f5b8ff81b8ad1d750 Mon Sep 17 00:00:00 2001 From: Lon Baker Date: Thu, 29 Oct 2026 20:23:47 -0700 Subject: [PATCH 3/4] Encode at the H.264 level the picture needs, with no B-frames Level 5.1 above 1080p, and 4.2 otherwise, on VAAPI and libx264. No B-frames, so the software fallback stays low-delay. A failed GPU open frees the codec before that fallback, including when the frame buffer or video stream cannot be created. --- README.md | 4 ++-- src/encoder.cpp | 45 +++++++++++++++++++++++++++++++++-------- src/encoder.h | 3 +++ tests/backend_tests.cpp | 33 ++++++++++++++++++++++++++++++ 4 files changed, 75 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 49bfeb0..abfbe9e 100644 --- a/README.md +++ b/README.md @@ -75,9 +75,9 @@ A disconnected source or encoder error stops the take and preserves its files. A ./bin/test ``` -Tests cover format ranking, camera JPEG decoding, pause timing, PCM levels, atomic saving, the edit model, undo, reopening an interrupted take, take deletion and locking, filmstrip generation, edited exports (duration and content on both sides of a cut), the first frame showing on entering the editor, theme file changes and symlink swaps, actual H.264/AAC encoding and decoding, and native QML keyboard/layout behavior with simulated sources. They run offscreen and do not activate a camera or microphone. The capture-independent backend uses an injected file picker for save and recovery tests. The QML tests write inspection screenshots to `/tmp/monologue-ui-*.png`. +Tests cover format ranking, camera JPEG decoding, pause timing, PCM levels, atomic saving, the edit model, undo, reopening an interrupted take, take deletion and locking, filmstrip generation, edited exports (duration and content on both sides of a cut), the first frame showing on entering the editor, theme file changes and symlink swaps, actual H.264/AAC encoding and decoding (High profile, level 4.2, no B-frames, including the libx264 fallback), and native QML keyboard/layout behavior with simulated sources. They run offscreen and do not activate a camera or microphone. The capture-independent backend uses an injected file picker for save and recovery tests. The QML tests write inspection screenshots to `/tmp/monologue-ui-*.png`. -The recording engine takes native camera frames and one shared microphone PCM stream and encodes them with FFmpeg on a worker thread, using the GPU's VAAPI H.264 encoder when available (libx264 at its `veryfast` preset otherwise), so 4K at 30 fps is sustained. Webcams deliver 4K30 as JPEG; each frame is decoded with libjpeg-turbo straight to 8-bit 4:2:0, so recordings are standard H.264 High profile that any player opens. Microphone timestamps account for the audio server's measured capture latency, including buffered samples. The writer removes paused intervals by capture time and accepts delayed samples from before Pause or Finish. Finish briefly waits for those samples before closing the recording. The audio encoder counts samples, so the writer pads genuine capture gaps and trims overlaps to maintain the common timeline. Preview and metering remain live while paused. Finish remuxes the MP4 without re-encoding to normalize packet durations and put playback metadata at the front of the file. +The recording engine takes native camera frames and one shared microphone PCM stream and encodes them with FFmpeg on a worker thread, using the GPU's VAAPI H.264 encoder when available (libx264 at its `veryfast` preset otherwise), so 4K at 30 fps is sustained. Webcams deliver 4K30 as JPEG; each frame is decoded with libjpeg-turbo straight to 8-bit 4:2:0. Recordings are H.264 High profile with no B-frames, level 5.1 at 4K and level 4.2 otherwise, so any player opens them. Microphone timestamps account for the audio server's measured capture latency, including buffered samples. The writer removes paused intervals by capture time and accepts delayed samples from before Pause or Finish. Finish briefly waits for those samples before closing the recording. The audio encoder counts samples, so the writer pads genuine capture gaps and trims overlaps to maintain the common timeline. Preview and metering remain live while paused. Finish remuxes the MP4 without re-encoding to normalize packet durations and put playback metadata at the front of the file. A synchronization regression test encodes and decodes matching flashes and audio clicks with delayed delivery, including events just before Pause and Finish. It checks event alignment within 10 ms, rather than only comparing stream durations. diff --git a/src/encoder.cpp b/src/encoder.cpp index cdf24dd..52e093d 100644 --- a/src/encoder.cpp +++ b/src/encoder.cpp @@ -5,6 +5,7 @@ extern "C" { #include #include #include +#include #include #include #include @@ -67,7 +68,10 @@ QString Encoder::open(const Settings &settings, std::functionpb, path.constData(), AVIO_FLAG_WRITE)) < 0) error = failure("Could not open the recording file", result); @@ -92,6 +96,11 @@ QString Encoder::openVideo(const Settings &settings, bool hardware) { m_video->time_base = av_inv_q(rate); // Keyframes every two seconds keep the editor's seeking quick. m_video->gop_size = std::max(1, int(settings.fps * 2)); + // No B-frames, so the libx264 fallback has the same low-delay timestamps as VAAPI. + m_video->max_b_frames = 0; + // 5.1 covers 4K30. 4.2 is enough at 1080p and below. + m_video->level = settings.size.height() > 1080 ? 51 : 42; + m_video->profile = AV_PROFILE_H264_HIGH; m_video->color_range = AVCOL_RANGE_MPEG; m_video->colorspace = AVCOL_SPC_BT470BG; if (m_format->oformat->flags & AVFMT_GLOBALHEADER) m_video->flags |= AV_CODEC_FLAG_GLOBAL_HEADER; @@ -112,7 +121,6 @@ QString Encoder::openVideo(const Settings &settings, bool hardware) { } m_video->hw_frames_ctx = frames; m_video->pix_fmt = AV_PIX_FMT_VAAPI; - m_video->profile = AV_PROFILE_H264_HIGH; // Constant QP 20 is close to the CRF 19 libx264 quality Qt used. m_video->global_quality = 20; av_dict_set(&options, "rc_mode", "CQP", 0); @@ -131,8 +139,15 @@ QString Encoder::openVideo(const Settings &settings, bool hardware) { m_software->format = software; m_software->width = m_video->width; m_software->height = m_video->height; - if ((result = av_frame_get_buffer(m_software, 0)) < 0) return failure("Could not allocate a video frame", result); + if ((result = av_frame_get_buffer(m_software, 0)) < 0) { + releaseVideo(); + return failure("Could not allocate a video frame", result); + } m_videoStream = avformat_new_stream(m_format, nullptr); + if (!m_videoStream) { + releaseVideo(); + return "Could not add the video stream."; + } m_videoStream->time_base = m_video->time_base; m_videoStream->avg_frame_rate = rate; avcodec_parameters_from_context(m_videoStream->codecpar, m_video); @@ -234,11 +249,20 @@ QString Encoder::encodeVideo(const QVideoFrame &source) { for (int i = 0; i < frame.planeCount(); ++i) { planes[i] = frame.bits(i); strides[i] = frame.bytesPerLine(i); } } const QSize size = image.isNull() ? frame.size() : image.size(); - m_scale = sws_getCachedContext(m_scale, size.width(), size.height(), format, - m_video->width, m_video->height, AVPixelFormat(m_software->format), - SWS_BILINEAR, nullptr, nullptr, nullptr); - int result = m_scale ? av_frame_make_writable(m_software) : AVERROR(EINVAL); - if (result >= 0) sws_scale(m_scale, planes, strides, 0, size.height(), m_software->data, m_software->linesize); + const bool sameSize = size.width() == m_video->width && size.height() == m_video->height; + const bool sameFormat = format == AVPixelFormat(m_software->format); + int result = av_frame_make_writable(m_software); + if (result >= 0 && sameSize && sameFormat) { + const uint8_t *sourcePlanes[4] = {planes[0], planes[1], planes[2], planes[3]}; + av_image_copy(m_software->data, m_software->linesize, sourcePlanes, strides, format, size.width(), size.height()); + } else if (result >= 0) { + // Same-size conversion is a shuffle (YUV420P to NV12), not a resample. + m_scale = sws_getCachedContext(m_scale, size.width(), size.height(), format, + m_video->width, m_video->height, AVPixelFormat(m_software->format), + sameSize ? SWS_POINT : SWS_BILINEAR, nullptr, nullptr, nullptr); + if (!m_scale || sws_scale(m_scale, planes, strides, 0, size.height(), m_software->data, m_software->linesize) <= 0) + result = AVERROR(EINVAL); + } if (image.isNull()) frame.unmap(); if (result < 0) return failure("Could not convert a camera frame", result); // Frame times are already quantized to the output rate. @@ -317,6 +341,11 @@ QString Encoder::close() { release(); return error; } +void Encoder::releaseVideo() { + avcodec_free_context(&m_video); + av_buffer_unref(&m_device); + av_frame_free(&m_software); +} void Encoder::release() { avcodec_free_context(&m_video); avcodec_free_context(&m_audio); diff --git a/src/encoder.h b/src/encoder.h index 209edba..500d560 100644 --- a/src/encoder.h +++ b/src/encoder.h @@ -45,6 +45,9 @@ class Encoder { struct Job { QVideoFrame frame; QByteArray pcm; }; QString openVideo(const Settings &settings, bool hardware); QString openAudio(const QAudioFormat &format); + // Frees the video codec and VAAPI device. The MP4 context stays open so a + // failed GPU open can fall back to libx264 before the header is written. + void releaseVideo(); void run(); QString encodeVideo(const QVideoFrame &source); QString encodeAudio(const QByteArray &pcm, bool flush); diff --git a/tests/backend_tests.cpp b/tests/backend_tests.cpp index 2bfe8c9..2767c23 100644 --- a/tests/backend_tests.cpp +++ b/tests/backend_tests.cpp @@ -204,6 +204,8 @@ private slots: // Standard 8-bit 4:2:0 High profile, which every player decodes. QCOMPARE(stream["pix_fmt"].toString(),QString("yuv420p")); QCOMPARE(stream["profile"].toString(),QString("High")); + QCOMPARE(stream["level"].toString().toInt(),42); + QCOMPARE(stream["has_b_frames"].toString().toInt(),0); } } if(sound) { @@ -430,6 +432,37 @@ private slots: QVERIFY2(std::abs(flashes[i]-clicks[i])<.01,qPrintable(QString("Event %1: video %2, audio %3").arg(i).arg(flashes[i]).arg(clicks[i]))); } } + void softwareEncoderMatches() { + // A missing VAAPI driver must fall back to libx264 with the same constraints. + struct Driver { + const QByteArray previous = qgetenv("LIBVA_DRIVER_NAME"); + const bool had = qEnvironmentVariableIsSet("LIBVA_DRIVER_NAME"); + Driver() { qputenv("LIBVA_DRIVER_NAME", "monologue-no-such-driver"); } + ~Driver() { if (had) qputenv("LIBVA_DRIVER_NAME", previous); else qunsetenv("LIBVA_DRIVER_NAME"); } + } driver; + Q_UNUSED(driver); + QTemporaryDir directory; + const auto path = directory.filePath("software.mp4"); + Writer writer; QSignalSpy ended(&writer, &Writer::finished), errors(&writer, &Writer::failed); + QVERIFY(writer.start(path, QVideoFrameFormat(QSize(64, 64), QVideoFrameFormat::Format_BGRA8888), 30, {}, 0)); + QImage image(64, 64, QImage::Format_RGB32); image.fill(Qt::gray); + for (int i = 0; i < 10; ++i) { + writer.video(QVideoFrame(image), qRound64(i * 1000000.0 / 30)); + QTest::qWait(5); + } + writer.finish(); QTRY_VERIFY_WITH_TIMEOUT(!ended.isEmpty(), 15000); + QVERIFY2(errors.isEmpty(), errors.isEmpty() ? "" : qPrintable(errors.first().first().toString())); + QProcess streams; streams.start("ffprobe", {"-v", "error", "-select_streams", "v:0", "-show_entries", "stream=pix_fmt,profile,level,has_b_frames,nb_frames:stream_tags=encoder", "-of", "json", path}); + QVERIFY(streams.waitForFinished()); + const auto stream = QJsonDocument::fromJson(streams.readAllStandardOutput()).object()["streams"].toArray().first().toObject(); + QCOMPARE(stream["pix_fmt"].toString(), QString("yuv420p")); + QCOMPARE(stream["profile"].toString(), QString("High")); + QCOMPARE(stream["level"].toString().toInt(), 42); + QCOMPARE(stream["has_b_frames"].toString().toInt(), 0); + QCOMPARE(stream["nb_frames"].toString().toInt(), 10); + const auto encoder = stream["tags"].toObject()["encoder"].toString(); + QVERIFY2(encoder.contains("libx264"), qPrintable(encoder)); + } }; int main(int argc,char**argv) { qputenv("QT_MEDIA_BACKEND","ffmpeg"); From d33f7e55cf2269c3a690dd18cac29ed352ab2572 Mon Sep 17 00:00:00 2001 From: Lon Baker Date: Thu, 29 Oct 2026 20:30:14 -0700 Subject: [PATCH 4/4] Decode camera JPEGs from a copy and skip a bad frame The capture thread copies the compressed bytes and decodes them on a pool of two threads, so a worker never maps the preview frame and a 4K decode cannot occupy every core. One bad JPEG is skipped. Fifteen in a row stop the take. VAAPI is not flushed when that stop happens before any frame, which faults. The file records the encoder name, and the level check reads ffprobe's numeric JSON field. --- README.md | 4 +-- src/encoder.cpp | 5 ++- src/encoder.h | 2 +- src/writer.cpp | 23 +++++++++--- src/writer.h | 8 +++++ tests/backend_tests.cpp | 79 ++++++++++++++++++++++++++++++++++++----- 6 files changed, 104 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index abfbe9e..bcdb7f2 100644 --- a/README.md +++ b/README.md @@ -75,9 +75,9 @@ A disconnected source or encoder error stops the take and preserves its files. A ./bin/test ``` -Tests cover format ranking, camera JPEG decoding, pause timing, PCM levels, atomic saving, the edit model, undo, reopening an interrupted take, take deletion and locking, filmstrip generation, edited exports (duration and content on both sides of a cut), the first frame showing on entering the editor, theme file changes and symlink swaps, actual H.264/AAC encoding and decoding (High profile, level 4.2, no B-frames, including the libx264 fallback), and native QML keyboard/layout behavior with simulated sources. They run offscreen and do not activate a camera or microphone. The capture-independent backend uses an injected file picker for save and recovery tests. The QML tests write inspection screenshots to `/tmp/monologue-ui-*.png`. +Tests cover format ranking, camera JPEG decoding, pause timing, PCM levels, atomic saving, the edit model, undo, reopening an interrupted take, take deletion and locking, filmstrip generation, edited exports (duration and content on both sides of a cut), the first frame showing on entering the editor, theme file changes and symlink swaps, actual H.264/AAC encoding and decoding (High profile, level 4.2, no B-frames, including the libx264 fallback), a skipped bad JPEG, and native QML keyboard/layout behavior with simulated sources. They run offscreen and do not activate a camera or microphone. The capture-independent backend uses an injected file picker for save and recovery tests. The QML tests write inspection screenshots to `/tmp/monologue-ui-*.png`. -The recording engine takes native camera frames and one shared microphone PCM stream and encodes them with FFmpeg on a worker thread, using the GPU's VAAPI H.264 encoder when available (libx264 at its `veryfast` preset otherwise), so 4K at 30 fps is sustained. Webcams deliver 4K30 as JPEG; each frame is decoded with libjpeg-turbo straight to 8-bit 4:2:0. Recordings are H.264 High profile with no B-frames, level 5.1 at 4K and level 4.2 otherwise, so any player opens them. Microphone timestamps account for the audio server's measured capture latency, including buffered samples. The writer removes paused intervals by capture time and accepts delayed samples from before Pause or Finish. Finish briefly waits for those samples before closing the recording. The audio encoder counts samples, so the writer pads genuine capture gaps and trims overlaps to maintain the common timeline. Preview and metering remain live while paused. Finish remuxes the MP4 without re-encoding to normalize packet durations and put playback metadata at the front of the file. +The recording engine takes native camera frames and one shared microphone PCM stream and encodes them with FFmpeg on a worker thread, using the GPU's VAAPI H.264 encoder when available (libx264 at its `veryfast` preset otherwise), so 4K at 30 fps is sustained. Webcams deliver 4K30 as JPEG. Each frame's bytes are copied on the capture thread and decoded with libjpeg-turbo, on at most two threads, straight to 8-bit 4:2:0. One bad JPEG is skipped. A run of them, about half a second, stops the take. Recordings are H.264 High profile with no B-frames, level 5.1 at 4K and level 4.2 otherwise, so any player opens them. Microphone timestamps account for the audio server's measured capture latency, including buffered samples. The writer removes paused intervals by capture time and accepts delayed samples from before Pause or Finish. Finish briefly waits for those samples before closing the recording. The audio encoder counts samples, so the writer pads genuine capture gaps and trims overlaps to maintain the common timeline. Preview and metering remain live while paused. Finish remuxes the MP4 without re-encoding to normalize packet durations and put playback metadata at the front of the file. A synchronization regression test encodes and decodes matching flashes and audio clicks with delayed delivery, including events just before Pause and Finish. It checks event alignment within 10 ms, rather than only comparing stream durations. diff --git a/src/encoder.cpp b/src/encoder.cpp index 52e093d..f34561a 100644 --- a/src/encoder.cpp +++ b/src/encoder.cpp @@ -152,6 +152,7 @@ QString Encoder::openVideo(const Settings &settings, bool hardware) { m_videoStream->avg_frame_rate = rate; avcodec_parameters_from_context(m_videoStream->codecpar, m_video); m_videoEncoderName = codec->name; + av_dict_set(&m_videoStream->metadata, "encoder", m_videoEncoderName.toUtf8().constData(), 0); return {}; } QString Encoder::openAudio(const QAudioFormat &format) { @@ -268,6 +269,7 @@ QString Encoder::encodeVideo(const QVideoFrame &source) { // Frame times are already quantized to the output rate. m_software->pts = av_rescale_q_rnd(frame.startTime(), {1, 1000000}, m_video->time_base, AVRounding(AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX)); + m_videoStarted = true; if (!m_device) return send(m_video, m_videoStream, m_software); AVFrame *upload = av_frame_alloc(); if ((result = av_hwframe_get_buffer(m_video->hw_frames_ctx, upload, 0)) < 0 || @@ -331,7 +333,8 @@ QString Encoder::send(AVCodecContext *codec, AVStream *stream, AVFrame *frame) { } QString Encoder::close() { QString error; - if (m_video) error = send(m_video, m_videoStream, nullptr); + // VAAPI faults if flushed before any frame. A take can stop on bad JPEGs first. + if (m_video && m_videoStarted) error = send(m_video, m_videoStream, nullptr); if (m_audio) { const auto audio = send(m_audio, m_audioStream, nullptr); if (error.isEmpty()) error = audio; } if (m_headerWritten) { const int result = av_write_trailer(m_format); diff --git a/src/encoder.h b/src/encoder.h index 500d560..b2b6219 100644 --- a/src/encoder.h +++ b/src/encoder.h @@ -65,7 +65,7 @@ class Encoder { QAudioFormat m_audioFormat; QString m_videoEncoderName; qint64 m_audioSamples = 0; - bool m_headerWritten = false; + bool m_headerWritten = false, m_videoStarted = false; std::thread m_thread; std::mutex m_mutex; std::condition_variable m_wake; diff --git a/src/writer.cpp b/src/writer.cpp index 5b4ecc7..feee175 100644 --- a/src/writer.cpp +++ b/src/writer.cpp @@ -75,6 +75,7 @@ QList TakeClock::joins() const { } Writer::Writer(QObject *parent) : QObject(parent) { + m_decodePool.setMaxThreadCount(2); m_flushTimeout.setSingleShot(true); m_flushTimeout.setInterval(10000); connect(&m_flushTimeout, &QTimer::timeout, this, [this] { @@ -132,15 +133,23 @@ void Writer::videoAt(QVideoFrame frame, qint64 position) { drain(); return; } - auto decoded = QtConcurrent::run([frame, stamp]() mutable { - if (!frame.map(QVideoFrame::ReadOnly)) return QVideoFrame(); - auto result = media::decodeJpeg(QByteArrayView(frame.bits(0), frame.mappedBytes(0))); - frame.unmap(); + // Copy the compressed bytes here. The worker must not map the preview's frame. + if (!frame.map(QVideoFrame::ReadOnly)) { noteBadFrame(); return; } + const QByteArray jpeg(reinterpret_cast(frame.bits(0)), frame.mappedBytes(0)); + frame.unmap(); + if (jpeg.isEmpty()) { noteBadFrame(); return; } + auto decoded = QtConcurrent::run(&m_decodePool, [jpeg, stamp] { + auto result = media::decodeJpeg(QByteArrayView(jpeg)); return result.isValid() ? stamp(result) : result; }); m_frames.enqueue(decoded); decoded.then(this, [this](const QVideoFrame &) { drain(); }); } +bool Writer::noteBadFrame() { + if (++m_badDecodes < 15) return false; + fail("A camera frame could not be decoded, so the take was stopped."); + return true; +} void Writer::audio(QByteArray data, qint64 capturedAt) { if (m_stopping || m_failed || !m_audioFormat.isValid() || data.isEmpty()) return; const qint64 end = capturedAt + m_audioFormat.durationForBytes(data.size()); @@ -185,7 +194,11 @@ void Writer::drain() { if (m_failed || m_closing) return; while (!m_frames.isEmpty() && m_frames.head().isFinished()) { const auto frame = m_frames.dequeue().result(); - if (!frame.isValid()) { fail("A camera frame could not be decoded, so the take was stopped."); return; } + if (!frame.isValid()) { + if (noteBadFrame()) return; + continue; + } + m_badDecodes = 0; m_encoder->video(frame); } if (m_stopping && m_frames.isEmpty()) close(false); diff --git a/src/writer.h b/src/writer.h index 1b7acec..f522de0 100644 --- a/src/writer.h +++ b/src/writer.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -56,14 +57,21 @@ class Writer : public QObject { void appendAudio(const QByteArray &data); void padAudioTo(qint64 time); void videoAt(QVideoFrame frame, qint64 position); + // A single bad JPEG is skipped. Several in a row stop the take. + bool noteBadFrame(); std::unique_ptr m_encoder; QAudioFormat m_audioFormat; + // At most two 4K decodes at once, so the pool cannot occupy every core. + // Declared before the queue: the queue is destroyed first, then the pool + // waits for a decode that is still running. + QThreadPool m_decodePool; // In capture order; JPEG frames resolve once decoded off the GUI thread. QQueue> m_frames; QVideoFrame m_tailFrame; QTimer m_flushTimeout; TakeClock m_clock; qint64 m_lastVideo = -1; + int m_badDecodes = 0; qint64 m_frameDuration = 33333; double m_fps = 30; qint64 m_audioFramesWritten = 0; diff --git a/tests/backend_tests.cpp b/tests/backend_tests.cpp index 2767c23..4658dd6 100644 --- a/tests/backend_tests.cpp +++ b/tests/backend_tests.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -18,6 +19,41 @@ #include "writer.h" #include "backend.h" +namespace { +// The plane size is the JPEG size. A frame created from QVideoFrameFormat +// alone can report a larger mapped buffer than the compressed bytes. +class JpegBytes : public QAbstractVideoBuffer { +public: + JpegBytes(QByteArray bytes, QSize size) : m_bytes(std::move(bytes)), m_format(size, QVideoFrameFormat::Format_Jpeg) {} + QVideoFrameFormat format() const override { return m_format; } + MapData map(QVideoFrame::MapMode) override { + MapData result; + result.planeCount = 1; + result.data[0] = reinterpret_cast(m_bytes.data()); + result.bytesPerLine[0] = int(m_bytes.size()); + result.dataSize[0] = int(m_bytes.size()); + return result; + } + void unmap() override {} +private: + QByteArray m_bytes; + QVideoFrameFormat m_format; +}; +QVideoFrame jpegFrame(const QByteArray &bytes) { + return QVideoFrame(std::make_unique(bytes, QSize(64, 32))); +} +QByteArray sampleJpeg() { + QImage image(64, 32, QImage::Format_RGB888); image.fill(Qt::red); + tjhandle tj = tj3Init(TJINIT_COMPRESS); + tj3Set(tj, TJPARAM_SUBSAMP, TJSAMP_420); tj3Set(tj, TJPARAM_QUALITY, 90); + unsigned char *jpeg = nullptr; size_t size = 0; + const int rc = tj3Compress8(tj, image.constBits(), 64, int(image.bytesPerLine()), 32, TJPF_RGB, &jpeg, &size); + QByteArray bytes; if (rc == 0) bytes = QByteArray(reinterpret_cast(jpeg), qsizetype(size)); + tj3Free(jpeg); tj3Destroy(tj); + return bytes; +} +} + class FakePicker : public FilePicker { public: QUrl suggestion; @@ -204,8 +240,8 @@ private slots: // Standard 8-bit 4:2:0 High profile, which every player decodes. QCOMPARE(stream["pix_fmt"].toString(),QString("yuv420p")); QCOMPARE(stream["profile"].toString(),QString("High")); - QCOMPARE(stream["level"].toString().toInt(),42); - QCOMPARE(stream["has_b_frames"].toString().toInt(),0); + QCOMPARE(stream["level"].toInt(),42); + QCOMPARE(stream["has_b_frames"].toInt(),0); } } if(sound) { @@ -432,6 +468,33 @@ private slots: QVERIFY2(std::abs(flashes[i]-clicks[i])<.01,qPrintable(QString("Event %1: video %2, audio %3").arg(i).arg(flashes[i]).arg(clicks[i]))); } } + void badJpegIsSkipped() { + const auto good = sampleJpeg(); + QVERIFY(!good.isEmpty()); + QTemporaryDir directory; + const auto path = directory.filePath("skip.mp4"); + { + Writer writer; QSignalSpy ended(&writer, &Writer::finished), errors(&writer, &Writer::failed); + QVERIFY(writer.start(path, QVideoFrameFormat(QSize(64, 32), QVideoFrameFormat::Format_YUV420P), 30, {}, 0)); + writer.video(jpegFrame(QByteArray("not a jpeg")), 0); + for (int i = 1; i <= 4; ++i) writer.video(jpegFrame(good), qRound64(i * 1000000.0 / 30)); + writer.finish(); QTRY_VERIFY_WITH_TIMEOUT(!ended.isEmpty(), 15000); + QVERIFY2(errors.isEmpty(), errors.isEmpty() ? "" : qPrintable(errors.first().first().toString())); + QProcess streams; streams.start("ffprobe", {"-v", "error", "-select_streams", "v:0", "-show_entries", "stream=nb_frames", "-of", "csv=p=0", path}); + QVERIFY(streams.waitForFinished()); + QCOMPARE(streams.readAllStandardOutput().trimmed().toInt(), 4); + } + Writer writer; QSignalSpy ended(&writer, &Writer::finished), errors(&writer, &Writer::failed); + QVERIFY(writer.start(directory.filePath("stop.mp4"), QVideoFrameFormat(QSize(64, 32), QVideoFrameFormat::Format_YUV420P), 30, {}, 0)); + for (int i = 0; i < 15; ++i) { + // Let each undecodable frame leave the eight-frame queue before the next. + writer.video(jpegFrame(QByteArray("not a jpeg")), qRound64(i * 1000000.0 / 30)); + QTest::qWait(50); + } + QTRY_COMPARE_WITH_TIMEOUT(errors.size(), 1, 5000); + QCOMPARE(errors.first().first().toString(), QString("A camera frame could not be decoded, so the take was stopped.")); + QTRY_VERIFY_WITH_TIMEOUT(!ended.isEmpty(), 5000); + } void softwareEncoderMatches() { // A missing VAAPI driver must fall back to libx264 with the same constraints. struct Driver { @@ -452,16 +515,16 @@ private slots: } writer.finish(); QTRY_VERIFY_WITH_TIMEOUT(!ended.isEmpty(), 15000); QVERIFY2(errors.isEmpty(), errors.isEmpty() ? "" : qPrintable(errors.first().first().toString())); - QProcess streams; streams.start("ffprobe", {"-v", "error", "-select_streams", "v:0", "-show_entries", "stream=pix_fmt,profile,level,has_b_frames,nb_frames:stream_tags=encoder", "-of", "json", path}); + QProcess streams; streams.start("ffprobe", {"-v", "error", "-select_streams", "v:0", "-show_streams", "-of", "json", path}); QVERIFY(streams.waitForFinished()); - const auto stream = QJsonDocument::fromJson(streams.readAllStandardOutput()).object()["streams"].toArray().first().toObject(); + const auto output = streams.readAllStandardOutput(); + const auto stream = QJsonDocument::fromJson(output).object()["streams"].toArray().first().toObject(); QCOMPARE(stream["pix_fmt"].toString(), QString("yuv420p")); QCOMPARE(stream["profile"].toString(), QString("High")); - QCOMPARE(stream["level"].toString().toInt(), 42); - QCOMPARE(stream["has_b_frames"].toString().toInt(), 0); + QCOMPARE(stream["level"].toInt(), 42); + QCOMPARE(stream["has_b_frames"].toInt(), 0); QCOMPARE(stream["nb_frames"].toString().toInt(), 10); - const auto encoder = stream["tags"].toObject()["encoder"].toString(); - QVERIFY2(encoder.contains("libx264"), qPrintable(encoder)); + QVERIFY2(stream["tags"].toObject()["encoder"].toString().contains("libx264"), output.constData()); } }; int main(int argc,char**argv) {