diff --git a/README.md b/README.md index 1168e5c..bcdb7f2 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ A simple webcam recorder for Omarchy. Choose your camera and microphone once, th ## Build and run -Install a C++17 compiler, `make`, `qt6-base`, `qt6-declarative`, `qt6-multimedia` (Qt 6.8 or newer), `libpulse`, and `ffmpeg`. Microphone capture requires a PulseAudio-compatible server, such as PipeWire-Pulse on Omarchy. Then: +Install a C++17 compiler, `make`, `qt6-base`, `qt6-declarative`, `qt6-multimedia` (Qt 6.8 or newer), `libpulse`, `libjpeg-turbo`, and `ffmpeg`. Microphone capture requires a PulseAudio-compatible server, such as PipeWire-Pulse on Omarchy. Then: ```sh ./bin/build @@ -75,9 +75,9 @@ A disconnected source or encoder error stops the take and preserves its files. A ./bin/test ``` -Tests cover format ranking, pause timing, PCM levels, atomic saving, the edit model, undo, reopening an interrupted take, take deletion and locking, filmstrip generation, edited exports (duration and content on both sides of a cut), the first frame showing on entering the editor, theme file changes and symlink swaps, actual H.264/AAC encoding and decoding, and native QML keyboard/layout behavior with simulated sources. They run offscreen and do not activate a camera or microphone. The capture-independent backend uses an injected file picker for save and recovery tests. The QML tests write inspection screenshots to `/tmp/monologue-ui-*.png`. +Tests cover format ranking, camera JPEG decoding, pause timing, PCM levels, atomic saving, the edit model, undo, reopening an interrupted take, take deletion and locking, filmstrip generation, edited exports (duration and content on both sides of a cut), the first frame showing on entering the editor, theme file changes and symlink swaps, actual H.264/AAC encoding and decoding (High profile, level 4.2, no B-frames, including the libx264 fallback), a skipped bad JPEG, and native QML keyboard/layout behavior with simulated sources. They run offscreen and do not activate a camera or microphone. The capture-independent backend uses an injected file picker for save and recovery tests. The QML tests write inspection screenshots to `/tmp/monologue-ui-*.png`. -The recording engine forwards native camera frames and one shared microphone PCM stream to timestamped Qt inputs. Microphone timestamps account for the audio server's measured capture latency, including buffered samples. The writer removes paused intervals by capture time and accepts delayed samples from before Pause or Finish. Finish briefly waits for those samples before closing the recording. Qt's audio encoder counts samples, so the writer pads genuine capture gaps and trims overlaps to maintain the common timeline. Preview and metering remain live while paused. Finish remuxes the MP4 without re-encoding to normalize packet durations and put playback metadata at the front of the file. +The recording engine takes native camera frames and one shared microphone PCM stream and encodes them with FFmpeg on a worker thread, using the GPU's VAAPI H.264 encoder when available (libx264 at its `veryfast` preset otherwise), so 4K at 30 fps is sustained. Webcams deliver 4K30 as JPEG. Each frame's bytes are copied on the capture thread and decoded with libjpeg-turbo, on at most two threads, straight to 8-bit 4:2:0. One bad JPEG is skipped. A run of them, about half a second, stops the take. Recordings are H.264 High profile with no B-frames, level 5.1 at 4K and level 4.2 otherwise, so any player opens them. Microphone timestamps account for the audio server's measured capture latency, including buffered samples. The writer removes paused intervals by capture time and accepts delayed samples from before Pause or Finish. Finish briefly waits for those samples before closing the recording. The audio encoder counts samples, so the writer pads genuine capture gaps and trims overlaps to maintain the common timeline. Preview and metering remain live while paused. Finish remuxes the MP4 without re-encoding to normalize packet durations and put playback metadata at the front of the file. A synchronization regression test encodes and decodes matching flashes and audio clicks with delayed delivery, including events just before Pause and Finish. It checks event alignment within 10 ms, rather than only comparing stream durations. diff --git a/pkgbuild/PKGBUILD b/pkgbuild/PKGBUILD index a971ac4..bb01621 100644 --- a/pkgbuild/PKGBUILD +++ b/pkgbuild/PKGBUILD @@ -5,7 +5,7 @@ pkgrel=1 pkgdesc='A simple, theme-synced webcam recorder for Omarchy' arch=('x86_64' 'aarch64') license=('MIT') -depends=('ffmpeg' 'libpulse' 'qt6-base' 'qt6-declarative' 'qt6-multimedia>=6.8' 'xdg-desktop-portal') +depends=('ffmpeg' 'libjpeg-turbo' 'libpulse' 'qt6-base' 'qt6-declarative' 'qt6-multimedia>=6.8' 'xdg-desktop-portal') makedepends=('gcc' 'make') source=() sha256sums=() diff --git a/src/common.pri b/src/common.pri index f7936b5..8140c55 100644 --- a/src/common.pri +++ b/src/common.pri @@ -1,8 +1,8 @@ CONFIG += link_pkgconfig -PKGCONFIG += libpulse -HEADERS += $$PWD/theme.h $$PWD/mediautils.h $$PWD/writer.h $$PWD/backend.h \ +PKGCONFIG += libpulse libturbojpeg libavcodec libavformat libavutil libswscale libswresample +HEADERS += $$PWD/theme.h $$PWD/mediautils.h $$PWD/encoder.h $$PWD/writer.h $$PWD/backend.h \ $$PWD/filepicker.h $$PWD/portalfilepicker.h $$PWD/audiocapture.h \ $$PWD/edit.h $$PWD/thumbnails.h -SOURCES += $$PWD/theme.cpp $$PWD/mediautils.cpp $$PWD/writer.cpp $$PWD/backend.cpp \ +SOURCES += $$PWD/theme.cpp $$PWD/mediautils.cpp $$PWD/encoder.cpp $$PWD/writer.cpp $$PWD/backend.cpp \ $$PWD/portalfilepicker.cpp $$PWD/audiocapture.cpp \ $$PWD/edit.cpp $$PWD/thumbnails.cpp diff --git a/src/encoder.cpp b/src/encoder.cpp new file mode 100644 index 0000000..f34561a --- /dev/null +++ b/src/encoder.cpp @@ -0,0 +1,366 @@ +#include "encoder.h" +#include +extern "C" { +#include +#include +#include +#include +#include +#include +#include +#include +} + +namespace { +QString failure(const char *what, int code) { + char reason[AV_ERROR_MAX_STRING_SIZE] = {}; + av_strerror(code, reason, sizeof reason); + return QString("%1: %2").arg(what, reason); +} +AVPixelFormat pixelFormat(QVideoFrameFormat::PixelFormat format) { + switch (format) { + case QVideoFrameFormat::Format_YUV420P: return AV_PIX_FMT_YUV420P; + case QVideoFrameFormat::Format_YUV422P: return AV_PIX_FMT_YUV422P; + case QVideoFrameFormat::Format_NV12: return AV_PIX_FMT_NV12; + case QVideoFrameFormat::Format_YUYV: return AV_PIX_FMT_YUYV422; + case QVideoFrameFormat::Format_UYVY: return AV_PIX_FMT_UYVY422; + case QVideoFrameFormat::Format_BGRA8888: return AV_PIX_FMT_BGRA; + case QVideoFrameFormat::Format_BGRX8888: return AV_PIX_FMT_BGR0; + case QVideoFrameFormat::Format_RGBA8888: return AV_PIX_FMT_RGBA; + case QVideoFrameFormat::Format_RGBX8888: return AV_PIX_FMT_RGB0; + case QVideoFrameFormat::Format_ARGB8888: return AV_PIX_FMT_ARGB; + case QVideoFrameFormat::Format_XRGB8888: return AV_PIX_FMT_0RGB; + case QVideoFrameFormat::Format_ABGR8888: return AV_PIX_FMT_ABGR; + case QVideoFrameFormat::Format_XBGR8888: return AV_PIX_FMT_0BGR; + default: return AV_PIX_FMT_NONE; + } +} +AVSampleFormat sampleFormat(QAudioFormat::SampleFormat format) { + switch (format) { + case QAudioFormat::UInt8: return AV_SAMPLE_FMT_U8; + case QAudioFormat::Int16: return AV_SAMPLE_FMT_S16; + case QAudioFormat::Int32: return AV_SAMPLE_FMT_S32; + case QAudioFormat::Float: return AV_SAMPLE_FMT_FLT; + default: return AV_SAMPLE_FMT_NONE; + } +} +} + +Encoder::~Encoder() { + { + std::lock_guard lock(m_mutex); + m_finishing = m_discard = true; + m_done = nullptr; m_failed = nullptr; + } + m_wake.notify_one(); + if (m_thread.joinable()) m_thread.join(); + release(); +} +bool Encoder::supported(bool audio) { + return av_guess_format("mp4", nullptr, nullptr) && + (avcodec_find_encoder_by_name("h264_vaapi") || avcodec_find_encoder_by_name("libx264")) && + (!audio || avcodec_find_encoder(AV_CODEC_ID_AAC)); +} +QString Encoder::open(const Settings &settings, std::function failed) { + m_failed = std::move(failed); + m_audioFormat = settings.audio; + const auto path = settings.path.toUtf8(); + int result = avformat_alloc_output_context2(&m_format, nullptr, "mp4", path.constData()); + if (result < 0) return failure("Could not create the MP4", result); + auto error = openVideo(settings, true); + if (!error.isEmpty()) { + releaseVideo(); + error = openVideo(settings, false); + } + if (error.isEmpty() && settings.audio.isValid()) error = openAudio(settings.audio); + if (error.isEmpty() && (result = avio_open(&m_format->pb, path.constData(), AVIO_FLAG_WRITE)) < 0) + error = failure("Could not open the recording file", result); + if (error.isEmpty() && (result = avformat_write_header(m_format, nullptr)) < 0) + error = failure("Could not start the MP4", result); + if (!error.isEmpty()) { release(); return error; } + m_headerWritten = true; + m_thread = std::thread(&Encoder::run, this); + return {}; +} +QString Encoder::openVideo(const Settings &settings, bool hardware) { + const AVCodec *codec = avcodec_find_encoder_by_name(hardware ? "h264_vaapi" : "libx264"); + if (!codec) return "No H.264 encoder is available."; + int result = 0; + if (hardware && (result = av_hwdevice_ctx_create(&m_device, AV_HWDEVICE_TYPE_VAAPI, nullptr, nullptr, 0)) < 0) + return failure("No VAAPI device", result); + const AVRational rate = av_d2q(settings.fps, 1001000); + m_video = avcodec_alloc_context3(codec); + m_video->width = settings.size.width(); + m_video->height = settings.size.height(); + m_video->framerate = rate; + m_video->time_base = av_inv_q(rate); + // Keyframes every two seconds keep the editor's seeking quick. + m_video->gop_size = std::max(1, int(settings.fps * 2)); + // No B-frames, so the libx264 fallback has the same low-delay timestamps as VAAPI. + m_video->max_b_frames = 0; + // 5.1 covers 4K30. 4.2 is enough at 1080p and below. + m_video->level = settings.size.height() > 1080 ? 51 : 42; + m_video->profile = AV_PROFILE_H264_HIGH; + m_video->color_range = AVCOL_RANGE_MPEG; + m_video->colorspace = AVCOL_SPC_BT470BG; + if (m_format->oformat->flags & AVFMT_GLOBALHEADER) m_video->flags |= AV_CODEC_FLAG_GLOBAL_HEADER; + AVDictionary *options = nullptr; + const AVPixelFormat software = hardware ? AV_PIX_FMT_NV12 : AV_PIX_FMT_YUV420P; + if (hardware) { + AVBufferRef *frames = av_hwframe_ctx_alloc(m_device); + auto *context = reinterpret_cast(frames->data); + context->format = AV_PIX_FMT_VAAPI; + context->sw_format = software; + context->width = m_video->width; + context->height = m_video->height; + context->initial_pool_size = 20; + if ((result = av_hwframe_ctx_init(frames)) < 0) { + av_buffer_unref(&frames); + avcodec_free_context(&m_video); av_buffer_unref(&m_device); + return failure("Could not allocate VAAPI frames", result); + } + m_video->hw_frames_ctx = frames; + m_video->pix_fmt = AV_PIX_FMT_VAAPI; + // Constant QP 20 is close to the CRF 19 libx264 quality Qt used. + m_video->global_quality = 20; + av_dict_set(&options, "rc_mode", "CQP", 0); + } else { + m_video->pix_fmt = software; + av_dict_set(&options, "preset", "veryfast", 0); + av_dict_set(&options, "crf", "19", 0); + } + result = avcodec_open2(m_video, codec, &options); + av_dict_free(&options); + if (result < 0) { + avcodec_free_context(&m_video); av_buffer_unref(&m_device); + return failure("Could not open the H.264 encoder", result); + } + m_software = av_frame_alloc(); + m_software->format = software; + m_software->width = m_video->width; + m_software->height = m_video->height; + if ((result = av_frame_get_buffer(m_software, 0)) < 0) { + releaseVideo(); + return failure("Could not allocate a video frame", result); + } + m_videoStream = avformat_new_stream(m_format, nullptr); + if (!m_videoStream) { + releaseVideo(); + return "Could not add the video stream."; + } + m_videoStream->time_base = m_video->time_base; + m_videoStream->avg_frame_rate = rate; + avcodec_parameters_from_context(m_videoStream->codecpar, m_video); + m_videoEncoderName = codec->name; + av_dict_set(&m_videoStream->metadata, "encoder", m_videoEncoderName.toUtf8().constData(), 0); + return {}; +} +QString Encoder::openAudio(const QAudioFormat &format) { + const AVCodec *codec = avcodec_find_encoder(AV_CODEC_ID_AAC); + const AVSampleFormat input = sampleFormat(format.sampleFormat()); + if (!codec || input == AV_SAMPLE_FMT_NONE) return "No AAC encoder is available for this microphone format."; + m_audio = avcodec_alloc_context3(codec); + m_audio->sample_fmt = AV_SAMPLE_FMT_FLTP; + m_audio->sample_rate = format.sampleRate(); + av_channel_layout_default(&m_audio->ch_layout, format.channelCount()); + m_audio->bit_rate = 192000; + m_audio->time_base = {1, format.sampleRate()}; + if (m_format->oformat->flags & AVFMT_GLOBALHEADER) m_audio->flags |= AV_CODEC_FLAG_GLOBAL_HEADER; + int result = avcodec_open2(m_audio, codec, nullptr); + if (result < 0) return failure("Could not open the AAC encoder", result); + if ((result = swr_alloc_set_opts2(&m_resample, &m_audio->ch_layout, AV_SAMPLE_FMT_FLTP, m_audio->sample_rate, + &m_audio->ch_layout, input, m_audio->sample_rate, 0, nullptr)) < 0 || + (result = swr_init(m_resample)) < 0) + return failure("Could not convert the microphone audio", result); + m_fifo = av_audio_fifo_alloc(AV_SAMPLE_FMT_FLTP, format.channelCount(), std::max(1, m_audio->frame_size)); + m_audioStream = avformat_new_stream(m_format, nullptr); + m_audioStream->time_base = m_audio->time_base; + avcodec_parameters_from_context(m_audioStream->codecpar, m_audio); + return {}; +} +void Encoder::video(const QVideoFrame &frame) { + { + std::lock_guard lock(m_mutex); + if (m_finishing) return; + m_jobs.push_back({frame, {}}); + ++m_queuedFrames; + } + m_wake.notify_one(); +} +void Encoder::audio(const QByteArray &pcm) { + { + std::lock_guard lock(m_mutex); + if (m_finishing || !m_audio) return; + m_jobs.push_back({{}, pcm}); + m_queuedAudioUs += m_audioFormat.durationForBytes(pcm.size()); + } + m_wake.notify_one(); +} +void Encoder::finish(bool discard, std::function done) { + { + std::lock_guard lock(m_mutex); + m_finishing = true; + m_discard = m_discard || discard; + m_done = std::move(done); + } + m_wake.notify_one(); +} +void Encoder::run() { + QString error; + for (;;) { + Job job; + { + std::unique_lock lock(m_mutex); + m_wake.wait(lock, [this] { return !m_jobs.empty() || m_finishing; }); + if (m_discard) { m_jobs.clear(); m_queuedFrames = 0; m_queuedAudioUs = 0; } + if (m_jobs.empty()) break; + job = std::move(m_jobs.front()); + m_jobs.pop_front(); + } + if (error.isEmpty()) { + error = job.frame.isValid() ? encodeVideo(job.frame) : encodeAudio(job.pcm, false); + if (!error.isEmpty()) { + std::lock_guard lock(m_mutex); + if (m_failed) m_failed(error); + } + } + if (job.frame.isValid()) --m_queuedFrames; + else m_queuedAudioUs -= m_audioFormat.durationForBytes(job.pcm.size()); + } + if (error.isEmpty() && m_audio) error = encodeAudio({}, true); + const auto closing = close(); + if (error.isEmpty()) error = closing; + // Hold the lock so the owner cannot be destroyed while it is notified. + std::lock_guard lock(m_mutex); + if (m_done) m_done(error); +} +QString Encoder::encodeVideo(const QVideoFrame &source) { + QVideoFrame frame = source; + AVPixelFormat format = pixelFormat(frame.pixelFormat()); + const uint8_t *planes[4] = {}; + int strides[4] = {}; + QImage image; + if (format == AV_PIX_FMT_NONE) { + image = frame.toImage().convertToFormat(QImage::Format_RGB32); + if (image.isNull()) return "A camera frame could not be converted for encoding."; + format = AV_PIX_FMT_BGR0; + planes[0] = image.constBits(); strides[0] = int(image.bytesPerLine()); + } else { + if (!frame.map(QVideoFrame::ReadOnly)) return "A camera frame could not be read for encoding."; + for (int i = 0; i < frame.planeCount(); ++i) { planes[i] = frame.bits(i); strides[i] = frame.bytesPerLine(i); } + } + const QSize size = image.isNull() ? frame.size() : image.size(); + const bool sameSize = size.width() == m_video->width && size.height() == m_video->height; + const bool sameFormat = format == AVPixelFormat(m_software->format); + int result = av_frame_make_writable(m_software); + if (result >= 0 && sameSize && sameFormat) { + const uint8_t *sourcePlanes[4] = {planes[0], planes[1], planes[2], planes[3]}; + av_image_copy(m_software->data, m_software->linesize, sourcePlanes, strides, format, size.width(), size.height()); + } else if (result >= 0) { + // Same-size conversion is a shuffle (YUV420P to NV12), not a resample. + m_scale = sws_getCachedContext(m_scale, size.width(), size.height(), format, + m_video->width, m_video->height, AVPixelFormat(m_software->format), + sameSize ? SWS_POINT : SWS_BILINEAR, nullptr, nullptr, nullptr); + if (!m_scale || sws_scale(m_scale, planes, strides, 0, size.height(), m_software->data, m_software->linesize) <= 0) + result = AVERROR(EINVAL); + } + if (image.isNull()) frame.unmap(); + if (result < 0) return failure("Could not convert a camera frame", result); + // Frame times are already quantized to the output rate. + m_software->pts = av_rescale_q_rnd(frame.startTime(), {1, 1000000}, m_video->time_base, + AVRounding(AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX)); + m_videoStarted = true; + if (!m_device) return send(m_video, m_videoStream, m_software); + AVFrame *upload = av_frame_alloc(); + if ((result = av_hwframe_get_buffer(m_video->hw_frames_ctx, upload, 0)) < 0 || + (result = av_hwframe_transfer_data(upload, m_software, 0)) < 0) { + av_frame_free(&upload); + return failure("Could not upload a frame to the GPU", result); + } + upload->pts = m_software->pts; + const auto error = send(m_video, m_videoStream, upload); + av_frame_free(&upload); + return error; +} +QString Encoder::encodeAudio(const QByteArray &pcm, bool flush) { + const int channels = m_audio->ch_layout.nb_channels; + if (!pcm.isEmpty()) { + const int samples = m_audioFormat.framesForBytes(pcm.size()); + uint8_t **converted = nullptr; + int result = av_samples_alloc_array_and_samples(&converted, nullptr, channels, samples, AV_SAMPLE_FMT_FLTP, 0); + if (result < 0) return failure("Could not convert microphone audio", result); + const uint8_t *input[1] = {reinterpret_cast(pcm.constData())}; + result = swr_convert(m_resample, converted, samples, input, samples); + if (result > 0) result = av_audio_fifo_write(m_fifo, reinterpret_cast(converted), result); + av_freep(&converted[0]); av_freep(&converted); + if (result < 0) return failure("Could not convert microphone audio", result); + } + const int frameSize = m_audio->frame_size > 0 ? m_audio->frame_size : 1024; + while (av_audio_fifo_size(m_fifo) >= frameSize || (flush && av_audio_fifo_size(m_fifo) > 0)) { + const int available = std::min(frameSize, av_audio_fifo_size(m_fifo)); + const bool padded = available < frameSize && !(m_audio->codec->capabilities & AV_CODEC_CAP_SMALL_LAST_FRAME); + AVFrame *frame = av_frame_alloc(); + frame->nb_samples = padded ? frameSize : available; + frame->format = AV_SAMPLE_FMT_FLTP; + frame->sample_rate = m_audio->sample_rate; + av_channel_layout_copy(&frame->ch_layout, &m_audio->ch_layout); + int result = av_frame_get_buffer(frame, 0); + if (result >= 0) { + if (padded) av_samples_set_silence(frame->data, 0, frameSize, channels, AV_SAMPLE_FMT_FLTP); + result = av_audio_fifo_read(m_fifo, reinterpret_cast(frame->data), available); + } + if (result < 0) { av_frame_free(&frame); return failure("Could not buffer microphone audio", result); } + frame->pts = m_audioSamples; + m_audioSamples += frame->nb_samples; + const auto error = send(m_audio, m_audioStream, frame); + av_frame_free(&frame); + if (!error.isEmpty()) return error; + } + return {}; +} +QString Encoder::send(AVCodecContext *codec, AVStream *stream, AVFrame *frame) { + int result = avcodec_send_frame(codec, frame); + if (result < 0) return failure("Encoding failed", result); + AVPacket *packet = av_packet_alloc(); + while ((result = avcodec_receive_packet(codec, packet)) >= 0) { + av_packet_rescale_ts(packet, codec->time_base, stream->time_base); + packet->stream_index = stream->index; + if ((result = av_interleaved_write_frame(m_format, packet)) < 0) break; + } + av_packet_free(&packet); + if (result == AVERROR(EAGAIN) || result == AVERROR_EOF) return {}; + return failure("Could not write the recording", result); +} +QString Encoder::close() { + QString error; + // VAAPI faults if flushed before any frame. A take can stop on bad JPEGs first. + if (m_video && m_videoStarted) error = send(m_video, m_videoStream, nullptr); + if (m_audio) { const auto audio = send(m_audio, m_audioStream, nullptr); if (error.isEmpty()) error = audio; } + if (m_headerWritten) { + const int result = av_write_trailer(m_format); + if (result < 0 && error.isEmpty()) error = failure("Could not finish the MP4", result); + m_headerWritten = false; + } + release(); + return error; +} +void Encoder::releaseVideo() { + avcodec_free_context(&m_video); + av_buffer_unref(&m_device); + av_frame_free(&m_software); +} +void Encoder::release() { + avcodec_free_context(&m_video); + avcodec_free_context(&m_audio); + av_buffer_unref(&m_device); + av_frame_free(&m_software); + sws_freeContext(m_scale); m_scale = nullptr; + swr_free(&m_resample); + if (m_fifo) { av_audio_fifo_free(m_fifo); m_fifo = nullptr; } + if (m_format) { + if (m_format->pb) avio_closep(&m_format->pb); + avformat_free_context(m_format); + m_format = nullptr; + } + m_videoStream = m_audioStream = nullptr; +} diff --git a/src/encoder.h b/src/encoder.h new file mode 100644 index 0000000..b2b6219 --- /dev/null +++ b/src/encoder.h @@ -0,0 +1,77 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct AVCodecContext; +struct AVFormatContext; +struct AVStream; +struct AVBufferRef; +struct AVFrame; +struct AVAudioFifo; +struct SwsContext; +struct SwrContext; + +// Encodes H.264/AAC into an MP4 on its own thread. Uses the GPU's VAAPI +// encoder when available, which sustains 4K30 where Qt's libx264 (always at +// its "medium" preset) cannot; otherwise libx264 at a realtime preset. +class Encoder { +public: + struct Settings { QString path; QSize size; double fps = 30; QAudioFormat audio; }; + ~Encoder(); + // Opens the file and codecs and starts the worker. Returns an error, or empty. + // failed runs on the worker thread at the first encoding error. + QString open(const Settings &settings, std::function failed); + // Queues a frame; its start time, quantized to the frame rate, sets its position. + void video(const QVideoFrame &frame); + // Queues PCM in the input format, contiguous with all earlier audio. + void audio(const QByteArray &pcm); + // Flushes and closes the file, dropping queued input first if discard is set. + // done runs on the worker thread with an error, or empty on success. + void finish(bool discard, std::function done); + int queuedFrames() const { return m_queuedFrames; } + qint64 queuedAudioUs() const { return m_queuedAudioUs; } + QString videoEncoder() const { return m_videoEncoderName; } + static bool supported(bool audio); +private: + struct Job { QVideoFrame frame; QByteArray pcm; }; + QString openVideo(const Settings &settings, bool hardware); + QString openAudio(const QAudioFormat &format); + // Frees the video codec and VAAPI device. The MP4 context stays open so a + // failed GPU open can fall back to libx264 before the header is written. + void releaseVideo(); + void run(); + QString encodeVideo(const QVideoFrame &source); + QString encodeAudio(const QByteArray &pcm, bool flush); + QString send(AVCodecContext *codec, AVStream *stream, AVFrame *frame); + QString close(); + void release(); + AVFormatContext *m_format = nullptr; + AVCodecContext *m_video = nullptr, *m_audio = nullptr; + AVStream *m_videoStream = nullptr, *m_audioStream = nullptr; + AVBufferRef *m_device = nullptr; + AVFrame *m_software = nullptr; + SwsContext *m_scale = nullptr; + SwrContext *m_resample = nullptr; + AVAudioFifo *m_fifo = nullptr; + QAudioFormat m_audioFormat; + QString m_videoEncoderName; + qint64 m_audioSamples = 0; + bool m_headerWritten = false, m_videoStarted = false; + std::thread m_thread; + std::mutex m_mutex; + std::condition_variable m_wake; + std::deque m_jobs; + bool m_finishing = false, m_discard = false; + std::function m_done, m_failed; + std::atomic m_queuedFrames{0}; + std::atomic m_queuedAudioUs{0}; +}; diff --git a/src/main.cpp b/src/main.cpp index 262c9e0..c338992 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -9,7 +9,7 @@ #include "thumbprovider.h" int main(int argc,char *argv[]) { - // Custom timestamped frame/buffer inputs require Qt's FFmpeg backend. + // Qt's FFmpeg backend delivers timestamped native camera frames. qputenv("QT_MEDIA_BACKEND","ffmpeg"); QGuiApplication app(argc,argv); app.setApplicationName("monologue"); diff --git a/src/mediautils.cpp b/src/mediautils.cpp index 190212f..dc68799 100644 --- a/src/mediautils.cpp +++ b/src/mediautils.cpp @@ -9,6 +9,9 @@ #include #include #include +#include +#include +#include double media::targetFps(double minimum, double maximum) { return std::clamp(30.0, minimum, maximum); } int media::bestFormat(const QList &formats) { @@ -35,6 +38,70 @@ QCameraFormat media::bestCameraFormat(const QCameraDevice &device) { const int index = bestFormat(choices); return index < 0 ? QCameraFormat() : formats[index]; } +QVideoFrame media::decodeJpeg(QByteArrayView jpeg) { + std::unique_ptr tj(tj3Init(TJINIT_DECOMPRESS), tj3Destroy); + const auto *data = reinterpret_cast(jpeg.data()); + if (!tj || tj3DecompressHeader(tj.get(), data, jpeg.size()) != 0) return {}; + const int width = tj3Get(tj.get(), TJPARAM_JPEGWIDTH), height = tj3Get(tj.get(), TJPARAM_JPEGHEIGHT); + const int sampling = tj3Get(tj.get(), TJPARAM_SUBSAMP), colorspace = tj3Get(tj.get(), TJPARAM_COLORSPACE); + if (width <= 0 || height <= 0 || width % 2 || height % 2 || sampling == TJSAMP_UNKNOWN || + (colorspace != TJCS_YCbCr && colorspace != TJCS_GRAY)) return {}; + // Decode to the JPEG's own planes, skipping libjpeg's RGB conversion. + const int planes = sampling == TJSAMP_GRAY ? 1 : 3; + std::vector source[3]; + unsigned char *destinations[3] = {}; + int strides[3] = {}, heights[3] = {}; + for (int i = 0; i < planes; ++i) { + strides[i] = tj3YUVPlaneWidth(i, width, sampling); + heights[i] = tj3YUVPlaneHeight(i, height, sampling); + source[i].resize(size_t(strides[i]) * heights[i]); + destinations[i] = source[i].data(); + } + if (tj3DecompressToYUVPlanes8(tj.get(), data, jpeg.size(), destinations, strides) != 0) return {}; + + QVideoFrameFormat format(QSize(width, height), QVideoFrameFormat::Format_YUV420P); + format.setColorSpace(QVideoFrameFormat::ColorSpace_BT601); + format.setColorRange(QVideoFrameFormat::ColorRange_Video); + QVideoFrame frame(format); + if (!frame.map(QVideoFrame::WriteOnly)) return {}; + // JPEG is full range; H.264 players expect video range. + unsigned char luma[256], chroma[256]; + for (int v = 0; v < 256; ++v) { luma[v] = 16 + (v * 219 + 127) / 255; chroma[v] = 16 + (v * 224 + 127) / 255; } + for (int y = 0; y < height; ++y) { + const unsigned char *in = source[0].data() + size_t(y) * strides[0]; + unsigned char *out = frame.bits(0) + qsizetype(y) * frame.bytesPerLine(0); + for (int x = 0; x < width; ++x) out[x] = luma[in[x]]; + } + // Box-filter each chroma plane, whatever its sampling, onto the 4:2:0 grid. + const int outWidth = width / 2, outHeight = height / 2; + std::vector columns(outWidth + 1); + for (int i = 1; i < 3; ++i) { + unsigned char *plane = frame.bits(i); + const int bytesPerLine = frame.bytesPerLine(i); + if (planes == 1) { + for (int y = 0; y < outHeight; ++y) std::memset(plane + qsizetype(y) * bytesPerLine, 128, outWidth); + continue; + } + for (int x = 0; x <= outWidth; ++x) columns[x] = qint64(x) * strides[i] / outWidth; + for (int y = 0; y < outHeight; ++y) { + const int top = qint64(y) * heights[i] / outHeight; + const int bottom = std::max(top + 1, int(qint64(y + 1) * heights[i] / outHeight)); + unsigned char *out = plane + qsizetype(y) * bytesPerLine; + for (int x = 0; x < outWidth; ++x) { + const int left = columns[x], right = std::max(left + 1, columns[x + 1]); + int sum = 0; + for (int row = top; row < bottom; ++row) { + const unsigned char *in = source[i].data() + size_t(row) * strides[i]; + for (int column = left; column < right; ++column) sum += in[column]; + } + const int count = (bottom - top) * (right - left); + out[x] = chroma[(sum + count / 2) / count]; + } + } + } + frame.unmap(); + return frame; +} double media::peak(const QByteArray &data, const QAudioFormat &format) { double result = 0; const int bytes = format.bytesPerSample(); diff --git a/src/mediautils.h b/src/mediautils.h index 17645d3..5045c93 100644 --- a/src/mediautils.h +++ b/src/mediautils.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include "edit.h" @@ -12,6 +13,10 @@ struct Format { QSize size; double minFps; double maxFps; int pixelFormat; }; int bestFormat(const QList &formats); QCameraFormat bestCameraFormat(const QCameraDevice &device); double targetFps(double minimum, double maximum); +// Decodes a camera JPEG to 8-bit 4:2:0 limited-range video, as delivered by +// webcams' MJPEG modes. Encoding Qt's RGB decode instead yields 10-bit 4:4:4 +// H.264, too slow to sustain 4K and unplayable in most players. +QVideoFrame decodeJpeg(QByteArrayView jpeg); double peak(const QByteArray &data, const QAudioFormat &format); QString copyAtomically(const QString &source, const QString &destination); QString normalizeMp4(const QString &path, double fps); diff --git a/src/writer.cpp b/src/writer.cpp index 3a29664..feee175 100644 --- a/src/writer.cpp +++ b/src/writer.cpp @@ -1,7 +1,7 @@ #include "writer.h" -#include -#include +#include "mediautils.h" #include +#include #include namespace { @@ -75,48 +75,27 @@ QList TakeClock::joins() const { } Writer::Writer(QObject *parent) : QObject(parent) { - m_session.setRecorder(&m_recorder); + m_decodePool.setMaxThreadCount(2); m_flushTimeout.setSingleShot(true); m_flushTimeout.setInterval(10000); - connect(&m_flushTimeout, &QTimer::timeout, this, [this] { fail("The encoder stopped responding, so the take was stopped."); }); - connect(&m_recorder, &QMediaRecorder::errorOccurred, this, [this](auto, const QString &message) { fail(message); }); - connect(&m_recorder, &QMediaRecorder::recorderStateChanged, this, [this](auto state) { - if (state == QMediaRecorder::StoppedState && m_stopping) { - m_flushTimeout.stop(); - emit finished(); - } + connect(&m_flushTimeout, &QTimer::timeout, this, [this] { + if (!m_failed) { m_failed = true; emit failed("The encoder stopped responding, so the take was stopped."); } + emitFinished(); }); } -static QMediaFormat recordingFormat(bool audio) { - QMediaFormat f(QMediaFormat::MPEG4); - f.setVideoCodec(QMediaFormat::VideoCodec::H264); - if (audio) f.setAudioCodec(QMediaFormat::AudioCodec::AAC); - return f; -} -bool Writer::supported(bool audio) { return recordingFormat(audio).isSupported(QMediaFormat::Encode); } +Writer::~Writer() = default; +bool Writer::supported(bool audio) { return Encoder::supported(audio); } bool Writer::start(const QString &path, const QVideoFrameFormat &format, double fps, const QAudioFormat &audioFormat, qint64 now) { m_clock.start(now); m_frameDuration = qRound64(1000000.0 / fps); m_fps = fps; m_audioFormat = audioFormat; - // Let the first frame initialize the encoder. Supplying a format hint can - // leave Qt 6.11's canPushFrame cache false before its worker starts. - m_video = new QVideoFrameInput(this); - m_session.setVideoFrameInput(m_video); - connect(m_video, &QVideoFrameInput::readyToSendVideoFrame, this, &Writer::drain); - if (audioFormat.isValid()) { - m_audio = new QAudioBufferInput(this); - m_session.setAudioBufferInput(m_audio); - connect(m_audio, &QAudioBufferInput::readyToSendAudioBuffer, this, &Writer::drain); - } - m_recorder.setMediaFormat(recordingFormat(audioFormat.isValid())); - m_recorder.setVideoResolution(format.frameSize()); - m_recorder.setVideoFrameRate(fps); - m_recorder.setQuality(QMediaRecorder::VeryHighQuality); - m_recorder.setAudioBitRate(192000); - m_recorder.setOutputLocation(QUrl::fromLocalFile(path)); - m_recorder.record(); + m_encoder = std::make_unique(); + const auto error = m_encoder->open({path, format.frameSize(), fps, audioFormat}, [this](const QString &message) { + QMetaObject::invokeMethod(this, [this, message] { fail(message); }, Qt::QueuedConnection); + }); + if (!error.isEmpty()) { m_encoder.reset(); fail(error); } return !m_failed; } void Writer::pause(qint64 now) { @@ -138,18 +117,41 @@ void Writer::videoAt(QVideoFrame frame, qint64 position) { const qint64 timestamp = qRound64(frameIndex * 1000000.0 / m_fps); if (timestamp <= m_lastVideo) return; m_tailFrame = frame; - frame = QVideoFrame(std::make_unique(frame)); - frame.setStartTime(timestamp); - frame.setEndTime(qRound64((frameIndex + 1) * 1000000.0 / m_fps)); - frame.setStreamFrameRate(m_fps); m_lastVideo = timestamp; // Bounded native-frame references; never build an unbounded 4K frame queue. - if (m_frames.size() >= 6) { fail("Video encoding cannot keep up at this camera's maximum resolution, so the take was stopped."); return; } - m_frames.enqueue(frame); - drain(); + if (m_frames.size() + m_encoder->queuedFrames() >= 8) { + fail("Video encoding cannot keep up at this camera's maximum resolution, so the take was stopped."); return; + } + const auto stamp = [timestamp, end = qRound64((frameIndex + 1) * 1000000.0 / m_fps), fps = m_fps](QVideoFrame frame) { + frame.setStartTime(timestamp); + frame.setEndTime(end); + frame.setStreamFrameRate(fps); + return frame; + }; + if (frame.pixelFormat() != QVideoFrameFormat::Format_Jpeg) { + m_frames.enqueue(QtFuture::makeReadyValueFuture(stamp(QVideoFrame(std::make_unique(frame))))); + drain(); + return; + } + // Copy the compressed bytes here. The worker must not map the preview's frame. + if (!frame.map(QVideoFrame::ReadOnly)) { noteBadFrame(); return; } + const QByteArray jpeg(reinterpret_cast(frame.bits(0)), frame.mappedBytes(0)); + frame.unmap(); + if (jpeg.isEmpty()) { noteBadFrame(); return; } + auto decoded = QtConcurrent::run(&m_decodePool, [jpeg, stamp] { + auto result = media::decodeJpeg(QByteArrayView(jpeg)); + return result.isValid() ? stamp(result) : result; + }); + m_frames.enqueue(decoded); + decoded.then(this, [this](const QVideoFrame &) { drain(); }); +} +bool Writer::noteBadFrame() { + if (++m_badDecodes < 15) return false; + fail("A camera frame could not be decoded, so the take was stopped."); + return true; } void Writer::audio(QByteArray data, qint64 capturedAt) { - if (m_stopping || m_failed || !m_audio || data.isEmpty()) return; + if (m_stopping || m_failed || !m_audioFormat.isValid() || data.isEmpty()) return; const qint64 end = capturedAt + m_audioFormat.durationForBytes(data.size()); const int frameBytes = m_audioFormat.bytesPerFrame(); // Select by capture time, including closed intervals. A late buffer may @@ -162,8 +164,8 @@ void Writer::audio(QByteArray data, qint64 capturedAt) { const qint64 last = sampleAtOrAfter(span.end - capturedAt); QByteArray part = data.mid(first * frameBytes, (last - first) * frameBytes); const auto position = span.position + m_audioFormat.durationForFrames(first) - (span.begin - capturedAt); - // Qt counts audio samples rather than honoring PTS. Materialize capture - // gaps as silence and trim overlaps, keeping the original event times. + // The encoder counts audio samples rather than honoring PTS. Materialize + // capture gaps as silence and trim overlaps, keeping the original event times. const qint64 desiredFrame = qRound64(position * m_audioFormat.sampleRate() / 1000000.0); if (desiredFrame < m_audioFramesWritten) { const auto overlap = (m_audioFramesWritten - desiredFrame) * frameBytes; @@ -173,7 +175,7 @@ void Writer::audio(QByteArray data, qint64 capturedAt) { } } void Writer::padAudioTo(qint64 time) { - if (!m_audio || m_failed) return; + if (!m_audioFormat.isValid() || m_failed) return; const qint64 missing = qRound64(time * m_audioFormat.sampleRate() / 1000000.0) - m_audioFramesWritten; if (missing <= 0) return; if (m_audioFormat.durationForFrames(missing) > 500000) { @@ -184,22 +186,22 @@ void Writer::padAudioTo(qint64 time) { } void Writer::appendAudio(const QByteArray &data) { if (m_failed) return; - QAudioBuffer buffer(data, m_audioFormat, m_audioFormat.durationForFrames(m_audioFramesWritten)); - m_audioFramesWritten += buffer.frameCount(); - m_queuedAudioUs += buffer.duration(); - if (m_queuedAudioUs > 500000) { fail("Audio encoding cannot keep up, so the take was stopped."); return; } - m_buffers.enqueue(buffer); - drain(); + if (m_encoder->queuedAudioUs() > 500000) { fail("Audio encoding cannot keep up, so the take was stopped."); return; } + m_audioFramesWritten += m_audioFormat.framesForBytes(data.size()); + m_encoder->audio(data); } void Writer::drain() { - if (m_draining || m_failed) return; - m_draining = true; - while (!m_frames.isEmpty() && m_video->sendVideoFrame(m_frames.head())) m_frames.dequeue(); - while (!m_buffers.isEmpty() && m_audio->sendAudioBuffer(m_buffers.head())) { - m_queuedAudioUs -= m_buffers.head().duration(); m_buffers.dequeue(); + if (m_failed || m_closing) return; + while (!m_frames.isEmpty() && m_frames.head().isFinished()) { + const auto frame = m_frames.dequeue().result(); + if (!frame.isValid()) { + if (noteBadFrame()) return; + continue; + } + m_badDecodes = 0; + m_encoder->video(frame); } - m_draining = false; - if (m_stopping && m_frames.isEmpty() && m_buffers.isEmpty()) m_recorder.stop(); + if (m_stopping && m_frames.isEmpty()) close(false); } void Writer::finish() { if (m_stopping) return; @@ -214,18 +216,30 @@ void Writer::finish() { m_stopping = true; m_flushTimeout.start(); drain(); - if (m_recorder.recorderState() == QMediaRecorder::StoppedState) { - m_flushTimeout.stop(); - QTimer::singleShot(0, this, &Writer::finished); - } +} +void Writer::close(bool discard) { + if (m_closing) return; + m_closing = true; + if (!m_encoder) { QTimer::singleShot(0, this, &Writer::emitFinished); return; } + m_encoder->finish(discard, [this](const QString &error) { + QMetaObject::invokeMethod(this, [this, error] { + if (!error.isEmpty() && !m_failed) { m_failed = true; emit failed(error); } + emitFinished(); + }, Qt::QueuedConnection); + }); +} +void Writer::emitFinished() { + m_flushTimeout.stop(); + if (m_finished) return; + m_finished = true; + emit finished(); } void Writer::fail(const QString &message) { if (m_failed) return; m_failed = true; m_stopping = true; emit failed(message); - m_frames.clear(); m_buffers.clear(); - m_recorder.stop(); - if (m_recorder.recorderState() == QMediaRecorder::StoppedState) - QTimer::singleShot(0, this, &Writer::finished); + m_frames.clear(); + // Keep what was already encoded: the file is still closed as a valid MP4. + close(true); } diff --git a/src/writer.h b/src/writer.h index 9838276..f522de0 100644 --- a/src/writer.h +++ b/src/writer.h @@ -1,13 +1,14 @@ #pragma once #include -#include -#include -#include -#include +#include #include +#include #include +#include #include +#include #include +#include "encoder.h" // Maps a monotonic capture clock onto a take with the paused intervals removed. class TakeClock { @@ -34,6 +35,7 @@ class Writer : public QObject { Q_OBJECT public: explicit Writer(QObject *parent = nullptr); + ~Writer() override; bool start(const QString &path, const QVideoFrameFormat &videoFormat, double fps, const QAudioFormat &audioFormat, qint64 now); void video(QVideoFrame frame, qint64 capturedAt); @@ -49,24 +51,29 @@ class Writer : public QObject { void failed(const QString &message); private: void drain(); + void close(bool discard); + void emitFinished(); void fail(const QString &message); void appendAudio(const QByteArray &data); void padAudioTo(qint64 time); void videoAt(QVideoFrame frame, qint64 position); - QMediaCaptureSession m_session; - QMediaRecorder m_recorder; - QVideoFrameInput *m_video = nullptr; - QAudioBufferInput *m_audio = nullptr; + // A single bad JPEG is skipped. Several in a row stop the take. + bool noteBadFrame(); + std::unique_ptr m_encoder; QAudioFormat m_audioFormat; - QQueue m_frames; + // At most two 4K decodes at once, so the pool cannot occupy every core. + // Declared before the queue: the queue is destroyed first, then the pool + // waits for a decode that is still running. + QThreadPool m_decodePool; + // In capture order; JPEG frames resolve once decoded off the GUI thread. + QQueue> m_frames; QVideoFrame m_tailFrame; - QQueue m_buffers; QTimer m_flushTimeout; TakeClock m_clock; qint64 m_lastVideo = -1; + int m_badDecodes = 0; qint64 m_frameDuration = 33333; double m_fps = 30; - qint64 m_queuedAudioUs = 0; qint64 m_audioFramesWritten = 0; - bool m_stopping = false, m_failed = false, m_draining = false; + bool m_stopping = false, m_failed = false, m_closing = false, m_finished = false; }; diff --git a/tests/backend_tests.cpp b/tests/backend_tests.cpp index 19745fe..4658dd6 100644 --- a/tests/backend_tests.cpp +++ b/tests/backend_tests.cpp @@ -9,14 +9,51 @@ #include #include #include +#include #include #include #include +#include #include "mediautils.h" #include "theme.h" #include "writer.h" #include "backend.h" +namespace { +// The plane size is the JPEG size. A frame created from QVideoFrameFormat +// alone can report a larger mapped buffer than the compressed bytes. +class JpegBytes : public QAbstractVideoBuffer { +public: + JpegBytes(QByteArray bytes, QSize size) : m_bytes(std::move(bytes)), m_format(size, QVideoFrameFormat::Format_Jpeg) {} + QVideoFrameFormat format() const override { return m_format; } + MapData map(QVideoFrame::MapMode) override { + MapData result; + result.planeCount = 1; + result.data[0] = reinterpret_cast(m_bytes.data()); + result.bytesPerLine[0] = int(m_bytes.size()); + result.dataSize[0] = int(m_bytes.size()); + return result; + } + void unmap() override {} +private: + QByteArray m_bytes; + QVideoFrameFormat m_format; +}; +QVideoFrame jpegFrame(const QByteArray &bytes) { + return QVideoFrame(std::make_unique(bytes, QSize(64, 32))); +} +QByteArray sampleJpeg() { + QImage image(64, 32, QImage::Format_RGB888); image.fill(Qt::red); + tjhandle tj = tj3Init(TJINIT_COMPRESS); + tj3Set(tj, TJPARAM_SUBSAMP, TJSAMP_420); tj3Set(tj, TJPARAM_QUALITY, 90); + unsigned char *jpeg = nullptr; size_t size = 0; + const int rc = tj3Compress8(tj, image.constBits(), 64, int(image.bytesPerLine()), 32, TJPF_RGB, &jpeg, &size); + QByteArray bytes; if (rc == 0) bytes = QByteArray(reinterpret_cast(jpeg), qsizetype(size)); + tj3Free(jpeg); tj3Destroy(tj); + return bytes; +} +} + class FakePicker : public FilePicker { public: QUrl suggestion; @@ -117,6 +154,38 @@ private slots: QVERIFY(write(response,"{\"int\":8}")); QTRY_COMPARE(theme.radius(),8); QVERIFY(write(response,"unavailable")); QTest::qWait(1200); QCOMPARE(theme.radius(),8); } + void jpegDecodesToVideoRange_data() { + QTest::addColumn("sampling"); + QTest::newRow("4:2:2")<("sound"); QTest::newRow("silent")<