Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion src/backend.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -170,8 +170,16 @@ void Backend::updateReady() {
m_state="ready"; m_message.clear(); emit changed();
}
}
void Backend::receiveVideo(const QVideoFrame &frame) {
void Backend::receiveVideo(QVideoFrame frame) {
if(!m_camera || !frame.isValid()) return;
// Qt maps V4L2's sRGB colorspace to BT.601 and ignores the YCbCr encoding the
// driver reports, so HDMI capture cards sending BT.709 decode with skewed colors.
if(!m_cameraHealthy) m_colorSpace=media::cameraColorSpace(m_camera->cameraDevice().id());
if(frame.surfaceFormat().colorSpace()==QVideoFrameFormat::ColorSpace_BT601 &&
m_colorSpace!=QVideoFrameFormat::ColorSpace_Undefined && m_colorSpace!=QVideoFrameFormat::ColorSpace_BT601) {
auto format=frame.surfaceFormat(); format.setColorSpace(m_colorSpace);
frame=media::withFormat(frame,format);
Comment thread
dhh marked this conversation as resolved.
Resolved
}
m_lastVideoAt=now();
if(frame.size()!=m_cameraFormat.resolution()) {
sourceFailed(tr("The camera did not provide its maximum resolution. Choose another source or Retry.")); return;
Expand Down
3 changes: 2 additions & 1 deletion src/backend.h
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ class Backend : public QObject {
void readAudio();
void receiveAudio(const QByteArray &data, qint64 capturedAt);
void finishWhenCaptured();
void receiveVideo(const QVideoFrame &frame);
void receiveVideo(QVideoFrame frame);
void tick();
void sourceFailed(const QString &message);
void startTake();
Expand All @@ -148,6 +148,7 @@ class Backend : public QObject {
QAudioFormat m_audioFormat;
QVideoFrame m_lastFrame;
QCameraFormat m_cameraFormat;
QVideoFrameFormat::ColorSpace m_colorSpace = QVideoFrameFormat::ColorSpace_Undefined;
QElapsedTimer m_wall;
QTimer m_tick;
QTimer m_finishTimeout;
Expand Down
55 changes: 55 additions & 0 deletions src/mediautils.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,62 @@
#include <limits>
#include <cstdio>
#include <sys/stat.h>
#include <QAbstractVideoBuffer>
#include <fcntl.h>
#include <sys/ioctl.h>
#include <unistd.h>
#include <linux/videodev2.h>

namespace {
class FrameBuffer : public QAbstractVideoBuffer {
public:
FrameBuffer(const QVideoFrame &source, const QVideoFrameFormat &format) : m_source(source), m_format(format) {}
~FrameBuffer() override { unmap(); }
QVideoFrameFormat format() const override { return m_format; }
MapData map(QVideoFrame::MapMode mode) override {
MapData result;
if(!m_source.map(mode)) return result;
m_mapped=true;
result.planeCount=m_source.planeCount();
for(int i=0;i<result.planeCount;++i) {
result.data[i]=m_source.bits(i);
result.bytesPerLine[i]=m_source.bytesPerLine(i);
result.dataSize[i]=m_source.mappedBytes(i);
}
return result;
}
void unmap() override { if(m_mapped) { m_source.unmap(); m_mapped=false; } }
private:
QVideoFrame m_source;
QVideoFrameFormat m_format;
bool m_mapped=false;
};
}

QVideoFrame media::withFormat(const QVideoFrame &source, const QVideoFrameFormat &format) {
QVideoFrame frame(std::make_unique<FrameBuffer>(source, format));
frame.setStartTime(source.startTime());
frame.setEndTime(source.endTime());
return frame;
}
QVideoFrameFormat::ColorSpace media::colorSpace(int v4l2Colorspace, int v4l2Encoding) {
if (v4l2Encoding == V4L2_YCBCR_ENC_DEFAULT) v4l2Encoding = V4L2_MAP_YCBCR_ENC_DEFAULT(v4l2Colorspace);
switch (v4l2Encoding) {
case V4L2_YCBCR_ENC_601: case V4L2_YCBCR_ENC_XV601: return QVideoFrameFormat::ColorSpace_BT601;
case V4L2_YCBCR_ENC_709: case V4L2_YCBCR_ENC_XV709: return QVideoFrameFormat::ColorSpace_BT709;
case V4L2_YCBCR_ENC_BT2020: case V4L2_YCBCR_ENC_BT2020_CONST_LUM: return QVideoFrameFormat::ColorSpace_BT2020;
default: return QVideoFrameFormat::ColorSpace_Undefined;
}
}
QVideoFrameFormat::ColorSpace media::cameraColorSpace(const QByteArray &device) {
const int fd = ::open(device.constData(), O_RDONLY | O_NONBLOCK | O_CLOEXEC);
if (fd < 0) return QVideoFrameFormat::ColorSpace_Undefined;
v4l2_format format{};
format.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
const bool ok = ::ioctl(fd, VIDIOC_G_FMT, &format) == 0;
::close(fd);
return ok ? colorSpace(format.fmt.pix.colorspace, format.fmt.pix.ycbcr_enc) : QVideoFrameFormat::ColorSpace_Undefined;
}

QString media::chooseSource(const QStringList &connected, const QString &preferred, const QString &remembered) {
if (connected.isEmpty() || connected.contains(remembered)) return remembered;
Expand Down
7 changes: 7 additions & 0 deletions src/mediautils.h
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,20 @@
#include <QJsonObject>
#include <QString>
#include <QImage>
#include <QVideoFrame>
#include <atomic>
#include "edit.h"

namespace media {
struct Format { QSize size; double minFps; double maxFps; int pixelFormat; };
int bestFormat(const QList<Format> &formats);
QCameraFormat bestCameraFormat(const QCameraDevice &device);
// V4L2 colorimetry to Qt: the YCbCr encoding the driver reports, falling back to its colorspace default.
QVideoFrameFormat::ColorSpace colorSpace(int v4l2Colorspace, int v4l2Encoding);
// Asks the V4L2 device (e.g. "/dev/video4") which YCbCr encoding it is delivering.
QVideoFrameFormat::ColorSpace cameraColorSpace(const QByteArray &device);
// Shares the source's image planes and capture times under a new format; later metadata changes stay independent.
QVideoFrame withFormat(const QVideoFrame &source, const QVideoFrameFormat &format);
double targetFps(double minimum, double maximum);
// The source to use: the remembered one while it's connected, else the default, else the first connected.
QString chooseSource(const QStringList &connected, const QString &preferred, const QString &remembered);
Expand Down
30 changes: 3 additions & 27 deletions src/writer.cpp
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
#include "writer.h"
#include <QMediaFormat>
#include <QUrl>
#include "mediautils.h"
#include <QAbstractVideoBuffer>
#include <QImage>
#include <limits>
Expand All @@ -9,32 +10,6 @@
#include <turbojpeg.h>

namespace {
// QVideoFrame copies share timestamp metadata as well as pixels. Give recording
// frames independent metadata without copying their image planes or altering
// the live preview (or an already queued copy of a pause's final frame).
class FrameBuffer : public QAbstractVideoBuffer {
public:
explicit FrameBuffer(const QVideoFrame &source) : m_source(source) {}
~FrameBuffer() override { unmap(); }
QVideoFrameFormat format() const override { return m_source.surfaceFormat(); }
MapData map(QVideoFrame::MapMode mode) override {
MapData result;
if(!m_source.map(mode)) return result;
m_mapped=true;
result.planeCount=m_source.planeCount();
for(int i=0;i<result.planeCount;++i) {
result.data[i]=m_source.bits(i);
result.bytesPerLine[i]=m_source.bytesPerLine(i);
result.dataSize[i]=m_source.mappedBytes(i);
}
return result;
}
void unmap() override { if(m_mapped) { m_source.unmap(); m_mapped=false; } }
private:
QVideoFrame m_source;
bool m_mapped=false;
};

// Qt chooses the encoder's input format from the source frame's. For RGB it
// asks for 12-bit 4:4:4 and for packed 4:2:2 for 10-bit 4:2:2, both of which
// H.264 NVENC rejects; Qt then falls back to software x264 at 4:4:4 or 4:2:2,
Expand Down Expand Up @@ -327,7 +302,8 @@ void Writer::videoAt(QVideoFrame frame, qint64 position) {
if (Yuv420FrameBuffer::converts(frame.pixelFormat()))
frame = QVideoFrame(std::make_unique<Yuv420FrameBuffer>(frame));
else
frame = QVideoFrame(std::make_unique<FrameBuffer>(frame));
// Independent timestamps for the recording copy, without touching the preview's frame.
frame = media::withFormat(frame, frame.surfaceFormat());
frame.setStartTime(timestamp);
frame.setEndTime(qRound64((frameIndex + 1) * 1000000.0 / m_fps));
frame.setStreamFrameRate(m_fps);
Expand Down
39 changes: 39 additions & 0 deletions tests/backend_tests.cpp
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
#include <linux/videodev2.h>
#include <QtTest>
#include <QScopeGuard>
#include <cstring>
Expand Down Expand Up @@ -63,6 +64,20 @@ private slots:
QCOMPARE(media::bestFormat({}),-1);
QCOMPARE(media::targetFps(15,15),15.0);
}
void cameraColorSpace() {
// Cam Link 4K: sRGB colorspace, explicit BT.709 encoding.
QCOMPARE(media::colorSpace(V4L2_COLORSPACE_SRGB,V4L2_YCBCR_ENC_709),QVideoFrameFormat::ColorSpace_BT709);
QCOMPARE(media::colorSpace(V4L2_COLORSPACE_SRGB,V4L2_YCBCR_ENC_DEFAULT),QVideoFrameFormat::ColorSpace_BT601);
QCOMPARE(media::colorSpace(V4L2_COLORSPACE_REC709,V4L2_YCBCR_ENC_DEFAULT),QVideoFrameFormat::ColorSpace_BT709);
QCOMPARE(media::cameraColorSpace("/nonexistent"),QVideoFrameFormat::ColorSpace_Undefined);
QImage image(4,4,QImage::Format_RGB32);
auto format=QVideoFrame(image).surfaceFormat(); format.setColorSpace(QVideoFrameFormat::ColorSpace_BT709);
QVideoFrame source(image); source.setStartTime(1234); source.setEndTime(5678);
const auto frame=media::withFormat(source,format);
QCOMPARE(frame.surfaceFormat().colorSpace(),QVideoFrameFormat::ColorSpace_BT709);
QCOMPARE(frame.startTime(),1234); QCOMPARE(frame.endTime(),5678);
QCOMPARE(frame.toImage().size(),QSize(4,4));
}
void clockExcludesPauses() {
TakeClock c; c.start(1000000);
QVERIFY(!c.position(999999));
Expand Down Expand Up @@ -455,6 +470,30 @@ private slots:
const auto rgb=pixels.readAllStandardOutput(); QCOMPARE(rgb.size(),3);
QVERIFY2(quint8(rgb[0])>200 && quint8(rgb[1])<60 && quint8(rgb[2])<60,qPrintable(rgb.toHex()));
}
void relabelledCameraRecordsItsColorSpace() {
// A Cam Link sends packed 4:2:2 that Qt labels BT.601; once relabelled from
// the driver's BT.709, the 4:2:0 recording must carry BT.709 too.
QTemporaryDir directory;
const auto path=directory.filePath("709.mp4");
QVideoFrameFormat yuyv(QSize(320,240),QVideoFrameFormat::Format_YUYV);
auto bt709=yuyv; bt709.setColorSpace(QVideoFrameFormat::ColorSpace_BT709);
QAudioFormat audio; audio.setSampleRate(48000); audio.setChannelCount(1); audio.setSampleFormat(QAudioFormat::Int16);
Writer writer; QSignalSpy ended(&writer,&Writer::finished), errors(&writer,&Writer::failed);
QVERIFY(writer.start(path,bt709,30,audio,0));
for(int i=0;i<30;++i) {
QVideoFrame frame(yuyv); QVERIFY(frame.map(QVideoFrame::WriteOnly));
for(int y=0;y<240;++y) { auto *row=frame.bits(0)+y*frame.bytesPerLine(0); for(int x=0;x<320*2;x+=4) { row[x]=row[x+2]=128; row[x+1]=60; row[x+3]=200; } }
frame.unmap();
writer.video(media::withFormat(frame,bt709),qRound64(i*1000000.0/30));
writer.audio(QByteArray(1600*2,0),qRound64(i*1000000.0/30));
QTest::qWait(33);
}
writer.finish(); QTRY_VERIFY_WITH_TIMEOUT(!ended.isEmpty(),15000);
QVERIFY2(errors.isEmpty(),errors.isEmpty()?"":qPrintable(errors.first().first().toString()));
QProcess probe; probe.start("ffprobe",{"-v","error","-select_streams","v:0","-show_entries","stream=pix_fmt,color_space","-of","csv=p=0",path});
QVERIFY(probe.waitForFinished());
QCOMPARE(QString::fromUtf8(probe.readAllStandardOutput()).trimmed(),QString("yuv420p,bt709"));
}
void delayedCaptureStaysInSync() {
QTemporaryDir directory;
const auto path=directory.filePath("sync.mp4");
Expand Down
Loading