From 3c5441114af9f9e6f93fadf94710269b27676b54 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 31 Oct 2026 17:17:49 +1100 Subject: [PATCH] media: apple: isp: report the frame gain and exposure Userspace has no way to tell how hard the sensor is working. Gain is the signal a camera pipeline needs to decide when the picture is noisy enough to denoise, and the firmware already writes it into every frame's metadata buffer, which the driver maps but never reads. The metadata buffer opens with a table of section offsets. On H17 firmware the fourth section starts with the frame's exposure: the integration time in microseconds, then the total sensor gain in 1/256 steps with 256 as unity. This was established on T6040 and T8140 by watching auto exposure converge on a fixed scene, where the gain climbed from 260 to its final value once the integration time hit the frame period, and by stepping the frame rate, which capped the integration time at the frame period. Read the section when a metadata buffer comes back, through the table and bounded by the buffer size, and report the last values as read-only, volatile V4L2_CID_GAIN and V4L2_CID_EXPOSURE_ABSOLUTE controls. Only hardware whose match data says its metadata carries the report gets the controls. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-drv.c | 2 + drivers/media/platform/apple/isp/isp-drv.h | 9 ++ .../media/platform/apple/isp/isp-v4l2-test.c | 46 ++++++++++ drivers/media/platform/apple/isp/isp-v4l2.c | 92 ++++++++++++++++++- 4 files changed, 148 insertions(+), 1 deletion(-) diff --git a/drivers/media/platform/apple/isp/isp-drv.c b/drivers/media/platform/apple/isp/isp-drv.c index 084cca4dc3707..15612805bf7fb 100644 --- a/drivers/media/platform/apple/isp/isp-drv.c +++ b/drivers/media/platform/apple/isp/isp-drv.c @@ -827,6 +827,7 @@ static const struct apple_isp_hw apple_isp_hw_t8140 = { .boot_mode = 1, .capture_meta_size = ISP_CAPTURE_META_SIZE_T8140, .resident_fw = true, + .meta_exposure = true, }; /* The same for T6040, from the J616s ADT's DART address filter. */ @@ -885,6 +886,7 @@ static const struct apple_isp_hw apple_isp_hw_t6040 = { .boot_mode = 1, .shared_size = 0xde600000, .resident_fw = true, + .meta_exposure = true, }; static const struct of_device_id apple_isp_of_match[] = { diff --git a/drivers/media/platform/apple/isp/isp-drv.h b/drivers/media/platform/apple/isp/isp-drv.h index dbe6b6ed5a2c4..0515570993ac6 100644 --- a/drivers/media/platform/apple/isp/isp-drv.h +++ b/drivers/media/platform/apple/isp/isp-drv.h @@ -201,6 +201,9 @@ struct apple_isp_hw { /* size of a capture metadata buffer, 0 for no such pool */ u32 capture_meta_size; + /* metadata buffers carry the frame's exposure report (H17 firmware) */ + bool meta_exposure; + /* * The firmware can be started only once per system boot, so it is * started at probe and kept running while the driver is bound. @@ -302,6 +305,12 @@ struct apple_isp { struct v4l2_device v4l2_dev; struct v4l2_ctrl_handler ctrl_handler; struct v4l2_ctrl *exposure_priority; + /* read-only exposure report of the last frame, NULL without one */ + struct v4l2_ctrl *gain; + struct v4l2_ctrl *exposure; + /* last reported total gain (1/256 steps) and exposure (us) */ + u32 frame_gain; + u32 frame_exposure_us; struct vb2_queue vbq; struct mutex video_lock; unsigned int sequence; diff --git a/drivers/media/platform/apple/isp/isp-v4l2-test.c b/drivers/media/platform/apple/isp/isp-v4l2-test.c index 398e8e92356ab..88f428b3b0c27 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2-test.c +++ b/drivers/media/platform/apple/isp/isp-v4l2-test.c @@ -348,7 +348,53 @@ static void isp_test_imx958_last_lease_requeue(struct kunit *test) } } +/* The exposure report is found through the section table, never past it. */ +static void isp_test_meta_exposure(struct kunit *test) +{ + static const struct apple_isp_hw hw = { .meta_exposure = true }; + static const struct apple_isp_hw hw_none = { }; + struct apple_isp *isp = isp_test_alloc(test); + struct isp_surf meta = { .size = 0x2000 }; + u8 *virt = kunit_kzalloc(test, meta.size, GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, virt); + meta.virt = virt; + isp->hw = &hw; + isp->frame_gain = 256; + + /* fourth table entry: the section at 0x1870 */ + put_unaligned_le32(0x1870, virt + 0x1c); + put_unaligned_le32(32475, virt + 0x1870); + put_unaligned_le16(0x0800, virt + 0x1874); + put_unaligned_le16(3253, virt + 0x1876); + isp_meta_read_exposure(isp, &meta); + KUNIT_EXPECT_EQ(test, isp->frame_exposure_us, 32475); + KUNIT_EXPECT_EQ(test, isp->frame_gain, 3253); + + /* a section that would end past the buffer is ignored */ + put_unaligned_le32(meta.size - 4, virt + 0x1c); + put_unaligned_le32(1000, virt + meta.size - 4); + isp_meta_read_exposure(isp, &meta); + KUNIT_EXPECT_EQ(test, isp->frame_gain, 3253); + + /* so is an unwritten section */ + put_unaligned_le32(0x1000, virt + 0x1c); + isp_meta_read_exposure(isp, &meta); + KUNIT_EXPECT_EQ(test, isp->frame_exposure_us, 32475); + + /* and firmware without the report */ + put_unaligned_le32(0x1870, virt + 0x1c); + put_unaligned_le16(600, virt + 0x1876); + isp->hw = &hw_none; + isp_meta_read_exposure(isp, &meta); + KUNIT_EXPECT_EQ(test, isp->frame_gain, 3253); + isp->hw = &hw; + isp_meta_read_exposure(isp, &meta); + KUNIT_EXPECT_EQ(test, isp->frame_gain, 600); +} + static struct kunit_case isp_v4l2_test_cases[] = { + KUNIT_CASE(isp_test_meta_exposure), KUNIT_CASE(isp_test_imx958_last_lease_requeue), KUNIT_CASE(isp_test_p010_report_defers_completion), KUNIT_CASE(isp_test_p010_mode_selection), diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c index 431c85c6695c9..2e858d392292d 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.c +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -3,6 +3,7 @@ #include #include +#include #include #include @@ -68,6 +69,45 @@ static bool isp_p010_buffer_matches(const struct isp_format *fmt, !desc->iovas[2] && !desc->iovas[3] && desc->tag == buf->tag; } +/* + * A metadata buffer starts with a table of section offsets. On H17 firmware + * the fourth section opens with the frame's exposure: the integration time in + * microseconds (le32 at 0) and the total sensor gain in 1/256 steps, 256 for + * unity (le16 at 6). Observed on T6040 and T8140 with AE converging under a + * fixed scene and with the frame rate capping the integration time. + */ +#define ISP_META_SECTIONS 0x10 +#define ISP_META_SECTION_EXPOSURE 3 +#define ISP_META_EXPOSURE_TIME 0x0 +#define ISP_META_EXPOSURE_GAIN 0x6 +#define ISP_META_EXPOSURE_SIZE 0x8 + +static void isp_meta_read_exposure(struct apple_isp *isp, + const struct isp_surf *meta) +{ + const u8 *virt = meta->virt; + u32 offset, time; + u16 gain; + + if (!isp->hw->meta_exposure || !virt) + return; + + offset = get_unaligned_le32(virt + ISP_META_SECTIONS + + 4 * ISP_META_SECTION_EXPOSURE); + if (offset < ISP_META_SECTIONS || offset > meta->size || + meta->size - offset < ISP_META_EXPOSURE_SIZE) + return; + + time = get_unaligned_le32(virt + offset + ISP_META_EXPOSURE_TIME); + gain = get_unaligned_le16(virt + offset + ISP_META_EXPOSURE_GAIN); + /* An unwritten section or one this layout does not describe */ + if (!time || !gain) + return; + + WRITE_ONCE(isp->frame_exposure_us, time); + WRITE_ONCE(isp->frame_gain, gain); +} + void ipc_bt_complete(struct apple_isp *isp) { struct isp_buffer *buf, *tmp; @@ -128,6 +168,7 @@ int ipc_bt_handle(struct apple_isp *isp, struct isp_channel *chan) struct isp_surf *meta = isp->meta_surfs[j]; if ((u32)bufd->iovas[0] == (u32)meta->iova) { WARN_ON(!meta->submitted); + isp_meta_read_exposure(isp, meta); if (isp_is_p010(isp)) __set_bit(j, &isp->meta_returned); else @@ -1062,6 +1103,53 @@ static int isp_vb2_queue_init(struct apple_isp *isp) return vb2_queue_init(vbq); } +static int isp_g_volatile_ctrl(struct v4l2_ctrl *ctrl) +{ + struct apple_isp *isp = + container_of(ctrl->handler, struct apple_isp, ctrl_handler); + + switch (ctrl->id) { + case V4L2_CID_GAIN: + ctrl->val = READ_ONCE(isp->frame_gain); + return 0; + case V4L2_CID_EXPOSURE_ABSOLUTE: + /* in 100 us units */ + ctrl->val = DIV_ROUND_CLOSEST(READ_ONCE(isp->frame_exposure_us), + 100); + return 0; + } + + return -EINVAL; +} + +static const struct v4l2_ctrl_ops isp_exposure_ctrl_ops = { + .g_volatile_ctrl = isp_g_volatile_ctrl, +}; + +/* + * Auto exposure picks the gain and exposure of each frame. Report the last + * frame's values read-only, so that userspace can follow the scene light, for + * example to decide when to denoise. The gain is the firmware's total sensor + * gain in 1/256 steps. + */ +static void isp_init_exposure_ctrls(struct apple_isp *isp) +{ + isp->frame_gain = 256; + isp->gain = v4l2_ctrl_new_std(&isp->ctrl_handler, + &isp_exposure_ctrl_ops, V4L2_CID_GAIN, + 0, U16_MAX, 1, 256); + isp->exposure = v4l2_ctrl_new_std(&isp->ctrl_handler, + &isp_exposure_ctrl_ops, + V4L2_CID_EXPOSURE_ABSOLUTE, + 0, 10000, 1, 0); + if (isp->gain) + isp->gain->flags |= V4L2_CTRL_FLAG_READ_ONLY | + V4L2_CTRL_FLAG_VOLATILE; + if (isp->exposure) + isp->exposure->flags |= V4L2_CTRL_FLAG_READ_ONLY | + V4L2_CTRL_FLAG_VOLATILE; +} + int apple_isp_setup_video(struct apple_isp *isp) { struct video_device *vdev = &isp->vdev; @@ -1116,10 +1204,12 @@ int apple_isp_setup_video(struct apple_isp *isp) * Auto exposure is always on. By default it holds the frame rate, as * V4L2 specifies; with exposure priority it may slow down in low light. */ - v4l2_ctrl_handler_init(&isp->ctrl_handler, 1); + v4l2_ctrl_handler_init(&isp->ctrl_handler, 3); isp->exposure_priority = v4l2_ctrl_new_std(&isp->ctrl_handler, NULL, V4L2_CID_EXPOSURE_AUTO_PRIORITY, 0, 1, 1, 0); + if (isp->hw->meta_exposure) + isp_init_exposure_ctrls(isp); if (isp->ctrl_handler.error) { err = isp->ctrl_handler.error; dev_err(isp->dev, "failed to create controls: %d\n", err);