diff --git a/autotest/gdrivers/vrtwarp.py b/autotest/gdrivers/vrtwarp.py
index b747fd7b6162..798adaa58e16 100755
--- a/autotest/gdrivers/vrtwarp.py
+++ b/autotest/gdrivers/vrtwarp.py
@@ -705,6 +705,28 @@ def test_vrtwarp_irasterio_optim_three_band():
     )
 
 
+###############################################################################
+# Test that VRTWarpedDataset::IRasterIO() honours the requested resampling
+# when reading the whole image at a reduced resolution
+
+
+def test_vrtwarp_irasterio_optim_rescaled_resampling():
+
+    src_ds = gdal.Translate("", "data/byte.tif", format="MEM", width=1000)
+    warped_vrt_ds = gdal.Warp("", src_ds, format="VRT")
+
+    with gdaltest.config_option("GDAL_VRT_WARP_USE_DATASET_RASTERIO", "NO"):
+        expected_data = warped_vrt_ds.ReadRaster(
+            buf_xsize=100, buf_ysize=10, resample_alg=gdal.GRIORA_Average
+        )
+    assert (
+        warped_vrt_ds.ReadRaster(
+            buf_xsize=100, buf_ysize=10, resample_alg=gdal.GRIORA_Average
+        )
+        == expected_data
+    )
+
+
 ###############################################################################
 # Test VRTWarpedDataset::IRasterIO() code path
 
diff --git a/frmts/vrt/vrtwarped.cpp b/frmts/vrt/vrtwarped.cpp
index 6a36503a8022..4bceba6461e7 100644
--- a/frmts/vrt/vrtwarped.cpp
+++ b/frmts/vrt/vrtwarped.cpp
@@ -1959,7 +1959,10 @@ CPLErr VRTWarpedDataset::IRasterIO(
 
     if (nBufXSize != nXSize || nBufYSize != nYSize)
     {
-        if (!bWholeImage || !GDALTransformHasFastClone(psWO->pTransformerArg))
+        // The rescaled warp below ignores the requested resampling
+        if (!bWholeImage ||
+            psExtraArg->eResampleAlg != GRIORA_NearestNeighbour ||
+            !GDALTransformHasFastClone(psWO->pTransformerArg))
         {
             return GDALDataset::IRasterIO(eRWFlag, nXOff, nYOff, nXSize, nYSize,
                                           pData, nBufXSize, nBufYSize, eBufType,