Compress Image if it is to large to fit into single OPUS packet

Author: TobiGr

app/src/main/java/org/schabi/newpipe/streams/OggFromWebMWriter.java

@@ -28,6 +28,8 @@
 import java.util.List;
 import java.util.stream.Collectors;
 
+import us.shandian.giga.postprocessing.ImageUtils;
+
 /**
  * <p>
  *     This class is used to convert a WebM stream containing Opus or Vorbis audio
@@ -50,16 +52,44 @@
  * @author tobigr
  */
 public class OggFromWebMWriter implements Closeable {
+    private static final String TAG = OggFromWebMWriter.class.getSimpleName();
+
+    /**
+     * No flags set.
+     */
     private static final byte FLAG_UNSET = 0x00;
-    //private static final byte FLAG_CONTINUED = 0x01;
+    /**
+     * The packet is continued from previous the previous page.
+     */
+    private static final byte FLAG_CONTINUED = 0x01;
+    /**
+     * BOS (beginning of stream).
+     */
     private static final byte FLAG_FIRST = 0x02;
+    /**
+     * EOS (end of stream).
+     */
     private static final byte FLAG_LAST = 0x04;
 
     private static final byte HEADER_CHECKSUM_OFFSET = 22;
     private static final byte HEADER_SIZE = 27;
 
     private static final int TIME_SCALE_NS = 1000000000;
 
+    private static final int MAX_SEGMENT_SIZE = 255;
+
+    private static final int OPUS_MAX_PACKETS_SIZE = 61_140;
+
+    /**
+     * <p>The maximum size of the compressed thumbnail image in bytes,
+     * to be included in the Opus metadata.</p>
+     *
+     * This is a safe size to avoid creating metadata tags that are too large for the Ogg page,
+     * since the metadata header and other tags can also take up space in the page.
+     */
+    private static final int MAX_THUMBNAIL_SIZE = OPUS_MAX_PACKETS_SIZE - 4500;
+
+
     private boolean done = false;
     private boolean parsed = false;
 
@@ -323,12 +353,12 @@ private int makePacketHeader(final long granPos, @NonNull final ByteBuffer buffe
      * @ImplNote See <a href="https://datatracker.ietf.org/doc/html/rfc7845.html#section-5.2">
      *     RFC7845 5.2</a>
      *
-     * @return
+     * @return The binary metadata header, or null if not implemented for the codec
      */
     @Nullable
     private byte[] makeMetadata() {
         if (DEBUG) {
-            Log.d("OggFromWebMWriter", "Downloading media with codec ID " + webmTrack.codecId);
+            Log.d(TAG, "Downloading media with codec ID " + webmTrack.codecId);
         }
 
         if ("A_OPUS".equals(webmTrack.codecId)) {
@@ -343,18 +373,21 @@ private byte[] makeMetadata() {
                         .getLocalDateTime()
                         .format(DateTimeFormatter.ISO_DATE)));
                  if (thumbnail != null) {
-                     metadata.add(makeOpusPictureTag(thumbnail));
+                     var pictureTag = makeOpusPictureTag(thumbnail, MAX_THUMBNAIL_SIZE);
+                     if (pictureTag != null) {
+                         metadata.add(pictureTag);
+                     }
                  }
             }
 
             if (DEBUG) {
-                Log.d("OggFromWebMWriter", "Creating metadata header with this data:");
-                metadata.forEach(p -> Log.d("OggFromWebMWriter", p.first + "=" + p.second));
+                Log.d(TAG, "Creating metadata header with this data:");
+                metadata.forEach(p -> Log.d(TAG, p.first + "=" + p.second));
             }
 
             return makeOpusTagsHeader(metadata);
         } else if ("A_VORBIS".equals(webmTrack.codecId)) {
-            /**
+            /*
              * See <a href="https://datatracker.ietf.org/doc/html/rfc7845.html#section-5.2">
              *  RFC7845 5.2</a>
              */
@@ -399,43 +432,68 @@ private static byte[] makeOpusMetadataTag(final Pair<String, String> pair) {
      * </p>
      *
      * @param bitmap The bitmap to use as cover art
+     * @param maxSize The maximum size of the compressed image in bytes.
+     *                If the compressed image exceeds this size,
+     *                it will be further compressed until it fits.
+     *                This is necessary to avoid creating metadata tags
+     *                that are too large for the Ogg page.
      * @return The key-value pair representing the tag
+     * or null if the image cannot be compressed to the maxSize
      */
-    private static Pair<String, String> makeOpusPictureTag(final Bitmap bitmap) {
+    @Nullable
+    private static Pair<String, String> makeOpusPictureTag(final Bitmap bitmap, final int maxSize) {
         // FLAC picture block format (big-endian):
         // uint32 picture_type
-        // uint32 mime_length, mime_string
-        // uint32 desc_length, desc_string
+        // uint32 mime_length,
+        //        mime_string
+        // uint32 desc_length,
+        //        desc_string
         // uint32 width
         // uint32 height
         // uint32 color_depth
         // uint32 colors_indexed
-        // uint32 data_length, data_bytes
+        // uint32 data_length,
+        //        data_bytes
 
         final ByteArrayOutputStream baos = new ByteArrayOutputStream();
-        bitmap.compress(Bitmap.CompressFormat.JPEG, 100, baos);
+
+        final var compressedThumbnail = ImageUtils.INSTANCE.compressToSize(bitmap, maxSize);
+        if (compressedThumbnail == null) {
+            if (DEBUG) {
+                Log.d(TAG, "failed to compress thumbnail to target size " + maxSize);
+            }
+            return null;
+        }
+        compressedThumbnail.getBitmap().compress(
+                Bitmap.CompressFormat.JPEG, compressedThumbnail.getQuality(), baos);
 
         final byte[] imageData = baos.toByteArray();
         final byte[] mimeBytes = "image/jpeg".getBytes(StandardCharsets.UTF_8);
         final byte[] descBytes = new byte[0]; // optional description
         // fixed ints + mime + desc
         final int headerSize = 4 * 8 + mimeBytes.length + descBytes.length;
         final ByteBuffer buf = ByteBuffer.allocate(headerSize + imageData.length);
-        buf.putInt(3); // picture type: 3 = Cover (front)
+        // See https://id3.org/id3v2.3.0#Attached_picture for a full list of picture types
+        // TODO: allow specifying other picture types, i.e. cover (front) for music albums;
+        //       but this info needs to be provided by the extractor first.
+        buf.putInt(3); // picture type: 0 = Other
         buf.putInt(mimeBytes.length);
         buf.put(mimeBytes);
         buf.putInt(descBytes.length);
-        // no description
         if (descBytes.length > 0) {
+            // currently no description available, might be added later.
             buf.put(descBytes);
         }
-        buf.putInt(bitmap.getWidth()); // width (unknown)
-        buf.putInt(bitmap.getHeight()); // height (unknown)
+        buf.putInt(compressedThumbnail.getBitmap().getWidth());
+        buf.putInt(compressedThumbnail.getBitmap().getHeight());
         buf.putInt(0); // color depth
         buf.putInt(0); // colors indexed
         buf.putInt(imageData.length);
         buf.put(imageData);
+
         final String b64 = Base64.getEncoder().encodeToString(buf.array());
+        Log.d(TAG, "Compressed thumbnail size: " + imageData.length
+                + " bytes, base64 metadata size: " + b64.length() + " characters");
         return Pair.create("METADATA_BLOCK_PICTURE", b64);
     }
 
@@ -457,7 +515,7 @@ private static byte[] makeOpusTagsHeader(final List<Pair<String, String>> keyVal
                 .stream()
                 .filter(p -> !p.second.isBlank())
                 .map(OggFromWebMWriter::makeOpusMetadataTag)
-                .collect(Collectors.toUnmodifiableList());
+                .toList();
 
         final var tagsBytes = tags.stream().collect(Collectors.summingInt(arr -> arr.length));
 

app/src/main/java/us/shandian/giga/get/DownloadMission.java

@@ -864,7 +864,7 @@ public void fetchThumbnail(@NonNull final List<Image> images) {
             // Alternative approaches are to either downscale a high res image or
             // to download the correct size depending on the chosen post-processing algorithm.
             final String thumbnailUrl = ImageStrategy.choosePreferredImage(
-                    images, PreferredImageQuality.MEDIUM);
+                    images, PreferredImageQuality.HIGH);
             // TODO: get context from somewhere else
             thumbnail = CoilHelper.INSTANCE.loadBitmapBlocking(App.getInstance(), thumbnailUrl);
             thumbnailFetched = true;

app/src/main/java/us/shandian/giga/postprocessing/ImageUtils.kt

@@ -0,0 +1,61 @@
+package us.shandian.giga.postprocessing
+
+import android.graphics.Bitmap
+import java.io.ByteArrayOutputStream
+import java.util.Base64
+import org.schabi.newpipe.ktx.scale
+import org.schabi.newpipe.util.image.PreferredImageQuality
+
+object ImageUtils {
+    fun getImageTypeFromUrl(url: String): String {
+        val extension = url.substringAfterLast('.', "")
+        return when (extension.lowercase()) {
+            "jpg", "jpeg" -> "image/jpeg"
+            "png" -> "image/png"
+            "gif" -> "image/gif"
+            "bmp" -> "image/bmp"
+            "webp" -> "image/webp"
+            else -> "application/octet-stream" // Default binary type
+        }
+    }
+
+    data class CompressedImage(
+        val bitmap: Bitmap,
+        val quality: Int,
+        val width: Int,
+        val height: Int
+    )
+
+    fun compressToSize(original: Bitmap, maxSizeBytes: Int): CompressedImage? {
+        var quality = 100
+        var scale = 1.0f
+        var width = original.width
+        var height = original.height
+        var compressedSize: Int
+
+        do {
+            var bitmap = original.copy(original.config ?: Bitmap.Config.ARGB_8888, false)
+            if (scale < 1.0f) {
+                bitmap = bitmap.scale(width = width, height = height)
+            }
+            do {
+                val outputStream = ByteArrayOutputStream()
+                bitmap.compress(Bitmap.CompressFormat.JPEG, quality, outputStream)
+                compressedSize = Base64.getEncoder().encodeToString(outputStream.toByteArray()).length
+                quality -= 5 // Decrease quality by 5% for the next iteration
+            } while (compressedSize > maxSizeBytes && quality > 70)
+            if (compressedSize <= maxSizeBytes) {
+                return CompressedImage(bitmap, quality, width, height)
+            }
+            if (scale > 0.5f) {
+                scale -= 0.1f
+            } else {
+                scale *= 0.9f
+            }
+            width = (original.width * scale).toInt()
+            height = (original.height * scale).toInt()
+            quality = 100 // Reset quality for the next size reduction
+        } while (width > 50 && height > 50) // Prevent too much downscaling
+        return null
+    }
+}