[YouTube] Workaround Shorts UI for playlists by using a continuation for initial items

YouTube don't return currently a continuation, if applicable, for the Shorts UI
in playlists, restricting access to the 100th first items. The reel items
returned don't give also upload date, uploader info and precise view count.

Using a continuation which requests the first page of the playlist allows
currently to get access to continuations, if applicable, and also standard
video elements instead of Shorts ones, making extraction of upload date,
uploader info and precise view count again possible.

This continuation is only requested and used when the data structure should be
the one of the Shorts UI.

A fallback to the standard response is made in the cases where fetching or
parsing the continuation response fails, or when no stream items are extracted.

This change required to:
- add a protocol buffer definition file for which its structure is based on
reverse engineering of playlists continuations sent by WEB InnerTube client,
received from InnerTube responses;
- add a dependency on Protobuf Java Lite;
- generate Java classes from the definition file with the protoc version
corresponding to the one of the Protobuf Java Lite dependency used (see
https://protobuf.dev/support/version-support/ for more details); they are
excluded from Checkstyle checks as they do not follow our style rules at all;
- add a dependency on Apache Commons Codec, as Java's Base64 class desugaring
for Android is not released yet.

Author: AudricV

extractor/build.gradle

@@ -18,6 +18,9 @@ checkstyle {
     toolVersion checkstyleVersion
 }
 
+// Exclude Protobuf generated files from Checkstyle
+checkstyleMain.exclude('org/schabi/newpipe/extractor/services/youtube/protos')
+
 checkstyleTest {
     enabled false // do not checkstyle test files
 }
@@ -28,6 +31,11 @@ dependencies {
     implementation "com.github.TeamNewPipe:nanojson:$nanojsonVersion"
     implementation 'org.jsoup:jsoup:1.16.1'
     implementation "com.github.spotbugs:spotbugs-annotations:$spotbugsVersion"
+    implementation "com.google.protobuf:protobuf-javalite:3.24.3"
+
+    // TODO: remove this dependency used for Base64 encoding and use Java's Base64 once its
+    //  Android desugarging support has been added
+    implementation 'commons-codec:commons-codec:1.16.0'
 
     // do not upgrade to 1.7.14, since in 1.7.14 Rhino uses the `SourceVersion` class, which is not
     // available on Android (even when using desugaring), and `NoClassDefFoundError` is thrown

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java

@@ -15,6 +15,7 @@
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonWriter;
 
+import org.apache.commons.codec.binary.Base64;
 import org.schabi.newpipe.extractor.Image;
 import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.StreamingService;
@@ -27,19 +28,43 @@
 import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
 import org.schabi.newpipe.extractor.playlist.PlaylistInfo;
 import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
+import org.schabi.newpipe.extractor.services.youtube.protos.playlist.PlaylistProtobufContinuation.ContinuationParams;
+import org.schabi.newpipe.extractor.services.youtube.protos.playlist.PlaylistProtobufContinuation.PlaylistContentFiltersParams;
+import org.schabi.newpipe.extractor.services.youtube.protos.playlist.PlaylistProtobufContinuation.PlaylistContinuation;
+import org.schabi.newpipe.extractor.services.youtube.protos.playlist.PlaylistProtobufContinuation.PlaylistContinuationProperties;
 import org.schabi.newpipe.extractor.stream.Description;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.IOException;
+import java.io.UnsupportedEncodingException;
 import java.nio.charset.StandardCharsets;
 import java.util.List;
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
 public class YoutubePlaylistExtractor extends PlaylistExtractor {
+
+    private static final String PLAYLIST_CONTINUATION_PROPERTIES_BASE64;
+
+    static {
+        try {
+            PLAYLIST_CONTINUATION_PROPERTIES_BASE64 = Utils.encodeUrlUtf8(
+                    Base64.encodeBase64String(
+                            PlaylistContinuationProperties.newBuilder()
+                                    .setRequestCount(0)
+                                    .setContentFilters(PlaylistContentFiltersParams.newBuilder()
+                                            .setHideUnavailableVideos(false)
+                                            .build())
+                                    .build()
+                                    .toByteArray()));
+        } catch (final UnsupportedEncodingException e) {
+            throw new RuntimeException("Couldn't encode playlist continuation properties", e);
+        }
+    }
+
     // Names of some objects in JSON response frequently used in this class
     private static final String PLAYLIST_VIDEO_RENDERER = "playlistVideoRenderer";
     private static final String PLAYLIST_VIDEO_LIST_RENDERER = "playlistVideoListRenderer";
@@ -50,6 +75,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
     private static final String VIDEO_OWNER_RENDERER = "videoOwnerRenderer";
 
     private JsonObject browseResponse;
+    private JsonObject initialContinuationResponse;
 
     private JsonObject playlistInfo;
     private JsonObject uploaderInfo;
@@ -330,6 +356,17 @@ public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, Extrac
             if (videoPlaylistObject.has(PLAYLIST_VIDEO_LIST_RENDERER)) {
                 renderer = videoPlaylistObject.getObject(PLAYLIST_VIDEO_LIST_RENDERER);
             } else if (videoPlaylistObject.has(RICH_GRID_RENDERER)) {
+                // richGridRenderer objects are returned for playlists with a Shorts UI
+                // As of 09/12/2023 (American English date format), an initial continuation allows
+                // to get continuations of playlists with a Shorts UI and regular playlist video
+                // renderers
+                final InfoItemsPage<StreamInfoItem> continuationPage = getInitialContinuationPage();
+                if (!continuationPage.getItems().isEmpty()) {
+                    return continuationPage;
+                }
+
+                // If no items could be extracted from the continuation, fall back to the shorts UI
+                // renderers, no continuation is provided
                 renderer = videoPlaylistObject.getObject(RICH_GRID_RENDERER);
             } else {
                 return new InfoItemsPage<>(collector, null);
@@ -375,25 +412,66 @@ private Page getNextPageFrom(final JsonArray contents)
 
         final JsonObject lastElement = contents.getObject(contents.size() - 1);
         if (lastElement.has("continuationItemRenderer")) {
-            final String continuation = lastElement
-                    .getObject("continuationItemRenderer")
+            return getPageFromContinuation(lastElement.getObject("continuationItemRenderer")
                     .getObject("continuationEndpoint")
                     .getObject("continuationCommand")
-                    .getString("token");
-
-            final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
-                            getExtractorLocalization(), getExtractorContentCountry())
-                            .value("continuation", continuation)
-                            .done())
-                    .getBytes(StandardCharsets.UTF_8);
-
-            return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey()
-                    + DISABLE_PRETTY_PRINT_PARAMETER, body);
+                    .getString("token"));
         } else {
             return null;
         }
     }
 
+    @Nonnull
+    private Page getPageFromContinuation(@Nonnull final String continuation)
+            throws IOException, ExtractionException {
+        final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
+                        getExtractorLocalization(), getExtractorContentCountry())
+                        .value("continuation", continuation)
+                        .done())
+                .getBytes(StandardCharsets.UTF_8);
+
+        return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey()
+                + DISABLE_PRETTY_PRINT_PARAMETER, body);
+    }
+
+    @Nonnull
+    private InfoItemsPage<StreamInfoItem> getInitialContinuationPage()
+            throws IOException, ExtractionException {
+        if (initialContinuationResponse == null) {
+            final String playlistId = getId();
+            final PlaylistContinuation playlistContinuation = PlaylistContinuation.newBuilder()
+                    .setParameters(ContinuationParams.newBuilder()
+                            .setBrowseId("VL" + playlistId)
+                            .setPlaylistId(playlistId)
+                            .setContinuationProperties(PLAYLIST_CONTINUATION_PROPERTIES_BASE64)
+                            .build())
+                    .build();
+
+            final String initialContinuation = Utils.encodeUrlUtf8(
+                    Base64.encodeBase64String(playlistContinuation.toByteArray()));
+            final Page page = getPageFromContinuation(initialContinuation);
+
+            try {
+                initialContinuationResponse = getJsonPostResponse("browse", page.getBody(),
+                        getExtractorLocalization());
+            } catch (final Exception e) {
+                return InfoItemsPage.emptyPage();
+            }
+        }
+
+        final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
+
+        final JsonArray initialItems = initialContinuationResponse
+                .getArray("onResponseReceivedActions")
+                .getObject(0)
+                .getObject("reloadContinuationItemsCommand")
+                .getArray("continuationItems");
+
+        collectStreamsFrom(collector, initialItems);
+
+        return new InfoItemsPage<>(collector, getNextPageFrom(initialItems));
+    }
+
     private void collectStreamsFrom(@Nonnull final StreamInfoItemsCollector collector,
                                     @Nonnull final JsonArray videos) {
         final TimeAgoParser timeAgoParser = getTimeAgoParser();