[YouTube] Workaround playlists' Shorts UI

YouTube doesn't return currently a continuation, if applicable, for
Shorts UI playlists, restricting access to the 100th first items. The
reel items returned don't give also upload date, uploader info and
precise view count.

Using a continuation which requests the first page of the playlist
allows currently to get access to continuations, if applicable, and
also standard video elements instead of Shorts ones, making extraction
of upload date, uploader info and precise view count again possible.

This method is used for all playlist types, the original request is
still made, but now only returns what we need.

It requires to add a protocol buffer definition file, for which its
structure is based on reverse engineering of playlists continuations
sent by WEB InnerTube client, received from InnerTube responses.

Java classes of this file are generated for the Java Lite runtime of
Protobuf with the Protobuf Gradle plugin, as the lite version is enough
for our use cases. This plugin ships in JARs Protobuf definitions,
which should be avoided.

As Protobuf classes are parsed by Checkstyle checks and do not follow
our style rules at all, an exclusion rule has been for them.

Author: AudricV

extractor/build.gradle

@@ -1,5 +1,6 @@
 plugins {
-    id 'checkstyle'
+    id "checkstyle"
+    id "com.google.protobuf" version "0.9.5"
 }
 
 test {
@@ -18,12 +19,16 @@ checkstyle {
     toolVersion checkstyleVersion
 }
 
+// Exclude Protobuf generated files from Checkstyle
+checkstyleMain.exclude("org/schabi/newpipe/extractor/services/youtube/protos")
+
 checkstyleTest {
     enabled false // do not checkstyle test files
 }
 
 ext {
     rhinoVersion = '1.8.0'
+    protobufVersion = '4.30.2'
 }
 
 dependencies {
@@ -32,6 +37,7 @@ dependencies {
     implementation "com.github.TeamNewPipe:nanojson:$nanojsonVersion"
     implementation 'org.jsoup:jsoup:1.19.1'
     implementation "com.google.code.findbugs:jsr305:$jsr305Version"
+    implementation "com.google.protobuf:protobuf-javalite:$protobufVersion"
 
     implementation "org.mozilla:rhino:$rhinoVersion"
     implementation "org.mozilla:rhino-engine:$rhinoVersion"
@@ -47,3 +53,19 @@ dependencies {
     testImplementation "com.squareup.okhttp3:okhttp:4.12.0"
     testImplementation 'com.google.code.gson:gson:2.12.1'
 }
+
+protobuf {
+    protoc {
+        artifact = "com.google.protobuf:protoc:$protobufVersion"
+    }
+
+    generateProtoTasks {
+        all().configureEach { task ->
+            task.builtins {
+                java {
+                    option "lite"
+                }
+            }
+        }
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java

@@ -8,6 +8,8 @@
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
+import static org.schabi.newpipe.extractor.services.youtube.protos.playlist.PlaylistProtobufContinuation.ContinuationParams;
+import static org.schabi.newpipe.extractor.services.youtube.protos.playlist.PlaylistProtobufContinuation.PlaylistContinuation;
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
 
 import com.grack.nanojson.JsonArray;
@@ -33,6 +35,7 @@
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.util.Base64;
 import java.util.List;
 
 import javax.annotation.Nonnull;
@@ -41,14 +44,17 @@
 public class YoutubePlaylistExtractor extends PlaylistExtractor {
     // Names of some objects in JSON response frequently used in this class
     private static final String PLAYLIST_VIDEO_RENDERER = "playlistVideoRenderer";
-    private static final String PLAYLIST_VIDEO_LIST_RENDERER = "playlistVideoListRenderer";
-    private static final String RICH_GRID_RENDERER = "richGridRenderer";
     private static final String RICH_ITEM_RENDERER = "richItemRenderer";
     private static final String REEL_ITEM_RENDERER = "reelItemRenderer";
     private static final String SIDEBAR = "sidebar";
+    private static final String HEADER = "header";
     private static final String VIDEO_OWNER_RENDERER = "videoOwnerRenderer";
+    private static final String MICROFORMAT = "microformat";
+    // Continuation properties requesting first page and showing unavailable videos
+    private static final String PLAYLIST_CONTINUATION_PROPERTIES_BASE64 = "CADCBgIIAA%3D%3D";
 
-    private JsonObject browseResponse;
+    private JsonObject browseMetadataResponse;
+    private JsonObject initialBrowseContinuationResponse;
 
     private JsonObject playlistInfo;
     private JsonObject uploaderInfo;
@@ -64,17 +70,40 @@ public YoutubePlaylistExtractor(final StreamingService service,
     @Override
     public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
             ExtractionException {
+        final String playlistId = getId();
+
         final Localization localization = getExtractorLocalization();
         final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(localization,
                         getExtractorContentCountry())
-                        .value("browseId", "VL" + getId())
+                        .value("browseId", "VL" + playlistId)
                         .value("params", "wgYCCAA%3D") // Show unavailable videos
                         .done())
                 .getBytes(StandardCharsets.UTF_8);
 
-        browseResponse = getJsonPostResponse("browse", body, localization);
-        YoutubeParsingHelper.defaultAlertsCheck(browseResponse);
+        browseMetadataResponse = getJsonPostResponse("browse",
+                List.of("$fields=" + SIDEBAR + "," + HEADER + "," + MICROFORMAT + ",alerts"),
+                body,
+                localization);
+
+        YoutubeParsingHelper.defaultAlertsCheck(browseMetadataResponse);
         isNewPlaylistInterface = checkIfResponseIsNewPlaylistInterface();
+
+        final PlaylistContinuation playlistContinuation = PlaylistContinuation.newBuilder()
+                .setParameters(ContinuationParams.newBuilder()
+                        .setBrowseId("VL" + playlistId)
+                        .setPlaylistId(playlistId)
+                        .setContinuationProperties(PLAYLIST_CONTINUATION_PROPERTIES_BASE64)
+                        .build())
+                .build();
+
+        initialBrowseContinuationResponse = getJsonPostResponse("browse",
+                JsonWriter.string(prepareDesktopJsonBuilder(localization,
+                        getExtractorContentCountry())
+                        .value("continuation", Utils.encodeUrlUtf8(Base64.getUrlEncoder()
+                                .encodeToString(playlistContinuation.toByteArray())))
+                        .done())
+                        .getBytes(StandardCharsets.UTF_8),
+                localization);
     }
 
     /**
@@ -93,13 +122,13 @@ public void onFetchPage(@Nonnull final Downloader downloader) throws IOException
      */
     private boolean checkIfResponseIsNewPlaylistInterface() {
         // The "old" playlist UI can be also returned with the new one
-        return browseResponse.has("header") && !browseResponse.has(SIDEBAR);
+        return browseMetadataResponse.has(HEADER) && !browseMetadataResponse.has(SIDEBAR);
     }
 
     @Nonnull
     private JsonObject getUploaderInfo() throws ParsingException {
         if (uploaderInfo == null) {
-            uploaderInfo = browseResponse.getObject(SIDEBAR)
+            uploaderInfo = browseMetadataResponse.getObject(SIDEBAR)
                     .getObject("playlistSidebarRenderer")
                     .getArray("items")
                     .stream()
@@ -121,7 +150,7 @@ private JsonObject getUploaderInfo() throws ParsingException {
     @Nonnull
     private JsonObject getPlaylistInfo() throws ParsingException {
         if (playlistInfo == null) {
-            playlistInfo = browseResponse.getObject(SIDEBAR)
+            playlistInfo = browseMetadataResponse.getObject(SIDEBAR)
                     .getObject("playlistSidebarRenderer")
                     .getArray("items")
                     .stream()
@@ -139,7 +168,7 @@ private JsonObject getPlaylistInfo() throws ParsingException {
     @Nonnull
     private JsonObject getPlaylistHeader() {
         if (playlistHeader == null) {
-            playlistHeader = browseResponse.getObject("header")
+            playlistHeader = browseMetadataResponse.getObject(HEADER)
                     .getObject("playlistHeaderRenderer");
         }
 
@@ -154,7 +183,7 @@ public String getName() throws ParsingException {
             return name;
         }
 
-        return browseResponse.getObject("microformat")
+        return browseMetadataResponse.getObject(MICROFORMAT)
                 .getObject("microformatDataRenderer")
                 .getString("title");
     }
@@ -180,7 +209,7 @@ public List<Image> getThumbnails() throws ParsingException {
         }
 
         // This data structure is returned in both layouts
-        final JsonArray microFormatThumbnailsArray = browseResponse.getObject("microformat")
+        final JsonArray microFormatThumbnailsArray = browseMetadataResponse.getObject(MICROFORMAT)
                     .getObject("microformatDataRenderer")
                     .getObject("thumbnail")
                     .getArray("thumbnails");
@@ -302,45 +331,16 @@ public Description getDescription() throws ParsingException {
     @Override
     public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
         final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
-        Page nextPage = null;
 
-        final JsonArray contents = browseResponse.getObject("contents")
-                .getObject("twoColumnBrowseResultsRenderer")
-                .getArray("tabs")
+        final JsonArray initialItems = initialBrowseContinuationResponse
+                .getArray("onResponseReceivedActions")
                 .getObject(0)
-                .getObject("tabRenderer")
-                .getObject("content")
-                .getObject("sectionListRenderer")
-                .getArray("contents");
-
-        final JsonObject videoPlaylistObject = contents.stream()
-                .filter(JsonObject.class::isInstance)
-                .map(JsonObject.class::cast)
-                .map(content -> content.getObject("itemSectionRenderer")
-                        .getArray("contents")
-                        .getObject(0))
-                .filter(content -> content.has(PLAYLIST_VIDEO_LIST_RENDERER)
-                        || content.has(RICH_GRID_RENDERER))
-                .findFirst()
-                .orElse(null);
-
-        if (videoPlaylistObject != null) {
-            final JsonObject renderer;
-            if (videoPlaylistObject.has(PLAYLIST_VIDEO_LIST_RENDERER)) {
-                renderer = videoPlaylistObject.getObject(PLAYLIST_VIDEO_LIST_RENDERER);
-            } else if (videoPlaylistObject.has(RICH_GRID_RENDERER)) {
-                renderer = videoPlaylistObject.getObject(RICH_GRID_RENDERER);
-            } else {
-                return new InfoItemsPage<>(collector, null);
-            }
-
-            final JsonArray videosArray = renderer.getArray("contents");
-            collectStreamsFrom(collector, videosArray);
+                .getObject("reloadContinuationItemsCommand")
+                .getArray("continuationItems");
 
-            nextPage = getNextPageFrom(videosArray);
-        }
+        collectStreamsFrom(collector, initialItems);
 
-        return new InfoItemsPage<>(collector, nextPage);
+        return new InfoItemsPage<>(collector, getNextPageFrom(initialItems));
     }
 
     @Override

extractor/src/main/proto/youtube/playlist/playlist_continuation_schema.proto

@@ -0,0 +1,20 @@
+syntax = "proto3";
+
+package youtube.playlists;
+
+option java_outer_classname = "PlaylistProtobufContinuation";
+option java_multiple_files = false;
+option java_package = "org.schabi.newpipe.extractor.services.youtube.protos.playlist";
+option optimize_for = LITE_RUNTIME;
+
+message PlaylistContinuation {
+  ContinuationParams parameters = 80226972;
+}
+
+message ContinuationParams {
+  // The playlist ID as a browse one (it should be "VL" + playlist ID)
+  string browseId = 2;
+  // A PlaylistContinuationProperties message safe-encoded as a Base64 string
+  string continuationProperties = 3;
+  string playlistId = 35;
+}