Merge pull request TeamNewPipe#1104 from AudricV/yt_continuations-playlists-shorts-ui

Author: Stypox

build.gradle

@@ -40,12 +40,20 @@ dependencies {
 }
 
 subprojects {
-    task sourcesJar(type: Jar, dependsOn: classes) {
+    tasks.register('sourcesJar', Jar) {
+        dependsOn classes
         archiveClassifier.set('sources')
         from sourceSets.main.allSource
     }
 
-    tasks.withType(Test) {
+    // Protobuf files would uselessly end up in the JAR otherwise, see
+    // https://github.com/google/protobuf-gradle-plugin/issues/390
+    tasks.withType(Jar).configureEach {
+        exclude '**/*.proto'
+        includeEmptyDirs false
+    }
+
+    tasks.withType(Test).configureEach {
         testLogging {
             events "skipped", "failed"
             showStandardStreams = true
@@ -59,8 +67,8 @@ subprojects {
 }
 
 // https://discuss.gradle.org/t/best-approach-gradle-multi-module-project-generate-just-one-global-javadoc/18657/21
-task aggregatedJavadocs(type: Javadoc, group: 'Documentation') {
-    destinationDir = file("$buildDir/docs/javadoc")
+tasks.register('aggregatedJavadocs', Javadoc) {
+    destinationDir = file("${layout.buildDirectory}/docs/javadoc")
     title = "$project.name $version"
     // options.memberLevel = JavadocMemberLevel.PRIVATE
     options.links 'https://docs.oracle.com/javase/8/docs/api/'

extractor/build.gradle

@@ -1,5 +1,6 @@
 plugins {
-    id 'checkstyle'
+    id "checkstyle"
+    id "com.google.protobuf" version "0.9.5"
 }
 
 test {
@@ -18,12 +19,16 @@ checkstyle {
     toolVersion checkstyleVersion
 }
 
+// Exclude Protobuf generated files from Checkstyle
+checkstyleMain.exclude("org/schabi/newpipe/extractor/services/youtube/protos")
+
 checkstyleTest {
     enabled false // do not checkstyle test files
 }
 
 ext {
     rhinoVersion = '1.8.0'
+    protobufVersion = '4.30.2'
 }
 
 dependencies {
@@ -32,6 +37,7 @@ dependencies {
     implementation "com.github.TeamNewPipe:nanojson:$nanojsonVersion"
     implementation 'org.jsoup:jsoup:1.21.1'
     implementation "com.google.code.findbugs:jsr305:$jsr305Version"
+    implementation "com.google.protobuf:protobuf-javalite:$protobufVersion"
 
     implementation "org.mozilla:rhino:$rhinoVersion"
     implementation "org.mozilla:rhino-engine:$rhinoVersion"
@@ -47,3 +53,19 @@ dependencies {
     testImplementation "com.squareup.okhttp3:okhttp:4.12.0"
     testImplementation 'com.google.code.gson:gson:2.13.1'
 }
+
+protobuf {
+    protoc {
+        artifact = "com.google.protobuf:protoc:$protobufVersion"
+    }
+
+    generateProtoTasks {
+        all().configureEach { task ->
+            task.builtins {
+                java {
+                    option "lite"
+                }
+            }
+        }
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -1017,9 +1017,9 @@ public static String getValidJsonResponseBody(@Nonnull final Response response)
         return responseBody;
     }
 
-    public static JsonObject getJsonPostResponse(final String endpoint,
+    public static JsonObject getJsonPostResponse(@Nonnull final String endpoint,
                                                  final byte[] body,
-                                                 final Localization localization)
+                                                 @Nonnull final Localization localization)
             throws IOException, ExtractionException {
         final var headers = getYouTubeHeaders();
 
@@ -1028,6 +1028,26 @@ public static JsonObject getJsonPostResponse(final String endpoint,
                         + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization)));
     }
 
+    public static JsonObject getJsonPostResponse(@Nonnull final String endpoint,
+                                                 @Nonnull final List<String> queryParameters,
+                                                 final byte[] body,
+                                                 @Nonnull final Localization localization)
+            throws IOException, ExtractionException {
+        final var headers = getYouTubeHeaders();
+
+        final String queryParametersString;
+        if (queryParameters.isEmpty()) {
+            queryParametersString = "?" + DISABLE_PRETTY_PRINT_PARAMETER;
+        } else {
+            queryParametersString = "?" + String.join("&", queryParameters)
+                    + "&" + DISABLE_PRETTY_PRINT_PARAMETER;
+        }
+
+        return JsonUtils.toJsonObject(getValidJsonResponseBody(
+                getDownloader().postWithContentTypeJson(YOUTUBEI_V1_URL + endpoint
+                        + queryParametersString, headers, body, localization)));
+    }
+
     @Nonnull
     public static JsonBuilder<JsonObject> prepareDesktopJsonBuilder(
             @Nonnull final Localization localization,

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java

@@ -8,6 +8,8 @@
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
+import static org.schabi.newpipe.extractor.services.youtube.protos.playlist.PlaylistProtobufContinuation.ContinuationParams;
+import static org.schabi.newpipe.extractor.services.youtube.protos.playlist.PlaylistProtobufContinuation.PlaylistContinuation;
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
 
 import com.grack.nanojson.JsonArray;
@@ -33,6 +35,7 @@
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.util.Base64;
 import java.util.List;
 
 import javax.annotation.Nonnull;
@@ -41,14 +44,17 @@
 public class YoutubePlaylistExtractor extends PlaylistExtractor {
     // Names of some objects in JSON response frequently used in this class
     private static final String PLAYLIST_VIDEO_RENDERER = "playlistVideoRenderer";
-    private static final String PLAYLIST_VIDEO_LIST_RENDERER = "playlistVideoListRenderer";
-    private static final String RICH_GRID_RENDERER = "richGridRenderer";
     private static final String RICH_ITEM_RENDERER = "richItemRenderer";
     private static final String REEL_ITEM_RENDERER = "reelItemRenderer";
     private static final String SIDEBAR = "sidebar";
+    private static final String HEADER = "header";
     private static final String VIDEO_OWNER_RENDERER = "videoOwnerRenderer";
+    private static final String MICROFORMAT = "microformat";
+    // Continuation properties requesting first page and showing unavailable videos
+    private static final String PLAYLIST_CONTINUATION_PROPERTIES_BASE64 = "CADCBgIIAA%3D%3D";
 
-    private JsonObject browseResponse;
+    private JsonObject browseMetadataResponse;
+    private JsonObject initialBrowseContinuationResponse;
 
     private JsonObject playlistInfo;
     private JsonObject uploaderInfo;
@@ -64,17 +70,40 @@ public YoutubePlaylistExtractor(final StreamingService service,
     @Override
     public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
             ExtractionException {
+        final String playlistId = getId();
+
         final Localization localization = getExtractorLocalization();
         final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(localization,
                         getExtractorContentCountry())
-                        .value("browseId", "VL" + getId())
+                        .value("browseId", "VL" + playlistId)
                         .value("params", "wgYCCAA%3D") // Show unavailable videos
                         .done())
                 .getBytes(StandardCharsets.UTF_8);
 
-        browseResponse = getJsonPostResponse("browse", body, localization);
-        YoutubeParsingHelper.defaultAlertsCheck(browseResponse);
+        browseMetadataResponse = getJsonPostResponse("browse",
+                List.of("$fields=" + SIDEBAR + "," + HEADER + "," + MICROFORMAT + ",alerts"),
+                body,
+                localization);
+
+        YoutubeParsingHelper.defaultAlertsCheck(browseMetadataResponse);
         isNewPlaylistInterface = checkIfResponseIsNewPlaylistInterface();
+
+        final PlaylistContinuation playlistContinuation = PlaylistContinuation.newBuilder()
+                .setParameters(ContinuationParams.newBuilder()
+                        .setBrowseId("VL" + playlistId)
+                        .setPlaylistId(playlistId)
+                        .setContinuationProperties(PLAYLIST_CONTINUATION_PROPERTIES_BASE64)
+                        .build())
+                .build();
+
+        initialBrowseContinuationResponse = getJsonPostResponse("browse",
+                JsonWriter.string(prepareDesktopJsonBuilder(localization,
+                        getExtractorContentCountry())
+                        .value("continuation", Utils.encodeUrlUtf8(Base64.getUrlEncoder()
+                                .encodeToString(playlistContinuation.toByteArray())))
+                        .done())
+                        .getBytes(StandardCharsets.UTF_8),
+                localization);
     }
 
     /**
@@ -93,13 +122,13 @@ public void onFetchPage(@Nonnull final Downloader downloader) throws IOException
      */
     private boolean checkIfResponseIsNewPlaylistInterface() {
         // The "old" playlist UI can be also returned with the new one
-        return browseResponse.has("header") && !browseResponse.has(SIDEBAR);
+        return browseMetadataResponse.has(HEADER) && !browseMetadataResponse.has(SIDEBAR);
     }
 
     @Nonnull
     private JsonObject getUploaderInfo() throws ParsingException {
         if (uploaderInfo == null) {
-            uploaderInfo = browseResponse.getObject(SIDEBAR)
+            uploaderInfo = browseMetadataResponse.getObject(SIDEBAR)
                     .getObject("playlistSidebarRenderer")
                     .getArray("items")
                     .stream()
@@ -121,7 +150,7 @@ private JsonObject getUploaderInfo() throws ParsingException {
     @Nonnull
     private JsonObject getPlaylistInfo() throws ParsingException {
         if (playlistInfo == null) {
-            playlistInfo = browseResponse.getObject(SIDEBAR)
+            playlistInfo = browseMetadataResponse.getObject(SIDEBAR)
                     .getObject("playlistSidebarRenderer")
                     .getArray("items")
                     .stream()
@@ -139,7 +168,7 @@ private JsonObject getPlaylistInfo() throws ParsingException {
     @Nonnull
     private JsonObject getPlaylistHeader() {
         if (playlistHeader == null) {
-            playlistHeader = browseResponse.getObject("header")
+            playlistHeader = browseMetadataResponse.getObject(HEADER)
                     .getObject("playlistHeaderRenderer");
         }
 
@@ -154,7 +183,7 @@ public String getName() throws ParsingException {
             return name;
         }
 
-        return browseResponse.getObject("microformat")
+        return browseMetadataResponse.getObject(MICROFORMAT)
                 .getObject("microformatDataRenderer")
                 .getString("title");
     }
@@ -180,7 +209,7 @@ public List<Image> getThumbnails() throws ParsingException {
         }
 
         // This data structure is returned in both layouts
-        final JsonArray microFormatThumbnailsArray = browseResponse.getObject("microformat")
+        final JsonArray microFormatThumbnailsArray = browseMetadataResponse.getObject(MICROFORMAT)
                     .getObject("microformatDataRenderer")
                     .getObject("thumbnail")
                     .getArray("thumbnails");
@@ -302,45 +331,16 @@ public Description getDescription() throws ParsingException {
     @Override
     public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
         final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
-        Page nextPage = null;
 
-        final JsonArray contents = browseResponse.getObject("contents")
-                .getObject("twoColumnBrowseResultsRenderer")
-                .getArray("tabs")
+        final JsonArray initialItems = initialBrowseContinuationResponse
+                .getArray("onResponseReceivedActions")
                 .getObject(0)
-                .getObject("tabRenderer")
-                .getObject("content")
-                .getObject("sectionListRenderer")
-                .getArray("contents");
-
-        final JsonObject videoPlaylistObject = contents.stream()
-                .filter(JsonObject.class::isInstance)
-                .map(JsonObject.class::cast)
-                .map(content -> content.getObject("itemSectionRenderer")
-                        .getArray("contents")
-                        .getObject(0))
-                .filter(content -> content.has(PLAYLIST_VIDEO_LIST_RENDERER)
-                        || content.has(RICH_GRID_RENDERER))
-                .findFirst()
-                .orElse(null);
-
-        if (videoPlaylistObject != null) {
-            final JsonObject renderer;
-            if (videoPlaylistObject.has(PLAYLIST_VIDEO_LIST_RENDERER)) {
-                renderer = videoPlaylistObject.getObject(PLAYLIST_VIDEO_LIST_RENDERER);
-            } else if (videoPlaylistObject.has(RICH_GRID_RENDERER)) {
-                renderer = videoPlaylistObject.getObject(RICH_GRID_RENDERER);
-            } else {
-                return new InfoItemsPage<>(collector, null);
-            }
-
-            final JsonArray videosArray = renderer.getArray("contents");
-            collectStreamsFrom(collector, videosArray);
+                .getObject("reloadContinuationItemsCommand")
+                .getArray("continuationItems");
 
-            nextPage = getNextPageFrom(videosArray);
-        }
+        collectStreamsFrom(collector, initialItems);
 
-        return new InfoItemsPage<>(collector, nextPage);
+        return new InfoItemsPage<>(collector, getNextPageFrom(initialItems));
     }
 
     @Override

extractor/src/main/proto/youtube/playlist/playlist_continuation_schema.proto

@@ -0,0 +1,20 @@
+syntax = "proto3";
+
+package youtube.playlists;
+
+option java_outer_classname = "PlaylistProtobufContinuation";
+option java_multiple_files = false;
+option java_package = "org.schabi.newpipe.extractor.services.youtube.protos.playlist";
+option optimize_for = LITE_RUNTIME;
+
+message PlaylistContinuation {
+  ContinuationParams parameters = 80226972;
+}
+
+message ContinuationParams {
+  // The playlist ID as a browse one (it should be "VL" + playlist ID)
+  string browseId = 2;
+  // A PlaylistContinuationProperties message safe-encoded as a Base64 string
+  string continuationProperties = 3;
+  string playlistId = 35;
+}

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java

@@ -481,17 +481,12 @@ public void testOriginalUrl() throws Exception {
                     extractor.getOriginalUrl());
         }
 
-        @Disabled("Known problem, see https://github.com/TeamNewPipe/NewPipeExtractor/issues/1273")
         @Test
         @Override
         public void testRelatedItems() throws Exception {
             defaultTestRelatedItems(extractor);
         }
 
-        // TODO: enable test when continuations are available
-        @Disabled("Shorts UI doesn't return any continuation, even if when there are more than 100 "
-                + "items: this is a bug on YouTube's side, which is not related to the requirement "
-                + "of a valid visitorData like it is for Shorts channel tab")
         @Test
         @Override
         public void testMoreRelatedItems() throws Exception {

extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channelTabs/ageRestrictedTabsShorts/generated_mock_0.json

@@ -44,10 +44,10 @@
         "same-origin; report-to\u003d\"youtube_main\""
       ],
       "date": [
-        "Sun, 10 Nov 2024 17:48:21 GMT"
+        "Fri, 18 Apr 2025 14:18:26 GMT"
       ],
       "expires": [
-        "Sun, 10 Nov 2024 17:48:21 GMT"
+        "Fri, 18 Apr 2025 14:18:26 GMT"
       ],
       "origin-trial": [
         "AmhMBR6zCLzDDxpW+HfpP67BqwIknWnyMOXOQGfzYswFmJe+fgaI6XZgAzcxOrzNtP7hEDsOo1jdjFnVr2IdxQ4AAAB4eyJvcmlnaW4iOiJodHRwczovL3lvdXR1YmUuY29tOjQ0MyIsImZlYXR1cmUiOiJXZWJWaWV3WFJlcXVlc3RlZFdpdGhEZXByZWNhdGlvbiIsImV4cGlyeSI6MTc1ODA2NzE5OSwiaXNTdWJkb21haW4iOnRydWV9"
@@ -65,8 +65,8 @@
         "ESF"
       ],
       "set-cookie": [
-        "YSC\u003d4hCTcf7rUXA; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
-        "VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dMon, 14-Feb-2022 17:48:21 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone"
+        "YSC\u003dQCKW-aKaIuY; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
+        "VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dSat, 23-Jul-2022 14:18:26 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone"
       ],
       "strict-transport-security": [
         "max-age\u003d31536000"