fix: extract YouTube trends from new "Videos" tab

Author: Theta-Dev

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java

@@ -41,12 +41,15 @@
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.util.stream.Stream;
 
 import javax.annotation.Nonnull;
 
 public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
     private JsonObject initialData;
 
+    private static final String VIDEOS_TAB_PARAMS = "4gIOGgxtb3N0X3BvcHVsYXI%3D";
+
     public YoutubeTrendingExtractor(final StreamingService service,
                                     final ListLinkHandler linkHandler,
                                     final String kioskId) {
@@ -60,6 +63,7 @@ public void onFetchPage(@Nonnull final Downloader downloader)
         final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(),
                 getExtractorContentCountry())
                 .value("browseId", "FEtrending")
+                .value("params", VIDEOS_TAB_PARAMS)
                 .done())
                 .getBytes(StandardCharsets.UTF_8);
         // @formatter:on
@@ -94,7 +98,10 @@ public String getName() throws ParsingException {
     public InfoItemsPage<StreamInfoItem> getInitialPage() throws ParsingException {
         final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
         final TimeAgoParser timeAgoParser = getTimeAgoParser();
-        final JsonObject tabContent = getTrendingTabContent();
+        final JsonObject tab = getTrendingTab();
+        final JsonObject tabContent = tab.getObject("content");
+        final boolean isVideoTab = tab.getObject("endpoint").getObject("browseEndpoint")
+                .getString("params", "").equals(VIDEOS_TAB_PARAMS);
 
         if (tabContent.has("richGridRenderer")) {
             tabContent.getObject("richGridRenderer")
@@ -110,7 +117,7 @@ public InfoItemsPage<StreamInfoItem> getInitialPage() throws ParsingException {
                     .forEachOrdered(videoRenderer -> collector.commit(
                             new YoutubeStreamInfoItemExtractor(videoRenderer, timeAgoParser)));
         } else if (tabContent.has("sectionListRenderer")) {
-            tabContent.getObject("sectionListRenderer")
+            final Stream<JsonObject> shelves = tabContent.getObject("sectionListRenderer")
                     .getArray("contents")
                     .stream()
                     .filter(JsonObject.class::isInstance)
@@ -120,11 +127,19 @@ public InfoItemsPage<StreamInfoItem> getInitialPage() throws ParsingException {
                             .stream())
                     .filter(JsonObject.class::isInstance)
                     .map(JsonObject.class::cast)
-                    .map(content -> content.getObject("shelfRenderer"))
-                    // Filter Trending shorts and Recently trending sections which have a title,
-                    // contrary to normal trends
-                    .filter(shelfRenderer -> !shelfRenderer.has("title"))
-                    .flatMap(shelfRenderer -> shelfRenderer.getObject("content")
+                    .map(content -> content.getObject("shelfRenderer"));
+
+            final Stream<JsonObject> items;
+            if (isVideoTab) {
+                // The first shelf of the Videos tab contains the normal trends
+                items = shelves.findFirst().stream();
+            } else {
+                // Filter Trending shorts and Recently trending sections which have a title,
+                // contrary to normal trends
+                items = shelves.filter(shelfRenderer -> !shelfRenderer.has("title"));
+            }
+
+            items.flatMap(shelfRenderer -> shelfRenderer.getObject("content")
                             .getObject("expandedShelfContentsRenderer")
                             .getArray("items")
                             .stream())
@@ -138,7 +153,7 @@ public InfoItemsPage<StreamInfoItem> getInitialPage() throws ParsingException {
         return new InfoItemsPage<>(collector, null);
     }
 
-    private JsonObject getTrendingTabContent() throws ParsingException {
+    private JsonObject getTrendingTab() throws ParsingException {
         return initialData.getObject("contents")
                 .getObject("twoColumnBrowseResultsRenderer")
                 .getArray("tabs")
@@ -150,7 +165,7 @@ private JsonObject getTrendingTabContent() throws ParsingException {
                 .filter(tabRenderer -> tabRenderer.has("content"))
                 // There should be at most one tab selected
                 .findFirst()
-                .orElseThrow(() -> new ParsingException("Could not get \"Now\" trending tab"))
-                .getObject("content");
+                .orElseThrow(() ->
+                        new ParsingException("Could not get \"Now\" or \"Videos\" trending tab"));
     }
 }

extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/kiosk/trending/generated_mock_0.json

@@ -29,7 +29,7 @@
         "https://www.youtube.com"
       ],
       "alt-svc": [
-        "h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
+        "h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000"
       ],
       "cache-control": [
         "private, max-age\u003d0"
@@ -41,10 +41,10 @@
         "same-origin; report-to\u003d\"youtube_main\""
       ],
       "date": [
-        "Tue, 22 Nov 2022 10:40:26 GMT"
+        "Sun, 02 Apr 2023 00:11:13 GMT"
       ],
       "expires": [
-        "Tue, 22 Nov 2022 10:40:26 GMT"
+        "Sun, 02 Apr 2023 00:11:13 GMT"
       ],
       "p3p": [
         "CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
@@ -59,9 +59,9 @@
         "ESF"
       ],
       "set-cookie": [
-        "YSC\u003daSSq4mC6HTI; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
-        "VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dWed, 26-Feb-2020 10:40:26 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
-        "CONSENT\u003dPENDING+953; expires\u003dThu, 21-Nov-2024 10:40:26 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
+        "YSC\u003dPYs50GAOR5M; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
+        "VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dMon, 06-Jul-2020 00:11:13 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
+        "CONSENT\u003dPENDING+445; expires\u003dTue, 01-Apr-2025 00:11:13 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
       ],
       "strict-transport-security": [
         "max-age\u003d31536000"