Merge pull request #1013 from Stypox/fix-music-mixes

[YouTube] Now music mixes can be treated as normal mixes

Author: Stypox

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -388,8 +388,7 @@ public static OffsetDateTime parseDateFrom(final String textualUploadDate)
      * @return Whether given id belongs to a YouTube Mix
      */
     public static boolean isYoutubeMixId(@Nonnull final String playlistId) {
-        return playlistId.startsWith("RD")
-                && !isYoutubeMusicMixId(playlistId);
+        return playlistId.startsWith("RD");
     }
 
     /**

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java

@@ -110,8 +110,7 @@ public ChannelExtractor getChannelExtractor(final ListLinkHandler linkHandler) {
 
     @Override
     public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) {
-        if (YoutubeParsingHelper.isYoutubeMixId(linkHandler.getId())
-                && !YoutubeParsingHelper.isYoutubeMusicMixId(linkHandler.getId())) {
+        if (YoutubeParsingHelper.isYoutubeMixId(linkHandler.getId())) {
             return new YoutubeMixPlaylistExtractor(this, linkHandler);
         } else {
             return new YoutubePlaylistExtractor(this, linkHandler);

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java

@@ -23,22 +23,22 @@
 import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
 import org.schabi.newpipe.extractor.playlist.PlaylistInfo;
 import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 
+@SuppressWarnings({"MismatchedQueryAndUpdateOfCollection", "NewClassNamingConvention"})
 public class YoutubeMixPlaylistExtractorTest {
 
     private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/mix/";
-    private static final Map<String, String> dummyCookie = new HashMap<>();
-
+    private static final Map<String, String> dummyCookie = Map.of(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
     private static YoutubeMixPlaylistExtractor extractor;
 
     public static class Mix {
@@ -50,7 +50,6 @@ public static void setUp() throws Exception {
             YoutubeTestsUtils.ensureStateless();
             YoutubeParsingHelper.setConsentAccepted(true);
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "mix"));
-            dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
             extractor = (YoutubeMixPlaylistExtractor) YouTube
                     .getPlaylistExtractor("https://www.youtube.com/watch?v=" + VIDEO_ID
                             + "&list=RD" + VIDEO_ID);
@@ -135,7 +134,6 @@ void getPlaylistType() throws ParsingException {
     }
 
     public static class MixWithIndex {
-
         private static final String VIDEO_ID = "FAqYW76GLPA";
         private static final String VIDEO_TITLE = "Mix – ";
         private static final int INDEX = 7; // YT starts the index with 1...
@@ -146,7 +144,6 @@ public static void setUp() throws Exception {
             YoutubeTestsUtils.ensureStateless();
             YoutubeParsingHelper.setConsentAccepted(true);
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "mixWithIndex"));
-            dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
             extractor = (YoutubeMixPlaylistExtractor) YouTube
                     .getPlaylistExtractor("https://www.youtube.com/watch?v=" + VIDEO_ID_AT_INDEX
                             + "&list=RD" + VIDEO_ID + "&index=" + INDEX);
@@ -233,7 +230,6 @@ public static void setUp() throws Exception {
             YoutubeTestsUtils.ensureStateless();
             YoutubeParsingHelper.setConsentAccepted(true);
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "myMix"));
-            dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
             extractor = (YoutubeMixPlaylistExtractor) YouTube
                     .getPlaylistExtractor("https://www.youtube.com/watch?v=" + VIDEO_ID
                             + "&list=RDMM" + VIDEO_ID);
@@ -316,15 +312,13 @@ void getPlaylistType() throws ParsingException {
     }
 
     public static class Invalid {
-
         private static final String VIDEO_ID = "QMVCAPd5cwBcg";
 
         @BeforeAll
         public static void setUp() throws IOException {
             YoutubeTestsUtils.ensureStateless();
             YoutubeParsingHelper.setConsentAccepted(true);
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "invalid"));
-            dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
         }
 
         @Test
@@ -348,7 +342,6 @@ void invalidVideoId() throws Exception {
     }
 
     public static class ChannelMix {
-
         private static final String CHANNEL_ID = "UCXuqSBlHAE6Xw-yeJA0Tunw";
         private static final String VIDEO_ID_OF_CHANNEL = "mnk6gnOBYIo";
         private static final String CHANNEL_TITLE = "Linus Tech Tips";
@@ -359,7 +352,6 @@ public static void setUp() throws Exception {
             YoutubeTestsUtils.ensureStateless();
             YoutubeParsingHelper.setConsentAccepted(true);
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "channelMix"));
-            dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
             extractor = (YoutubeMixPlaylistExtractor) YouTube
                     .getPlaylistExtractor("https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL
                             + "&list=RDCM" + CHANNEL_ID);
@@ -424,7 +416,6 @@ public static void setUp() throws Exception {
             YoutubeTestsUtils.ensureStateless();
             YoutubeParsingHelper.setConsentAccepted(true);
             NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "genreMix"));
-            dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
             extractor = (YoutubeMixPlaylistExtractor) YouTube
                     .getPlaylistExtractor("https://www.youtube.com/watch?v=" + VIDEO_ID
                             + "&list=RDGMEMYH9CUrFO7CfLJpaD7UR85w");
@@ -504,4 +495,93 @@ void getPlaylistType() throws ParsingException {
             assertEquals(PlaylistInfo.PlaylistType.MIX_GENRE, extractor.getPlaylistType());
         }
     }
+
+    public static class Music {
+        private static final String VIDEO_ID = "dQw4w9WgXcQ";
+        private static final String MIX_TITLE = "Mix – Rick Astley - Never Gonna Give You Up (Official Music Video)";
+
+        @BeforeAll
+        public static void setUp() throws Exception {
+            YoutubeTestsUtils.ensureStateless();
+            YoutubeParsingHelper.setConsentAccepted(true);
+            NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "musicMix"));
+            extractor = (YoutubeMixPlaylistExtractor)
+                    YouTube.getPlaylistExtractor("https://m.youtube.com/watch?v=" + VIDEO_ID
+                            + "&list=RDAMVM" + VIDEO_ID);
+            extractor.fetchPage();
+        }
+
+        @Test
+        void getServiceId() {
+            assertEquals(YouTube.getServiceId(), extractor.getServiceId());
+        }
+
+        @Test
+        void getName() throws Exception {
+            assertEquals(MIX_TITLE, extractor.getName());
+        }
+
+        @Test
+        void getThumbnailUrl() throws Exception {
+            final String thumbnailUrl = extractor.getThumbnailUrl();
+            assertIsSecureUrl(thumbnailUrl);
+            ExtractorAsserts.assertContains("yt", thumbnailUrl);
+            ExtractorAsserts.assertContains(VIDEO_ID, thumbnailUrl);
+        }
+
+        @Test
+        void getInitialPage() throws Exception {
+            final InfoItemsPage<StreamInfoItem> streams = extractor.getInitialPage();
+            assertFalse(streams.getItems().isEmpty());
+            assertTrue(streams.hasNextPage());
+        }
+
+        @Test
+        void getPage() throws Exception {
+            final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(
+                            NewPipe.getPreferredLocalization(), NewPipe.getPreferredContentCountry())
+                            .value("videoId", VIDEO_ID)
+                            .value("playlistId", "RD" + VIDEO_ID)
+                            .value("params", "OAE%3D")
+                            .done())
+                    .getBytes(StandardCharsets.UTF_8);
+
+            final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
+                    YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, dummyCookie, body));
+            assertFalse(streams.getItems().isEmpty());
+            assertTrue(streams.hasNextPage());
+        }
+
+        @Test
+        void getContinuations() throws Exception {
+            InfoItemsPage<StreamInfoItem> streams = extractor.getInitialPage();
+            final Set<String> urls = new HashSet<>();
+
+            // Should work infinitely, but for testing purposes only 3 times
+            for (int i = 0; i < 3; i++) {
+                assertTrue(streams.hasNextPage());
+                assertFalse(streams.getItems().isEmpty());
+
+                for (final StreamInfoItem item : streams.getItems()) {
+                    // TODO Duplicates are appearing
+                    // assertFalse(urls.contains(item.getUrl()));
+                    urls.add(item.getUrl());
+                }
+
+                streams = extractor.getPage(streams.getNextPage());
+            }
+            assertTrue(streams.hasNextPage());
+            assertFalse(streams.getItems().isEmpty());
+        }
+
+        @Test
+        void getStreamCount() throws ParsingException {
+            assertEquals(ListExtractor.ITEM_COUNT_INFINITE, extractor.getStreamCount());
+        }
+
+        @Test
+        void getPlaylistType() throws ParsingException {
+            assertEquals(PlaylistInfo.PlaylistType.MIX_MUSIC, extractor.getPlaylistType());
+        }
+    }
 }

extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/mix/musicMix/generated_mock_0.json

@@ -0,0 +1,82 @@
+{
+  "request": {
+    "httpMethod": "GET",
+    "url": "https://www.youtube.com/sw.js",
+    "headers": {
+      "Origin": [
+        "https://www.youtube.com"
+      ],
+      "Referer": [
+        "https://www.youtube.com"
+      ],
+      "Accept-Language": [
+        "en-GB, en;q\u003d0.9"
+      ]
+    },
+    "localization": {
+      "languageCode": "en",
+      "countryCode": "GB"
+    }
+  },
+  "response": {
+    "responseCode": 200,
+    "responseMessage": "",
+    "responseHeaders": {
+      "access-control-allow-credentials": [
+        "true"
+      ],
+      "access-control-allow-origin": [
+        "https://www.youtube.com"
+      ],
+      "alt-svc": [
+        "h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
+      ],
+      "cache-control": [
+        "private, max-age\u003d0"
+      ],
+      "content-type": [
+        "text/javascript; charset\u003dutf-8"
+      ],
+      "cross-origin-opener-policy-report-only": [
+        "same-origin; report-to\u003d\"youtube_main\""
+      ],
+      "date": [
+        "Sun, 15 Jan 2023 22:30:06 GMT"
+      ],
+      "expires": [
+        "Sun, 15 Jan 2023 22:30:06 GMT"
+      ],
+      "p3p": [
+        "CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
+      ],
+      "permissions-policy": [
+        "ch-ua-arch\u003d*, ch-ua-bitness\u003d*, ch-ua-full-version\u003d*, ch-ua-full-version-list\u003d*, ch-ua-model\u003d*, ch-ua-wow64\u003d*, ch-ua-platform\u003d*, ch-ua-platform-version\u003d*"
+      ],
+      "report-to": [
+        "{\"group\":\"youtube_main\",\"max_age\":2592000,\"endpoints\":[{\"url\":\"https://csp.withgoogle.com/csp/report-to/youtube_main\"}]}"
+      ],
+      "server": [
+        "ESF"
+      ],
+      "set-cookie": [
+        "YSC\u003d8TYEubKDjFA; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
+        "VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dMon, 20-Apr-2020 22:30:06 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
+        "CONSENT\u003dPENDING+835; expires\u003dTue, 14-Jan-2025 22:30:06 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
+      ],
+      "strict-transport-security": [
+        "max-age\u003d31536000"
+      ],
+      "x-content-type-options": [
+        "nosniff"
+      ],
+      "x-frame-options": [
+        "SAMEORIGIN"
+      ],
+      "x-xss-protection": [
+        "0"
+      ]
+    },
+    "responseBody": "\n  self.addEventListener(\u0027install\u0027, event \u003d\u003e {\n    event.waitUntil(self.skipWaiting());\n  });\n  self.addEventListener(\u0027activate\u0027, event \u003d\u003e {\n    event.waitUntil(\n      self.clients.claim().then(() \u003d\u003e self.registration.unregister()));\n  });\n  ",
+    "latestUrl": "https://www.youtube.com/sw.js"
+  }
+}