Merge pull request #465 from XiangRongLin/playlist_continuation

[YouTube] Fix playlist continuations extraction

Author: TobiGr

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java

@@ -190,9 +190,10 @@ public InfoItemsPage<StreamInfoItem> getInitialPage() {
             return new InfoItemsPage<>(collector, null);
         } else if (contents.getObject(0).has("playlistVideoListRenderer")) {
             final JsonObject videos = contents.getObject(0).getObject("playlistVideoListRenderer");
-            collectStreamsFrom(collector, videos.getArray("contents"));
+            final JsonArray videosArray = videos.getArray("contents");
+            collectStreamsFrom(collector, videosArray);
 
-            nextPage = getNextPageFrom(videos.getArray("continuations"));
+            nextPage = getNextPageFrom(videosArray);
         }
 
         return new InfoItemsPage<>(collector, nextPage);
@@ -207,24 +208,34 @@ public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException
         final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
         final JsonArray ajaxJson = getJsonResponse(page.getUrl(), getExtractorLocalization());
 
-        final JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response")
-                .getObject("continuationContents").getObject("playlistVideoListContinuation");
+        final JsonArray continuation = ajaxJson.getObject(1)
+            .getObject("response")
+            .getArray("onResponseReceivedActions")
+            .getObject(0)
+            .getObject("appendContinuationItemsAction")
+            .getArray("continuationItems");
 
-        collectStreamsFrom(collector, sectionListContinuation.getArray("contents"));
+        collectStreamsFrom(collector, continuation);
 
-        return new InfoItemsPage<>(collector, getNextPageFrom(sectionListContinuation.getArray("continuations")));
+        return new InfoItemsPage<>(collector, getNextPageFrom(continuation));
     }
 
-    private Page getNextPageFrom(final JsonArray continuations) {
-        if (isNullOrEmpty(continuations)) {
+    private Page getNextPageFrom(final JsonArray contents) {
+        if (isNullOrEmpty(contents)) {
             return null;
         }
 
-        final JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData");
-        final String continuation = nextContinuationData.getString("continuation");
-        final String clickTrackingParams = nextContinuationData.getString("clickTrackingParams");
-        return new Page("https://www.youtube.com/browse_ajax?ctoken=" + continuation + "&continuation=" + continuation
-                + "&itct=" + clickTrackingParams);
+        final JsonObject lastElement = contents.getObject(contents.size() - 1);
+        if (lastElement.has("continuationItemRenderer")) {
+            final String continuation = lastElement
+                .getObject("continuationItemRenderer")
+                .getObject("continuationEndpoint")
+                .getObject("continuationCommand")
+                .getString("token");
+            return new Page("https://www.youtube.com/browse_ajax?continuation=" + continuation);
+        } else {
+            return null;
+        }
     }
 
     private void collectStreamsFrom(final StreamInfoItemsCollector collector, final JsonArray videos) {

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java

@@ -3,6 +3,9 @@
 import org.junit.BeforeClass;
 import org.junit.Ignore;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Suite;
+import org.junit.runners.Suite.SuiteClasses;
 import org.schabi.newpipe.DownloaderTestImpl;
 import org.schabi.newpipe.extractor.ListExtractor;
 import org.schabi.newpipe.extractor.NewPipe;
@@ -11,10 +14,17 @@
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
 import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest;
+import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.ContinuationsTests;
+import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.HugePlaylist;
+import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.LearningPlaylist;
+import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.NotAvailable;
+import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTest.TimelessPopHits;
 import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 
+import static junit.framework.TestCase.assertFalse;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
 import static org.schabi.newpipe.extractor.ServiceList.YouTube;
@@ -23,6 +33,9 @@
 /**
  * Test for {@link YoutubePlaylistExtractor}
  */
+@RunWith(Suite.class)
+@SuiteClasses({NotAvailable.class, TimelessPopHits.class, HugePlaylist.class,
+    LearningPlaylist.class, ContinuationsTests.class})
 public class YoutubePlaylistExtractorTest {
 
     public static class NotAvailable {
@@ -114,7 +127,7 @@ public void testThumbnailUrl() throws Exception {
 
         @Ignore
         @Test
-        public void testBannerUrl() throws Exception {
+        public void testBannerUrl() {
             final String bannerUrl = extractor.getBannerUrl();
             assertIsSecureUrl(bannerUrl);
             assertTrue(bannerUrl, bannerUrl.contains("yt"));
@@ -227,7 +240,7 @@ public void testThumbnailUrl() throws Exception {
 
         @Ignore
         @Test
-        public void testBannerUrl() throws Exception {
+        public void testBannerUrl() {
             final String bannerUrl = extractor.getBannerUrl();
             assertIsSecureUrl(bannerUrl);
             assertTrue(bannerUrl, bannerUrl.contains("yt"));
@@ -324,7 +337,7 @@ public void testThumbnailUrl() throws Exception {
 
         @Ignore
         @Test
-        public void testBannerUrl() throws Exception {
+        public void testBannerUrl() {
             final String bannerUrl = extractor.getBannerUrl();
             assertIsSecureUrl(bannerUrl);
             assertTrue(bannerUrl, bannerUrl.contains("yt"));
@@ -352,4 +365,34 @@ public void testStreamCount() throws Exception {
             assertTrue("Error in the streams count", extractor.getStreamCount() > 40);
         }
     }
+
+    public static class ContinuationsTests {
+
+        @BeforeClass
+        public static void setUp() {
+            NewPipe.init(DownloaderTestImpl.getInstance());
+        }
+
+        @Test
+        public void testNoContinuations() throws Exception {
+            final YoutubePlaylistExtractor extractor = (YoutubePlaylistExtractor) YouTube
+                .getPlaylistExtractor(
+                    "https://www.youtube.com/playlist?list=PLXJg25X-OulsVsnvZ7RVtSDW-id9_RzAO");
+            extractor.fetchPage();
+
+            assertNoMoreItems(extractor);
+        }
+
+        @Test
+        public void testOnlySingleContinuation() throws Exception {
+            final YoutubePlaylistExtractor extractor = (YoutubePlaylistExtractor) YouTube
+                .getPlaylistExtractor(
+                    "https://www.youtube.com/playlist?list=PLjgwFL8urN2DFRuRkFTkmtHjyoNWHHdZX");
+            extractor.fetchPage();
+
+            final ListExtractor.InfoItemsPage<StreamInfoItem> page = defaultTestMoreItems(
+                extractor);
+            assertFalse("More items available when it shouldn't", page.hasNextPage());
+        }
+    }
 }