Merge pull request #540 from B0pol/fix-tests

fix tests, fix links in YouTube description

Author: TobiGr

extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java

@@ -50,6 +50,9 @@ public String getUrl(String id, String baseUrl) throws ParsingException {
      * @return a {@link LinkHandler} complete with information
      */
     public LinkHandler fromUrl(final String url) throws ParsingException {
+        if (Utils.isNullOrEmpty(url)) {
+            throw new IllegalArgumentException("The url is null or empty");
+        }
         final String polishedUrl = Utils.followGoogleRedirectIfNeeded(url);
         final String baseUrl = Utils.getBaseUrl(polishedUrl);
         return fromUrl(polishedUrl, baseUrl);

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -429,10 +429,17 @@ public static String[] getYoutubeMusicKeys() throws IOException, ReCaptchaExcept
         return youtubeMusicKeys = new String[]{key, clientName, clientVersion};
     }
 
+
+
     @Nullable
     public static String getUrlFromNavigationEndpoint(JsonObject navigationEndpoint) throws ParsingException {
         if (navigationEndpoint.has("urlEndpoint")) {
             String internUrl = navigationEndpoint.getObject("urlEndpoint").getString("url");
+            if (internUrl.startsWith("https://www.youtube.com/redirect?")) {
+                // remove https://www.youtube.com part to fall in the next if block
+                internUrl = internUrl.substring(23);
+            }
+
             if (internUrl.startsWith("/redirect?")) {
                 // q parameter can be the first parameter
                 internUrl = internUrl.substring(10);
@@ -450,6 +457,8 @@ public static String getUrlFromNavigationEndpoint(JsonObject navigationEndpoint)
                 }
             } else if (internUrl.startsWith("http")) {
                 return internUrl;
+            } else if (internUrl.startsWith("/channel") || internUrl.startsWith("/user") || internUrl.startsWith("/watch")) {
+                return "https://www.youtube.com" + internUrl;
             }
         } else if (navigationEndpoint.has("browseEndpoint")) {
             final JsonObject browseEndpoint = navigationEndpoint.getObject("browseEndpoint");

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java

@@ -224,7 +224,7 @@ public String getThumbnailUrl() throws ParsingException {
 
     @Nonnull
     @Override
-    public Description getDescription() {
+    public Description getDescription() throws ParsingException {
         assertPageFetched();
         // description with more info on links
         try {
@@ -234,8 +234,15 @@ public Description getDescription() {
             // age-restricted videos cause a ParsingException here
         }
 
+        String description = playerResponse.getObject("videoDetails").getString("shortDescription");
+        if (description == null) {
+            final JsonObject descriptionObject = playerResponse.getObject("microformat")
+                    .getObject("playerMicroformatRenderer").getObject("description");
+            description = getTextFromObject(descriptionObject);
+        }
+
         // raw non-html description
-        return new Description(playerResponse.getObject("videoDetails").getString("shortDescription"), Description.PLAIN_TEXT);
+        return new Description(description, Description.PLAIN_TEXT);
     }
 
     @Override

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java

@@ -69,7 +69,15 @@ public InfoItemsPage<StreamInfoItem> getPage(final Page page) {
     @Nonnull
     @Override
     public String getName() throws ParsingException {
-        String name = getTextFromObject(initialData.getObject("header").getObject("feedTabbedHeaderRenderer").getObject("title"));
+        final JsonObject header = initialData.getObject("header");
+        JsonObject title = null;
+        if (header.has("feedTabbedHeaderRenderer")) {
+            title = header.getObject("feedTabbedHeaderRenderer").getObject("title");
+        } else if (header.has("c4TabbedHeaderRenderer")) {
+            title = header.getObject("c4TabbedHeaderRenderer").getObject("title");
+        }
+
+        String name = getTextFromObject(title);
         if (!isNullOrEmpty(name)) {
             return name;
         }

extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCRecentListExtractorTest.java

@@ -26,16 +26,13 @@ public static void setUpClass() throws Exception {
     }
 
     @Test
-    @Ignore("TODO fix")
     public void testStreamList() throws Exception {
         final List<StreamInfoItem> items = extractor.getInitialPage().getItems();
         assertEquals(100, items.size());
         for (final StreamInfoItem item: items) {
             assertFalse(isNullOrEmpty(item.getName()));
             assertTrue(item.getDuration() > 0);
-            assertTrue(isNullOrEmpty(item.getUploaderName())); // we do not get the uploader name
             assertTrue(item.getUploadDate().offsetDateTime().isBefore(OffsetDateTime.now()));
-            assertTrue(item.getUploadDate().offsetDateTime().isAfter(OffsetDateTime.now().minusYears(1)));
         }
     }
 

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeAccountExtractorTest.java

@@ -101,7 +101,6 @@ public void testFeedUrl() throws ParsingException {
         }
 
         @Test
-        @Ignore("TODO fix")
         public void testSubscriberCount() throws ParsingException {
             assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 500);
         }

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistExtractorTest.java

@@ -4,6 +4,7 @@
 import org.junit.Ignore;
 import org.junit.Test;
 import org.schabi.newpipe.downloader.DownloaderTestImpl;
+import org.schabi.newpipe.extractor.ExtractorAsserts;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubePlaylistExtractor;
@@ -51,9 +52,8 @@ public void testGetUploaderName() throws ParsingException {
         }
 
         @Test
-        @Ignore("TODO fix")
         public void testGetStreamCount() throws ParsingException {
-            assertEquals(35, extractor.getStreamCount());
+            ExtractorAsserts.assertAtLeast(39, extractor.getStreamCount());
         }
 
         @Test
@@ -62,7 +62,6 @@ public void testGetSubChannelUrl() throws ParsingException {
         }
 
         @Test
-        @Ignore("TODO fix")
         public void testGetSubChannelName() throws ParsingException {
             assertEquals("SHOCKING !", extractor.getSubChannelName());
         }

extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java

@@ -94,13 +94,6 @@ public void testGetLanguageInformation() throws ParsingException {
         @Override public Locale expectedLanguageInfo() { return Locale.forLanguageTag("en"); }
         @Override public List<String> expectedTags() { return Arrays.asList("framasoft", "peertube"); }
         @Override public int expectedStreamSegmentsCount() { return 0; }
-
-        @Override
-        @Test
-        @Ignore("TODO fix")
-        public void testSubChannelName() throws Exception {
-            super.testSubChannelName();
-        }
     }
 
     @Ignore("TODO fix")

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractorTest.java

@@ -42,7 +42,6 @@ public void testServiceId() {
         }
 
         @Test
-        @Ignore("TODO fix")
         public void testName() {
             assertEquals("THE PERFECT LUV TAPE®️", extractor.getName());
         }

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamLinkHandlerFactoryTest.java

@@ -26,7 +26,6 @@ public static void setUp() throws Exception {
     }
 
     @Test(expected = IllegalArgumentException.class)
-    @Ignore("TODO fix")
     public void getIdWithNullAsUrl() throws ParsingException {
         linkHandler.fromUrl(null).getId();
     }