Merge branch 'dev' into yt-webm-opus

Author: Stypox

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java

@@ -82,6 +82,11 @@ public String getUrl() throws ParsingException {
     @Nonnull
     @Override
     public String getId() throws ParsingException {
+        try {
+            return doc.select("meta[itemprop=\"channelId\"]").first().attr("content");
+        } catch (Exception ignored) {}
+
+        // fallback method; does not work with channels that have no "Subscribe" button (e.g. EminemVEVO)
         try {
             Element element = doc.getElementsByClass("yt-uix-subscription-button").first();
             if (element == null) element = doc.getElementsByClass("yt-uix-subscription-preferences-button").first();
@@ -135,11 +140,12 @@ public String getFeedUrl() throws ParsingException {
 
     @Override
     public long getSubscriberCount() throws ParsingException {
-        final String el = doc.select("span[class*=\"yt-subscription-button-subscriber-count\"]")
-                .first().attr("title");
+
+        final Element el = doc.select("span[class*=\"yt-subscription-button-subscriber-count\"]").first();
         if (el != null) {
+            String elTitle = el.attr("title");
             try {
-                return Utils.mixedNumberWordToLong(el);
+                return Utils.mixedNumberWordToLong(elTitle);
             } catch (NumberFormatException e) {
                 throw new ParsingException("Could not get subscriber count", e);
             }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java

@@ -56,19 +56,25 @@ public String getName() throws ParsingException {
 
     @Override
     public String getUrl() throws ParsingException {
-        String buttonTrackingUrl = el.select("button[class*=\"yt-uix-button\"]").first()
-                .attr("abs:data-href");
+        try {
+            String buttonTrackingUrl = el.select("button[class*=\"yt-uix-button\"]").first()
+                    .attr("abs:data-href");
 
-        Pattern channelIdPattern = Pattern.compile("(?:.*?)\\%252Fchannel\\%252F([A-Za-z0-9\\-\\_]+)(?:.*)");
-        Matcher match = channelIdPattern.matcher(buttonTrackingUrl);
+            Pattern channelIdPattern = Pattern.compile("(?:.*?)\\%252Fchannel\\%252F([A-Za-z0-9\\-\\_]+)(?:.*)");
+            Matcher match = channelIdPattern.matcher(buttonTrackingUrl);
 
-        if (match.matches()) {
-            return YoutubeChannelExtractor.CHANNEL_URL_BASE + match.group(1);
-        } else {
-            // fallback method just in case youtube changes things; it should never run and tests will fail
-            // provides an url with "/user/NAME", that is inconsistent with stream and channel extractor
+            if (match.matches()) {
+                return YoutubeChannelExtractor.CHANNEL_URL_BASE + match.group(1);
+            }
+        } catch(Exception ignored) {}
+
+        // fallback method for channels without "Subscribe" button (or just in case yt changes things)
+        // provides an url with "/user/NAME", inconsistent with stream and channel extractor: tests will fail
+        try {
             return el.select("a[class*=\"yt-uix-tile-link\"]").first()
                     .attr("abs:href");
+        } catch (Exception e) {
+            throw new ParsingException("Could not get channel url", e);
         }
     }
 

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java

@@ -575,21 +575,26 @@ public StreamInfoItemsCollector getRelatedStreams() throws IOException, Extracti
      */
     @Override
     public String getErrorMessage() {
-        String errorMessage = doc.select("h1[id=\"unavailable-message\"]").first().text();
         StringBuilder errorReason;
+        Element errorElement = doc.select("h1[id=\"unavailable-message\"]").first();
 
-        if (errorMessage == null || errorMessage.isEmpty()) {
+        if (errorElement == null) {
             errorReason = null;
-        } else if (errorMessage.contains("GEMA")) {
-            // Gema sometimes blocks youtube music content in germany:
-            // https://www.gema.de/en/
-            // Detailed description:
-            // https://en.wikipedia.org/wiki/GEMA_%28German_organization%29
-            errorReason = new StringBuilder("GEMA");
         } else {
-            errorReason = new StringBuilder(errorMessage);
-            errorReason.append("  ");
-            errorReason.append(doc.select("[id=\"unavailable-submessage\"]").first().text());
+            String errorMessage = errorElement.text();
+            if (errorMessage == null || errorMessage.isEmpty()) {
+                errorReason = null;
+            } else if (errorMessage.contains("GEMA")) {
+                // Gema sometimes blocks youtube music content in germany:
+                // https://www.gema.de/en/
+                // Detailed description:
+                // https://en.wikipedia.org/wiki/GEMA_%28German_organization%29
+                errorReason = new StringBuilder("GEMA");
+            } else {
+                errorReason = new StringBuilder(errorMessage);
+                errorReason.append("  ");
+                errorReason.append(doc.select("[id=\"unavailable-submessage\"]").first().text());
+            }
         }
 
         return errorReason != null ? errorReason.toString() : null;

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java

@@ -392,6 +392,100 @@ public void testSubscriberCount() throws Exception {
         }
     }
 
+    // this channel has no "Subscribe" button
+    public static class EminemVEVO implements BaseChannelExtractorTest {
+        private static YoutubeChannelExtractor extractor;
+
+        @BeforeClass
+        public static void setUp() throws Exception {
+            NewPipe.init(Downloader.getInstance(), new Localization("GB", "en"));
+            extractor = (YoutubeChannelExtractor) YouTube
+                    .getChannelExtractor("https://www.youtube.com/user/EminemVEVO/");
+            extractor.fetchPage();
+        }
+
+        /*//////////////////////////////////////////////////////////////////////////
+        // Extractor
+        //////////////////////////////////////////////////////////////////////////*/
+
+        @Test
+        public void testServiceId() {
+            assertEquals(YouTube.getServiceId(), extractor.getServiceId());
+        }
+
+        @Test
+        public void testName() throws Exception {
+            assertEquals("EminemVEVO", extractor.getName());
+        }
+
+        @Test
+        public void testId() throws Exception {
+            assertEquals("UC20vb-R_px4CguHzzBPhoyQ", extractor.getId());
+        }
+
+        @Test
+        public void testUrl() throws ParsingException {
+            assertEquals("https://www.youtube.com/channel/UC20vb-R_px4CguHzzBPhoyQ", extractor.getUrl());
+        }
+
+        @Test
+        public void testOriginalUrl() throws ParsingException {
+            assertEquals("https://www.youtube.com/user/EminemVEVO/", extractor.getOriginalUrl());
+        }
+
+        /*//////////////////////////////////////////////////////////////////////////
+        // ListExtractor
+        //////////////////////////////////////////////////////////////////////////*/
+
+        @Test
+        public void testRelatedItems() throws Exception {
+            defaultTestRelatedItems(extractor, YouTube.getServiceId());
+        }
+
+        @Test
+        public void testMoreRelatedItems() throws Exception {
+            defaultTestMoreItems(extractor, YouTube.getServiceId());
+        }
+
+         /*//////////////////////////////////////////////////////////////////////////
+         // ChannelExtractor
+         //////////////////////////////////////////////////////////////////////////*/
+
+        @Test
+        public void testDescription() throws Exception {
+            final String description = extractor.getDescription();
+            assertTrue(description, description.contains("Eminem on Vevo"));
+        }
+
+        @Test
+        public void testAvatarUrl() throws Exception {
+            String avatarUrl = extractor.getAvatarUrl();
+            assertIsSecureUrl(avatarUrl);
+            assertTrue(avatarUrl, avatarUrl.contains("yt3"));
+        }
+
+        @Test
+        public void testBannerUrl() throws Exception {
+            String bannerUrl = extractor.getBannerUrl();
+            assertIsSecureUrl(bannerUrl);
+            assertTrue(bannerUrl, bannerUrl.contains("yt3"));
+        }
+
+        @Test
+        public void testFeedUrl() throws Exception {
+            assertEquals("https://www.youtube.com/feeds/videos.xml?channel_id=UC20vb-R_px4CguHzzBPhoyQ", extractor.getFeedUrl());
+        }
+
+        @Test
+        public void testSubscriberCount() throws Exception {
+            // there is no "Subscribe" button
+            long subscribers = extractor.getSubscriberCount();
+            assertEquals("Wrong subscriber count", -1, subscribers);
+        }
+    }
+
+
+
     public static class RandomChannel implements BaseChannelExtractorTest {
         private static YoutubeChannelExtractor extractor;
 
@@ -483,8 +577,9 @@ public void testFeedUrl() throws Exception {
 
         @Test
         public void testSubscriberCount() throws Exception {
-            assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 50);
+            long subscribers = extractor.getSubscriberCount();
+            assertTrue("Wrong subscriber count: " + subscribers, subscribers >= 50);
         }
     }
-};
+}