Merge pull request #580 from TeamNewPipe/accountTerminated

Add AccountTerminatedException for terminated channels

Author: TobiGr

extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/AccountTerminatedException.java

@@ -0,0 +1,31 @@
+package org.schabi.newpipe.extractor.exceptions;
+
+public class AccountTerminatedException extends ContentNotAvailableException {
+
+    private Reason reason = Reason.UNKNOWN;
+
+    public AccountTerminatedException(final String message) {
+        super(message);
+    }
+
+    public AccountTerminatedException(final String message, final Reason reason) {
+        super(message);
+        this.reason = reason;
+    }
+
+    public AccountTerminatedException(final String message, final Throwable cause) {
+        super(message, cause);
+    }
+
+    /**
+     * The reason for the violation. There should also be more info in the exception's message.
+     */
+    public Reason getReason() {
+        return reason;
+    }
+
+    public enum Reason {
+        UNKNOWN,
+        VIOLATION
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -10,10 +10,7 @@
 import org.schabi.newpipe.extractor.MetaInfo;
 import org.schabi.newpipe.extractor.Page;
 import org.schabi.newpipe.extractor.downloader.Response;
-import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
-import org.schabi.newpipe.extractor.exceptions.ExtractionException;
-import org.schabi.newpipe.extractor.exceptions.ParsingException;
-import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
+import org.schabi.newpipe.extractor.exceptions.*;
 import org.schabi.newpipe.extractor.localization.Localization;
 import org.schabi.newpipe.extractor.stream.Description;
 import org.schabi.newpipe.extractor.utils.JsonUtils;
@@ -762,6 +759,23 @@ public static void defaultAlertsCheck(final JsonObject initialData) throws Parsi
             final String alertText = getTextFromObject(alertRenderer.getObject("text"));
             final String alertType = alertRenderer.getString("type", EMPTY_STRING);
             if (alertType.equalsIgnoreCase("ERROR")) {
+                if (alertText != null && alertText.contains("This account has been terminated")) {
+                    if (alertText.contains("violation") || alertText.contains("violating")
+                            || alertText.contains("infringement")) {
+                        // possible error messages:
+                        // "This account has been terminated for a violation of YouTube's Terms of Service."
+                        // "This account has been terminated due to multiple or severe violations of YouTube's policy prohibiting hate speech."
+                        // "This account has been terminated due to multiple or severe violations of YouTube's policy prohibiting content designed to harass, bully or threaten."
+                        // "This account has been terminated due to multiple or severe violations of YouTube's policy against spam, deceptive practices and misleading content or other Terms of Service violations."
+                        // "This account has been terminated due to multiple or severe violations of YouTube's policy on nudity or sexual content."
+                        // "This account has been terminated for violating YouTube's Community Guidelines."
+                        // "This account has been terminated because we received multiple third-party claims of copyright infringement regarding material that the user posted."
+                        // "This account has been terminated because it is linked to an account that received multiple third-party claims of copyright infringement."
+                        throw new AccountTerminatedException(alertText, AccountTerminatedException.Reason.VIOLATION);
+                    } else {
+                        throw new AccountTerminatedException(alertText);
+                    }
+                }
                 throw new ContentNotAvailableException("Got error: \"" + alertText + "\"");
             }
         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeFeedExtractor.java

@@ -9,6 +9,7 @@
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.downloader.Response;
+import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.feed.FeedExtractor;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
@@ -33,6 +34,9 @@ public void onFetchPage(@Nonnull Downloader downloader) throws IOException, Extr
         final String feedUrl = YoutubeParsingHelper.getFeedUrlFrom(channelIdOrUser);
 
         final Response response = downloader.get(feedUrl);
+        if (response.responseCode() == 404) {
+            throw new ContentNotAvailableException("Could not get feed: 404 - not found");
+        }
         document = Jsoup.parse(response.responseBody());
     }
 

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java

@@ -6,6 +6,7 @@
 import org.schabi.newpipe.downloader.DownloaderTestImpl;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.channel.ChannelExtractor;
+import org.schabi.newpipe.extractor.exceptions.AccountTerminatedException;
 import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
 import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
@@ -50,6 +51,90 @@ public void nonExistentFetch() throws Exception {
                     YouTube.getChannelExtractor("https://www.youtube.com/channel/DOESNT-EXIST");
             extractor.fetchPage();
         }
+
+        @Test(expected = AccountTerminatedException.class)
+        public void accountTerminatedTOSFetch() throws Exception {
+            // "This account has been terminated for a violation of YouTube's Terms of Service."
+            final ChannelExtractor extractor =
+                    YouTube.getChannelExtractor("https://www.youtube.com/channel/UCTGjY2I-ZUGnwVoWAGRd7XQ");
+            try {
+                extractor.fetchPage();
+            } catch (AccountTerminatedException e) {
+                assertEquals(e.getReason(), AccountTerminatedException.Reason.VIOLATION);
+                throw e;
+            }
+        }
+
+        @Test(expected = AccountTerminatedException.class)
+        public void accountTerminatedCommunityFetch() throws Exception {
+            // "This account has been terminated for violating YouTube's Community Guidelines."
+            final ChannelExtractor extractor =
+                    YouTube.getChannelExtractor("https://www.youtube.com/channel/UC0AuOxCr9TZ0TtEgL1zpIgA");
+            try {
+                extractor.fetchPage();
+            } catch (AccountTerminatedException e) {
+                assertEquals(e.getReason(), AccountTerminatedException.Reason.VIOLATION);
+                throw e;
+            }
+        }
+
+        @Test(expected = AccountTerminatedException.class)
+        public void accountTerminatedHateFetch() throws Exception {
+            // "This account has been terminated due to multiple or severe violations
+            // of YouTube's policy prohibiting hate speech."
+            final ChannelExtractor extractor =
+                    YouTube.getChannelExtractor("https://www.youtube.com/channel/UCPWXIOPK-9myzek6jHR5yrg");
+            try {
+                extractor.fetchPage();
+            } catch (AccountTerminatedException e) {
+                assertEquals(e.getReason(), AccountTerminatedException.Reason.VIOLATION);
+                throw e;
+            }
+        }
+
+        @Test(expected = AccountTerminatedException.class)
+        public void accountTerminatedBullyFetch() throws Exception {
+            // "This account has been terminated due to multiple or severe violations
+            // of YouTube's policy prohibiting content designed to harass, bully or threaten."
+            final ChannelExtractor extractor =
+                    YouTube.getChannelExtractor("https://youtube.com/channel/UCB1o7_gbFp2PLsamWxFenBg");
+            try {
+                extractor.fetchPage();
+            } catch (AccountTerminatedException e) {
+                assertEquals(e.getReason(), AccountTerminatedException.Reason.VIOLATION);
+                throw e;
+            }
+        }
+
+        @Test(expected = AccountTerminatedException.class)
+        public void accountTerminatedSpamFetch() throws Exception {
+            // "This account has been terminated due to multiple or severe violations
+            // of YouTube's policy against spam, deceptive practices and misleading content
+            // or other Terms of Service violations."
+            final ChannelExtractor extractor =
+                    YouTube.getChannelExtractor("https://www.youtube.com/channel/UCoaO4U_p7G7AwalqSbGCZOA");
+            try {
+                extractor.fetchPage();
+            } catch (AccountTerminatedException e) {
+                assertEquals(e.getReason(), AccountTerminatedException.Reason.VIOLATION);
+                throw e;
+            }
+        }
+
+        @Test(expected = AccountTerminatedException.class)
+        public void accountTerminatedCopyrightFetch() throws Exception {
+            // "This account has been terminated because we received multiple third-party claims
+            // of copyright infringement regarding material that the user posted."
+            final ChannelExtractor extractor =
+                    YouTube.getChannelExtractor("https://www.youtube.com/channel/UCpExuV8qJMfCaSQNL1YG6bQ");
+            try {
+                extractor.fetchPage();
+            } catch (AccountTerminatedException e) {
+                assertEquals(e.getReason(), AccountTerminatedException.Reason.VIOLATION);
+                throw e;
+            }
+        }
+
     }
 
     public static class NotSupported {

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeFeedExtractorTest.java

@@ -4,10 +4,12 @@
 import org.junit.Test;
 import org.schabi.newpipe.downloader.DownloaderFactory;
 import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.services.BaseListExtractorTest;
 import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeFeedExtractor;
 
+import java.io.IOException;
 import java.util.Random;
 
 import static org.junit.Assert.assertEquals;
@@ -77,4 +79,20 @@ public void testMoreRelatedItems() throws Exception {
             assertNoMoreItems(extractor);
         }
     }
+
+    public static class NotAvailable {
+
+        @BeforeClass
+        public static void setUp() throws IOException {
+            NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "notAvailable/"));
+        }
+
+        @Test(expected = ContentNotAvailableException.class)
+        public void AccountTerminatedFetch() throws Exception {
+            YoutubeFeedExtractor extractor = (YoutubeFeedExtractor) YouTube
+                    .getFeedExtractor("https://www.youtube.com/channel/UCTGjY2I-ZUGnwVoWAGRd7XQ");
+            extractor.fetchPage();
+        }
+
+    }
 }