Merge pull request #296 from mauriciocolli/fix-search-errors-detection

Fix search errors detection and refactor search tests

Author: TobiGr

extractor/src/main/java/org/schabi/newpipe/extractor/search/InfoItemsSearchCollector.java

@@ -11,6 +11,10 @@
 import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
 /*
  * Created by Christian Schabesberger on 12.02.17.
  *
@@ -48,13 +52,31 @@ public class InfoItemsSearchCollector extends InfoItemsCollector<InfoItem, InfoI
     private final ChannelInfoItemsCollector userCollector;
     private final PlaylistInfoItemsCollector playlistCollector;
 
-    InfoItemsSearchCollector(int serviceId) {
+    public InfoItemsSearchCollector(int serviceId) {
         super(serviceId);
         streamCollector = new StreamInfoItemsCollector(serviceId);
         userCollector = new ChannelInfoItemsCollector(serviceId);
         playlistCollector = new PlaylistInfoItemsCollector(serviceId);
     }
 
+    @Override
+    public List<Throwable> getErrors() {
+        final List<Throwable> errors = new ArrayList<>(super.getErrors());
+        errors.addAll(streamCollector.getErrors());
+        errors.addAll(userCollector.getErrors());
+        errors.addAll(playlistCollector.getErrors());
+
+        return Collections.unmodifiableList(errors);
+    }
+
+    @Override
+    public void reset() {
+        super.reset();
+        streamCollector.reset();
+        userCollector.reset();
+        playlistCollector.reset();
+    }
+
     @Override
     public InfoItem extract(InfoItemExtractor extractor) throws ParsingException {
         // Use the corresponding collector for each item extractor type

extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java

@@ -17,11 +17,8 @@ public NothingFoundException(String message) {
         }
     }
 
-    private final InfoItemsSearchCollector collector;
-
     public SearchExtractor(StreamingService service, SearchQueryHandler linkHandler) {
         super(service, linkHandler);
-        collector = new InfoItemsSearchCollector(service.getServiceId());
     }
 
     public String getSearchString() {
@@ -30,10 +27,6 @@ public String getSearchString() {
 
     public abstract String getSearchSuggestion() throws ParsingException;
 
-    protected InfoItemsSearchCollector getInfoItemSearchCollector() {
-        return collector;
-    }
-
     @Override
     public SearchQueryHandler getLinkHandler() {
         return (SearchQueryHandler) super.getLinkHandler();

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java

@@ -47,8 +47,7 @@ public String getSearchSuggestion() throws ParsingException {
     @Nonnull
     @Override
     public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
-        InfoItemsSearchCollector searchItems = getInfoItemSearchCollector();
-        searchItems.reset();
+        final InfoItemsSearchCollector searchItems = new InfoItemsSearchCollector(getServiceId());
 
         if (getLinkHandler().getContentFilters().contains(CONFERENCES)
                 || getLinkHandler().getContentFilters().contains(ALL)

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java

@@ -47,8 +47,7 @@ public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionEx
     }
 
     private InfoItemsCollector<InfoItem, InfoItemExtractor> collectStreamsFrom(JsonObject json) throws ParsingException {
-
-        final InfoItemsSearchCollector collector = getInfoItemSearchCollector();
+        final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
 
         JsonArray contents;
         try {

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchExtractor.java

@@ -78,8 +78,7 @@ public void onFetchPage(@Nonnull Downloader downloader) throws IOException, Extr
     }
 
     private InfoItemsCollector<InfoItem, InfoItemExtractor> collectItems(JsonArray searchCollection) {
-        final InfoItemsSearchCollector collector = getInfoItemSearchCollector();
-        collector.reset();
+        final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
 
         for (Object result : searchCollection) {
             if (!(result instanceof JsonObject)) continue;

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java

@@ -70,8 +70,14 @@ public String getUrl() throws ParsingException {
     @Override
     public long getSubscriberCount() throws ParsingException {
         try {
-            String subscribers = getTextFromObject(channelInfoItem.getObject("subscriberCountText"));
-            return Utils.mixedNumberWordToLong(subscribers);
+            final JsonObject subscriberCountObject = channelInfoItem.getObject("subscriberCountText");
+
+            if (subscriberCountObject == null) {
+                // Subscription count is not available for this channel item.
+                return -1;
+            }
+
+            return Utils.mixedNumberWordToLong(getTextFromObject(subscriberCountObject));
         } catch (Exception e) {
             throw new ParsingException("Could not get subscriber count", e);
         }
@@ -80,7 +86,14 @@ public long getSubscriberCount() throws ParsingException {
     @Override
     public long getStreamCount() throws ParsingException {
         try {
-            return Long.parseLong(Utils.removeNonDigitCharacters(getTextFromObject(channelInfoItem.getObject("videoCountText"))));
+            final JsonObject videoCountObject = channelInfoItem.getObject("videoCountText");
+
+            if (videoCountObject == null) {
+                // Video count is not available, channel probably has no public uploads.
+                return -1;
+            }
+
+            return Long.parseLong(Utils.removeNonDigitCharacters(getTextFromObject(videoCountObject)));
         } catch (Exception e) {
             throw new ParsingException("Could not get stream count", e);
         }
@@ -89,7 +102,14 @@ public long getStreamCount() throws ParsingException {
     @Override
     public String getDescription() throws ParsingException {
         try {
-            return getTextFromObject(channelInfoItem.getObject("descriptionSnippet"));
+            final JsonObject descriptionObject = channelInfoItem.getObject("descriptionSnippet");
+
+            if (descriptionObject == null) {
+                // Channel have no description.
+                return null;
+            }
+
+            return getTextFromObject(descriptionObject);
         } catch (Exception e) {
             throw new ParsingException("Could not get description", e);
         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java

@@ -79,7 +79,7 @@ public String getSearchSuggestion() throws ParsingException {
     @Nonnull
     @Override
     public InfoItemsPage<InfoItem> getInitialPage() throws ExtractionException {
-        InfoItemsSearchCollector collector = getInfoItemSearchCollector();
+        final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
         JsonArray sections = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer")
                 .getObject("primaryContents").getObject("sectionListRenderer").getArray("contents");
 
@@ -103,7 +103,7 @@ public InfoItemsPage<InfoItem> getPage(String pageUrl) throws IOException, Extra
             throw new ExtractionException(new IllegalArgumentException("Page url is empty or null"));
         }
 
-        InfoItemsSearchCollector collector = getInfoItemSearchCollector();
+        final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
         final JsonArray ajaxJson = getJsonResponse(pageUrl, getExtractorLocalization());
 
         JsonObject itemSectionRenderer = ajaxJson.getObject(1).getObject("response")

extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseSearchExtractorTest.java

@@ -0,0 +1,7 @@
+package org.schabi.newpipe.extractor.services;
+
+@SuppressWarnings("unused")
+public interface BaseSearchExtractorTest extends BaseListExtractorTest {
+    void testSearchString() throws Exception;
+    void testSearchSuggestion() throws Exception;
+}

extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultExtractorTest.java

@@ -0,0 +1,53 @@
+package org.schabi.newpipe.extractor.services;
+
+import org.junit.Test;
+import org.schabi.newpipe.extractor.Extractor;
+import org.schabi.newpipe.extractor.StreamingService;
+
+import static org.hamcrest.CoreMatchers.*;
+import static org.junit.Assert.*;
+import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
+
+public abstract class DefaultExtractorTest<T extends Extractor> implements BaseExtractorTest {
+    public abstract T extractor() throws Exception;
+
+    public abstract StreamingService expectedService() throws Exception;
+    public abstract String expectedName() throws Exception;
+    public abstract String expectedId() throws Exception;
+    public abstract String expectedUrlContains() throws Exception;
+    public abstract String expectedOriginalUrlContains() throws Exception;
+
+    @Test
+    @Override
+    public void testServiceId() throws Exception {
+        assertEquals(expectedService().getServiceId(), extractor().getServiceId());
+    }
+
+    @Test
+    @Override
+    public void testName() throws Exception {
+        assertEquals(expectedName(), extractor().getName());
+    }
+
+    @Test
+    @Override
+    public void testId() throws Exception {
+        assertEquals(expectedId(), extractor().getId());
+    }
+
+    @Test
+    @Override
+    public void testUrl() throws Exception {
+        final String url = extractor().getUrl();
+        assertIsSecureUrl(url);
+        assertThat(url, containsString(expectedUrlContains()));
+    }
+
+    @Test
+    @Override
+    public void testOriginalUrl() throws Exception {
+        final String originalUrl = extractor().getOriginalUrl();
+        assertIsSecureUrl(originalUrl);
+        assertThat(originalUrl, containsString(expectedOriginalUrlContains()));
+    }
+}

extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultListExtractorTest.java

@@ -0,0 +1,50 @@
+package org.schabi.newpipe.extractor.services;
+
+import org.junit.Test;
+import org.schabi.newpipe.extractor.InfoItem;
+import org.schabi.newpipe.extractor.ListExtractor;
+
+import javax.annotation.Nullable;
+
+import static org.schabi.newpipe.extractor.services.DefaultTests.*;
+
+public abstract class DefaultListExtractorTest<T extends ListExtractor<? extends InfoItem>> extends DefaultExtractorTest<T>
+        implements BaseListExtractorTest {
+
+    @Nullable
+    public InfoItem.InfoType expectedInfoItemType() {
+        return null;
+    }
+
+    public boolean expectedHasMoreItems() {
+        return true;
+    }
+
+    @Test
+    @Override
+    public void testRelatedItems() throws Exception {
+        final ListExtractor<? extends InfoItem> extractor = extractor();
+
+        final InfoItem.InfoType expectedType = expectedInfoItemType();
+        final ListExtractor.InfoItemsPage<? extends InfoItem> items = defaultTestRelatedItems(extractor);
+        if (expectedType != null) {
+            assertOnlyContainsType(items, expectedType);
+        }
+    }
+
+    @Test
+    @Override
+    public void testMoreRelatedItems() throws Exception {
+        final ListExtractor<? extends InfoItem> extractor = extractor();
+
+        if (expectedHasMoreItems()) {
+            final InfoItem.InfoType expectedType = expectedInfoItemType();
+            final ListExtractor.InfoItemsPage<? extends InfoItem> items = defaultTestMoreItems(extractor);
+            if (expectedType != null) {
+                assertOnlyContainsType(items, expectedType);
+            }
+        } else {
+            assertNoMoreItems(extractor);
+        }
+    }
+}