Merge pull request #505 from TeamNewPipe/meida.ccc.de_search_fix

[media.ccc.de] Fix NPE in search results if they contain a future talk

Author: Stypox

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java

@@ -1,7 +1,6 @@
 package org.schabi.newpipe.extractor.services.media_ccc.extractors;
 
 import com.grack.nanojson.JsonArray;
-import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
 import com.grack.nanojson.JsonParserException;
 import org.schabi.newpipe.extractor.downloader.Downloader;
@@ -13,9 +12,11 @@
 import java.io.IOException;
 import java.time.OffsetDateTime;
 import java.time.format.DateTimeParseException;
+import java.util.Locale;
 import java.util.regex.Pattern;
 
 public final class MediaCCCParsingHelper {
+    private static final Pattern LIVE_STREAM_ID_PATTERN = Pattern.compile("\\w+/\\w+"); // {conference_slug}/{room_slug}
     private static JsonArray liveStreams = null;
 
     private MediaCCCParsingHelper() { }
@@ -28,12 +29,28 @@ public static OffsetDateTime parseDateFrom(final String textualUploadDate) throw
         }
     }
 
-    public static boolean isLiveStreamId(final String url) {
-        final String pattern = "\\w+/\\w+";
-        return Pattern.matches(pattern, url); // {conference_slug}/{room_slug}
+    /**
+     * Check whether an id is a live stream id
+     * @param id the {@code id} to check
+     * @return returns {@code true} if the {@code id} is formatted like {@code {conference_slug}/{room_slug}};
+     *         {@code false} otherwise
+     */
+    public static boolean isLiveStreamId(final String id) {
+        return LIVE_STREAM_ID_PATTERN.matcher(id).find();
     }
 
-    public static JsonArray getLiveStreams(final Downloader downloader, final Localization localization) throws ExtractionException {
+    /**
+     * Get currently available live streams from
+     * <a href="https://streaming.media.ccc.de/streams/v2.json">https://streaming.media.ccc.de/streams/v2.json</a>.
+     * Use this method to cache requests, because they can get quite big.
+     * TODO: implement better caching policy (max-age: 3 min)
+     * @param downloader The downloader to use for making the request
+     * @param localization The localization to be used. Will most likely be ignored.
+     * @return {@link JsonArray} containing current conferences and info about their rooms and streams.
+     * @throws ExtractionException if the data could not be fetched or the retrieved data could not be parsed to a {@link JsonArray}
+     */
+    public static JsonArray getLiveStreams(final Downloader downloader, final Localization localization)
+            throws ExtractionException {
         if (liveStreams == null) {
             try {
                 final String site = downloader.get("https://streaming.media.ccc.de/streams/v2.json",

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java

@@ -81,8 +81,13 @@ public InfoItemsPage<InfoItem> getInitialPage() {
                 || getLinkHandler().getContentFilters().isEmpty()) {
             JsonArray events = doc.getArray("events");
             for (int i = 0; i < events.size(); i++) {
-                searchItems.commit(new MediaCCCStreamInfoItemExtractor(
-                        events.getObject(i)));
+                // Ensure only uploaded talks are shown in the search results.
+                // If the release date is null, the talk has not been held or uploaded yet
+                // and no streams are going to be available anyway.
+                if (events.getObject(i).getString("release_date") != null) {
+                    searchItems.commit(new MediaCCCStreamInfoItemExtractor(
+                            events.getObject(i)));
+                }
             }
         }
         return new InfoItemsPage<>(searchItems, null);
@@ -105,7 +110,7 @@ public void onFetchPage(@Nonnull final Downloader downloader)
             try {
                 doc = JsonParser.object().from(site);
             } catch (JsonParserException jpe) {
-                throw new ExtractionException("Could not parse json.", jpe);
+                throw new ExtractionException("Could not parse JSON.", jpe);
             }
         }
         if (getLinkHandler().getContentFilters().contains(CONFERENCES)

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/infoItems/MediaCCCStreamInfoItemExtractor.java

@@ -55,7 +55,11 @@ public String getTextualUploadDate() {
     @Nullable
     @Override
     public DateWrapper getUploadDate() throws ParsingException {
-        return new DateWrapper(MediaCCCParsingHelper.parseDateFrom(getTextualUploadDate()));
+        final String date = getTextualUploadDate();
+        if (date == null) {
+            return null; // event is in the future...
+        }
+        return new DateWrapper(MediaCCCParsingHelper.parseDateFrom(date));
     }
 
     @Override

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java

@@ -33,7 +33,7 @@ public String getUrl(final String query, final List<String> contentFilter,
             return "https://media.ccc.de/public/events/search?q="
                     + URLEncoder.encode(query, "UTF-8");
         } catch (UnsupportedEncodingException e) {
-            throw new ParsingException("Could not create search string with querry: " + query, e);
+            throw new ParsingException("Could not create search string with query: " + query, e);
         }
     }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCLiveStreamListExtractorTest.java

@@ -27,7 +27,6 @@ public static void setUpClass() throws Exception {
     @Test
     public void getConferencesListTest() throws Exception {
         final List<InfoItem> a = extractor.getInitialPage().getItems();
-        assertTrue(a.size() > 2);
         for (int i = 0; i < a.size(); i++) {
             final InfoItem b = a.get(i);
             assertNotNull(a.get(i).getName());