Merge pull request #73 from karyogamy/live

HLS Livestream Extraction

Author: theScrabi

src/main/java/org/schabi/newpipe/extractor/search/SearchResult.java

@@ -44,23 +44,10 @@ public SearchResult(int serviceId, String suggestion, List<InfoItem> results, Li
         this.errors = Collections.unmodifiableList(new ArrayList<>(errors));
     }
 
-    public static SearchResult getSearchResult(SearchEngine engine, String query, int page, String languageCode, SearchEngine.Filter filter)
+    public static SearchResult getSearchResult(@Nonnull final SearchEngine engine, final String query, final int page,
+                                               final String languageCode, final SearchEngine.Filter filter)
             throws IOException, ExtractionException {
-
-        SearchResult result = engine
-                .search(query, page, languageCode, filter)
-                .getSearchResult();
-        if (result.resultList.isEmpty()) {
-            if (result.suggestion.isEmpty()) {
-                if (result.errors.isEmpty()) {
-                    throw new ExtractionException("Empty result despite no error");
-                }
-            } else {
-                // This is used as a fallback. Do not relay on it !!!
-                throw new SearchEngine.NothingFoundException(result.suggestion);
-            }
-        }
-        return result;
+        return engine.search(query, page, languageCode, filter).getSearchResult();
     }
 
     public String getSuggestion() {

src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java

@@ -116,9 +116,16 @@ public String getUploaderAvatarUrl() {
         return SoundcloudParsingHelper.getAvatarUrl(track);
     }
 
+    @Nonnull
     @Override
     public String getDashMpdUrl() {
-        return null;
+        return "";
+    }
+
+    @Nonnull
+    @Override
+    public String getHlsUrl() throws ParsingException {
+        return "";
     }
 
     @Override

src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java

@@ -66,12 +66,6 @@ public class GemaException extends ContentNotAvailableException {
         }
     }
 
-    public class LiveStreamException extends ContentNotAvailableException {
-        LiveStreamException(String message) {
-            super(message);
-        }
-    }
-
     public class SubtitlesException extends ContentNotAvailableException {
         SubtitlesException(String message, Throwable cause) {
             super(message, cause);
@@ -338,6 +332,7 @@ public String getUploaderAvatarUrl() throws ParsingException {
         }
     }
 
+    @Nonnull
     @Override
     public String getDashMpdUrl() throws ParsingException {
         assertPageFetched();
@@ -365,6 +360,24 @@ public String getDashMpdUrl() throws ParsingException {
         }
     }
 
+    @Nonnull
+    @Override
+    public String getHlsUrl() throws ParsingException {
+        assertPageFetched();
+        try {
+            String hlsvp;
+            if (playerArgs != null && playerArgs.isString("hlsvp")) {
+                hlsvp = playerArgs.getString("hlsvp", "");
+            } else {
+                return "";
+            }
+
+            return hlsvp;
+        } catch (Exception e) {
+            throw new ParsingException("Could not get hls manifest url", e);
+        }
+    }
+
     @Override
     public List<AudioStream> getAudioStreams() throws IOException, ExtractionException {
         assertPageFetched();
@@ -428,7 +441,7 @@ public List<VideoStream> getVideoOnlyStreams() throws IOException, ExtractionExc
     @Override
     @Nonnull
     public List<Subtitles> getSubtitlesDefault() throws IOException, ExtractionException {
-        return getSubtitles(SubtitlesFormat.VTT);
+        return getSubtitles(SubtitlesFormat.TTML);
     }
 
     @Override
@@ -444,7 +457,15 @@ public List<Subtitles> getSubtitles(final SubtitlesFormat format) throws IOExcep
 
     @Override
     public StreamType getStreamType() throws ParsingException {
-        //todo: if implementing livestream support this value should be generated dynamically
+        assertPageFetched();
+        try {
+            if (playerArgs != null && (playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live") ||
+                    playerArgs.get(URL_ENCODED_FMT_STREAM_MAP).toString().isEmpty())) {
+                return StreamType.LIVE_STREAM;
+            }
+        } catch (Exception e) {
+            throw new ParsingException("Could not get hls manifest url", e);
+        }
         return StreamType.VIDEO_STREAM;
     }
 
@@ -517,13 +538,16 @@ public String getErrorMessage() {
     private static final String CONTENT = "content";
     private static final String DECRYPTION_FUNC_NAME = "decrypt";
 
+    private static final String VERIFIED_URL_PARAMS = "&has_verified=1&bpctr=9999999999";
+
     private volatile String decryptionCode = "";
 
     private String pageHtml = null;
 
-    private String getPageHtml(Downloader downloader) throws IOException, ExtractionException{
+    private String getPageHtml(Downloader downloader) throws IOException, ExtractionException {
+        final String verifiedUrl = getCleanUrl() + VERIFIED_URL_PARAMS;
         if (pageHtml == null) {
-            pageHtml = downloader.download(getCleanUrl());
+            pageHtml = downloader.download(verifiedUrl);
         }
         return pageHtml;
     }
@@ -534,7 +558,6 @@ public void onFetchPage(@Nonnull Downloader downloader) throws IOException, Extr
         doc = Jsoup.parse(pageContent, getCleanUrl());
 
         final String playerUrl;
-        // TODO: use embedded videos to fetch DASH manifest for all videos
         // Check if the video is age restricted
         if (pageContent.contains("<meta property=\"og:restrictions:age")) {
             final EmbeddedInfo info = getEmbeddedInfo();
@@ -582,21 +605,11 @@ private JsonObject getPlayerArgs(JsonObject playerConfig) throws ParsingExceptio
         JsonObject playerArgs;
 
         //attempt to load the youtube js player JSON arguments
-        boolean isLiveStream = false; //used to determine if this is a livestream or not
         try {
             playerArgs = playerConfig.getObject("args");
-
-            // check if we have a live stream. We need to filter it, since its not yet supported.
-            if ((playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live"))
-                    || (playerArgs.get(URL_ENCODED_FMT_STREAM_MAP).toString().isEmpty())) {
-                isLiveStream = true;
-            }
         } catch (Exception e) {
             throw new ParsingException("Could not parse yt player config", e);
         }
-        if (isLiveStream) {
-            throw new LiveStreamException("This is a Live stream. Can't use those right now.");
-        }
 
         return playerArgs;
     }
@@ -806,11 +819,6 @@ private static String getVideoInfoUrl(final String id, final String sts) {
                 "&sts=" + sts + "&ps=default&gl=US&hl=en";
     }
 
-    @Nonnull
-    private static String getSubtitleFormatUrl(final String baseUrl, final SubtitlesFormat format) {
-        return baseUrl.replaceAll("&fmt=[^&]*", "") + "&fmt=" + format.getExtension();
-    }
-
     private Map<String, ItagItem> getItags(String encodedUrlMapKey, ItagItem.ItagType itagTypeWanted) throws ParsingException {
         Map<String, ItagItem> urlAndItags = new LinkedHashMap<>();
 

src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java

@@ -126,7 +126,8 @@ protected long getTimestampSeconds(String regexPattern) throws ParsingException
      * @return the url as a string or an empty string
      * @throws ParsingException if an error occurs while reading
      */
-    public abstract String getDashMpdUrl() throws ParsingException;
+    @Nonnull public abstract String getDashMpdUrl() throws ParsingException;
+    @Nonnull public abstract String getHlsUrl() throws ParsingException;
     public abstract List<AudioStream> getAudioStreams() throws IOException, ExtractionException;
     public abstract List<VideoStream> getVideoStreams() throws IOException, ExtractionException;
     public abstract List<VideoStream> getVideoOnlyStreams() throws IOException, ExtractionException;

src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java

@@ -126,6 +126,10 @@ public String getDashMpdUrl() {
         return dashMpdUrl;
     }
 
+    public String getHlsUrl() {
+        return hlsUrl;
+    }
+
     public StreamInfoItem getNextVideo() {
         return next_video;
     }
@@ -206,6 +210,10 @@ public void setDashMpdUrl(String dashMpdUrl) {
         this.dashMpdUrl = dashMpdUrl;
     }
 
+    public void setHlsUrl(String hlsUrl) {
+        this.hlsUrl = hlsUrl;
+    }
+
     public void setNextVideo(StreamInfoItem next_video) {
         this.next_video = next_video;
     }
@@ -298,6 +306,12 @@ private static StreamInfo extractStreams(StreamInfo streamInfo, StreamExtractor
             streamInfo.addError(new ExtractionException("Couldn't get Dash manifest", e));
         }
 
+        try {
+            streamInfo.setHlsUrl(extractor.getHlsUrl());
+        } catch (Exception e) {
+            streamInfo.addError(new ExtractionException("Couldn't get HLS manifest", e));
+        }
+
         /*  Load and extract audio */
         try {
             streamInfo.setAudioStreams(extractor.getAudioStreams());
@@ -447,6 +461,7 @@ private static StreamInfo extractOptionalData(StreamInfo streamInfo, StreamExtra
     // crawling such a file is not service dependent. Therefore getting audio only streams by yust
     // providing the dash mpd file will be possible in the future.
     public String dashMpdUrl;
+    public String hlsUrl;
 
     public StreamInfoItem next_video;
     public List<InfoItem> related_streams;

src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorControversialTest.java

@@ -0,0 +1,125 @@
+package org.schabi.newpipe.extractor.services.youtube;
+
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.schabi.newpipe.Downloader;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.stream.StreamExtractor;
+import org.schabi.newpipe.extractor.stream.SubtitlesFormat;
+import org.schabi.newpipe.extractor.stream.VideoStream;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.*;
+import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
+import static org.schabi.newpipe.extractor.ServiceList.YouTube;
+
+/**
+ * Test for {@link YoutubeStreamUrlIdHandler}
+ */
+public class YoutubeStreamExtractorControversialTest {
+    private static YoutubeStreamExtractor extractor;
+
+    @BeforeClass
+    public static void setUp() throws Exception {
+        NewPipe.init(Downloader.getInstance());
+        extractor = (YoutubeStreamExtractor) YouTube
+                .getStreamExtractor("https://www.youtube.com/watch?v=T4XJQO3qol8");
+        extractor.fetchPage();
+    }
+
+    @Test
+    public void testGetInvalidTimeStamp() throws ParsingException {
+        assertTrue(extractor.getTimeStamp() + "", extractor.getTimeStamp() <= 0);
+    }
+
+    @Test
+    public void testGetValidTimeStamp() throws IOException, ExtractionException {
+        StreamExtractor extractor = YouTube.getStreamExtractor("https://youtu.be/FmG385_uUys?t=174");
+        assertEquals(extractor.getTimeStamp() + "", "174");
+    }
+
+    @Test
+    @Ignore
+    public void testGetAgeLimit() throws ParsingException {
+        assertEquals(18, extractor.getAgeLimit());
+    }
+
+    @Test
+    public void testGetName() throws ParsingException {
+        assertNotNull("name is null", extractor.getName());
+        assertFalse("name is empty", extractor.getName().isEmpty());
+    }
+
+    @Test
+    public void testGetDescription() throws ParsingException {
+        assertNotNull(extractor.getDescription());
+        assertFalse(extractor.getDescription().isEmpty());
+    }
+
+    @Test
+    public void testGetUploaderName() throws ParsingException {
+        assertNotNull(extractor.getUploaderName());
+        assertFalse(extractor.getUploaderName().isEmpty());
+    }
+
+    @Ignore // Currently there is no way get the length from restricted videos
+    @Test
+    public void testGetLength() throws ParsingException {
+        assertTrue(extractor.getLength() > 0);
+    }
+
+    @Test
+    public void testGetViews() throws ParsingException {
+        assertTrue(extractor.getViewCount() > 0);
+    }
+
+    @Test
+    public void testGetUploadDate() throws ParsingException {
+        assertTrue(extractor.getUploadDate().length() > 0);
+    }
+
+    @Test
+    public void testGetThumbnailUrl() throws ParsingException {
+        assertIsSecureUrl(extractor.getThumbnailUrl());
+    }
+
+    @Test
+    public void testGetUploaderAvatarUrl() throws ParsingException {
+        assertIsSecureUrl(extractor.getUploaderAvatarUrl());
+    }
+
+    // FIXME: 25.11.17 Are there no streams or are they not listed?
+    @Ignore
+    @Test
+    public void testGetAudioStreams() throws IOException, ExtractionException {
+        // audio streams are not always necessary
+        assertFalse(extractor.getAudioStreams().isEmpty());
+    }
+
+    @Test
+    public void testGetVideoStreams() throws IOException, ExtractionException {
+        List<VideoStream> streams = new ArrayList<>();
+        streams.addAll(extractor.getVideoStreams());
+        streams.addAll(extractor.getVideoOnlyStreams());
+        assertTrue(streams.size() > 0);
+    }
+
+
+    @Test
+    public void testGetSubtitlesListDefault() throws IOException, ExtractionException {
+        // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null
+        assertTrue(!extractor.getSubtitlesDefault().isEmpty());
+    }
+
+    @Test
+    public void testGetSubtitlesList() throws IOException, ExtractionException {
+        // Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null
+        assertTrue(!extractor.getSubtitles(SubtitlesFormat.TTML).isEmpty());
+    }
+}