Don't call fetch page in constructor

coffeemakr · coffeemakr · commit 9a469922850d · 2017-11-30T10:49:46.000+01:00
diff --git a/src/main/java/org/schabi/newpipe/extractor/Extractor.java b/src/main/java/org/schabi/newpipe/extractor/Extractor.java
@@ -33,12 +33,16 @@ public abstract class Extractor {
      */
     @Nullable
     private String cleanUrl;
+    private boolean pageFetched = false;
+    private final Downloader downloader;
 
     public Extractor(StreamingService service, String url) throws ExtractionException {
         if(service == null) throw new NullPointerException("service is null");
         if(url == null) throw new NullPointerException("url is null");
         this.service = service;
         this.originalUrl = url;
+        this.downloader = NewPipe.getDownloader();
+        if(downloader == null) throw new NullPointerException("downloader is null");
     }
 
     /**
@@ -49,8 +53,26 @@ public Extractor(StreamingService service, String url) throws ExtractionExceptio
 
     /**
      * Fetch the current page.
+     * @throws IOException if the page can not be loaded
+     * @throws ExtractionException if the pages content is not understood
      */
-    public abstract void fetchPage() throws IOException, ExtractionException;
+    public void fetchPage() throws IOException, ExtractionException {
+        if(pageFetched) return;
+        onFetchPage(downloader);
+        pageFetched = true;
+    }
+
+    protected void assertPageFetched() {
+        if(!pageFetched) throw new IllegalStateException("Page is not fetched. Make sure you call fetchPage()");
+    }
+
+    /**
+     * Fetch the current page.
+     * @param downloader the download to use
+     * @throws IOException if the page can not be loaded
+     * @throws ExtractionException if the pages content is not understood
+     */
+    public abstract void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException;
 
     @Nonnull
     public abstract String getId() throws ParsingException;
diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelExtractor.java
@@ -24,14 +24,13 @@ public SoundcloudChannelExtractor(StreamingService service, String url, String n
     }
 
     @Override
-    public void fetchPage() throws IOException, ExtractionException {
-        Downloader dl = NewPipe.getDownloader();
+    public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
 
         userId = getUrlIdHandler().getId(getOriginalUrl());
         String apiUrl = "https://api.soundcloud.com/users/" + userId +
                 "?client_id=" + SoundcloudParsingHelper.clientId();
 
-        String response = dl.download(apiUrl);
+        String response = downloader.download(apiUrl);
         try {
             user = JsonParser.object().from(response);
         } catch (JsonParserException e) {
diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java
@@ -4,6 +4,7 @@
 import java.util.Arrays;
 import java.util.List;
 
+import org.schabi.newpipe.extractor.Downloader;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.UrlIdHandler;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@@ -23,7 +24,7 @@ public SoundcloudChartsExtractor(StreamingService service, String url, String ne
     }
 
     @Override
-    public void fetchPage() {
+    public void onFetchPage(@Nonnull Downloader downloader) {
     }
 
     @Nonnull
diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java
@@ -66,13 +66,13 @@ public static String toDateString(String time) throws ParsingException {
      * Call the endpoint "/resolve" of the api.<br/>
      * See https://developers.soundcloud.com/docs/api/reference#resolve
      */
-    public static JsonObject resolveFor(String url) throws IOException, ReCaptchaException, ParsingException {
+    public static JsonObject resolveFor(Downloader downloader, String url) throws IOException, ReCaptchaException, ParsingException {
         String apiUrl = "https://api.soundcloud.com/resolve"
                 + "?url=" + URLEncoder.encode(url, "UTF-8")
                 + "&client_id=" + clientId();
 
         try {
-            return JsonParser.object().from(NewPipe.getDownloader().download(apiUrl));
+            return JsonParser.object().from(downloader.download(apiUrl));
         } catch (JsonParserException e) {
             throw new ParsingException("Could not parse json response", e);
         }
diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java
@@ -24,15 +24,14 @@ public SoundcloudPlaylistExtractor(StreamingService service, String url, String
     }
 
     @Override
-    public void fetchPage() throws IOException, ExtractionException {
-        Downloader dl = NewPipe.getDownloader();
+    public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
 
         playlistId = getUrlIdHandler().getId(getOriginalUrl());
         String apiUrl = "https://api.soundcloud.com/playlists/" + playlistId +
                 "?client_id=" + SoundcloudParsingHelper.clientId() +
                 "&representation=compact";
 
-        String response = dl.download(apiUrl);
+        String response = downloader.download(apiUrl);
         try {
             playlist = JsonParser.object().from(response);
         } catch (JsonParserException e) {
diff --git a/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java
@@ -21,12 +21,11 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
 
     public SoundcloudStreamExtractor(StreamingService service, String url) throws IOException, ExtractionException {
         super(service, url);
-        fetchPage();
     }
 
     @Override
-    public void fetchPage() throws IOException, ExtractionException {
-        track = SoundcloudParsingHelper.resolveFor(getOriginalUrl());
+    public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
+        track = SoundcloudParsingHelper.resolveFor(downloader, getOriginalUrl());
 
         String policy = track.getString("policy", "");
         if (!policy.equals("ALLOW") && !policy.equals("MONETIZE")) {
diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractor.java
@@ -59,9 +59,7 @@ public YoutubeChannelExtractor(StreamingService service, String url, String next
     }
 
     @Override
-    public void fetchPage() throws IOException, ExtractionException {
-        Downloader downloader = NewPipe.getDownloader();
-
+    public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
         String channelUrl = super.getCleanUrl() + CHANNEL_URL_PARAMETERS;
         String pageContent = downloader.download(channelUrl);
         doc = Jsoup.parse(pageContent, channelUrl);
diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java
@@ -36,9 +36,7 @@ public YoutubePlaylistExtractor(StreamingService service, String url, String nex
     }
 
     @Override
-    public void fetchPage() throws IOException, ExtractionException {
-        Downloader downloader = NewPipe.getDownloader();
-
+    public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
         String pageContent = downloader.download(getCleanUrl());
         doc = Jsoup.parse(pageContent, getCleanUrl());
 
diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java
@@ -86,7 +86,6 @@ public class LiveStreamException extends ContentNotAvailableException {
 
     public YoutubeStreamExtractor(StreamingService service, String url) throws IOException, ExtractionException {
         super(service, url);
-        fetchPage();
     }
 
     /*//////////////////////////////////////////////////////////////////////////
@@ -415,7 +414,8 @@ public List<Subtitles> getSubtitles(SubtitlesFormat format) throws IOException,
             // If the video is age restricted getPlayerConfig will fail
             return null;
         }
-        JsonObject playerConfig = getPlayerConfig(getPageHtml());
+        // TODO: This should be done in onFetchPage()
+        JsonObject playerConfig = getPlayerConfig(getPageHtml(NewPipe.getDownloader()));
         String playerResponse = playerConfig.getObject("args").getString("player_response");
 
         JsonObject captions;
@@ -530,26 +530,23 @@ public String getErrorMessage() {
 
     private static String pageHtml = null;
 
-    private String getPageHtml() throws IOException, ExtractionException{
+    private String getPageHtml(Downloader downloader) throws IOException, ExtractionException{
         if (pageHtml == null) {
-            Downloader dl = NewPipe.getDownloader();
-            pageHtml = dl.download(getCleanUrl());
+            pageHtml = downloader.download(getCleanUrl());
         }
         return pageHtml;
     }
 
     @Override
-    public void fetchPage() throws IOException, ExtractionException {
-        Downloader dl = NewPipe.getDownloader();
-
-        String pageContent = getPageHtml();
+    public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
+        String pageContent = getPageHtml(downloader);
         doc = Jsoup.parse(pageContent, getCleanUrl());
 
 
         String playerUrl;
         // Check if the video is age restricted
         if (pageContent.contains("<meta property=\"og:restrictions:age")) {
-            String infoPageResponse = dl.download(String.format(GET_VIDEO_INFO_URL, getId()));
+            String infoPageResponse = downloader.download(String.format(GET_VIDEO_INFO_URL, getId()));
             videoInfoPage.putAll(Parser.compatParseMap(infoPageResponse));
             playerUrl = getPlayerUrlFromRestrictedVideo();
             isAgeRestricted = true;
diff --git a/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java b/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeTrendingExtractor.java
@@ -42,9 +42,7 @@ public YoutubeTrendingExtractor(StreamingService service, String url, String nex
     }
 
     @Override
-    public void fetchPage() throws IOException, ExtractionException {
-        Downloader downloader = NewPipe.getDownloader();
-
+    public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
         final String contentCountry = getContentCountry();
         String url = getCleanUrl();
         if(contentCountry != null && !contentCountry.isEmpty()) {
diff --git a/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java
@@ -236,7 +236,8 @@ public static StreamInfo getInfo(StreamingService service, String url) throws IO
      * Fills out the video info fields which are common to all services.
      * Probably needs to be overridden by subclasses
      */
-    public static StreamInfo getInfo(StreamExtractor extractor) throws ExtractionException {
+    private static StreamInfo getInfo(StreamExtractor extractor) throws ExtractionException, IOException {
+        extractor.fetchPage();
         StreamInfo streamInfo;
         try {
             streamInfo = extractImportantData(extractor);
diff --git a/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java b/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorDefaultTest.java
@@ -29,6 +29,7 @@ public class SoundcloudStreamExtractorDefaultTest {
     public void setUp() throws Exception {
         NewPipe.init(Downloader.getInstance());
         extractor = (SoundcloudStreamExtractor) SoundCloud.getService().getStreamExtractor("https://soundcloud.com/liluzivert/do-what-i-want-produced-by-maaly-raw-don-cannon");
+        extractor.fetchPage();
     }
 
     @Test
diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java
@@ -41,12 +41,14 @@
  */
 public class YoutubeStreamExtractorDefaultTest {
     public static final String HTTPS = "https://";
-    private StreamExtractor extractor;
+    private YoutubeStreamExtractor extractor;
 
     @Before
     public void setUp() throws Exception {
         NewPipe.init(Downloader.getInstance());
-        extractor = YouTube.getService().getStreamExtractor("https://www.youtube.com/watch?v=rYEDA3JcQqw");
+        extractor = (YoutubeStreamExtractor) YouTube.getService()
+                .getStreamExtractor("https://www.youtube.com/watch?v=rYEDA3JcQqw");
+        extractor.fetchPage();
     }
 
     @Test
diff --git a/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java b/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorRestrictedTest.java
@@ -28,6 +28,7 @@ public void setUp() throws Exception {
         NewPipe.init(Downloader.getInstance());
         extractor = (YoutubeStreamExtractor) YouTube.getService()
                 .getStreamExtractor("https://www.youtube.com/watch?v=i6JTvzrpBy0");
+        extractor.fetchPage();
     }
 
     @Test

Original file line number	Diff line number	Diff line change
`@@ -4,6 +4,7 @@`
`4`	`4`	`import java.util.Arrays;`
`5`	`5`	`import java.util.List;`
`6`	`6`
	`7`	`+import org.schabi.newpipe.extractor.Downloader;`
`7`	`8`	`import org.schabi.newpipe.extractor.StreamingService;`
`8`	`9`	`import org.schabi.newpipe.extractor.UrlIdHandler;`
`9`	`10`	`import org.schabi.newpipe.extractor.exceptions.ExtractionException;`
`@@ -23,7 +24,7 @@ public SoundcloudChartsExtractor(StreamingService service, String url, String ne`
`23`	`24`	`}`
`24`	`25`
`25`	`26`	`@Override`
`26`		`- public void fetchPage() {`
	`27`	`+ public void onFetchPage(@Nonnull Downloader downloader) {`
`27`	`28`	`}`
`28`	`29`
`29`	`30`	`@Nonnull`
Original file line number	Diff line number	Diff line change
`@@ -36,9 +36,7 @@ public YoutubePlaylistExtractor(StreamingService service, String url, String nex`
`36`	`36`	`}`
`37`	`37`
`38`	`38`	`@Override`
`39`		`- public void fetchPage() throws IOException, ExtractionException {`
`40`		`- Downloader downloader = NewPipe.getDownloader();`
`41`		`-`
	`39`	`+ public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {`
`42`	`40`	`String pageContent = downloader.download(getCleanUrl());`
`43`	`41`	`doc = Jsoup.parse(pageContent, getCleanUrl());`
`44`	`42`