Merge pull request #322 from wb9688/upgrade-jsoup

TobiGr · web-flow · commit 0f8057ae56f5 · 2020-05-28T21:16:33.000+02:00
Upgrade dependencies
diff --git a/extractor/build.gradle b/extractor/build.gradle
@@ -2,10 +2,10 @@ dependencies {
     implementation project(':timeago-parser')
 
     implementation 'com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751'
-    implementation 'org.jsoup:jsoup:1.9.2'
-    implementation 'org.mozilla:rhino:1.7.7.1'
-    implementation 'com.github.spotbugs:spotbugs-annotations:3.1.0'
-    implementation 'org.nibor.autolink:autolink:0.8.0'
+    implementation 'org.jsoup:jsoup:1.13.1'
+    implementation 'org.mozilla:rhino:1.7.12'
+    implementation 'com.github.spotbugs:spotbugs-annotations:4.0.2'
+    implementation 'org.nibor.autolink:autolink:0.10.0'
 
-    testImplementation 'junit:junit:4.12'
+    testImplementation 'junit:junit:4.13'
 }
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeInstance.java
@@ -3,13 +3,14 @@
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
 import com.grack.nanojson.JsonParserException;
-import org.jsoup.helper.StringUtil;
+
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.downloader.Response;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
 import org.schabi.newpipe.extractor.utils.JsonUtils;
+import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.IOException;
 
@@ -43,7 +44,7 @@ public void fetchInstanceMetaData() throws Exception {
             throw new Exception("unable to configure instance " + url, e);
         }
 
-        if (response == null || StringUtil.isBlank(response.responseBody())) {
+        if (response == null || Utils.isBlank(response.responseBody())) {
             throw new Exception("unable to configure instance " + url);
         }
 
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java
@@ -1,10 +1,11 @@
 package org.schabi.newpipe.extractor.services.peertube;
 
 import com.grack.nanojson.JsonObject;
-import org.jsoup.helper.StringUtil;
+
 import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.utils.Parser;
+import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
@@ -24,7 +25,7 @@ private PeertubeParsingHelper() {
 
     public static void validate(JsonObject json) throws ContentNotAvailableException {
         String error = json.getString("error");
-        if (!StringUtil.isBlank(error)) {
+        if (!Utils.isBlank(error)) {
             throw new ContentNotAvailableException(error);
         }
     }
@@ -51,7 +52,7 @@ public static String getNextPageUrl(String prevPageUrl, long total) {
         } catch (Parser.RegexException e) {
             return "";
         }
-        if (StringUtil.isBlank(prevStart)) return "";
+        if (Utils.isBlank(prevStart)) return "";
         long nextStart = 0;
         try {
             nextStart = Long.parseLong(prevStart) + ITEMS_PER_PAGE;
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java
@@ -4,7 +4,7 @@
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
 import com.grack.nanojson.JsonParserException;
-import org.jsoup.helper.StringUtil;
+
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.channel.ChannelExtractor;
 import org.schabi.newpipe.extractor.downloader.Downloader;
@@ -16,6 +16,7 @@
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 import org.schabi.newpipe.extractor.utils.JsonUtils;
+import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.IOException;
 
@@ -119,7 +120,7 @@ public String getNextPageUrl() throws IOException, ExtractionException {
     public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
         Response response = getDownloader().get(pageUrl);
         JsonObject json = null;
-        if (response != null && !StringUtil.isBlank(response.responseBody())) {
+        if (response != null && !Utils.isBlank(response.responseBody())) {
             try {
                 json = JsonParser.object().from(response.responseBody());
             } catch (Exception e) {
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeChannelExtractor.java
@@ -4,7 +4,7 @@
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
 import com.grack.nanojson.JsonParserException;
-import org.jsoup.helper.StringUtil;
+
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.channel.ChannelExtractor;
 import org.schabi.newpipe.extractor.downloader.Downloader;
@@ -18,6 +18,7 @@
 import org.schabi.newpipe.extractor.utils.JsonUtils;
 import org.schabi.newpipe.extractor.utils.Parser;
 import org.schabi.newpipe.extractor.utils.Parser.RegexException;
+import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.IOException;
 
@@ -127,7 +128,7 @@ public String getNextPageUrl() throws IOException, ExtractionException {
     public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
         Response response = getDownloader().get(pageUrl);
         JsonObject json = null;
-        if (null != response && !StringUtil.isBlank(response.responseBody())) {
+        if (response != null && !Utils.isBlank(response.responseBody())) {
             try {
                 json = JsonParser.object().from(response.responseBody());
             } catch (Exception e) {
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java
@@ -3,7 +3,7 @@
 import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
-import org.jsoup.helper.StringUtil;
+
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.comments.CommentsExtractor;
 import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
@@ -17,6 +17,7 @@
 import org.schabi.newpipe.extractor.utils.JsonUtils;
 import org.schabi.newpipe.extractor.utils.Parser;
 import org.schabi.newpipe.extractor.utils.Parser.RegexException;
+import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.IOException;
 
@@ -65,7 +66,7 @@ public String getNextPageUrl() throws IOException, ExtractionException {
     public InfoItemsPage<CommentsInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
         Response response = getDownloader().get(pageUrl);
         JsonObject json = null;
-        if (null != response && !StringUtil.isBlank(response.responseBody())) {
+        if (response != null && !Utils.isBlank(response.responseBody())) {
             try {
                 json = JsonParser.object().from(response.responseBody());
             } catch (Exception e) {
@@ -88,5 +89,4 @@ public InfoItemsPage<CommentsInfoItem> getPage(String pageUrl) throws IOExceptio
     public void onFetchPage(Downloader downloader) throws IOException, ExtractionException {
         this.initPage = getPage(getUrl() + "?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE);
     }
-
 }
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java
@@ -3,7 +3,7 @@
 import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
-import org.jsoup.helper.StringUtil;
+
 import org.schabi.newpipe.extractor.InfoItem;
 import org.schabi.newpipe.extractor.InfoItemExtractor;
 import org.schabi.newpipe.extractor.InfoItemsCollector;
@@ -19,6 +19,7 @@
 import org.schabi.newpipe.extractor.utils.JsonUtils;
 import org.schabi.newpipe.extractor.utils.Parser;
 import org.schabi.newpipe.extractor.utils.Parser.RegexException;
+import org.schabi.newpipe.extractor.utils.Utils;
 
 import javax.annotation.Nonnull;
 import java.io.IOException;
@@ -84,7 +85,7 @@ public String getNextPageUrl() throws IOException, ExtractionException {
     public InfoItemsPage<InfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
         Response response = getDownloader().get(pageUrl);
         JsonObject json = null;
-        if (null != response && !StringUtil.isBlank(response.responseBody())) {
+        if (null != response && !Utils.isBlank(response.responseBody())) {
             try {
                 json = JsonParser.object().from(response.responseBody());
             } catch (Exception e) {
@@ -104,5 +105,4 @@ public InfoItemsPage<InfoItem> getPage(String pageUrl) throws IOException, Extra
     public void onFetchPage(Downloader downloader) throws IOException, ExtractionException {
         initPage = getPage(getUrl() + "&" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE);
     }
-
 }
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java
@@ -4,7 +4,7 @@
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
 import com.grack.nanojson.JsonParserException;
-import org.jsoup.helper.StringUtil;
+
 import org.schabi.newpipe.extractor.MediaFormat;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.StreamingService;
@@ -17,10 +17,18 @@
 import org.schabi.newpipe.extractor.localization.DateWrapper;
 import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
 import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory;
-import org.schabi.newpipe.extractor.stream.*;
+import org.schabi.newpipe.extractor.stream.AudioStream;
+import org.schabi.newpipe.extractor.stream.Description;
+import org.schabi.newpipe.extractor.stream.Stream;
+import org.schabi.newpipe.extractor.stream.StreamExtractor;
+import org.schabi.newpipe.extractor.stream.StreamInfoItem;
+import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
+import org.schabi.newpipe.extractor.stream.StreamType;
+import org.schabi.newpipe.extractor.stream.SubtitlesStream;
+import org.schabi.newpipe.extractor.stream.VideoStream;
 import org.schabi.newpipe.extractor.utils.JsonUtils;
+import org.schabi.newpipe.extractor.utils.Utils;
 
-import javax.annotation.Nonnull;
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
 import java.net.URLEncoder;
@@ -29,6 +37,8 @@
 import java.util.List;
 import java.util.Locale;
 
+import javax.annotation.Nonnull;
+
 public class PeertubeStreamExtractor extends StreamExtractor {
 
 
@@ -255,7 +265,7 @@ public StreamInfoItemsCollector getRelatedStreams() throws IOException, Extracti
         } else {
             apiUrl = getUploaderUrl() + "/videos?start=0&count=8";
         }
-        if (!StringUtil.isBlank(apiUrl)) getStreamsFromApi(collector, apiUrl);
+        if (!Utils.isBlank(apiUrl)) getStreamsFromApi(collector, apiUrl);
         return collector;
     }
 
@@ -292,7 +302,7 @@ private String getRelatedStreamsUrl(List<String> tags) throws UnsupportedEncodin
     private void getStreamsFromApi(StreamInfoItemsCollector collector, String apiUrl) throws ReCaptchaException, IOException, ParsingException {
         Response response = getDownloader().get(apiUrl);
         JsonObject relatedVideosJson = null;
-        if (null != response && !StringUtil.isBlank(response.responseBody())) {
+        if (null != response && !Utils.isBlank(response.responseBody())) {
             try {
                 relatedVideosJson = JsonParser.object().from(response.responseBody());
             } catch (JsonParserException e) {
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeTrendingExtractor.java
@@ -3,7 +3,7 @@
 import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
-import org.jsoup.helper.StringUtil;
+
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.downloader.Response;
@@ -15,6 +15,7 @@
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 import org.schabi.newpipe.extractor.utils.JsonUtils;
+import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.IOException;
 
@@ -69,7 +70,7 @@ public String getNextPageUrl() throws IOException, ExtractionException {
     public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
         Response response = getDownloader().get(pageUrl);
         JsonObject json = null;
-        if (null != response && !StringUtil.isBlank(response.responseBody())) {
+        if (response != null && !Utils.isBlank(response.responseBody())) {
             try {
                 json = JsonParser.object().from(response.responseBody());
             } catch (Exception e) {
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
@@ -1,6 +1,5 @@
 package org.schabi.newpipe.extractor.services.youtube;
 
-
 import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
@@ -111,9 +110,7 @@ public static boolean isInvidioURL(URL url) {
 
     public static long parseDurationString(String input)
             throws ParsingException, NumberFormatException {
-
         // If time separator : is not detected, try . instead
-
         final String[] splitInput = input.contains(":")
                 ? input.split(":")
                 : input.split("\\.");
@@ -145,10 +142,10 @@ public static long parseDurationString(String input)
             default:
                 throw new ParsingException("Error duration string with unknown format: " + input);
         }
-        return ((((Long.parseLong(days) * 24)
-                + Long.parseLong(hours) * 60)
-                + Long.parseLong(minutes)) * 60)
-                + Long.parseLong(seconds);
+        return (((Long.parseLong(Utils.removeNonDigitCharacters(days)) * 24)
+                + Long.parseLong(Utils.removeNonDigitCharacters(hours)) * 60)
+                + Long.parseLong(Utils.removeNonDigitCharacters(minutes)) * 60)
+                + Long.parseLong(Utils.removeNonDigitCharacters(seconds));
     }
 
     public static String getFeedUrlFrom(final String channelIdOrUser) {
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java
@@ -203,4 +203,23 @@ public static boolean isNullOrEmpty(final Collection<?> collection) {
     public static boolean isNullOrEmpty(final Map map) {
         return map == null || map.isEmpty();
     }
-}
+
+    public static boolean isWhitespace(final int c){
+        return c == ' ' || c == '\t' || c == '\n' || c == '\f' || c == '\r';
+    }
+
+    public static boolean isBlank(final String string) {
+        if (string == null || string.isEmpty()) {
+            return true;
+        }
+
+        final int length = string.length();
+        for (int i = 0; i < length; i++) {
+            if (!isWhitespace(string.codePointAt(i))) {
+                return false;
+            }
+        }
+
+        return true;
+    }
+}
diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java
@@ -1,6 +1,5 @@
 package org.schabi.newpipe.extractor.services.peertube;
 
-import org.jsoup.helper.StringUtil;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.schabi.newpipe.DownloaderTestImpl;
@@ -10,6 +9,7 @@
 import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeCommentsExtractor;
+import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.IOException;
 import java.util.List;
@@ -50,7 +50,7 @@ public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionExce
         result = findInComments(commentsInfo.getRelatedItems(), "Loved it!!!");
 
         String nextPage = commentsInfo.getNextPageUrl();
-        while (!StringUtil.isBlank(nextPage) && !result) {
+        while (!Utils.isBlank(nextPage) && !result) {
             InfoItemsPage<CommentsInfoItem> moreItems = CommentsInfo.getMoreItems(PeerTube, commentsInfo, nextPage);
             result = findInComments(moreItems.getItems(), "Loved it!!!");
             nextPage = moreItems.getNextPageUrl();
@@ -63,15 +63,15 @@ public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionExce
     public void testGetCommentsAllData() throws IOException, ExtractionException {
         InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
         for (CommentsInfoItem c : comments.getItems()) {
-            assertFalse(StringUtil.isBlank(c.getUploaderUrl()));
-            assertFalse(StringUtil.isBlank(c.getUploaderName()));
-            assertFalse(StringUtil.isBlank(c.getUploaderAvatarUrl()));
-            assertFalse(StringUtil.isBlank(c.getCommentId()));
-            assertFalse(StringUtil.isBlank(c.getCommentText()));
-            assertFalse(StringUtil.isBlank(c.getName()));
-            assertFalse(StringUtil.isBlank(c.getTextualUploadDate()));
-            assertFalse(StringUtil.isBlank(c.getThumbnailUrl()));
-            assertFalse(StringUtil.isBlank(c.getUrl()));
+            assertFalse(Utils.isBlank(c.getUploaderUrl()));
+            assertFalse(Utils.isBlank(c.getUploaderName()));
+            assertFalse(Utils.isBlank(c.getUploaderAvatarUrl()));
+            assertFalse(Utils.isBlank(c.getCommentId()));
+            assertFalse(Utils.isBlank(c.getCommentText()));
+            assertFalse(Utils.isBlank(c.getName()));
+            assertFalse(Utils.isBlank(c.getTextualUploadDate()));
+            assertFalse(Utils.isBlank(c.getThumbnailUrl()));
+            assertFalse(Utils.isBlank(c.getUrl()));
             assertFalse(c.getLikeCount() != -1);
         }
     }
diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsExtractorTest.java
diff --git a/timeago-parser/build.gradle b/timeago-parser/build.gradle
