Add Utils methods for URL encoding/decoding using UTF-8.

Isira-Seneviratne · Isira-Seneviratne · commit ddbce3b83d54 · 2022-11-12T07:29:15.000+05:30
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java
@@ -13,9 +13,9 @@
 import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
+import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.IOException;
-import java.net.URLEncoder;
 import java.util.Collections;
 import java.util.List;
 import java.util.stream.Collectors;
@@ -33,7 +33,7 @@ public List<String> suggestionList(final String query) throws IOException, Extra
 
         try {
             final JsonObject fuzzyResults = JsonParser.object().from(downloader
-                    .get(AUTOCOMPLETE_URL + URLEncoder.encode(query, "UTF-8")).responseBody());
+                    .get(AUTOCOMPLETE_URL + Utils.encodeUrlUtf8(query)).responseBody());
 
             return fuzzyResults.getObject("auto").getArray("results").stream()
                     .filter(JsonObject.class::isInstance)
@@ -44,6 +44,5 @@ public List<String> suggestionList(final String query) throws IOException, Extra
         } catch (final JsonParserException e) {
             return Collections.emptyList();
         }
-
     }
 }
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/linkHandler/BandcampSearchQueryHandlerFactory.java
@@ -2,24 +2,22 @@
 
 package org.schabi.newpipe.extractor.services.bandcamp.linkHandler;
 
+import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.BASE_URL;
+
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
+import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.UnsupportedEncodingException;
-import java.net.URLEncoder;
 import java.util.List;
 
-import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.BASE_URL;
-
 public class BandcampSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
-
-
     @Override
     public String getUrl(final String query,
                          final List<String> contentFilter,
                          final String sortFilter) throws ParsingException {
         try {
-            return BASE_URL + "/search?q=" + URLEncoder.encode(query, "UTF-8") + "&page=1";
+            return BASE_URL + "/search?q=" + Utils.encodeUrlUtf8(query) + "&page=1";
         } catch (final UnsupportedEncodingException e) {
             throw new ParsingException("query \"" + query + "\" could not be encoded", e);
         }
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCSearchQueryHandlerFactory.java
@@ -2,10 +2,9 @@
 
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
+import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.UnsupportedEncodingException;
-import java.net.URLEncoder;
-import java.nio.charset.StandardCharsets;
 import java.util.List;
 
 public class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
@@ -31,8 +30,7 @@ public String[] getAvailableSortFilter() {
     public String getUrl(final String query, final List<String> contentFilter,
                          final String sortFilter) throws ParsingException {
         try {
-            return "https://media.ccc.de/public/events/search?q="
-                    + URLEncoder.encode(query, StandardCharsets.UTF_8.name());
+            return "https://media.ccc.de/public/events/search?q=" + Utils.encodeUrlUtf8(query);
         } catch (final UnsupportedEncodingException e) {
             throw new ParsingException("Could not create search string with query: " + query, e);
         }
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java
@@ -35,8 +35,6 @@
 
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
-import java.net.URLEncoder;
-import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -327,8 +325,7 @@ private String getRelatedItemsUrl(@Nonnull final List<String> tags)
         final StringBuilder params = new StringBuilder();
         params.append("start=0&count=8&sort=-createdAt");
         for (final String tag : tags) {
-            params.append("&tagsOneOf=");
-            params.append(URLEncoder.encode(tag, StandardCharsets.UTF_8.name()));
+            params.append("&tagsOneOf=").append(Utils.encodeUrlUtf8(tag));
         }
         return url + "?" + params;
     }
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/linkHandler/PeertubeSearchQueryHandlerFactory.java
@@ -3,10 +3,9 @@
 import org.schabi.newpipe.extractor.ServiceList;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
+import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.UnsupportedEncodingException;
-import java.net.URLEncoder;
-import java.nio.charset.StandardCharsets;
 import java.util.List;
 
 public final class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
@@ -42,8 +41,7 @@ public String getUrl(final String searchString,
                          final String sortFilter,
                          final String baseUrl) throws ParsingException {
         try {
-            return baseUrl + SEARCH_ENDPOINT + "?search=" + URLEncoder.encode(searchString,
-                    StandardCharsets.UTF_8.name());
+            return baseUrl + SEARCH_ENDPOINT + "?search=" + Utils.encodeUrlUtf8(searchString);
         } catch (final UnsupportedEncodingException e) {
             throw new ParsingException("Could not encode query", e);
         }
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java
@@ -31,8 +31,6 @@
 import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
-import java.net.URLEncoder;
-import java.nio.charset.StandardCharsets;
 import java.time.OffsetDateTime;
 import java.time.format.DateTimeFormatter;
 import java.time.format.DateTimeParseException;
@@ -109,7 +107,7 @@ public static OffsetDateTime parseDateFrom(final String textualUploadDate)
     public static JsonObject resolveFor(@Nonnull final Downloader downloader, final String url)
             throws IOException, ExtractionException {
         final String apiUrl = SOUNDCLOUD_API_V2_URL + "resolve"
-                + "?url=" + URLEncoder.encode(url, StandardCharsets.UTF_8.name())
+                + "?url=" + Utils.encodeUrlUtf8(url)
                 + "&client_id=" + clientId();
 
         try {
@@ -131,8 +129,7 @@ public static String resolveUrlWithEmbedPlayer(final String apiUrl) throws IOExc
             ReCaptchaException {
 
         final String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url="
-                + URLEncoder.encode(apiUrl, StandardCharsets.UTF_8.name()),
-                SoundCloud.getLocalization()).responseBody();
+                + Utils.encodeUrlUtf8(apiUrl), SoundCloud.getLocalization()).responseBody();
 
         return Jsoup.parse(response).select("link[rel=\"canonical\"]").first()
                 .attr("abs:href");
@@ -163,7 +160,7 @@ public static String resolveIdWithWidgetApi(final String urlString) throws IOExc
 
         try {
             final String widgetUrl = "https://api-widget.soundcloud.com/resolve?url="
-                    + URLEncoder.encode(url.toString(), StandardCharsets.UTF_8.name())
+                    + Utils.encodeUrlUtf8(url.toString())
                     + "&format=json&client_id=" + SoundcloudParsingHelper.clientId();
             final String response = NewPipe.getDownloader().get(widgetUrl,
                     SoundCloud.getLocalization()).responseBody();
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java
@@ -31,11 +31,10 @@
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 import org.schabi.newpipe.extractor.stream.StreamType;
 import org.schabi.newpipe.extractor.stream.VideoStream;
+import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
-import java.net.URLEncoder;
-import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -320,7 +319,7 @@ public void extractDownloadableFileIfAvailable(final List<AudioStream> audioStre
 
     private static String urlEncode(final String value) {
         try {
-            return URLEncoder.encode(value, StandardCharsets.UTF_8.name());
+            return Utils.encodeUrlUtf8(value);
         } catch (final UnsupportedEncodingException e) {
             throw new IllegalStateException(e);
         }
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSuggestionExtractor.java
@@ -14,10 +14,9 @@
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
 import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
+import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.IOException;
-import java.net.URLEncoder;
-import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -32,9 +31,9 @@ public List<String> suggestionList(final String query) throws IOException,
             ExtractionException {
         final List<String> suggestions = new ArrayList<>();
         final Downloader dl = NewPipe.getDownloader();
-        final String url = SOUNDCLOUD_API_V2_URL + "search/queries" + "?q="
-                + URLEncoder.encode(query, StandardCharsets.UTF_8.name()) + "&client_id="
-                + SoundcloudParsingHelper.clientId() + "&limit=10";
+        final String url = SOUNDCLOUD_API_V2_URL + "search/queries?q="
+                + Utils.encodeUrlUtf8(query) + "&client_id=" + SoundcloudParsingHelper.clientId()
+                + "&limit=10";
         final String response = dl.get(url, getExtractorLocalization()).responseBody();
 
         try {
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudSearchQueryHandlerFactory.java
@@ -7,11 +7,10 @@
 import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
 import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
 import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
+import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
-import java.net.URLEncoder;
-import java.nio.charset.StandardCharsets;
 import java.util.List;
 
 public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
@@ -48,7 +47,7 @@ public String getUrl(final String id,
                 }
             }
 
-            return url + "?q=" + URLEncoder.encode(id, StandardCharsets.UTF_8.name())
+            return url + "?q=" + Utils.encodeUrlUtf8(id)
                     + "&client_id=" + SoundcloudParsingHelper.clientId()
                     + "&limit=" + ITEMS_PER_PAGE + "&offset=0";
 
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
@@ -54,7 +54,6 @@
 import java.io.UnsupportedEncodingException;
 import java.net.MalformedURLException;
 import java.net.URL;
-import java.net.URLDecoder;
 import java.nio.charset.StandardCharsets;
 import java.security.SecureRandom;
 import java.time.LocalDate;
@@ -891,8 +890,7 @@ public static String getUrlFromNavigationEndpoint(@Nonnull final JsonObject navi
                 for (final String param : params) {
                     if (param.split("=")[0].equals("q")) {
                         try {
-                            return URLDecoder.decode(param.split("=")[1],
-                                    StandardCharsets.UTF_8.name());
+                            return Utils.decodeUrlUtf8(param.split("=")[1]);
                         } catch (final UnsupportedEncodingException e) {
                             return null;
                         }
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSuggestionExtractor.java
@@ -12,10 +12,9 @@
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
+import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.IOException;
-import java.net.URLEncoder;
-import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -54,9 +53,8 @@ public List<String> suggestionList(final String query) throws IOException, Extra
                 + "?client=" + "youtube" //"firefox" for JSON, 'toolbar' for xml
                 + "&jsonp=" + "JP"
                 + "&ds=" + "yt"
-                + "&gl=" + URLEncoder.encode(getExtractorContentCountry().getCountryCode(),
-                StandardCharsets.UTF_8.name())
-                + "&q=" + URLEncoder.encode(query, StandardCharsets.UTF_8.name());
+                + "&gl=" + Utils.encodeUrlUtf8(getExtractorContentCountry().getCountryCode())
+                + "&q=" + Utils.encodeUrlUtf8(query);
 
         String response = dl.get(url, getCookieHeader(), getExtractorLocalization()).responseBody();
         // trim JSONP part "JP(...)"
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java
@@ -1,22 +1,4 @@
-/*
- * Created by Christian Schabesberger on 12.08.17.
- *
- * Copyright (C) Christian Schabesberger 2018 <chris.schabesberger@mailbox.org>
- * YoutubeTrendingExtractor.java is part of NewPipe Extractor.
- *
- * NewPipe Extractor is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * NewPipe Extractor is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with NewPipe Extractor. If not, see <https://www.gnu.org/licenses/>.
- */
+package org.schabi.newpipe.extractor.services.youtube.extractors;
 
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextAtKey;
@@ -42,6 +24,26 @@
 
 import javax.annotation.Nonnull;
 
+/*
+ * Created by Christian Schabesberger on 12.08.17.
+ *
+ * Copyright (C) Christian Schabesberger 2018 <chris.schabesberger@mailbox.org>
+ * YoutubeTrendingExtractor.java is part of NewPipe Extractor.
+ *
+ * NewPipe Extractor is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * NewPipe Extractor is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NewPipe Extractor. If not, see <https://www.gnu.org/licenses/>.
+ */
+
 public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
     private JsonObject initialData;
 
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java
@@ -1,13 +1,12 @@
 package org.schabi.newpipe.extractor.services.youtube.linkHandler;
 
+import static org.schabi.newpipe.extractor.utils.Utils.encodeUrlUtf8;
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
 
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
 
 import java.io.UnsupportedEncodingException;
-import java.net.URLEncoder;
-import java.nio.charset.StandardCharsets;
 import java.util.List;
 
 import javax.annotation.Nonnull;
@@ -45,28 +44,21 @@ public String getUrl(final String searchString,
                     default:
                         break;
                     case VIDEOS:
-                        return SEARCH_URL + URLEncoder.encode(searchString,
-                                StandardCharsets.UTF_8.name())
-                                + "&sp=EgIQAQ%253D%253D";
+                        return SEARCH_URL + encodeUrlUtf8(searchString) + "&sp=EgIQAQ%253D%253D";
                     case CHANNELS:
-                        return SEARCH_URL + URLEncoder.encode(searchString,
-                                StandardCharsets.UTF_8.name())
-                                + "&sp=EgIQAg%253D%253D";
+                        return SEARCH_URL + encodeUrlUtf8(searchString) + "&sp=EgIQAg%253D%253D";
                     case PLAYLISTS:
-                        return SEARCH_URL + URLEncoder.encode(searchString,
-                                StandardCharsets.UTF_8.name())
-                                + "&sp=EgIQAw%253D%253D";
+                        return SEARCH_URL + encodeUrlUtf8(searchString) + "&sp=EgIQAw%253D%253D";
                     case MUSIC_SONGS:
                     case MUSIC_VIDEOS:
                     case MUSIC_ALBUMS:
                     case MUSIC_PLAYLISTS:
                     case MUSIC_ARTISTS:
-                        return MUSIC_SEARCH_URL + URLEncoder.encode(searchString,
-                                StandardCharsets.UTF_8.name());
+                        return MUSIC_SEARCH_URL + encodeUrlUtf8(searchString);
                 }
             }
 
-            return SEARCH_URL + URLEncoder.encode(searchString, StandardCharsets.UTF_8.name());
+            return SEARCH_URL + encodeUrlUtf8(searchString);
         } catch (final UnsupportedEncodingException e) {
             throw new ParsingException("Could not encode query", e);
         }
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Parser.java
@@ -20,15 +20,9 @@
 
 package org.schabi.newpipe.extractor.utils;
 
-import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
-
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 
 import java.io.UnsupportedEncodingException;
-import java.net.URLDecoder;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.regex.Matcher;
@@ -102,7 +96,7 @@ public static Map<String, String> compatParseMap(@Nonnull final String input)
         for (final String arg : input.split("&")) {
             final String[] splitArg = arg.split("=");
             if (splitArg.length > 1) {
-                map.put(splitArg[0], URLDecoder.decode(splitArg[1], StandardCharsets.UTF_8.name()));
+                map.put(splitArg[0], Utils.decodeUrlUtf8(splitArg[1]));
             } else {
                 map.put(splitArg[0], "");
             }
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java
diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java
