[SoundCloud] Fix SoundCloud ID extraction

resolveIdWithEmbedPlayer() does not work anymore because the JSON data has been extracted to an API call. For this reason, replace resolveIdWithEmbedPlayer() with resolveIdWithWidgetApi)( which performs the API call.

Author: TobiGr

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java

@@ -19,6 +19,7 @@
 import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudStreamExtractor;
 import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudStreamInfoItemExtractor;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
+import org.schabi.newpipe.extractor.utils.JsonUtils;
 import org.schabi.newpipe.extractor.utils.Parser;
 import org.schabi.newpipe.extractor.utils.Parser.RegexException;
 import org.schabi.newpipe.extractor.utils.Utils;
@@ -140,27 +141,37 @@ public static String resolveUrlWithEmbedPlayer(String apiUrl) throws IOException
     }
 
     /**
-     * Fetch the embed player with the url and return the id (like the id from the json api).
+     * Fetch the widget API with the url and return the id (like the id from the json api).
      *
      * @return the resolved id
      */
-    public static String resolveIdWithEmbedPlayer(String urlString) throws IOException, ReCaptchaException, ParsingException {
+    public static String resolveIdWithWidgetApi(String urlString) throws IOException, ReCaptchaException, ParsingException {
         // Remove the tailing slash from URLs due to issues with the SoundCloud API
         if (urlString.charAt(urlString.length() - 1) == '/') urlString = urlString.substring(0, urlString.length() - 1);
+        // Make URL lower case and remove www. if it exists.
+        // Without doing this, the widget API does not recognize the URL.
+        urlString = Utils.removeWWWFromUrl(urlString.toLowerCase());
 
-        URL url;
+        final URL url;
         try {
             url = Utils.stringToURL(urlString);
         } catch (MalformedURLException e) {
             throw new IllegalArgumentException("The given URL is not valid");
         }
 
-        String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url="
-                + URLEncoder.encode(url.toString(), UTF_8), SoundCloud.getLocalization()).responseBody();
-        // handle playlists / sets different and get playlist id via uir field in JSON
-        if (url.getPath().contains("/sets/") && !url.getPath().endsWith("/sets"))
-            return Parser.matchGroup1("\"uri\":\\s*\"https:\\/\\/api\\.soundcloud\\.com\\/playlists\\/((\\d)*?)\"", response);
-        return Parser.matchGroup1(",\"id\":(([^}\\n])*?),", response);
+        try {
+            final String widgetUrl = "https://api-widget.soundcloud.com/resolve?url="
+                    + URLEncoder.encode(url.toString(), UTF_8)
+                    + "&format=json&client_id=" + SoundcloudParsingHelper.clientId();
+            final String response = NewPipe.getDownloader().get(widgetUrl,
+                    SoundCloud.getLocalization()).responseBody();
+            final JsonObject o = JsonParser.object().from(response);
+            return String.valueOf(JsonUtils.getValue(o, "id"));
+        } catch (JsonParserException e) {
+            throw new ParsingException("Could not parse JSON response", e);
+        } catch (ExtractionException e) {
+            throw new ParsingException("Could not resolve id with embedded player. ClientId not extracted", e);
+        }
     }
 
     /**

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudChannelLinkHandlerFactory.java

@@ -23,7 +23,7 @@ public String getId(String url) throws ParsingException {
         Utils.checkUrl(URL_PATTERN, url);
 
         try {
-            return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url);
+            return SoundcloudParsingHelper.resolveIdWithWidgetApi(url);
         } catch (Exception e) {
             throw new ParsingException(e.getMessage(), e);
         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudPlaylistLinkHandlerFactory.java

@@ -22,7 +22,7 @@ public String getId(String url) throws ParsingException {
         Utils.checkUrl(URL_PATTERN, url);
 
         try {
-            return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url);
+            return SoundcloudParsingHelper.resolveIdWithWidgetApi(url);
         } catch (Exception e) {
             throw new ParsingException("Could not get id of url: " + url + " " + e.getMessage(), e);
         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java

@@ -32,7 +32,7 @@ public String getId(String url) throws ParsingException {
         Utils.checkUrl(URL_PATTERN, url);
 
         try {
-            return SoundcloudParsingHelper.resolveIdWithEmbedPlayer(url);
+            return SoundcloudParsingHelper.resolveIdWithWidgetApi(url);
         } catch (Exception e) {
             throw new ParsingException(e.getMessage(), e);
         }

extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java

@@ -7,13 +7,15 @@
 import java.net.URL;
 import java.net.URLDecoder;
 import java.util.*;
+import java.util.regex.Pattern;
 
 public class Utils {
 
     public static final String HTTP = "http://";
     public static final String HTTPS = "https://";
     public static final String UTF_8 = "UTF-8";
     public static final String EMPTY_STRING = "";
+    private static final Pattern WWW_PATTERN = Pattern.compile("(https?)?:\\/\\/www\\.");
 
     private Utils() {
         //no instance
@@ -170,6 +172,13 @@ public static boolean isHTTP(URL url) {
         return setsNoPort || usesDefaultPort;
     }
 
+    public static String removeWWWFromUrl(String url) {
+        if (WWW_PATTERN.matcher(url).find()) {
+            return url.replace("www.", "");
+        }
+        return url;
+    }
+
     public static String removeUTF8BOM(String s) {
         if (s.startsWith("\uFEFF")) {
             s = s.substring(1);

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java

@@ -29,9 +29,9 @@ public void resolveUrlWithEmbedPlayerTest() throws Exception {
     }
 
     @Test
-    public void resolveIdWithEmbedPlayerTest() throws Exception {
-        Assert.assertEquals("26057743", SoundcloudParsingHelper.resolveIdWithEmbedPlayer("https://soundcloud.com/trapcity"));
-        Assert.assertEquals("16069159", SoundcloudParsingHelper.resolveIdWithEmbedPlayer("https://soundcloud.com/nocopyrightsounds"));
+    public void resolveIdWithWidgetApiTest() throws Exception {
+        Assert.assertEquals("26057743", SoundcloudParsingHelper.resolveIdWithWidgetApi("https://soundcloud.com/trapcity"));
+        Assert.assertEquals("16069159", SoundcloudParsingHelper.resolveIdWithWidgetApi("https://soundcloud.com/nocopyrightsounds"));
 
     }