merged upstream/dev

Author: yausername

extractor/src/main/java/org/schabi/newpipe/extractor/DownloadResponse.java

@@ -7,15 +7,21 @@
 import javax.annotation.Nonnull;
 
 public class DownloadResponse {
+    private final int responseCode;
     private final String responseBody;
     private final Map<String, List<String>> responseHeaders;
 
-    public DownloadResponse(String responseBody, Map<String, List<String>> headers) {
+    public DownloadResponse(int responseCode, String responseBody, Map<String, List<String>> headers) {
         super();
+        this.responseCode = responseCode;
         this.responseBody = responseBody;
         this.responseHeaders = headers;
     }
 
+    public int getResponseCode() {
+        return responseCode;
+    }
+
     public String getResponseBody() {
         return responseBody;
     }
@@ -33,5 +39,4 @@ public List<String> getResponseCookies(){
         else
             return cookies;
     }
-
 }

extractor/src/main/java/org/schabi/newpipe/extractor/Downloader.java

@@ -60,6 +60,11 @@ public interface Downloader {
      */
     String download(String siteUrl) throws IOException, ReCaptchaException;
 
+    DownloadResponse head(String siteUrl) throws IOException, ReCaptchaException;
+
+    DownloadResponse get(String siteUrl, Localization localization)
+            throws IOException, ReCaptchaException;
+
     DownloadResponse get(String siteUrl, DownloadRequest request)
             throws IOException, ReCaptchaException;
 

extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/ReCaptchaException.java

@@ -21,7 +21,14 @@
  */
 
 public class ReCaptchaException extends ExtractionException {
-    public ReCaptchaException(String message) {
+    private String url;
+
+    public ReCaptchaException(String message, String url) {
         super(message);
+        this.url = url;
+    }
+
+    public String getUrl() {
+        return url;
     }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChannelInfoItemExtractor.java

@@ -24,7 +24,9 @@ public String getUrl() {
 
     @Override
     public String getThumbnailUrl() {
-        return itemObject.getString("avatar_url", "");
+        String avatarUrl = itemObject.getString("avatar_url", "");
+        String avatarUrlBetterResolution = avatarUrl.replace("large.jpg", "crop.jpg");
+        return avatarUrlBetterResolution;
     }
 
     @Override

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java

@@ -7,9 +7,12 @@
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+import org.schabi.newpipe.extractor.DownloadResponse;
 import org.schabi.newpipe.extractor.Downloader;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.channel.ChannelInfoItemsCollector;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
@@ -21,40 +24,59 @@
 import java.net.URLEncoder;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
+import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
 
 import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
 
 public class SoundcloudParsingHelper {
+    private static final String HARDCODED_CLIENT_ID = "LHzSAKe8eP9Yy3FgBugfBapRPLncO6Ng"; // Updated on 22/10/19
     private static String clientId;
 
     private SoundcloudParsingHelper() {
     }
 
-    public static String clientId() throws ReCaptchaException, IOException, RegexException {
+    public static String clientId() throws ExtractionException, IOException {
         if (clientId != null && !clientId.isEmpty()) return clientId;
 
         Downloader dl = NewPipe.getDownloader();
-        String response = dl.download("https://soundcloud.com");
-
-        Document doc = Jsoup.parse(response);
-        Element jsElement = doc.select("script[src^=https://a-v2.sndcdn.com/assets/app]").first();
+        clientId = HARDCODED_CLIENT_ID;
+        if (checkIfHardcodedClientIdIsValid(dl)) {
+            return clientId;
+        }
 
+        final DownloadResponse download = dl.get("https://soundcloud.com");
+        String response = download.getResponseBody();
         final String clientIdPattern = ",client_id:\"(.*?)\"";
 
-        try {
-            final HashMap<String, String> headers = new HashMap<>();
-            headers.put("Range", "bytes=0-16384");
-            String js = dl.download(jsElement.attr("src"), headers);
-
-            return clientId = Parser.matchGroup1(clientIdPattern, js);
-        } catch (IOException | RegexException ignored) {
-            // Ignore it and proceed to download the whole js file
+        Document doc = Jsoup.parse(response);
+        final Elements possibleScripts = doc.select("script[src*=\"sndcdn.com/assets/\"][src$=\".js\"]");
+        // The one containing the client id will likely be the last one
+        Collections.reverse(possibleScripts);
+
+        final HashMap<String, String> headers = new HashMap<>();
+        headers.put("Range", "bytes=0-16384");
+
+        for (Element element : possibleScripts) {
+            final String srcUrl = element.attr("src");
+            if (srcUrl != null && !srcUrl.isEmpty()) {
+                try {
+                    return clientId = Parser.matchGroup1(clientIdPattern, dl.download(srcUrl, headers));
+                } catch (RegexException ignored) {
+                    // Ignore it and proceed to try searching other script
+                }
+            }
         }
 
-        String js = dl.download(jsElement.attr("src"));
-        return clientId = Parser.matchGroup1(clientIdPattern, js);
+        // Officially give up
+        throw new ExtractionException("Couldn't extract client id");
+    }
+
+    static boolean checkIfHardcodedClientIdIsValid(Downloader dl) throws IOException, ReCaptchaException {
+        final String apiUrl = "https://api.soundcloud.com/connect?client_id=" + HARDCODED_CLIENT_ID;
+        // Should return 200 to indicate that the client id is valid, a 401 is returned otherwise.
+        return dl.head(apiUrl).getResponseCode() == 200;
     }
 
     public static String toDateString(String time) throws ParsingException {
@@ -79,7 +101,7 @@ public static String toDateString(String time) throws ParsingException {
      * 
      * See https://developers.soundcloud.com/docs/api/reference#resolve
      */
-    public static JsonObject resolveFor(Downloader downloader, String url) throws IOException, ReCaptchaException, ParsingException {
+    public static JsonObject resolveFor(Downloader downloader, String url) throws IOException, ExtractionException {
         String apiUrl = "https://api.soundcloud.com/resolve"
                 + "?url=" + URLEncoder.encode(url, "UTF-8")
                 + "&client_id=" + clientId();
@@ -113,7 +135,10 @@ public static String resolveIdWithEmbedPlayer(String url) throws IOException, Re
 
         String response = NewPipe.getDownloader().download("https://w.soundcloud.com/player/?url="
                 + URLEncoder.encode(url, "UTF-8"));
-        return Parser.matchGroup1(",\"id\":(.*?),", response);
+        // handle playlists / sets different and get playlist id via uir field in JSON
+        if (url.contains("sets") && !url.endsWith("sets") && !url.endsWith("sets/"))
+            return Parser.matchGroup1("\"uri\":\\s*\"https:\\/\\/api\\.soundcloud\\.com\\/playlists\\/((\\d)*?)\"", response);
+        return Parser.matchGroup1(",\"id\":(([^}\\n])*?),", response);
     }
 
     /**

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java

@@ -71,13 +71,15 @@ public String getThumbnailUrl() {
                     final String thumbnailUrl = item.getThumbnailUrl();
                     if (thumbnailUrl == null || thumbnailUrl.isEmpty()) continue;
 
-                    return thumbnailUrl;
+                    String thumbnailUrlBetterResolution = thumbnailUrl.replace("large.jpg", "crop.jpg");
+                    return thumbnailUrlBetterResolution;
                 }
             } catch (Exception ignored) {
             }
         }
 
-        return artworkUrl;
+        String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg");
+        return artworkUrlBetterResolution;
     }
 
     @Override

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistInfoItemExtractor.java

@@ -32,7 +32,10 @@ public String getThumbnailUrl() throws ParsingException {
         // Over-engineering at its finest
         if (itemObject.isString(ARTWORK_URL_KEY)) {
             final String artworkUrl = itemObject.getString(ARTWORK_URL_KEY, "");
-            if (!artworkUrl.isEmpty()) return artworkUrl;
+            if (!artworkUrl.isEmpty()) {
+                String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg");
+                return artworkUrlBetterResolution;
+            }
         }
 
         try {
@@ -42,8 +45,11 @@ public String getThumbnailUrl() throws ParsingException {
 
                 // First look for track artwork url
                 if (trackObject.isString(ARTWORK_URL_KEY)) {
-                    final String url = trackObject.getString(ARTWORK_URL_KEY, "");
-                    if (!url.isEmpty()) return url;
+                    String artworkUrl = trackObject.getString(ARTWORK_URL_KEY, "");
+                    if (!artworkUrl.isEmpty()) {
+                        String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg");
+                        return artworkUrlBetterResolution;
+                    }
                 }
 
                 // Then look for track creator avatar url

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSearchQueryHandlerFactory.java

@@ -1,5 +1,6 @@
 package org.schabi.newpipe.extractor.services.soundcloud;
 
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
 import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
@@ -48,10 +49,10 @@ public String getUrl(String id, List<String> contentFilter, String sortFilter) t
 
         } catch (UnsupportedEncodingException e) {
             throw new ParsingException("Could not encode query", e);
-        } catch (IOException e) {
-            throw new ParsingException("Could not get client id", e);
         } catch (ReCaptchaException e) {
             throw new ParsingException("ReCaptcha required", e);
+        } catch (IOException | ExtractionException e) {
+            throw new ParsingException("Could not get client id", e);
         }
     }
 

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java

@@ -95,6 +95,7 @@ public KioskExtractor createNewKiosk(StreamingService streamingService,
         try {
             list.addKioskEntry(chartsFactory, h, "Top 50");
             list.addKioskEntry(chartsFactory, h, "New & hot");
+            list.setDefaultKiosk("New & hot");
         } catch (Exception e) {
             throw new ExtractionException(e);
         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java

@@ -57,7 +57,12 @@ public String getUploadDate() throws ParsingException {
     @Nonnull
     @Override
     public String getThumbnailUrl() {
-        return track.getString("artwork_url", "");
+        String artworkUrl = track.getString("artwork_url", "");
+        if (artworkUrl.isEmpty()) {
+            artworkUrl = track.getObject("user").getString("avatar_url", "");
+        }
+        String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg");
+        return artworkUrlBetterResolution;
     }
 
     @Nonnull