Merge pull request #1269 from AudricV/snd_no_drm_streams

[Soundcloud] Remove DRM-protected and downloadable formats extraction

Author: Stypox

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java

@@ -6,7 +6,6 @@
 import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.getAllImagesFromTrackObject;
 import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.getAvatarUrl;
 import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.parseDateFrom;
-import static org.schabi.newpipe.extractor.stream.AudioStream.UNKNOWN_BITRATE;
 import static org.schabi.newpipe.extractor.stream.Stream.ID_UNKNOWN;
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
 
@@ -170,34 +169,29 @@ public List<AudioStream> getAudioStreams() throws ExtractionException {
         }
 
         try {
-            final JsonArray transcodings = track.getObject("media").getArray("transcodings");
+            final JsonArray transcodings = track.getObject("media")
+                    .getArray("transcodings");
             if (!isNullOrEmpty(transcodings)) {
                 // Get information about what stream formats are available
-                extractAudioStreams(transcodings, checkMp3ProgressivePresence(transcodings),
-                        audioStreams);
+                extractAudioStreams(transcodings, audioStreams);
             }
-
-            extractDownloadableFileIfAvailable(audioStreams);
         } catch (final NullPointerException e) {
             throw new ExtractionException("Could not get audio streams", e);
         }
 
         return audioStreams;
     }
 
-    private static boolean checkMp3ProgressivePresence(@Nonnull final JsonArray transcodings) {
-        return transcodings.stream()
-                .filter(JsonObject.class::isInstance)
-                .map(JsonObject.class::cast)
-                .anyMatch(transcodingJsonObject -> transcodingJsonObject.getString("preset")
-                        .contains("mp3") && transcodingJsonObject.getObject("format")
-                        .getString("protocol").equals("progressive"));
-    }
-
     @Nonnull
     private String getTranscodingUrl(final String endpointUrl)
             throws IOException, ExtractionException {
-        final String apiStreamUrl = endpointUrl + "?client_id=" + clientId();
+        String apiStreamUrl = endpointUrl + "?client_id=" + clientId();
+
+        final String trackAuthorization = track.getString("track_authorization");
+        if (!isNullOrEmpty(trackAuthorization)) {
+            apiStreamUrl += "&track_authorization=" + trackAuthorization;
+        }
+
         final String response = NewPipe.getDownloader().get(apiStreamUrl).responseBody();
         final JsonObject urlObject;
         try {
@@ -209,27 +203,7 @@ private String getTranscodingUrl(final String endpointUrl)
         return urlObject.getString("url");
     }
 
-    @Nullable
-    private String getDownloadUrl(@Nonnull final String trackId)
-            throws IOException, ExtractionException {
-        final String response = NewPipe.getDownloader().get(SOUNDCLOUD_API_V2_URL + "tracks/"
-                + trackId + "/download" + "?client_id=" + clientId()).responseBody();
-
-        final JsonObject downloadJsonObject;
-        try {
-            downloadJsonObject = JsonParser.object().from(response);
-        } catch (final JsonParserException e) {
-            throw new ParsingException("Could not parse download URL", e);
-        }
-        final String redirectUri = downloadJsonObject.getString("redirectUri");
-        if (!isNullOrEmpty(redirectUri)) {
-            return redirectUri;
-        }
-        return null;
-    }
-
     private void extractAudioStreams(@Nonnull final JsonArray transcodings,
-                                     final boolean mp3ProgressiveInStreams,
                                      final List<AudioStream> audioStreams) {
         transcodings.stream()
                 .filter(JsonObject.class::isInstance)
@@ -244,23 +218,23 @@ private void extractAudioStreams(@Nonnull final JsonArray transcodings,
                         final String preset = transcoding.getString("preset", ID_UNKNOWN);
                         final String protocol = transcoding.getObject("format")
                                 .getString("protocol");
+
+                        if (protocol.contains("encrypted")) {
+                            // Skip DRM-protected streams, which have encrypted in their protocol
+                            // name
+                            return;
+                        }
+
                         final AudioStream.Builder builder = new AudioStream.Builder()
                                 .setId(preset);
 
-                        final boolean isHls = protocol.equals("hls");
-                        if (isHls) {
+                        if (protocol.equals("hls")) {
                             builder.setDeliveryMethod(DeliveryMethod.HLS);
                         }
 
                         builder.setContent(getTranscodingUrl(url), true);
 
                         if (preset.contains("mp3")) {
-                            // Don't add the MP3 HLS stream if there is a progressive stream
-                            // present because both have the same bitrate
-                            if (mp3ProgressiveInStreams && isHls) {
-                                return;
-                            }
-
                             builder.setMediaFormat(MediaFormat.MP3);
                             builder.setAverageBitrate(128);
                         } else if (preset.contains("opus")) {
@@ -283,39 +257,6 @@ private void extractAudioStreams(@Nonnull final JsonArray transcodings,
                 });
     }
 
-    /**
-     * Add the downloadable format if it is available.
-     *
-     * <p>
-     * A track can have the {@code downloadable} boolean set to {@code true}, but it doesn't mean
-     * we can download it.
-     * </p>
-     *
-     * <p>
-     * If the value of the {@code has_download_left} boolean is {@code true}, the track can be
-     * downloaded, and not otherwise.
-     * </p>
-     *
-     * @param audioStreams the audio streams to which the downloadable file is added
-     */
-    public void extractDownloadableFileIfAvailable(final List<AudioStream> audioStreams) {
-        if (track.getBoolean("downloadable") && track.getBoolean("has_downloads_left")) {
-            try {
-                final String downloadUrl = getDownloadUrl(getId());
-                if (!isNullOrEmpty(downloadUrl)) {
-                    audioStreams.add(new AudioStream.Builder()
-                            .setId("original-format")
-                            .setContent(downloadUrl, true)
-                            .setAverageBitrate(UNKNOWN_BITRATE)
-                            .build());
-                }
-            } catch (final Exception ignored) {
-                // If something went wrong when trying to get the download URL, ignore the
-                // exception throw because this "stream" is not necessary to play the track
-            }
-        }
-    }
-
     @Override
     public List<VideoStream> getVideoStreams() {
         return Collections.emptyList();

extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractorTest.java

@@ -24,6 +24,7 @@
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.fail;
 import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
 
 public class SoundcloudStreamExtractorTest {
@@ -188,26 +189,33 @@ public static void setUp() throws Exception {
         public void testAudioStreams() throws Exception {
             super.testAudioStreams();
             final List<AudioStream> audioStreams = extractor.getAudioStreams();
-            assertEquals(2, audioStreams.size());
+            assertEquals(3, audioStreams.size()); // 2 MP3 streams (1 progressive, 1 HLS) and 1 OPUS
             audioStreams.forEach(audioStream -> {
                 final DeliveryMethod deliveryMethod = audioStream.getDeliveryMethod();
                 final String mediaUrl = audioStream.getContent();
                 if (audioStream.getFormat() == MediaFormat.OPUS) {
-                    // Assert that it's an OPUS 64 kbps media URL with a single range which comes
-                    // from an HLS SoundCloud CDN
-                    ExtractorAsserts.assertContains("-hls-opus-media.sndcdn.com", mediaUrl);
-                    ExtractorAsserts.assertContains(".64.opus", mediaUrl);
                     assertSame(DeliveryMethod.HLS, deliveryMethod,
                             "Wrong delivery method for stream " + audioStream.getId() + ": "
                                     + deliveryMethod);
-                } else if (audioStream.getFormat() == MediaFormat.MP3) {
-                    // Assert that it's a MP3 128 kbps media URL which comes from a progressive
+                    // Assert it's an OPUS 64 kbps media playlist URL which comes from an HLS
                     // SoundCloud CDN
-                    ExtractorAsserts.assertContains("-media.sndcdn.com/bKOA7Pwbut93.128.mp3",
-                            mediaUrl);
-                    assertSame(DeliveryMethod.PROGRESSIVE_HTTP, deliveryMethod,
-                            "Wrong delivery method for stream " + audioStream.getId() + ": "
-                                    + deliveryMethod);
+                    ExtractorAsserts.assertContains("-hls-opus-media.sndcdn.com", mediaUrl);
+                    ExtractorAsserts.assertContains(".64.opus", mediaUrl);
+                } else if (audioStream.getFormat() == MediaFormat.MP3) {
+                    if (deliveryMethod == DeliveryMethod.PROGRESSIVE_HTTP) {
+                        // Assert it's a MP3 128 kbps media URL which comes from a progressive
+                        // SoundCloud CDN
+                        ExtractorAsserts.assertContains("-media.sndcdn.com/bKOA7Pwbut93.128.mp3",
+                                mediaUrl);
+                    } else if (deliveryMethod == DeliveryMethod.HLS) {
+                        // Assert it's a MP3 128 kbps media HLS playlist URL which comes from an HLS
+                        // SoundCloud CDN
+                        ExtractorAsserts.assertContains("-hls-media.sndcdn.com", mediaUrl);
+                        ExtractorAsserts.assertContains(".128.mp3", mediaUrl);
+                    } else {
+                        fail("Wrong delivery method for stream " + audioStream.getId() + ": "
+                                + deliveryMethod);
+                    }
                 }
             });
         }