Skip to content

Commit 6f03c6e

Browse files
TobiGrTobiGr
authored
Merge pull request #293 from Stypox/fix-soundcloud
[SoundCloud] Fix by migrating to `api-v2` and other improvements
2 parents 9eca7df + 222d659 commit 6f03c6e

8 files changed

Lines changed: 211 additions & 75 deletions

File tree

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudChartsExtractor.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) throws IOException,
4444
}
4545

4646

47-
private void computNextPageAndStreams() throws IOException, ExtractionException {
47+
private void computeNextPageAndStreams() throws IOException, ExtractionException {
4848
collector = new StreamInfoItemsCollector(getServiceId());
4949

5050
String apiUrl = "https://api-v2.soundcloud.com/charts" +
@@ -69,7 +69,7 @@ private void computNextPageAndStreams() throws IOException, ExtractionException
6969
@Override
7070
public String getNextPageUrl() throws IOException, ExtractionException {
7171
if (nextPageUrl == null) {
72-
computNextPageAndStreams();
72+
computeNextPageAndStreams();
7373
}
7474
return nextPageUrl;
7575
}
@@ -78,7 +78,7 @@ public String getNextPageUrl() throws IOException, ExtractionException {
7878
@Override
7979
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
8080
if (collector == null) {
81-
computNextPageAndStreams();
81+
computeNextPageAndStreams();
8282
}
8383
return new InfoItemsPage<>(collector, getNextPageUrl());
8484
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ static Calendar parseDate(String textualUploadDate) throws ParsingException {
108108
* See https://developers.soundcloud.com/docs/api/reference#resolve
109109
*/
110110
public static JsonObject resolveFor(Downloader downloader, String url) throws IOException, ExtractionException {
111-
String apiUrl = "https://api.soundcloud.com/resolve"
111+
String apiUrl = "https://api-v2.soundcloud.com/resolve"
112112
+ "?url=" + URLEncoder.encode(url, "UTF-8")
113113
+ "&client_id=" + clientId();
114114

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java‎

Lines changed: 74 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,11 @@
11
package org.schabi.newpipe.extractor.services.soundcloud;
22

3+
import com.grack.nanojson.JsonArray;
34
import com.grack.nanojson.JsonObject;
45
import com.grack.nanojson.JsonParser;
56
import com.grack.nanojson.JsonParserException;
7+
8+
import org.schabi.newpipe.extractor.NewPipe;
69
import org.schabi.newpipe.extractor.StreamingService;
710
import org.schabi.newpipe.extractor.downloader.Downloader;
811
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@@ -12,16 +15,20 @@
1215
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
1316
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
1417

15-
import javax.annotation.Nonnull;
1618
import java.io.IOException;
1719

20+
import javax.annotation.Nonnull;
21+
import javax.annotation.Nullable;
22+
1823
@SuppressWarnings("WeakerAccess")
1924
public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
25+
private static final int streamsPerRequestedPage = 15;
26+
2027
private String playlistId;
2128
private JsonObject playlist;
2229

23-
private StreamInfoItemsCollector streamInfoItemsCollector = null;
24-
private String nextPageUrl = null;
30+
private StreamInfoItemsCollector streamInfoItemsCollector;
31+
private String nextPageUrl;
2532

2633
public SoundcloudPlaylistExtractor(StreamingService service, ListLinkHandler linkHandler) {
2734
super(service, linkHandler);
@@ -31,7 +38,7 @@ public SoundcloudPlaylistExtractor(StreamingService service, ListLinkHandler lin
3138
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
3239

3340
playlistId = getLinkHandler().getId();
34-
String apiUrl = "https://api.soundcloud.com/playlists/" + playlistId +
41+
String apiUrl = "https://api-v2.soundcloud.com/playlists/" + playlistId +
3542
"?client_id=" + SoundcloudParsingHelper.clientId() +
3643
"&representation=compact";
3744

@@ -55,6 +62,7 @@ public String getName() {
5562
return playlist.getString("title");
5663
}
5764

65+
@Nullable
5866
@Override
5967
public String getThumbnailUrl() {
6068
String artworkUrl = playlist.getString("artwork_url");
@@ -64,21 +72,20 @@ public String getThumbnailUrl() {
6472
// if it also fails, return null
6573
try {
6674
final InfoItemsPage<StreamInfoItem> infoItems = getInitialPage();
67-
if (infoItems.getItems().isEmpty()) return null;
6875

6976
for (StreamInfoItem item : infoItems.getItems()) {
70-
final String thumbnailUrl = item.getThumbnailUrl();
71-
if (thumbnailUrl == null || thumbnailUrl.isEmpty()) continue;
72-
73-
String thumbnailUrlBetterResolution = thumbnailUrl.replace("large.jpg", "crop.jpg");
74-
return thumbnailUrlBetterResolution;
77+
artworkUrl = item.getThumbnailUrl();
78+
if (artworkUrl != null && !artworkUrl.isEmpty()) break;
7579
}
7680
} catch (Exception ignored) {
7781
}
82+
83+
if (artworkUrl == null) {
84+
return null;
85+
}
7886
}
7987

80-
String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg");
81-
return artworkUrlBetterResolution;
88+
return artworkUrl.replace("large.jpg", "crop.jpg");
8289
}
8390

8491
@Override
@@ -110,27 +117,42 @@ public long getStreamCount() {
110117
@Override
111118
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
112119
if (streamInfoItemsCollector == null) {
113-
computeStreamsAndNextPageUrl();
120+
computeInitialTracksAndNextPageUrl();
114121
}
115-
return new InfoItemsPage<>(streamInfoItemsCollector, getNextPageUrl());
122+
return new InfoItemsPage<>(streamInfoItemsCollector, nextPageUrl);
116123
}
117124

118-
private void computeStreamsAndNextPageUrl() throws ExtractionException, IOException {
125+
private void computeInitialTracksAndNextPageUrl() throws IOException, ExtractionException {
119126
streamInfoItemsCollector = new StreamInfoItemsCollector(getServiceId());
127+
StringBuilder nextPageUrlBuilder = new StringBuilder("https://api-v2.soundcloud.com/tracks?client_id=");
128+
nextPageUrlBuilder.append(SoundcloudParsingHelper.clientId());
129+
nextPageUrlBuilder.append("&ids=");
130+
131+
JsonArray tracks = playlist.getArray("tracks");
132+
for (Object o : tracks) {
133+
if (o instanceof JsonObject) {
134+
JsonObject track = (JsonObject) o;
135+
if (track.has("title")) { // i.e. if full info is available
136+
streamInfoItemsCollector.commit(new SoundcloudStreamInfoItemExtractor(track));
137+
} else {
138+
// %09d would be enough, but a 0 before the number does not create problems, so let's be sure
139+
nextPageUrlBuilder.append(String.format("%010d,", track.getInt("id")));
140+
}
141+
}
142+
}
120143

121-
// Note the "api", NOT "api-v2"
122-
String apiUrl = "https://api.soundcloud.com/playlists/" + getId() + "/tracks"
123-
+ "?client_id=" + SoundcloudParsingHelper.clientId()
124-
+ "&limit=20"
125-
+ "&linked_partitioning=1";
126-
127-
nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, streamInfoItemsCollector, apiUrl);
144+
nextPageUrlBuilder.setLength(nextPageUrlBuilder.length() - 1); // remove trailing ,
145+
nextPageUrl = nextPageUrlBuilder.toString();
146+
if (nextPageUrl.endsWith("&ids")) {
147+
// there are no other videos
148+
nextPageUrl = "";
149+
}
128150
}
129151

130152
@Override
131153
public String getNextPageUrl() throws IOException, ExtractionException {
132154
if (nextPageUrl == null) {
133-
computeStreamsAndNextPageUrl();
155+
computeInitialTracksAndNextPageUrl();
134156
}
135157
return nextPageUrl;
136158
}
@@ -141,9 +163,36 @@ public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) throws IOException,
141163
throw new ExtractionException(new IllegalArgumentException("Page url is empty or null"));
142164
}
143165

166+
// see computeInitialTracksAndNextPageUrl
167+
final int lengthFirstPartOfUrl = ("https://api-v2.soundcloud.com/tracks?client_id="
168+
+ SoundcloudParsingHelper.clientId()
169+
+ "&ids=").length();
170+
final int lengthOfEveryStream = 11;
171+
172+
String currentPageUrl, nextUrl;
173+
int lengthMaxStreams = lengthFirstPartOfUrl + lengthOfEveryStream * streamsPerRequestedPage;
174+
if (pageUrl.length() <= lengthMaxStreams) {
175+
currentPageUrl = pageUrl; // fetch every remaining video, there are less than the max
176+
nextUrl = ""; // afterwards the list is complete
177+
} else {
178+
currentPageUrl = pageUrl.substring(0, lengthMaxStreams);
179+
nextUrl = pageUrl.substring(0, lengthFirstPartOfUrl) + pageUrl.substring(lengthMaxStreams);
180+
}
181+
144182
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
145-
String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, pageUrl);
183+
String response = NewPipe.getDownloader().get(currentPageUrl, getExtractorLocalization()).responseBody();
184+
185+
try {
186+
JsonArray tracks = JsonParser.array().from(response);
187+
for (Object track : tracks) {
188+
if (track instanceof JsonObject) {
189+
collector.commit(new SoundcloudStreamInfoItemExtractor((JsonObject) track));
190+
}
191+
}
192+
} catch (JsonParserException e) {
193+
throw new ParsingException("Could not parse json response", e);
194+
}
146195

147-
return new InfoItemsPage<>(collector, nextPageUrl);
196+
return new InfoItemsPage<>(collector, nextUrl);
148197
}
149198
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java‎

Lines changed: 16 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import com.grack.nanojson.JsonObject;
55
import com.grack.nanojson.JsonParser;
66
import com.grack.nanojson.JsonParserException;
7+
78
import org.schabi.newpipe.extractor.MediaFormat;
89
import org.schabi.newpipe.extractor.NewPipe;
910
import org.schabi.newpipe.extractor.StreamingService;
@@ -13,9 +14,15 @@
1314
import org.schabi.newpipe.extractor.exceptions.ParsingException;
1415
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
1516
import org.schabi.newpipe.extractor.localization.DateWrapper;
16-
import org.schabi.newpipe.extractor.stream.*;
17+
import org.schabi.newpipe.extractor.stream.AudioStream;
18+
import org.schabi.newpipe.extractor.stream.Description;
19+
import org.schabi.newpipe.extractor.stream.StreamExtractor;
20+
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
21+
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
22+
import org.schabi.newpipe.extractor.stream.StreamType;
23+
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
24+
import org.schabi.newpipe.extractor.stream.VideoStream;
1725

18-
import javax.annotation.Nonnull;
1926
import java.io.IOException;
2027
import java.io.UnsupportedEncodingException;
2128
import java.net.URLEncoder;
@@ -24,6 +31,8 @@
2431
import java.util.List;
2532
import java.util.Locale;
2633

34+
import javax.annotation.Nonnull;
35+
2736
public class SoundcloudStreamExtractor extends StreamExtractor {
2837
private JsonObject track;
2938

@@ -55,14 +64,14 @@ public String getName() {
5564

5665
@Nonnull
5766
@Override
58-
public String getTextualUploadDate() {
59-
return track.getString("created_at");
67+
public String getTextualUploadDate() throws ParsingException {
68+
return track.getString("created_at").replace("T"," ").replace("Z", "");
6069
}
6170

6271
@Nonnull
6372
@Override
6473
public DateWrapper getUploadDate() throws ParsingException {
65-
return new DateWrapper(SoundcloudParsingHelper.parseDate(getTextualUploadDate()));
74+
return new DateWrapper(SoundcloudParsingHelper.parseDate(track.getString("created_at")));
6675
}
6776

6877
@Nonnull
@@ -146,24 +155,13 @@ public List<AudioStream> getAudioStreams() throws IOException, ExtractionExcepti
146155
List<AudioStream> audioStreams = new ArrayList<>();
147156
Downloader dl = NewPipe.getDownloader();
148157

149-
String apiUrl = "https://api-v2.soundcloud.com/tracks/" + urlEncode(getId())
150-
+ "?client_id=" + urlEncode(SoundcloudParsingHelper.clientId());
151-
152-
String response = dl.get(apiUrl, getExtractorLocalization()).responseBody();
153-
JsonObject responseObject;
154-
try {
155-
responseObject = JsonParser.object().from(response);
156-
} catch (JsonParserException e) {
157-
throw new ParsingException("Could not parse json response", e);
158-
}
159-
160158
// Streams can be streamable and downloadable - or explicitly not.
161159
// For playing the track, it is only necessary to have a streamable track.
162160
// If this is not the case, this track might not be published yet.
163-
if (!responseObject.getBoolean("streamable")) return audioStreams;
161+
if (!track.getBoolean("streamable")) return audioStreams;
164162

165163
try {
166-
JsonArray transcodings = responseObject.getObject("media").getArray("transcodings");
164+
JsonArray transcodings = track.getObject("media").getArray("transcodings");
167165

168166
// get information about what stream formats are available
169167
for (Object transcoding : transcodings) {

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudSubscriptionExtractor.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ public List<SubscriptionItem> fromChannelUrl(String channelUrl) throws IOExcepti
3636
throw new InvalidSourceException(e);
3737
}
3838

39-
String apiUrl = "https://api.soundcloud.com/users/" + id + "/followings"
39+
String apiUrl = "https://api-v2.soundcloud.com/users/" + id + "/followings"
4040
+ "?client_id=" + SoundcloudParsingHelper.clientId()
4141
+ "&limit=200";
4242
ChannelInfoItemsCollector collector = new ChannelInfoItemsCollector(service.getServiceId());

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ public void assertThatHardcodedClientIdIsValid() throws Exception {
2424
public void resolveUrlWithEmbedPlayerTest() throws Exception {
2525
Assert.assertEquals("https://soundcloud.com/trapcity", SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/users/26057743"));
2626
Assert.assertEquals("https://soundcloud.com/nocopyrightsounds", SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api.soundcloud.com/users/16069159"));
27+
Assert.assertEquals("https://soundcloud.com/trapcity", SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api-v2.soundcloud.com/users/26057743"));
28+
Assert.assertEquals("https://soundcloud.com/nocopyrightsounds", SoundcloudParsingHelper.resolveUrlWithEmbedPlayer("https://api-v2.soundcloud.com/users/16069159"));
2729
}
2830

2931
@Test

0 commit comments

Comments
 (0)