Skip to content

Commit ff00512

Browse files
TobiGrTobiGr
authored
Merge pull request #627 from TiA4f8R/use-snd-api-v2-everywhere
[SoundCloud] Use a lightweight request to check if the hardcoded client_id is valid, fix the extraction of mobile URLs and more
2 parents 636e273 + a00fdcb commit ff00512

21 files changed

Lines changed: 407 additions & 309 deletions

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java‎

Lines changed: 113 additions & 80 deletions
Large diffs are not rendered by default.

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java‎

Lines changed: 12 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
55
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
66
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
7-
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
87
import org.schabi.newpipe.extractor.kiosk.KioskList;
98
import org.schabi.newpipe.extractor.linkhandler.*;
109
import org.schabi.newpipe.extractor.localization.ContentCountry;
@@ -23,7 +22,7 @@
2322

2423
public class SoundcloudService extends StreamingService {
2524

26-
public SoundcloudService(int id) {
25+
public SoundcloudService(final int id) {
2726
super(id, "SoundCloud", asList(AUDIO, COMMENTS));
2827
}
2928

@@ -54,29 +53,29 @@ public ListLinkHandlerFactory getPlaylistLHFactory() {
5453

5554
@Override
5655
public List<ContentCountry> getSupportedCountries() {
57-
//Country selector here https://soundcloud.com/charts/top?genre=all-music
56+
// Country selector here: https://soundcloud.com/charts/top?genre=all-music
5857
return ContentCountry.listFrom(
5958
"AU", "CA", "DE", "FR", "GB", "IE", "NL", "NZ", "US"
6059
);
6160
}
6261

6362
@Override
64-
public StreamExtractor getStreamExtractor(LinkHandler LinkHandler) {
65-
return new SoundcloudStreamExtractor(this, LinkHandler);
63+
public StreamExtractor getStreamExtractor(final LinkHandler linkHandler) {
64+
return new SoundcloudStreamExtractor(this, linkHandler);
6665
}
6766

6867
@Override
69-
public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) {
68+
public ChannelExtractor getChannelExtractor(final ListLinkHandler linkHandler) {
7069
return new SoundcloudChannelExtractor(this, linkHandler);
7170
}
7271

7372
@Override
74-
public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) {
73+
public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) {
7574
return new SoundcloudPlaylistExtractor(this, linkHandler);
7675
}
7776

7877
@Override
79-
public SearchExtractor getSearchExtractor(SearchQueryHandler queryHandler) {
78+
public SearchExtractor getSearchExtractor(final SearchQueryHandler queryHandler) {
8079
return new SoundcloudSearchExtractor(this, queryHandler);
8180
}
8281

@@ -87,26 +86,19 @@ public SoundcloudSuggestionExtractor getSuggestionExtractor() {
8786

8887
@Override
8988
public KioskList getKioskList() throws ExtractionException {
90-
KioskList.KioskExtractorFactory chartsFactory = new KioskList.KioskExtractorFactory() {
91-
@Override
92-
public KioskExtractor createNewKiosk(StreamingService streamingService,
93-
String url,
94-
String id)
95-
throws ExtractionException {
96-
return new SoundcloudChartsExtractor(SoundcloudService.this,
89+
final KioskList.KioskExtractorFactory chartsFactory = (streamingService, url, id) ->
90+
new SoundcloudChartsExtractor(SoundcloudService.this,
9791
new SoundcloudChartsLinkHandlerFactory().fromUrl(url), id);
98-
}
99-
};
10092

101-
KioskList list = new KioskList(this);
93+
final KioskList list = new KioskList(this);
10294

10395
// add kiosks here e.g.:
10496
final SoundcloudChartsLinkHandlerFactory h = new SoundcloudChartsLinkHandlerFactory();
10597
try {
10698
list.addKioskEntry(chartsFactory, h, "Top 50");
10799
list.addKioskEntry(chartsFactory, h, "New & hot");
108100
list.setDefaultKiosk("New & hot");
109-
} catch (Exception e) {
101+
} catch (final Exception e) {
110102
throw new ExtractionException(e);
111103
}
112104

@@ -124,9 +116,8 @@ public ListLinkHandlerFactory getCommentsLHFactory() {
124116
}
125117

126118
@Override
127-
public CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler)
119+
public CommentsExtractor getCommentsExtractor(final ListLinkHandler linkHandler)
128120
throws ExtractionException {
129121
return new SoundcloudCommentsExtractor(this, linkHandler);
130122
}
131-
132123
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java‎

Lines changed: 22 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -17,29 +17,33 @@
1717
import javax.annotation.Nonnull;
1818
import java.io.IOException;
1919

20+
import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL;
2021
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
2122
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
2223

2324
@SuppressWarnings("WeakerAccess")
2425
public class SoundcloudChannelExtractor extends ChannelExtractor {
2526
private String userId;
2627
private JsonObject user;
28+
private static final String USERS_ENDPOINT = SOUNDCLOUD_API_V2_URL + "users/";
2729

28-
public SoundcloudChannelExtractor(final StreamingService service, final ListLinkHandler linkHandler) {
30+
public SoundcloudChannelExtractor(final StreamingService service,
31+
final ListLinkHandler linkHandler) {
2932
super(service, linkHandler);
3033
}
3134

3235
@Override
33-
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException {
36+
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
37+
ExtractionException {
3438

3539
userId = getLinkHandler().getId();
36-
final String apiUrl = "https://api-v2.soundcloud.com/users/" + userId +
37-
"?client_id=" + SoundcloudParsingHelper.clientId();
40+
final String apiUrl = USERS_ENDPOINT + userId + "?client_id="
41+
+ SoundcloudParsingHelper.clientId();
3842

3943
final String response = downloader.get(apiUrl, getExtractorLocalization()).responseBody();
4044
try {
4145
user = JsonParser.object().from(response);
42-
} catch (JsonParserException e) {
46+
} catch (final JsonParserException e) {
4347
throw new ParsingException("Could not parse json response", e);
4448
}
4549
}
@@ -63,7 +67,8 @@ public String getAvatarUrl() {
6367

6468
@Override
6569
public String getBannerUrl() {
66-
return user.getObject("visuals").getArray("visuals").getObject(0).getString("visual_url");
70+
return user.getObject("visuals").getArray("visuals").getObject(0)
71+
.getString("visual_url");
6772
}
6873

6974
@Override
@@ -105,29 +110,31 @@ public boolean isVerified() throws ParsingException {
105110
@Override
106111
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException {
107112
try {
108-
final StreamInfoItemsCollector streamInfoItemsCollector = new StreamInfoItemsCollector(getServiceId());
113+
final StreamInfoItemsCollector streamInfoItemsCollector =
114+
new StreamInfoItemsCollector(getServiceId());
109115

110-
final String apiUrl = "https://api-v2.soundcloud.com/users/" + getId() + "/tracks"
111-
+ "?client_id=" + SoundcloudParsingHelper.clientId()
112-
+ "&limit=20"
113-
+ "&linked_partitioning=1";
116+
final String apiUrl = USERS_ENDPOINT + getId() + "/tracks" + "?client_id="
117+
+ SoundcloudParsingHelper.clientId() + "&limit=20" + "&linked_partitioning=1";
114118

115-
final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, streamInfoItemsCollector, apiUrl);
119+
final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15,
120+
streamInfoItemsCollector, apiUrl);
116121

117122
return new InfoItemsPage<>(streamInfoItemsCollector, new Page(nextPageUrl));
118-
} catch (Exception e) {
123+
} catch (final Exception e) {
119124
throw new ExtractionException("Could not get next page", e);
120125
}
121126
}
122127

123128
@Override
124-
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException, ExtractionException {
129+
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException,
130+
ExtractionException {
125131
if (page == null || isNullOrEmpty(page.getUrl())) {
126132
throw new IllegalArgumentException("Page doesn't contain an URL");
127133
}
128134

129135
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
130-
final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, page.getUrl());
136+
final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector,
137+
page.getUrl());
131138

132139
return new InfoItemsPage<>(collector, new Page(nextPageUrl));
133140
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelInfoItemExtractor.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtractor {
1010
private final JsonObject itemObject;
1111

12-
public SoundcloudChannelInfoItemExtractor(JsonObject itemObject) {
12+
public SoundcloudChannelInfoItemExtractor(final JsonObject itemObject) {
1313
this.itemObject = itemObject;
1414
}
1515

@@ -26,8 +26,8 @@ public String getUrl() {
2626
@Override
2727
public String getThumbnailUrl() {
2828
String avatarUrl = itemObject.getString("avatar_url", EMPTY_STRING);
29-
String avatarUrlBetterResolution = avatarUrl.replace("large.jpg", "crop.jpg");
30-
return avatarUrlBetterResolution;
29+
// An avatar URL with a better resolution
30+
return avatarUrl.replace("large.jpg", "crop.jpg");
3131
}
3232

3333
@Override

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChartsExtractor.java‎

Lines changed: 19 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -15,17 +15,18 @@
1515
import java.io.IOException;
1616

1717
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
18+
import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL;
1819
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
1920

2021
public class SoundcloudChartsExtractor extends KioskExtractor<StreamInfoItem> {
21-
public SoundcloudChartsExtractor(StreamingService service,
22-
ListLinkHandler linkHandler,
23-
String kioskId) {
22+
public SoundcloudChartsExtractor(final StreamingService service,
23+
final ListLinkHandler linkHandler,
24+
final String kioskId) {
2425
super(service, linkHandler, kioskId);
2526
}
2627

2728
@Override
28-
public void onFetchPage(@Nonnull Downloader downloader) {
29+
public void onFetchPage(@Nonnull final Downloader downloader) {
2930
}
3031

3132
@Nonnull
@@ -35,13 +36,15 @@ public String getName() {
3536
}
3637

3738
@Override
38-
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException, ExtractionException {
39+
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException,
40+
ExtractionException {
3941
if (page == null || isNullOrEmpty(page.getUrl())) {
4042
throw new IllegalArgumentException("Page doesn't contain an URL");
4143
}
4244

4345
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
44-
final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, page.getUrl(), true);
46+
final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector,
47+
page.getUrl(), true);
4548

4649
return new InfoItemsPage<>(collector, new Page(nextPageUrl));
4750
}
@@ -51,9 +54,8 @@ public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException
5154
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
5255
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
5356

54-
String apiUrl = "https://api-v2.soundcloud.com/charts" +
55-
"?genre=soundcloud:genres:all-music" +
56-
"&client_id=" + SoundcloudParsingHelper.clientId();
57+
String apiUrl = SOUNDCLOUD_API_V2_URL + "charts" + "?genre=soundcloud:genres:all-music"
58+
+ "&client_id=" + SoundcloudParsingHelper.clientId();
5759

5860
if (getId().equals("Top 50")) {
5961
apiUrl += "&kind=top";
@@ -64,15 +66,18 @@ public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, Extrac
6466
final ContentCountry contentCountry = SoundCloud.getContentCountry();
6567
String apiUrlWithRegion = null;
6668
if (getService().getSupportedCountries().contains(contentCountry)) {
67-
apiUrlWithRegion = apiUrl + "&region=soundcloud:regions:" + contentCountry.getCountryCode();
69+
apiUrlWithRegion = apiUrl + "&region=soundcloud:regions:"
70+
+ contentCountry.getCountryCode();
6871
}
6972

7073
String nextPageUrl;
7174
try {
72-
nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrlWithRegion == null ? apiUrl : apiUrlWithRegion, true);
73-
} catch (IOException e) {
74-
// Request to other region may be geo-restricted. See https://github.com/TeamNewPipe/NewPipeExtractor/issues/537
75-
// we retry without the specified region.
75+
nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector,
76+
apiUrlWithRegion == null ? apiUrl : apiUrlWithRegion, true);
77+
} catch (final IOException e) {
78+
// Request to other region may be geo-restricted.
79+
// See https://github.com/TeamNewPipe/NewPipeExtractor/issues/537.
80+
// We retry without the specified region.
7681
nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl, true);
7782
}
7883

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -24,32 +24,36 @@
2424
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
2525

2626
public class SoundcloudCommentsExtractor extends CommentsExtractor {
27-
public SoundcloudCommentsExtractor(final StreamingService service, final ListLinkHandler uiHandler) {
27+
public SoundcloudCommentsExtractor(final StreamingService service,
28+
final ListLinkHandler uiHandler) {
2829
super(service, uiHandler);
2930
}
3031

3132
@Nonnull
3233
@Override
33-
public InfoItemsPage<CommentsInfoItem> getInitialPage() throws ExtractionException, IOException {
34+
public InfoItemsPage<CommentsInfoItem> getInitialPage() throws ExtractionException,
35+
IOException {
3436
final Downloader downloader = NewPipe.getDownloader();
3537
final Response response = downloader.get(getUrl());
3638

3739
final JsonObject json;
3840
try {
3941
json = JsonParser.object().from(response.responseBody());
40-
} catch (JsonParserException e) {
42+
} catch (final JsonParserException e) {
4143
throw new ParsingException("Could not parse json", e);
4244
}
4345

44-
final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId());
46+
final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(
47+
getServiceId());
4548

4649
collectStreamsFrom(collector, json.getArray("collection"));
4750

4851
return new InfoItemsPage<>(collector, new Page(json.getString("next_href")));
4952
}
5053

5154
@Override
52-
public InfoItemsPage<CommentsInfoItem> getPage(final Page page) throws ExtractionException, IOException {
55+
public InfoItemsPage<CommentsInfoItem> getPage(final Page page) throws ExtractionException,
56+
IOException {
5357
if (page == null || isNullOrEmpty(page.getUrl())) {
5458
throw new IllegalArgumentException("Page doesn't contain an URL");
5559
}
@@ -60,11 +64,12 @@ public InfoItemsPage<CommentsInfoItem> getPage(final Page page) throws Extractio
6064
final JsonObject json;
6165
try {
6266
json = JsonParser.object().from(response.responseBody());
63-
} catch (JsonParserException e) {
67+
} catch (final JsonParserException e) {
6468
throw new ParsingException("Could not parse json", e);
6569
}
6670

67-
final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId());
71+
final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(
72+
getServiceId());
6873

6974
collectStreamsFrom(collector, json.getArray("collection"));
7075

@@ -74,9 +79,10 @@ public InfoItemsPage<CommentsInfoItem> getPage(final Page page) throws Extractio
7479
@Override
7580
public void onFetchPage(@Nonnull final Downloader downloader) { }
7681

77-
private void collectStreamsFrom(final CommentsInfoItemsCollector collector, final JsonArray entries) throws ParsingException {
82+
private void collectStreamsFrom(final CommentsInfoItemsCollector collector,
83+
final JsonArray entries) throws ParsingException {
7884
final String url = getUrl();
79-
for (Object comment : entries) {
85+
for (final Object comment : entries) {
8086
collector.commit(new SoundcloudCommentsInfoItemExtractor((JsonObject) comment, url));
8187
}
8288
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,10 @@
1010
import java.util.Objects;
1111

1212
public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtractor {
13-
private JsonObject json;
14-
private String url;
13+
private final JsonObject json;
14+
private final String url;
1515

16-
public SoundcloudCommentsInfoItemExtractor(JsonObject json, String url) {
16+
public SoundcloudCommentsInfoItemExtractor(final JsonObject json, final String url) {
1717
this.json = json;
1818
this.url = url;
1919
}

0 commit comments

Comments
 (0)