Skip to content

Commit 8c96545

Browse files
litetexlitetex
committed
Merge branch 'TeamNewPipe:dev' into fix-broken-yt-liked-comments
2 parents 12fb18c + ff00512 commit 8c96545

61 files changed

Lines changed: 740 additions & 486 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java‎

Lines changed: 29 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -211,12 +211,37 @@ public List<AudioStream> getAudioStreams() {
211211
public List<VideoStream> getVideoStreams() throws ExtractionException {
212212
assertPageFetched();
213213
final List<VideoStream> videoStreams = new ArrayList<>();
214+
// mp4
214215
try {
215-
final JsonArray streams = json.getArray("files");
216+
videoStreams.addAll(getVideoStreamsFromArray(json.getArray("files")));
217+
} catch (Exception ignored) { }
218+
// HLS
219+
try {
220+
final JsonArray streamingPlaylists = json.getArray("streamingPlaylists");
221+
for (final Object p : streamingPlaylists) {
222+
if (!(p instanceof JsonObject)) continue;
223+
final JsonObject playlist = (JsonObject) p;
224+
videoStreams.addAll(getVideoStreamsFromArray(playlist.getArray("files")));
225+
}
226+
} catch (Exception e) {
227+
throw new ParsingException("Could not get video streams", e);
228+
}
229+
230+
return videoStreams;
231+
}
232+
233+
private List<VideoStream> getVideoStreamsFromArray(final JsonArray streams) throws ParsingException {
234+
try {
235+
final List<VideoStream> videoStreams = new ArrayList<>();
216236
for (final Object s : streams) {
217237
if (!(s instanceof JsonObject)) continue;
218238
final JsonObject stream = (JsonObject) s;
219-
final String url = JsonUtils.getString(stream, "fileUrl");
239+
final String url;
240+
if (stream.has("fileDownloadUrl")) {
241+
url = JsonUtils.getString(stream, "fileDownloadUrl");
242+
} else {
243+
url = JsonUtils.getString(stream, "fileUrl");
244+
}
220245
final String torrentUrl = JsonUtils.getString(stream, "torrentUrl");
221246
final String resolution = JsonUtils.getString(stream, "resolution.label");
222247
final String extension = url.substring(url.lastIndexOf(".") + 1);
@@ -226,14 +251,13 @@ public List<VideoStream> getVideoStreams() throws ExtractionException {
226251
videoStreams.add(videoStream);
227252
}
228253
}
254+
return videoStreams;
229255
} catch (Exception e) {
230-
throw new ParsingException("Could not get video streams", e);
256+
throw new ParsingException("Could not get video streams from array");
231257
}
232258

233-
return videoStreams;
234259
}
235260

236-
237261
@Override
238262
public List<VideoStream> getVideoOnlyStreams() {
239263
return Collections.emptyList();

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java‎

Lines changed: 113 additions & 80 deletions
Large diffs are not rendered by default.

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudService.java‎

Lines changed: 12 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
55
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
66
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
7-
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
87
import org.schabi.newpipe.extractor.kiosk.KioskList;
98
import org.schabi.newpipe.extractor.linkhandler.*;
109
import org.schabi.newpipe.extractor.localization.ContentCountry;
@@ -23,7 +22,7 @@
2322

2423
public class SoundcloudService extends StreamingService {
2524

26-
public SoundcloudService(int id) {
25+
public SoundcloudService(final int id) {
2726
super(id, "SoundCloud", asList(AUDIO, COMMENTS));
2827
}
2928

@@ -54,29 +53,29 @@ public ListLinkHandlerFactory getPlaylistLHFactory() {
5453

5554
@Override
5655
public List<ContentCountry> getSupportedCountries() {
57-
//Country selector here https://soundcloud.com/charts/top?genre=all-music
56+
// Country selector here: https://soundcloud.com/charts/top?genre=all-music
5857
return ContentCountry.listFrom(
5958
"AU", "CA", "DE", "FR", "GB", "IE", "NL", "NZ", "US"
6059
);
6160
}
6261

6362
@Override
64-
public StreamExtractor getStreamExtractor(LinkHandler LinkHandler) {
65-
return new SoundcloudStreamExtractor(this, LinkHandler);
63+
public StreamExtractor getStreamExtractor(final LinkHandler linkHandler) {
64+
return new SoundcloudStreamExtractor(this, linkHandler);
6665
}
6766

6867
@Override
69-
public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) {
68+
public ChannelExtractor getChannelExtractor(final ListLinkHandler linkHandler) {
7069
return new SoundcloudChannelExtractor(this, linkHandler);
7170
}
7271

7372
@Override
74-
public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) {
73+
public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) {
7574
return new SoundcloudPlaylistExtractor(this, linkHandler);
7675
}
7776

7877
@Override
79-
public SearchExtractor getSearchExtractor(SearchQueryHandler queryHandler) {
78+
public SearchExtractor getSearchExtractor(final SearchQueryHandler queryHandler) {
8079
return new SoundcloudSearchExtractor(this, queryHandler);
8180
}
8281

@@ -87,26 +86,19 @@ public SoundcloudSuggestionExtractor getSuggestionExtractor() {
8786

8887
@Override
8988
public KioskList getKioskList() throws ExtractionException {
90-
KioskList.KioskExtractorFactory chartsFactory = new KioskList.KioskExtractorFactory() {
91-
@Override
92-
public KioskExtractor createNewKiosk(StreamingService streamingService,
93-
String url,
94-
String id)
95-
throws ExtractionException {
96-
return new SoundcloudChartsExtractor(SoundcloudService.this,
89+
final KioskList.KioskExtractorFactory chartsFactory = (streamingService, url, id) ->
90+
new SoundcloudChartsExtractor(SoundcloudService.this,
9791
new SoundcloudChartsLinkHandlerFactory().fromUrl(url), id);
98-
}
99-
};
10092

101-
KioskList list = new KioskList(this);
93+
final KioskList list = new KioskList(this);
10294

10395
// add kiosks here e.g.:
10496
final SoundcloudChartsLinkHandlerFactory h = new SoundcloudChartsLinkHandlerFactory();
10597
try {
10698
list.addKioskEntry(chartsFactory, h, "Top 50");
10799
list.addKioskEntry(chartsFactory, h, "New & hot");
108100
list.setDefaultKiosk("New & hot");
109-
} catch (Exception e) {
101+
} catch (final Exception e) {
110102
throw new ExtractionException(e);
111103
}
112104

@@ -124,9 +116,8 @@ public ListLinkHandlerFactory getCommentsLHFactory() {
124116
}
125117

126118
@Override
127-
public CommentsExtractor getCommentsExtractor(ListLinkHandler linkHandler)
119+
public CommentsExtractor getCommentsExtractor(final ListLinkHandler linkHandler)
128120
throws ExtractionException {
129121
return new SoundcloudCommentsExtractor(this, linkHandler);
130122
}
131-
132123
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelExtractor.java‎

Lines changed: 22 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -17,29 +17,33 @@
1717
import javax.annotation.Nonnull;
1818
import java.io.IOException;
1919

20+
import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL;
2021
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
2122
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
2223

2324
@SuppressWarnings("WeakerAccess")
2425
public class SoundcloudChannelExtractor extends ChannelExtractor {
2526
private String userId;
2627
private JsonObject user;
28+
private static final String USERS_ENDPOINT = SOUNDCLOUD_API_V2_URL + "users/";
2729

28-
public SoundcloudChannelExtractor(final StreamingService service, final ListLinkHandler linkHandler) {
30+
public SoundcloudChannelExtractor(final StreamingService service,
31+
final ListLinkHandler linkHandler) {
2932
super(service, linkHandler);
3033
}
3134

3235
@Override
33-
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException {
36+
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
37+
ExtractionException {
3438

3539
userId = getLinkHandler().getId();
36-
final String apiUrl = "https://api-v2.soundcloud.com/users/" + userId +
37-
"?client_id=" + SoundcloudParsingHelper.clientId();
40+
final String apiUrl = USERS_ENDPOINT + userId + "?client_id="
41+
+ SoundcloudParsingHelper.clientId();
3842

3943
final String response = downloader.get(apiUrl, getExtractorLocalization()).responseBody();
4044
try {
4145
user = JsonParser.object().from(response);
42-
} catch (JsonParserException e) {
46+
} catch (final JsonParserException e) {
4347
throw new ParsingException("Could not parse json response", e);
4448
}
4549
}
@@ -63,7 +67,8 @@ public String getAvatarUrl() {
6367

6468
@Override
6569
public String getBannerUrl() {
66-
return user.getObject("visuals").getArray("visuals").getObject(0).getString("visual_url");
70+
return user.getObject("visuals").getArray("visuals").getObject(0)
71+
.getString("visual_url");
6772
}
6873

6974
@Override
@@ -105,29 +110,31 @@ public boolean isVerified() throws ParsingException {
105110
@Override
106111
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException {
107112
try {
108-
final StreamInfoItemsCollector streamInfoItemsCollector = new StreamInfoItemsCollector(getServiceId());
113+
final StreamInfoItemsCollector streamInfoItemsCollector =
114+
new StreamInfoItemsCollector(getServiceId());
109115

110-
final String apiUrl = "https://api-v2.soundcloud.com/users/" + getId() + "/tracks"
111-
+ "?client_id=" + SoundcloudParsingHelper.clientId()
112-
+ "&limit=20"
113-
+ "&linked_partitioning=1";
116+
final String apiUrl = USERS_ENDPOINT + getId() + "/tracks" + "?client_id="
117+
+ SoundcloudParsingHelper.clientId() + "&limit=20" + "&linked_partitioning=1";
114118

115-
final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, streamInfoItemsCollector, apiUrl);
119+
final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15,
120+
streamInfoItemsCollector, apiUrl);
116121

117122
return new InfoItemsPage<>(streamInfoItemsCollector, new Page(nextPageUrl));
118-
} catch (Exception e) {
123+
} catch (final Exception e) {
119124
throw new ExtractionException("Could not get next page", e);
120125
}
121126
}
122127

123128
@Override
124-
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException, ExtractionException {
129+
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException,
130+
ExtractionException {
125131
if (page == null || isNullOrEmpty(page.getUrl())) {
126132
throw new IllegalArgumentException("Page doesn't contain an URL");
127133
}
128134

129135
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
130-
final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector, page.getUrl());
136+
final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApiMinItems(15, collector,
137+
page.getUrl());
131138

132139
return new InfoItemsPage<>(collector, new Page(nextPageUrl));
133140
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChannelInfoItemExtractor.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtractor {
1010
private final JsonObject itemObject;
1111

12-
public SoundcloudChannelInfoItemExtractor(JsonObject itemObject) {
12+
public SoundcloudChannelInfoItemExtractor(final JsonObject itemObject) {
1313
this.itemObject = itemObject;
1414
}
1515

@@ -26,8 +26,8 @@ public String getUrl() {
2626
@Override
2727
public String getThumbnailUrl() {
2828
String avatarUrl = itemObject.getString("avatar_url", EMPTY_STRING);
29-
String avatarUrlBetterResolution = avatarUrl.replace("large.jpg", "crop.jpg");
30-
return avatarUrlBetterResolution;
29+
// An avatar URL with a better resolution
30+
return avatarUrl.replace("large.jpg", "crop.jpg");
3131
}
3232

3333
@Override

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudChartsExtractor.java‎

Lines changed: 19 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -15,17 +15,18 @@
1515
import java.io.IOException;
1616

1717
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
18+
import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper.SOUNDCLOUD_API_V2_URL;
1819
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
1920

2021
public class SoundcloudChartsExtractor extends KioskExtractor<StreamInfoItem> {
21-
public SoundcloudChartsExtractor(StreamingService service,
22-
ListLinkHandler linkHandler,
23-
String kioskId) {
22+
public SoundcloudChartsExtractor(final StreamingService service,
23+
final ListLinkHandler linkHandler,
24+
final String kioskId) {
2425
super(service, linkHandler, kioskId);
2526
}
2627

2728
@Override
28-
public void onFetchPage(@Nonnull Downloader downloader) {
29+
public void onFetchPage(@Nonnull final Downloader downloader) {
2930
}
3031

3132
@Nonnull
@@ -35,13 +36,15 @@ public String getName() {
3536
}
3637

3738
@Override
38-
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException, ExtractionException {
39+
public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException,
40+
ExtractionException {
3941
if (page == null || isNullOrEmpty(page.getUrl())) {
4042
throw new IllegalArgumentException("Page doesn't contain an URL");
4143
}
4244

4345
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
44-
final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, page.getUrl(), true);
46+
final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector,
47+
page.getUrl(), true);
4548

4649
return new InfoItemsPage<>(collector, new Page(nextPageUrl));
4750
}
@@ -51,9 +54,8 @@ public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException
5154
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
5255
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
5356

54-
String apiUrl = "https://api-v2.soundcloud.com/charts" +
55-
"?genre=soundcloud:genres:all-music" +
56-
"&client_id=" + SoundcloudParsingHelper.clientId();
57+
String apiUrl = SOUNDCLOUD_API_V2_URL + "charts" + "?genre=soundcloud:genres:all-music"
58+
+ "&client_id=" + SoundcloudParsingHelper.clientId();
5759

5860
if (getId().equals("Top 50")) {
5961
apiUrl += "&kind=top";
@@ -64,15 +66,18 @@ public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, Extrac
6466
final ContentCountry contentCountry = SoundCloud.getContentCountry();
6567
String apiUrlWithRegion = null;
6668
if (getService().getSupportedCountries().contains(contentCountry)) {
67-
apiUrlWithRegion = apiUrl + "&region=soundcloud:regions:" + contentCountry.getCountryCode();
69+
apiUrlWithRegion = apiUrl + "&region=soundcloud:regions:"
70+
+ contentCountry.getCountryCode();
6871
}
6972

7073
String nextPageUrl;
7174
try {
72-
nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrlWithRegion == null ? apiUrl : apiUrlWithRegion, true);
73-
} catch (IOException e) {
74-
// Request to other region may be geo-restricted. See https://github.com/TeamNewPipe/NewPipeExtractor/issues/537
75-
// we retry without the specified region.
75+
nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector,
76+
apiUrlWithRegion == null ? apiUrl : apiUrlWithRegion, true);
77+
} catch (final IOException e) {
78+
// Request to other region may be geo-restricted.
79+
// See https://github.com/TeamNewPipe/NewPipeExtractor/issues/537.
80+
// We retry without the specified region.
7681
nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl, true);
7782
}
7883

0 commit comments

Comments
 (0)