Skip to content

Commit c4f521f

Browse files
mauriciocollimauriciocolli
committed
Fix bugs and improve InfoItemExtractors
- Improve livestream detection
1 parent 5bf2e95 commit c4f521f

15 files changed

Lines changed: 252 additions & 489 deletions

‎src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudPlaylistExtractor.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ public String getPlaylistId() {
4848

4949
@Override
5050
public String getPlaylistName() {
51-
return playlist.getString("title");
51+
return playlist.optString("title");
5252
}
5353

5454
@Override

‎src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudStreamExtractor.java‎

Lines changed: 25 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
package org.schabi.newpipe.extractor.services.soundcloud;
22

3-
import org.json.JSONArray;
43
import org.json.JSONObject;
54
import org.schabi.newpipe.extractor.Downloader;
65
import org.schabi.newpipe.extractor.MediaFormat;
@@ -27,8 +26,9 @@ public SoundcloudStreamExtractor(StreamingService service, String url) throws IO
2726
public void fetchPage() throws IOException, ExtractionException {
2827
track = SoundcloudParsingHelper.resolveFor(getOriginalUrl());
2928

30-
if (!track.getString("policy").equals("ALLOW") && !track.getString("policy").equals("MONETIZE")) {
31-
throw new ContentNotAvailableException("Content not available: policy " + track.getString("policy"));
29+
String policy = track.getString("policy");
30+
if (!policy.equals("ALLOW") && !policy.equals("MONETIZE")) {
31+
throw new ContentNotAvailableException("Content not available: policy " + policy);
3232
}
3333
}
3434

@@ -48,12 +48,12 @@ public String getId() {
4848

4949
@Override
5050
public String getTitle() {
51-
return track.getString("title");
51+
return track.optString("title");
5252
}
5353

5454
@Override
5555
public String getDescription() {
56-
return track.getString("description");
56+
return track.optString("description");
5757
}
5858

5959
@Override
@@ -62,28 +62,33 @@ public String getUploaderName() {
6262
}
6363

6464
@Override
65-
public int getLength() {
66-
return track.getInt("duration") / 1000;
65+
public String getUploaderUrl() {
66+
return track.getJSONObject("user").getString("permalink_url");
6767
}
6868

6969
@Override
70-
public long getViewCount() {
71-
return track.getLong("playback_count");
70+
public String getUploaderAvatarUrl() {
71+
return track.getJSONObject("user").optString("avatar_url");
7272
}
7373

7474
@Override
75-
public String getUploadDate() throws ParsingException {
76-
return SoundcloudParsingHelper.toDateString(track.getString("created_at"));
75+
public String getThumbnailUrl() {
76+
return track.optString("artwork_url");
7777
}
7878

7979
@Override
80-
public String getThumbnailUrl() {
81-
return track.optString("artwork_url");
80+
public long getLength() {
81+
return track.getLong("duration") / 1000L;
8282
}
8383

8484
@Override
85-
public String getUploaderAvatarUrl() {
86-
return track.getJSONObject("user").getString("avatar_url");
85+
public long getViewCount() {
86+
return track.getLong("playback_count");
87+
}
88+
89+
@Override
90+
public String getUploadDate() throws ParsingException {
91+
return SoundcloudParsingHelper.toDateString(track.getString("created_at"));
8792
}
8893

8994
@Override
@@ -171,44 +176,31 @@ public int getAgeLimit() {
171176
}
172177

173178
@Override
174-
public int getLikeCount() {
175-
return track.getInt("likes_count");
179+
public long getLikeCount() {
180+
return track.getLong("likes_count");
176181
}
177182

178183
@Override
179-
public int getDislikeCount() {
184+
public long getDislikeCount() {
180185
return 0;
181186
}
182187

183188
@Override
184-
public StreamInfoItemExtractor getNextVideo() throws IOException, ExtractionException {
189+
public StreamInfoItem getNextVideo() throws IOException, ExtractionException {
185190
return null;
186191
}
187192

188193
@Override
189194
public StreamInfoItemCollector getRelatedVideos() throws IOException, ExtractionException {
190195
StreamInfoItemCollector collector = new StreamInfoItemCollector(getServiceId());
191-
Downloader dl = NewPipe.getDownloader();
192196

193197
String apiUrl = "https://api-v2.soundcloud.com/tracks/" + getId() + "/related"
194198
+ "?client_id=" + SoundcloudParsingHelper.clientId();
195199

196-
String response = dl.download(apiUrl);
197-
JSONObject responseObject = new JSONObject(response);
198-
JSONArray responseCollection = responseObject.getJSONArray("collection");
199-
200-
for (int i = 0; i < responseCollection.length(); i++) {
201-
JSONObject relatedVideo = responseCollection.getJSONObject(i);
202-
collector.commit(new SoundcloudStreamInfoItemExtractor(relatedVideo));
203-
}
200+
SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl);
204201
return collector;
205202
}
206203

207-
@Override
208-
public String getUploaderUrl() {
209-
return track.getJSONObject("user").getString("permalink_url");
210-
}
211-
212204
@Override
213205
public StreamType getStreamType() {
214206
return StreamType.AUDIO_STREAM;

‎src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudUserExtractor.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ public String getUserName() {
5353

5454
@Override
5555
public String getAvatarUrl() {
56-
return user.getString("avatar_url");
56+
return user.optString("avatar_url");
5757
}
5858

5959
@Override
@@ -67,7 +67,7 @@ public String getBannerUrl() throws ParsingException {
6767

6868
@Override
6969
public long getSubscriberCount() {
70-
return user.getLong("followers_count");
70+
return user.optLong("followers_count", 0L);
7171
}
7272

7373
@Override
@@ -102,6 +102,6 @@ public NextItemsResult getNextStreams() throws IOException, ExtractionException
102102

103103
@Override
104104
public String getDescription() throws ParsingException {
105-
return user.getString("description");
105+
return user.optString("description");
106106
}
107107
}

‎src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudUserInfoItemExtractor.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ public String getWebPageUrl() {
2727

2828
@Override
2929
public long getSubscriberCount() {
30-
return searchResult.getLong("followers_count");
30+
return searchResult.optLong("followers_count", 0L);
3131
}
3232

3333
@Override

‎src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractor.java‎

Lines changed: 13 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@
1414
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
1515
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
1616
import org.schabi.newpipe.extractor.stream.StreamInfoItemCollector;
17-
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
1817
import org.schabi.newpipe.extractor.stream.StreamType;
1918
import org.schabi.newpipe.extractor.utils.Parser;
2019
import org.schabi.newpipe.extractor.utils.Utils;
@@ -199,10 +198,10 @@ private void collectStreamsFrom(StreamInfoItemCollector collector, Element eleme
199198

200199
final UrlIdHandler streamUrlIdHandler = getService().getStreamUrlIdHandler();
201200
for (final Element li : element.children()) {
202-
collector.commit(new StreamInfoItemExtractor() {
201+
collector.commit(new YoutubeStreamInfoItemExtractor(li) {
203202
@Override
204-
public StreamType getStreamType() throws ParsingException {
205-
return StreamType.VIDEO_STREAM;
203+
public boolean isAd() throws ParsingException {
204+
return false;
206205
}
207206

208207
@Override
@@ -226,15 +225,18 @@ public String getTitle() throws ParsingException {
226225
@Override
227226
public int getDuration() throws ParsingException {
228227
try {
229-
return YoutubeParsingHelper.parseDurationString(
230-
li.select("div[class=\"timestamp\"] span").first().text().trim());
231-
} catch (Exception e) {
232-
if (isLiveStream(li)) {
233-
// -1 for no duration
228+
if (getStreamType() == StreamType.LIVE_STREAM) return -1;
229+
230+
Element first = li.select("div[class=\"timestamp\"] span").first();
231+
if (first == null) {
232+
// Video unavailable (private, deleted, etc.), this is a thing that happens specifically with playlists,
233+
// because in other cases, those videos don't even show up
234234
return -1;
235-
} else {
236-
throw new ParsingException("Could not get Duration: " + getTitle(), e);
237235
}
236+
237+
return YoutubeParsingHelper.parseDurationString(first.text());
238+
} catch (Exception e) {
239+
throw new ParsingException("Could not get Duration: " + getTitle(), e);
238240
}
239241
}
240242

@@ -261,24 +263,6 @@ public String getThumbnailUrl() throws ParsingException {
261263
throw new ParsingException("Could not get thumbnail url", e);
262264
}
263265
}
264-
265-
@Override
266-
public boolean isAd() throws ParsingException {
267-
return false;
268-
}
269-
270-
private boolean isLiveStream(Element item) {
271-
Element bla = item.select("span[class*=\"yt-badge-live\"]").first();
272-
273-
if (bla == null) {
274-
// sometimes livestreams dont have badges but sill are live streams
275-
// if video time is not available we most likly have an offline livestream
276-
if (item.select("span[class*=\"video-time\"]").first() == null) {
277-
return true;
278-
}
279-
}
280-
return bla != null;
281-
}
282266
});
283267
}
284268
}

0 commit comments

Comments
 (0)