Skip to content

Commit b9afc98

Browse files
TobiGrTobiGr
authored
Merge pull request #233 from Stypox/yt-stream-ext
Youtube stream extractor improvements by using playerResponse json
2 parents 82eff77 + 1cfdc45 commit b9afc98

4 files changed

Lines changed: 143 additions & 83 deletions

File tree

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java‎

Lines changed: 100 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -106,19 +106,21 @@ public YoutubeStreamExtractor(StreamingService service, LinkHandler linkHandler)
106106
@Override
107107
public String getName() throws ParsingException {
108108
assertPageFetched();
109-
String name = getStringFromMetaData("title");
110-
if(name == null) {
111-
// Fallback to HTML method
109+
try {
110+
return playerResponse.getObject("videoDetails").getString("title");
111+
112+
} catch (Exception e) {
113+
// fallback HTML method
114+
String name = null;
112115
try {
113116
name = doc.select("meta[name=title]").attr(CONTENT);
114-
} catch (Exception e) {
115-
throw new ParsingException("Could not get the title", e);
117+
} catch (Exception ignored) {}
118+
119+
if (name == null) {
120+
throw new ParsingException("Could not get name", e);
116121
}
122+
return name;
117123
}
118-
if(name == null || name.isEmpty()) {
119-
throw new ParsingException("Could not get the title");
120-
}
121-
return name;
122124
}
123125

124126
@Override
@@ -128,9 +130,17 @@ public String getTextualUploadDate() throws ParsingException {
128130
}
129131

130132
try {
131-
return doc.select("meta[itemprop=datePublished]").attr(CONTENT);
132-
} catch (Exception e) {//todo: add fallback method
133-
throw new ParsingException("Could not get upload date", e);
133+
return playerResponse.getObject("microformat").getObject("playerMicroformatRenderer").getString("publishDate");
134+
} catch (Exception e) {
135+
String uploadDate = null;
136+
try {
137+
uploadDate = doc.select("meta[itemprop=datePublished]").attr(CONTENT);
138+
} catch (Exception ignored) {}
139+
140+
if (uploadDate == null) {
141+
throw new ParsingException("Could not get upload date", e);
142+
}
143+
return uploadDate;
134144
}
135145
}
136146

@@ -149,34 +159,39 @@ public DateWrapper getUploadDate() throws ParsingException {
149159
@Override
150160
public String getThumbnailUrl() throws ParsingException {
151161
assertPageFetched();
152-
// Try to get high resolution thumbnail first, if it fails, use low res from the player instead
153162
try {
154-
return doc.select("link[itemprop=\"thumbnailUrl\"]").first().attr("abs:href");
155-
} catch (Exception ignored) {
156-
// Try other method...
157-
}
163+
JsonArray thumbnails = playerResponse.getObject("videoDetails").getObject("thumbnail").getArray("thumbnails");
164+
// the last thumbnail is the one with the highest resolution
165+
return thumbnails.getObject(thumbnails.size() - 1).getString("url");
158166

159-
try {
160-
if (playerArgs != null && playerArgs.isString("thumbnail_url")) return playerArgs.getString("thumbnail_url");
161-
} catch (Exception ignored) {
162-
// Try other method...
163-
}
164-
165-
try {
166-
return videoInfoPage.get("thumbnail_url");
167167
} catch (Exception e) {
168-
throw new ParsingException("Could not get thumbnail url", e);
168+
String url = null;
169+
try {
170+
url = doc.select("link[itemprop=\"thumbnailUrl\"]").first().attr("abs:href");
171+
} catch (Exception ignored) {}
172+
173+
if (url == null) {
174+
throw new ParsingException("Could not get thumbnail url", e);
175+
}
176+
return url;
169177
}
178+
170179
}
171180

172181
@Nonnull
173182
@Override
174183
public String getDescription() throws ParsingException {
175184
assertPageFetched();
176185
try {
186+
// first try to get html-formatted description
177187
return parseHtmlAndGetFullLinks(doc.select("p[id=\"eow-description\"]").first().html());
178188
} catch (Exception e) {
179-
throw new ParsingException("Could not get the description", e);
189+
try {
190+
// fallback to raw non-html description
191+
return playerResponse.getObject("videoDetails").getString("shortDescription");
192+
} catch (Exception ignored) {
193+
throw new ParsingException("Could not get the description", e);
194+
}
180195
}
181196
}
182197

@@ -269,25 +284,22 @@ public int getAgeLimit() throws ParsingException {
269284
public long getLength() throws ParsingException {
270285
assertPageFetched();
271286

272-
// try getting duration from playerargs
273-
try {
274-
String durationMs = playerResponse
275-
.getObject("streamingData")
276-
.getArray("formats")
277-
.getObject(0)
278-
.getString("approxDurationMs");
279-
return Long.parseLong(durationMs)/1000;
280-
} catch (Exception e) {
281-
}
282-
283-
//try getting value from age gated video
284287
try {
285288
String duration = playerResponse
286289
.getObject("videoDetails")
287290
.getString("lengthSeconds");
288291
return Long.parseLong(duration);
289292
} catch (Exception e) {
290-
throw new ParsingException("Every methode to get the duration has failed: ", e);
293+
try {
294+
String durationMs = playerResponse
295+
.getObject("streamingData")
296+
.getArray("formats")
297+
.getObject(0)
298+
.getString("approxDurationMs");
299+
return Math.round(Long.parseLong(durationMs) / 1000f);
300+
} catch (Exception ignored) {
301+
throw new ParsingException("Could not get duration", e);
302+
}
291303
}
292304
}
293305

@@ -307,11 +319,15 @@ public long getViewCount() throws ParsingException {
307319
try {
308320
if (getStreamType().equals(StreamType.LIVE_STREAM)) {
309321
return getLiveStreamWatchingCount();
322+
} else {
323+
return Long.parseLong(playerResponse.getObject("videoDetails").getString("viewCount"));
324+
}
325+
} catch (Exception e) {
326+
try {
327+
return Long.parseLong(doc.select("meta[itemprop=interactionCount]").attr(CONTENT));
328+
} catch (Exception ignored) {
329+
throw new ParsingException("Could not get view count", e);
310330
}
311-
312-
return Long.parseLong(doc.select("meta[itemprop=interactionCount]").attr(CONTENT));
313-
} catch (Exception e) {//todo: find fallback method
314-
throw new ParsingException("Could not get number of views", e);
315331
}
316332
}
317333

@@ -373,7 +389,10 @@ public long getLikeCount() throws ParsingException {
373389
try {
374390
likesString = button.select("span.yt-uix-button-content").first().text();
375391
} catch (NullPointerException e) {
376-
//if this kicks in our button has no content and therefore likes/dislikes are disabled
392+
//if this kicks in our button has no content and therefore ratings must be disabled
393+
if (playerResponse.getObject("videoDetails").getBoolean("allowRatings")) {
394+
throw new ParsingException("Ratings are enabled even though the like button is missing", e);
395+
}
377396
return -1;
378397
}
379398
return Integer.parseInt(Utils.removeNonDigitCharacters(likesString));
@@ -393,7 +412,10 @@ public long getDislikeCount() throws ParsingException {
393412
try {
394413
dislikesString = button.select("span.yt-uix-button-content").first().text();
395414
} catch (NullPointerException e) {
396-
//if this kicks in our button has no content and therefore likes/dislikes are disabled
415+
//if this kicks in our button has no content and therefore ratings must be disabled
416+
if (playerResponse.getObject("videoDetails").getBoolean("allowRatings")) {
417+
throw new ParsingException("Ratings are enabled even though the dislike button is missing", e);
418+
}
397419
return -1;
398420
}
399421
return Integer.parseInt(Utils.removeNonDigitCharacters(dislikesString));
@@ -409,60 +431,59 @@ public long getDislikeCount() throws ParsingException {
409431
public String getUploaderUrl() throws ParsingException {
410432
assertPageFetched();
411433
try {
412-
return doc.select("div[class=\"yt-user-info\"]").first().children()
413-
.select("a").first().attr("abs:href");
434+
return "https://www.youtube.com/channel/" +
435+
playerResponse.getObject("videoDetails").getString("channelId");
414436
} catch (Exception e) {
415-
throw new ParsingException("Could not get channel link", e);
416-
}
417-
}
418-
437+
String uploaderUrl = null;
438+
try {
439+
uploaderUrl = doc.select("div[class=\"yt-user-info\"]").first().children()
440+
.select("a").first().attr("abs:href");
441+
} catch (Exception ignored) {}
419442

420-
@Nullable
421-
private String getStringFromMetaData(String field) {
422-
assertPageFetched();
423-
String value = null;
424-
if(playerArgs != null) {
425-
// This can not fail
426-
value = playerArgs.getString(field);
427-
}
428-
if(value == null) {
429-
// This can not fail too
430-
value = videoInfoPage.get(field);
443+
if (uploaderUrl == null) {
444+
throw new ParsingException("Could not get channel link", e);
445+
}
446+
return uploaderUrl;
431447
}
432-
return value;
433448
}
434449

435450
@Nonnull
436451
@Override
437452
public String getUploaderName() throws ParsingException {
438453
assertPageFetched();
439-
String name = getStringFromMetaData("author");
440-
441-
if(name == null) {
454+
try {
455+
return playerResponse.getObject("videoDetails").getString("author");
456+
} catch (Exception e) {
457+
String name = null;
442458
try {
443-
// Fallback to HTML method
444459
name = doc.select("div.yt-user-info").first().text();
445-
} catch (Exception e) {
446-
throw new ParsingException("Could not get uploader name", e);
460+
} catch (Exception ignored) {}
461+
462+
if (name == null) {
463+
throw new ParsingException("Could not get uploader name");
447464
}
465+
return name;
448466
}
449-
if(name == null || name.isEmpty()) {
450-
throw new ParsingException("Could not get uploader name");
451-
}
452-
return name;
453467
}
454468

455469
@Nonnull
456470
@Override
457471
public String getUploaderAvatarUrl() throws ParsingException {
458472
assertPageFetched();
473+
474+
String uploaderAvatarUrl = null;
459475
try {
460-
return doc.select("a[class*=\"yt-user-photo\"]").first()
476+
uploaderAvatarUrl = doc.select("a[class*=\"yt-user-photo\"]").first()
461477
.select("img").first()
462478
.attr("abs:data-thumb");
463479
} catch (Exception e) {//todo: add fallback method
464-
throw new ParsingException("Could not get uploader thumbnail URL.", e);
480+
throw new ParsingException("Could not get uploader avatar url", e);
481+
}
482+
483+
if (uploaderAvatarUrl == null) {
484+
throw new ParsingException("Could not get uploader avatar url");
465485
}
486+
return uploaderAvatarUrl;
466487
}
467488

468489
@Nonnull
@@ -590,12 +611,12 @@ public List<SubtitlesStream> getSubtitles(final MediaFormat format) throws IOExc
590611
public StreamType getStreamType() throws ParsingException {
591612
assertPageFetched();
592613
try {
593-
if (playerArgs != null && (playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live") ||
594-
(!playerResponse.getObject("streamingData").has(FORMATS)))) {
614+
if (!playerResponse.getObject("streamingData").has(FORMATS) ||
615+
(playerArgs != null && playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live"))) {
595616
return StreamType.LIVE_STREAM;
596617
}
597618
} catch (Exception e) {
598-
throw new ParsingException("Could not get hls manifest url", e);
619+
throw new ParsingException("Could not get stream type", e);
599620
}
600621
return StreamType.VIDEO_STREAM;
601622
}

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ public void testGetUploaderName() throws ParsingException {
7575

7676
@Test
7777
public void testGetLength() throws ParsingException {
78-
assertEquals(1789, extractor.getLength());
78+
assertEquals(1790, extractor.getLength());
7979
}
8080

8181
@Test

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ public void testGetName() throws ParsingException {
6565
@Test
6666
public void testGetDescription() throws ParsingException {
6767
assertNotNull(extractor.getDescription());
68-
// assertFalse(extractor.getDescription().isEmpty());
68+
assertFalse(extractor.getDescription().isEmpty());
6969
}
7070

7171
@Test

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java‎

Lines changed: 41 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ public void testGetUploaderName() throws ParsingException {
101101

102102
@Test
103103
public void testGetLength() throws ParsingException {
104-
assertEquals(366, extractor.getLength());
104+
assertEquals(367, extractor.getLength());
105105
}
106106

107107
@Test
@@ -124,7 +124,11 @@ public void testGetUploadDate() throws ParsingException, ParseException {
124124

125125
@Test
126126
public void testGetUploaderUrl() throws ParsingException {
127-
assertEquals("https://www.youtube.com/channel/UCsRM0YB_dabtEPGPTKo-gcw", extractor.getUploaderUrl());
127+
String url = extractor.getUploaderUrl();
128+
if (!url.equals("https://www.youtube.com/channel/UCsRM0YB_dabtEPGPTKo-gcw") &&
129+
!url.equals("https://www.youtube.com/channel/UComP_epzeKzvBX156r6pm1Q")) {
130+
fail("Uploader url is neither the music channel one nor the Vevo one");
131+
}
128132
}
129133

130134
@Test
@@ -183,6 +187,18 @@ public void testGetSubtitlesList() throws IOException, ExtractionException {
183187
// Video (/view?v=YQHsXMglC9A) set in the setUp() method has no captions => null
184188
assertTrue(extractor.getSubtitles(MediaFormat.TTML).isEmpty());
185189
}
190+
191+
@Test
192+
public void testGetLikeCount() throws ParsingException {
193+
long likeCount = extractor.getLikeCount();
194+
assertTrue("" + likeCount, likeCount >= 15000000);
195+
}
196+
197+
@Test
198+
public void testGetDislikeCount() throws ParsingException {
199+
long dislikeCount = extractor.getDislikeCount();
200+
assertTrue("" + dislikeCount, dislikeCount >= 818000);
201+
}
186202
}
187203

188204
public static class DescriptionTestPewdiepie {
@@ -245,6 +261,29 @@ public void testGetFullLinksInDescription() throws ParsingException {
245261
}
246262
}
247263

264+
public static class RatingsDisabledTest {
265+
private static YoutubeStreamExtractor extractor;
266+
267+
@BeforeClass
268+
public static void setUp() throws Exception {
269+
NewPipe.init(DownloaderTestImpl.getInstance());
270+
extractor = (YoutubeStreamExtractor) YouTube
271+
.getStreamExtractor("https://www.youtube.com/watch?v=HRKu0cvrr_o");
272+
extractor.fetchPage();
273+
}
274+
275+
@Test
276+
public void testGetLikeCount() throws ParsingException {
277+
assertEquals(-1, extractor.getLikeCount());
278+
}
279+
280+
@Test
281+
public void testGetDislikeCount() throws ParsingException {
282+
assertEquals(-1, extractor.getDislikeCount());
283+
}
284+
285+
}
286+
248287
public static class FramesTest {
249288
private static YoutubeStreamExtractor extractor;
250289

0 commit comments

Comments
 (0)