Skip to content

Commit bcfe7be

Browse files
B0polB0pol
authored
Merge branch 'dev' into peertube
2 parents f403490 + b9afc98 commit bcfe7be

11 files changed

Lines changed: 171 additions & 135 deletions

File tree

‎.travis.yml‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
language: java
2+
jdk:
3+
- openjdk8
24

3-
script: ./gradlew check
4-
after_success: ./gradlew aggregatedJavadocs
5+
script:
6+
- ./gradlew check
7+
- ./gradlew aggregatedJavadocs
58

69
deploy:
710
provider: pages

‎build.gradle‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ allprojects {
55
sourceCompatibility = 1.7
66
targetCompatibility = 1.7
77

8-
version 'v0.13.0'
8+
version 'v0.18.0'
99
group 'com.github.TeamNewPipe'
1010

1111
repositories {
@@ -43,7 +43,8 @@ task aggregatedJavadocs(type: Javadoc, group: 'Documentation') {
4343
title = "$project.name $version"
4444
// options.memberLevel = JavadocMemberLevel.PRIVATE
4545
options.links 'https://docs.oracle.com/javase/7/docs/api/'
46-
46+
options.encoding 'UTF-8'
47+
4748
subprojects.each { project ->
4849
project.tasks.withType(Javadoc).each { javadocTask ->
4950
source += javadocTask.source

‎extractor/src/main/java/org/schabi/newpipe/extractor/linkhandler/LinkHandlerFactory.java‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ public abstract class LinkHandlerFactory {
3434
public abstract String getUrl(String id) throws ParsingException;
3535
public abstract boolean onAcceptUrl(final String url) throws ParsingException;
3636

37-
public String getUrl(String id, String baseUrl) throws ParsingException{
37+
public String getUrl(String id, String baseUrl) throws ParsingException {
3838
return getUrl(id);
3939
}
4040

@@ -43,13 +43,14 @@ public String getUrl(String id, String baseUrl) throws ParsingException{
4343
///////////////////////////////////
4444

4545
public LinkHandler fromUrl(String url) throws ParsingException {
46+
if (url == null) throw new IllegalArgumentException("url can not be null");
4647
final String baseUrl = Utils.getBaseUrl(url);
4748
return fromUrl(url, baseUrl);
4849
}
4950

5051
public LinkHandler fromUrl(String url, String baseUrl) throws ParsingException {
51-
if(url == null) throw new IllegalArgumentException("url can not be null");
52-
if(!acceptUrl(url)) {
52+
if (url == null) throw new IllegalArgumentException("url can not be null");
53+
if (!acceptUrl(url)) {
5354
throw new ParsingException("Malformed unacceptable url: " + url);
5455
}
5556

@@ -58,13 +59,13 @@ public LinkHandler fromUrl(String url, String baseUrl) throws ParsingException {
5859
}
5960

6061
public LinkHandler fromId(String id) throws ParsingException {
61-
if(id == null) throw new IllegalArgumentException("id can not be null");
62+
if (id == null) throw new IllegalArgumentException("id can not be null");
6263
final String url = getUrl(id);
6364
return new LinkHandler(url, url, id);
6465
}
6566

6667
public LinkHandler fromId(String id, String baseUrl) throws ParsingException {
67-
if(id == null) throw new IllegalArgumentException("id can not be null");
68+
if (id == null) throw new IllegalArgumentException("id can not be null");
6869
final String url = getUrl(id, baseUrl);
6970
return new LinkHandler(url, url, id);
7071
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java‎

Lines changed: 100 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -106,19 +106,21 @@ public YoutubeStreamExtractor(StreamingService service, LinkHandler linkHandler)
106106
@Override
107107
public String getName() throws ParsingException {
108108
assertPageFetched();
109-
String name = getStringFromMetaData("title");
110-
if(name == null) {
111-
// Fallback to HTML method
109+
try {
110+
return playerResponse.getObject("videoDetails").getString("title");
111+
112+
} catch (Exception e) {
113+
// fallback HTML method
114+
String name = null;
112115
try {
113116
name = doc.select("meta[name=title]").attr(CONTENT);
114-
} catch (Exception e) {
115-
throw new ParsingException("Could not get the title", e);
117+
} catch (Exception ignored) {}
118+
119+
if (name == null) {
120+
throw new ParsingException("Could not get name", e);
116121
}
122+
return name;
117123
}
118-
if(name == null || name.isEmpty()) {
119-
throw new ParsingException("Could not get the title");
120-
}
121-
return name;
122124
}
123125

124126
@Override
@@ -128,9 +130,17 @@ public String getTextualUploadDate() throws ParsingException {
128130
}
129131

130132
try {
131-
return doc.select("meta[itemprop=datePublished]").attr(CONTENT);
132-
} catch (Exception e) {//todo: add fallback method
133-
throw new ParsingException("Could not get upload date", e);
133+
return playerResponse.getObject("microformat").getObject("playerMicroformatRenderer").getString("publishDate");
134+
} catch (Exception e) {
135+
String uploadDate = null;
136+
try {
137+
uploadDate = doc.select("meta[itemprop=datePublished]").attr(CONTENT);
138+
} catch (Exception ignored) {}
139+
140+
if (uploadDate == null) {
141+
throw new ParsingException("Could not get upload date", e);
142+
}
143+
return uploadDate;
134144
}
135145
}
136146

@@ -149,34 +159,39 @@ public DateWrapper getUploadDate() throws ParsingException {
149159
@Override
150160
public String getThumbnailUrl() throws ParsingException {
151161
assertPageFetched();
152-
// Try to get high resolution thumbnail first, if it fails, use low res from the player instead
153162
try {
154-
return doc.select("link[itemprop=\"thumbnailUrl\"]").first().attr("abs:href");
155-
} catch (Exception ignored) {
156-
// Try other method...
157-
}
163+
JsonArray thumbnails = playerResponse.getObject("videoDetails").getObject("thumbnail").getArray("thumbnails");
164+
// the last thumbnail is the one with the highest resolution
165+
return thumbnails.getObject(thumbnails.size() - 1).getString("url");
158166

159-
try {
160-
if (playerArgs != null && playerArgs.isString("thumbnail_url")) return playerArgs.getString("thumbnail_url");
161-
} catch (Exception ignored) {
162-
// Try other method...
163-
}
164-
165-
try {
166-
return videoInfoPage.get("thumbnail_url");
167167
} catch (Exception e) {
168-
throw new ParsingException("Could not get thumbnail url", e);
168+
String url = null;
169+
try {
170+
url = doc.select("link[itemprop=\"thumbnailUrl\"]").first().attr("abs:href");
171+
} catch (Exception ignored) {}
172+
173+
if (url == null) {
174+
throw new ParsingException("Could not get thumbnail url", e);
175+
}
176+
return url;
169177
}
178+
170179
}
171180

172181
@Nonnull
173182
@Override
174183
public String getDescription() throws ParsingException {
175184
assertPageFetched();
176185
try {
186+
// first try to get html-formatted description
177187
return parseHtmlAndGetFullLinks(doc.select("p[id=\"eow-description\"]").first().html());
178188
} catch (Exception e) {
179-
throw new ParsingException("Could not get the description", e);
189+
try {
190+
// fallback to raw non-html description
191+
return playerResponse.getObject("videoDetails").getString("shortDescription");
192+
} catch (Exception ignored) {
193+
throw new ParsingException("Could not get the description", e);
194+
}
180195
}
181196
}
182197

@@ -269,25 +284,22 @@ public int getAgeLimit() throws ParsingException {
269284
public long getLength() throws ParsingException {
270285
assertPageFetched();
271286

272-
// try getting duration from playerargs
273-
try {
274-
String durationMs = playerResponse
275-
.getObject("streamingData")
276-
.getArray("formats")
277-
.getObject(0)
278-
.getString("approxDurationMs");
279-
return Long.parseLong(durationMs)/1000;
280-
} catch (Exception e) {
281-
}
282-
283-
//try getting value from age gated video
284287
try {
285288
String duration = playerResponse
286289
.getObject("videoDetails")
287290
.getString("lengthSeconds");
288291
return Long.parseLong(duration);
289292
} catch (Exception e) {
290-
throw new ParsingException("Every methode to get the duration has failed: ", e);
293+
try {
294+
String durationMs = playerResponse
295+
.getObject("streamingData")
296+
.getArray("formats")
297+
.getObject(0)
298+
.getString("approxDurationMs");
299+
return Math.round(Long.parseLong(durationMs) / 1000f);
300+
} catch (Exception ignored) {
301+
throw new ParsingException("Could not get duration", e);
302+
}
291303
}
292304
}
293305

@@ -307,11 +319,15 @@ public long getViewCount() throws ParsingException {
307319
try {
308320
if (getStreamType().equals(StreamType.LIVE_STREAM)) {
309321
return getLiveStreamWatchingCount();
322+
} else {
323+
return Long.parseLong(playerResponse.getObject("videoDetails").getString("viewCount"));
324+
}
325+
} catch (Exception e) {
326+
try {
327+
return Long.parseLong(doc.select("meta[itemprop=interactionCount]").attr(CONTENT));
328+
} catch (Exception ignored) {
329+
throw new ParsingException("Could not get view count", e);
310330
}
311-
312-
return Long.parseLong(doc.select("meta[itemprop=interactionCount]").attr(CONTENT));
313-
} catch (Exception e) {//todo: find fallback method
314-
throw new ParsingException("Could not get number of views", e);
315331
}
316332
}
317333

@@ -373,7 +389,10 @@ public long getLikeCount() throws ParsingException {
373389
try {
374390
likesString = button.select("span.yt-uix-button-content").first().text();
375391
} catch (NullPointerException e) {
376-
//if this kicks in our button has no content and therefore likes/dislikes are disabled
392+
//if this kicks in our button has no content and therefore ratings must be disabled
393+
if (playerResponse.getObject("videoDetails").getBoolean("allowRatings")) {
394+
throw new ParsingException("Ratings are enabled even though the like button is missing", e);
395+
}
377396
return -1;
378397
}
379398
return Integer.parseInt(Utils.removeNonDigitCharacters(likesString));
@@ -393,7 +412,10 @@ public long getDislikeCount() throws ParsingException {
393412
try {
394413
dislikesString = button.select("span.yt-uix-button-content").first().text();
395414
} catch (NullPointerException e) {
396-
//if this kicks in our button has no content and therefore likes/dislikes are disabled
415+
//if this kicks in our button has no content and therefore ratings must be disabled
416+
if (playerResponse.getObject("videoDetails").getBoolean("allowRatings")) {
417+
throw new ParsingException("Ratings are enabled even though the dislike button is missing", e);
418+
}
397419
return -1;
398420
}
399421
return Integer.parseInt(Utils.removeNonDigitCharacters(dislikesString));
@@ -409,60 +431,59 @@ public long getDislikeCount() throws ParsingException {
409431
public String getUploaderUrl() throws ParsingException {
410432
assertPageFetched();
411433
try {
412-
return doc.select("div[class=\"yt-user-info\"]").first().children()
413-
.select("a").first().attr("abs:href");
434+
return "https://www.youtube.com/channel/" +
435+
playerResponse.getObject("videoDetails").getString("channelId");
414436
} catch (Exception e) {
415-
throw new ParsingException("Could not get channel link", e);
416-
}
417-
}
418-
437+
String uploaderUrl = null;
438+
try {
439+
uploaderUrl = doc.select("div[class=\"yt-user-info\"]").first().children()
440+
.select("a").first().attr("abs:href");
441+
} catch (Exception ignored) {}
419442

420-
@Nullable
421-
private String getStringFromMetaData(String field) {
422-
assertPageFetched();
423-
String value = null;
424-
if(playerArgs != null) {
425-
// This can not fail
426-
value = playerArgs.getString(field);
427-
}
428-
if(value == null) {
429-
// This can not fail too
430-
value = videoInfoPage.get(field);
443+
if (uploaderUrl == null) {
444+
throw new ParsingException("Could not get channel link", e);
445+
}
446+
return uploaderUrl;
431447
}
432-
return value;
433448
}
434449

435450
@Nonnull
436451
@Override
437452
public String getUploaderName() throws ParsingException {
438453
assertPageFetched();
439-
String name = getStringFromMetaData("author");
440-
441-
if(name == null) {
454+
try {
455+
return playerResponse.getObject("videoDetails").getString("author");
456+
} catch (Exception e) {
457+
String name = null;
442458
try {
443-
// Fallback to HTML method
444459
name = doc.select("div.yt-user-info").first().text();
445-
} catch (Exception e) {
446-
throw new ParsingException("Could not get uploader name", e);
460+
} catch (Exception ignored) {}
461+
462+
if (name == null) {
463+
throw new ParsingException("Could not get uploader name");
447464
}
465+
return name;
448466
}
449-
if(name == null || name.isEmpty()) {
450-
throw new ParsingException("Could not get uploader name");
451-
}
452-
return name;
453467
}
454468

455469
@Nonnull
456470
@Override
457471
public String getUploaderAvatarUrl() throws ParsingException {
458472
assertPageFetched();
473+
474+
String uploaderAvatarUrl = null;
459475
try {
460-
return doc.select("a[class*=\"yt-user-photo\"]").first()
476+
uploaderAvatarUrl = doc.select("a[class*=\"yt-user-photo\"]").first()
461477
.select("img").first()
462478
.attr("abs:data-thumb");
463479
} catch (Exception e) {//todo: add fallback method
464-
throw new ParsingException("Could not get uploader thumbnail URL.", e);
480+
throw new ParsingException("Could not get uploader avatar url", e);
481+
}
482+
483+
if (uploaderAvatarUrl == null) {
484+
throw new ParsingException("Could not get uploader avatar url");
465485
}
486+
return uploaderAvatarUrl;
466487
}
467488

468489
@Nonnull
@@ -590,12 +611,12 @@ public List<SubtitlesStream> getSubtitles(final MediaFormat format) throws IOExc
590611
public StreamType getStreamType() throws ParsingException {
591612
assertPageFetched();
592613
try {
593-
if (playerArgs != null && (playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live") ||
594-
(!playerResponse.getObject("streamingData").has(FORMATS)))) {
614+
if (!playerResponse.getObject("streamingData").has(FORMATS) ||
615+
(playerArgs != null && playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live"))) {
595616
return StreamType.LIVE_STREAM;
596617
}
597618
} catch (Exception e) {
598-
throw new ParsingException("Could not get hls manifest url", e);
619+
throw new ParsingException("Could not get stream type", e);
599620
}
600621
return StreamType.VIDEO_STREAM;
601622
}

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ public void testAudioStreams() throws Exception {
8989

9090
@Test
9191
public void testGetTextualUploadDate() throws ParsingException {
92-
Assert.assertEquals("2018-05-11", extractor.getTextualUploadDate());
92+
Assert.assertEquals("2018-05-11T02:00:00.000+02:00", extractor.getTextualUploadDate());
9393
}
9494

9595
@Test

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamLinkHandlerFactoryTest.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -77,8 +77,8 @@ public void getIdfromYt() throws Exception {
7777
assertEquals("jZViOEv90dI", linkHandler.fromUrl("http://www.Youtube.com/embed/jZViOEv90dI").getId());
7878
assertEquals("jZViOEv90dI", linkHandler.fromUrl("http://www.youtube-nocookie.com/embed/jZViOEv90dI").getId());
7979
assertEquals("EhxJLojIE_o", linkHandler.fromUrl("http://www.youtube.com/attribution_link?a=JdfC0C9V6ZI&u=%2Fwatch%3Fv%3DEhxJLojIE_o%26feature%3Dshare").getId());
80-
assertEquals("jZViOEv90dI", linkHandler.fromUrl("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI").getId());
81-
assertEquals("jZViOEv90dI", linkHandler.fromUrl("vnd.youtube:jZViOEv90dI").getId());
80+
assertEquals("jZViOEv90dI", linkHandler.fromUrl("vnd.youtube://www.youtube.com/watch?v=jZViOEv90dI", "youtube.com").getId());
81+
assertEquals("jZViOEv90dI", linkHandler.fromUrl("vnd.youtube:jZViOEv90dI", "youtube.com").getId());
8282
assertEquals("O0EDx9WAelc", linkHandler.fromUrl("https://music.youtube.com/watch?v=O0EDx9WAelc").getId());
8383
}
8484

0 commit comments

Comments
 (0)